GenLocationIndex.h

Go to the documentation of this file.

#ifndef OSMSCOUT_IMPORT_GENLOCATIONINDEX_H
#define OSMSCOUT_IMPORT_GENLOCATIONINDEX_H

/*
  This source is part of the libosmscout library
  Copyright (C) 2009  Tim Teulings

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
*/

#include <map>
#include <memory>
#include <unordered_map>
#include <unordered_set>

#include <osmscout/Node.h>
#include <osmscout/Area.h>
#include <osmscout/Way.h>

#include <osmscout/ObjectRef.h>

#include <osmscout/TypeInfoSet.h>

#include <osmscout/import/Import.h>

#include <osmscout/system/Compiler.h>

namespace osmscout {

  class LocationIndexGenerator CLASS_FINAL : public ImportModule
  {
  public:
    static const char* const FILENAME_LOCATION_REGION_TXT;
    static const char* const FILENAME_LOCATION_FULL_TXT;
    static const char* const FILENAME_LOCATION_METRICS_TXT;

  private:
    struct RegionMetrics CLASS_FINAL
    {
      uint32_t minRegionChars;
      uint32_t maxRegionChars;
      uint32_t minRegionWords;
      uint32_t maxRegionWords;
      uint32_t maxPOIWords;
      uint32_t minLocationChars;
      uint32_t maxLocationChars;
      uint32_t minLocationWords;
      uint32_t maxLocationWords;
      uint32_t maxAddressWords;

      RegionMetrics();
    };

    struct RegionAlias CLASS_FINAL
    {
      FileOffset  reference; 
      std::string name;      
      std::string altName;
    };

    struct RegionPOI CLASS_FINAL
    {
      std::string   name;   
      ObjectFileRef object; 

      RegionPOI(const std::string& name,
               const ObjectFileRef& object)
      : name(name),
        object(object)
      {
        // no code
      }

      bool operator<(const RegionPOI& other) const
      {
        return object.GetFileOffset()<other.object.GetFileOffset();
      }
    };

    struct RegionAddress CLASS_FINAL
    {
      std::string   name;       
      ObjectFileRef object;     

      RegionAddress(const std::string& name,
                    const ObjectFileRef& object)
      : name(name),
        object(object)
      {
        // no code
      }

      bool operator<(const RegionAddress& other) const
      {
        return object.GetFileOffset()<other.object.GetFileOffset();
      }
    };

    struct RegionLocation CLASS_FINAL
    {
      std::unordered_map<std::string,
                         size_t>      names;            
      FileOffset                      dataOffsetOffset; 
      std::list<ObjectFileRef>        objects;          
      std::list<RegionAddress>        addresses;        

      std::string GetName() const;
    };

    class Region;

    typedef std::shared_ptr<Region> RegionRef;

    struct PostalArea CLASS_FINAL
    {
      std::string                          name;             
      FileOffset                           dataOffsetOffset; 
      std::map<std::string,RegionLocation> locations;        

      explicit PostalArea(const std::string& name)
      : name(name)
      {
        // no code
      }

      void AddLocationObject(const std::string& name,
                             const ObjectFileRef& objectRef);
    };

    class Region CLASS_FINAL
    {
    public:
      typedef std::unordered_map<std::string,PostalArea> PostalAreaMap;

    private:
      std::vector<GeoBox>                boundingBoxes;      
      GeoBox                             boundingBox;        
      std::vector<GeoCoord>              probePoints;        

    public:
      FileOffset                         indexOffset;        
      FileOffset                         dataOffset;         

      ObjectFileRef                      reference;          
      std::string                        name;               
      std::string                        altName;
      std::string                        isIn;               
      std::list<RegionAlias>             aliases;            
      int8_t                             level{-1};          

      std::vector<std::vector<GeoCoord>> areas;              
      std::list<RegionPOI>               pois;               
      PostalAreaMap                      postalAreas;        
      PostalAreaMap::iterator            defaultPostalArea;  

      std::list<RegionRef>               regions;            

    public:
      Region();

      void CalculateMinMax();
      bool CouldContain(const GeoBox& boundingBox) const;
      bool CouldContain(const Region& region, bool strict) const;

      bool Contains(Region& child) const;                 

      inline GeoBox GetBoundingBox() const
      {
        return boundingBox;
      }

      inline const std::vector<GeoBox> GetAreaBoundingBoxes() const
      {
        return boundingBoxes;
      }

      void AddLocationObject(const std::string& name,
                             const std::string& postalCode,
                             const ObjectFileRef& objectRef);

    protected:
      void CalculateProbePoints();                           
      void CalculateProbePointsForArea(size_t areaIndex,     
                                       size_t refinement=0);
    };

    class RegionIndex CLASS_FINAL
    {
    public:
      std::map<Pixel,std::list<RegionRef> > index;
      double                                cellWidth;
      double                                cellHeight;

    public:
      RegionRef GetRegionForNode(RegionRef& rootRegion,
                                 const GeoCoord& coord) const;
    };

  private:
    uint8_t                bytesForNodeFileOffset;
    uint8_t                bytesForAreaFileOffset;
    uint8_t                bytesForWayFileOffset;

    ImportErrorReporterRef errorReporter;

  private:
    void Write(FileWriter& writer,
               const ObjectFileRef& object);

    void AnalyseStringForIgnoreTokens(const std::string& string,
                                      std::unordered_map<std::string,size_t>& ignoreTokens,
                                      std::unordered_set<std::string>& blacklist);

    void CalculateRegionNameIgnoreTokens(const Region& parent,
                                         std::unordered_map<std::string,size_t>& ignoreTokens,
                                         std::unordered_set<std::string>& blacklist);

    void CalculatePOINameIgnoreTokens(const Region& parent,
                                      std::unordered_map<std::string,size_t>& ignoreTokens,
                                      std::unordered_set<std::string>& blacklist);

    void CalculateLocationNameIgnoreTokens(const Region& parent,
                                           std::unordered_map<std::string,size_t>& ignoreTokens,
                                           std::unordered_set<std::string>& blacklist);

    bool CalculateIgnoreTokens(const Region& rootRegion,
                               std::list<std::string>& regionTokens,
                               std::list<std::string>& poiTokens,
                               std::list<std::string>& locationTokens);


    void CalculateRegionMetrics(const Region& region,
                                RegionMetrics& metrics);

    void DumpRegion(const Region& parent,
                    size_t indent,
                    std::ostream& out);

    void DumpRegionAndData(const Region& parent,
                           size_t indent,
                           std::ostream& out);

    bool DumpRegionTree(Progress& progress,
                        const Region& rootRegion,
                        const std::string& filename);

    bool DumpLocationTree(Progress& progress,
                          const Region& rootRegion,
                          const std::string& filename);

    bool DumpLocationMetrics(Progress& progress,
                             const std::string& filename,
                             const LocationIndexGenerator::RegionMetrics& metrics,
                             const std::list<std::string>& regionIgnoreTokens,
                             const std::list<std::string>& poiIgnoreTokens,
                             const std::list<std::string>& locationIgnoreTokens);

    bool AddRegion(Region& parent,
                   const RegionRef& region,
                   bool assume_contains=true);

    bool GetBoundaryAreas(const ImportParameter& parameter,
                          Progress& progress,
                          const TypeConfigRef& typeConfig,
                          const TypeInfoSet& boundaryTypes,
                          std::vector<std::list<RegionRef>>& boundaryAreas);

    void SortInBoundaries(Progress& progress,
                          Region& rootRegion,
                          std::list<RegionRef>& boundaryAreas);

    bool GetRegionAreas(const TypeConfig& typeConfig,
                        const ImportParameter& parameter,
                        Progress& progress,
                        std::list<RegionRef>& regionAreas);

    bool SortInRegionAreas(Progress& progress,
                           Region& rootRegion,
                           std::list<RegionRef>& regionAreas);

    void SortInRegion(RegionRef& area,
                      std::vector<std::list<RegionRef> >& regionTree,
                      unsigned long level);

    unsigned long GetRegionTreeDepth(const Region& rootRegion);

    void IndexRegions(const std::vector<std::list<RegionRef> >& regionTree,
                      RegionIndex& regionIndex);

    void AddAliasToRegion(Region& region,
                          const RegionAlias& location,
                          const GeoCoord& node);

    bool IndexRegionNodes(const TypeConfigRef& typeConfig,
                          const ImportParameter& parameter,
                          Progress& progress,
                          RegionRef& rootRegion,
                          const RegionIndex& regionIndex);

    bool AddLocationAreaToRegion(Region& region,
                                 const Area& area,
                                 const std::vector<Point>& nodes,
                                 const std::string& name,
                                 const std::string& postalCode,
                                 const GeoBox& boundingBox);

    void AddLocationAreaToRegion(RegionRef& rootRegion,
                                 const Area& area,
                                 const Area::Ring& ring,
                                 const std::string& name,
                                 const std::string& postalCode,
                                 const RegionIndex& regionIndex);

    bool IndexLocationAreas(const TypeConfig& typeConfig,
                            const ImportParameter& parameter,
                            Progress& progress,
                            RegionRef& rootRegion,
                            const RegionIndex& regionIndex);

    bool AddLocationWayToRegion(Region& region,
                                const Way& way,
                                const std::string& name,
                                const std::string& postalCode,
                                const GeoBox& boundingBox);

    bool IndexLocationWays(const TypeConfig& typeConfig,
                           const ImportParameter& parameter,
                           Progress& progress,
                           RegionRef& rootRegion,
                           const RegionIndex& regionIndex);

    void AddAddressToRegion(Progress& progress,
                            Region& region,
                            const ObjectFileRef& object,
                            const std::string& location,
                            const std::string& address,
                            const std::string &postalCode,
                            bool allowDuplicates,
                            bool& added);

    void AddAddressAreaToRegion(Progress& progress,
                                Region& region,
                                const FileOffset& fileOffset,
                                const std::string& location,
                                const std::string& address,
                                const std::string &postalCode,
                                const std::vector<Point>& nodes,
                                const GeoBox& boundingBox,
                                bool& added);

    void AddPOIAreaToRegion(Progress& progress,
                            Region& region,
                            const FileOffset& fileOffset,
                            const std::string& name,
                            const std::vector<Point>& nodes,
                            const GeoBox& boundingBox,
                            bool& added);

    bool IndexAddressAreas(const TypeConfig& typeConfig,
                           const ImportParameter& parameter,
                           Progress& progress,
                           RegionRef& rootRegion,
                           const RegionIndex& regionIndex);

    bool AddAddressWayToRegion(Progress& progress,
                               Region& region,
                               const FileOffset& fileOffset,
                               const std::string& location,
                               const std::string& address,
                               const std::vector<Point>& nodes,
                               const GeoBox& boundingBox,
                               bool& added);

    bool AddPOIWayToRegion(Progress& progress,
                           Region& region,
                           const FileOffset& fileOffset,
                           const std::string& name,
                           const std::vector<Point>& nodes,
                           const GeoBox& boundingBox,
                           bool& added);

    bool IndexAddressWays(const TypeConfig& typeConfig,
                          const ImportParameter& parameter,
                          Progress& progress,
                          RegionRef& rootRegion,
                          const RegionIndex& regionIndex);

    std::map<std::string,RegionLocation>::iterator FindLocation(Progress& progress,
                                                                Region& region,
                                                                PostalArea& postalArea,
                                                                const std::string &locationName);

    void AddAddressNodeToRegion(Progress& progress,
                                Region& region,
                                const FileOffset& fileOffset,
                                const std::string& location,
                                const std::string& address,
                                const std::string& postalCode,
                                bool& added);

    void AddPOINodeToRegion(Region& region,
                            const FileOffset& fileOffset,
                            const std::string& name,
                            bool& added);

    bool IndexAddressNodes(const TypeConfig& typeConfig,
                           const ImportParameter& parameter,
                           Progress& progress,
                           RegionRef& rootRegion,
                           const RegionIndex& regionIndex);

    void CleanupPostalAreas(Region& region);

    void WriteIgnoreTokens(FileWriter& writer,
                           const std::list<std::string>& regionIgnoreTokens,
                           const std::list<std::string>& poiIgnoreTokens,
                           const std::list<std::string>& locationIgnoreTokens);

    void WriteRegionMetrics(FileWriter& writer,
                            const RegionMetrics& metrics);

    void WriteRegionIndexEntry(FileWriter& writer,
                               const Region& parentRegion,
                               Region& region);

    void WriteRegionIndex(FileWriter& writer,
                          Region& root);

    void WriteRegionDataEntry(FileWriter& writer,
                              Region& region);

    void WriteRegionData(FileWriter& writer,
                         Region& root);

    void WritePostalArea(FileWriter& writer,
                         PostalArea& postalArea);

    void WriteAddressDataEntry(FileWriter& writer,
                               Region& region);

    void WriteAddressData(FileWriter& writer,
                          Region& root);

  public:
    void GetDescription(const ImportParameter& parameter,
                        ImportModuleDescription& description) const override;

    bool Import(const TypeConfigRef& typeConfig,
                const ImportParameter& parameter,
                Progress& progress) override;
  };
}

#endif