GenCoordDat.h

Go to the documentation of this file.

#ifndef OSMSCOUT_IMPORT_GENCOORDDATA_H
#define OSMSCOUT_IMPORT_GENCOORDDATA_H

/*
  This source is part of the libosmscout library
  Copyright (C) 2016  Tim Teulings

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
*/

#include <map>
#include <vector>

#include <osmscout/import/Import.h>
#include <osmscout/import/RawNode.h>

#include <osmscout/system/Compiler.h>

namespace osmscout {

  class SerialIdManager CLASS_FINAL
  {
  private:
    // It seems, like a map is faster in this case then a default-initialized unordered_map
    std::map<Id,uint8_t> idToSerialMap;

  public:
    void MarkIdAsDuplicate(Id id);
    uint8_t GetNextSerialForId(Id id);
    size_t Size() const;
  };

  template<typename I, typename E>
  class PageManager
  {
  private:
    std::map<I,std::vector<E>> pages;
    uint32_t                   maximumEntriesInMemory;
    uint32_t                   pageSize;
    uint32_t                   overallEntryCount;         // number of entries in file
    std::function<void(const I&,std::vector<E>&&)> processor;
    I                          minPageId;
    I                          maxPageId;
    uint32_t                   processedPagesCount;
    uint32_t                   processedPagesEntryCount;  // number of entries in completely processed pages
    uint32_t                   handledPagesEntryCount;    // number entries in current pages

  public:
    PageManager(uint32_t maximumEntriesInMemory,
                uint32_t pageSize,
                uint32_t overallEntryCount,
                std::function<void(const I&,std::vector<E>&&)> processor)
    : maximumEntriesInMemory(maximumEntriesInMemory),
      pageSize(pageSize),
      overallEntryCount(overallEntryCount),
      processor(processor),
      minPageId(std::numeric_limits<I>::min()/static_cast<I>(pageSize)),
      maxPageId(std::numeric_limits<I>::max()/static_cast<I>(pageSize)),
      processedPagesCount(0u),
      processedPagesEntryCount(0u),
      handledPagesEntryCount(0u)
    {
    }

    I GetMinPageId() const {
      return minPageId;
    }

    uint32_t GetProcessedPagesCount() const
    {
      return processedPagesCount;
    }

    uint32_t GetProcessedPagesEntryCount() const
    {
      return processedPagesEntryCount;
    }

    bool IsACurrentlyHandledPage(const I& id) const
    {
      I pageId=id/pageSize;

      return pageId>=minPageId && pageId<=maxPageId;
    }

    void AddEntry(const I& id, const E& entry)
    {
      I pageId=id/pageSize;

      pages[pageId].push_back(entry);
      handledPagesEntryCount++;

      if (handledPagesEntryCount<maximumEntriesInMemory ||
          pages.size()<=1) {
        return;
      }

      [[maybe_unused]] I oldMaxPageId=maxPageId;

      while (handledPagesEntryCount>maximumEntriesInMemory
             && pages.size()>1) {
        auto pageEntry=pages.rbegin();
        I    currentMaxPageId=pageEntry->first;

        handledPagesEntryCount-=(uint32_t)pageEntry->second.size();
        pages.erase(currentMaxPageId);
        maxPageId=currentMaxPageId-1;
      }

      assert(maxPageId<=oldMaxPageId);
    }

    bool AreAllEntriesLoaded() const
    {
      return processedPagesEntryCount+handledPagesEntryCount==overallEntryCount;
    }

    void FileCompletelyScanned()
    {
      processedPagesEntryCount+=handledPagesEntryCount;

      for (auto& pageEntry : pages) {
        processor(pageEntry.first,std::move(pageEntry.second));
        processedPagesCount++;
      }

      handledPagesEntryCount=0;
      pages.clear();
      minPageId=maxPageId+1;
      maxPageId=std::numeric_limits<OSMId>::max()/pageSize;
    }
  };

  template<typename I, typename E>
  class PageSplitter
  {
  private:
    size_t                                        pageSize;      //< Size of page
    I                                             currentPageId; //< Id of current page
    size_t                                        entryCount;    //< Number of entries in the current page
    size_t                                        processedPagesCount;     //< Number of processed pages
    std::vector<E>                                page;          //< Page data itself
    std::function<void(const I&,std::vector<E>&)> processor; //< Callback for processing a complete page

  private:
    void FlushPage()
    {
      processor(currentPageId,page);
      InitializePage();

      processedPagesCount++;
    }

    void InitializePage()
    {
      std::for_each(page.begin(),page.end(),[](E& entry) {
        entry.isSet=false;
      });

      entryCount=0;
    }

  public:
    PageSplitter(size_t pageSize,
                 std::function<void(const I&,std::vector<E>&)> processor)
    : pageSize(pageSize),
      currentPageId(std::numeric_limits<I>::min()),
      entryCount(0),
      processedPagesCount(0),
      page(pageSize),
      processor(processor)
    {
    }

    void Set(const I& id, const E& value)
    {
      I relatedId=std::numeric_limits<I>::min()+id;
      I pageId=relatedId/pageSize;
      I index=relatedId%pageSize;

      if (pageId!=currentPageId) {
        if (entryCount>0) {
          FlushPage();
        }

        currentPageId=pageId;
      }

      page[index]=value;
      entryCount++;
    }

    void FileCompletelyScanned()
    {
      if (entryCount>0) {
        FlushPage();
      }
    }

    size_t GetProcessedPagesCount() const
    {
      return processedPagesCount;
    }
  };

  class CoordDataGenerator CLASS_FINAL : public ImportModule
  {
  private:
    bool FindDuplicateCoordinates(const TypeConfig& typeConfig,
                                  const ImportParameter& parameter,
                                  Progress& progress,
                                  SerialIdManager& serialIdManager) const;

    bool StoreCoordinates(const TypeConfig& typeConfig,
                          const ImportParameter& parameter,
                          Progress& progress,
                          SerialIdManager& serialIdManager) const;

  public:
    void GetDescription(const ImportParameter& parameter,
                        ImportModuleDescription& description) const override;

    bool Import(const TypeConfigRef& typeConfig,
                const ImportParameter& parameter,
                Progress& progress) override;
  };
}

#endif