GenNumericIndex.h

Go to the documentation of this file.

#ifndef OSMSCOUT_IMPORT_GENNUMERICINDEX_H
#define OSMSCOUT_IMPORT_GENNUMERICINDEX_H
 
/*
  This source is part of the libosmscout library
  Copyright (C) 2009  Tim Teulings
 
  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.
 
  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.
 
  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
*/
 
#include <vector>
 
#include <osmscout/io/File.h>
#include <osmscout/io/FileScanner.h>
#include <osmscout/io/FileWriter.h>
 
#include <osmscout/util/Cache.h>
#include <osmscout/util/Number.h>
#include <osmscout/util/Progress.h>
 
#include <osmscoutimport/Import.h>
 
namespace osmscout {
 
  template <class N,class T>
  class NumericIndexGenerator : public ImportModule
  {
  private:
    std::string description;
    std::string datafile;
    std::string indexfile;
 
  private:
    void ReadData(const TypeConfig& typeConfig,
                  FileScanner& scanner,
                  T& data) const;
 
  public:
    NumericIndexGenerator(const std::string& description,
                          const std::string& datafile,
                          const std::string& indexfile);
 
    ~NumericIndexGenerator() override;
 
    bool Import(const TypeConfigRef& typeConfig,
                const ImportParameter& parameter,
                Progress& progress) override;
  };
 
  template <class N,class T>
  NumericIndexGenerator<N,T>::NumericIndexGenerator(const std::string& description,
                                                    const std::string& datafile,
                                                    const std::string& indexfile)
   : description(description),
     datafile(datafile),
     indexfile(indexfile)
  {
    // no code
  }
 
  template <class N,class T>
  NumericIndexGenerator<N,T>::~NumericIndexGenerator()
  {
    // no code
  }
 
  template <class N,class T>
  void NumericIndexGenerator<N,T>::ReadData(const TypeConfig& typeConfig,
                                            FileScanner& scanner,
                                            T& data) const
  {
    data.Read(typeConfig,
              scanner);
  }
 
  template <class N,class T>
  bool NumericIndexGenerator<N,T>::Import(const TypeConfigRef& typeConfig,
                                          const ImportParameter& parameter,
                                          Progress& progress)
  {
    FileScanner             scanner;
    FileWriter              writer;
 
    uint32_t                dataCount;
 
    std::vector<N>          startingIds;
    std::vector<FileOffset> pageStarts;
 
    std::vector<uint32_t>   indexPageCounts;
 
    FileOffset              levelsOffset;
    FileOffset              lastLevelPageStartOffset;
 
    FileOffset              indexPageCountsOffset;
    uint32_t                pageSize=(uint32_t)parameter.GetNumericIndexPageSize();
 
    //
    // Writing index file
    //
 
    progress.SetAction(std::string("Generating '")+indexfile+"'");
 
    try {
      writer.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                  indexfile));
 
      scanner.Open(AppendFileToDir(parameter.GetDestinationDirectory(),
                                   datafile),
                   FileScanner::Sequential,true);
 
      dataCount=scanner.ReadUInt32();
 
      writer.WriteNumber(pageSize);       // Size of one index page in bytes
      writer.WriteNumber(dataCount);      // Number of entries in data file
 
      levelsOffset=writer.GetPos();
      writer.Write((uint32_t)0);        // Number of levels
 
      lastLevelPageStartOffset=writer.GetPos();
      writer.WriteFileOffset((FileOffset)0);        // Write the starting position of the last page
 
      indexPageCountsOffset=writer.GetPos();
      writer.WriteFileOffset((FileOffset)0);        // Write the starting position of list of sizes of each index level
 
      writer.FlushCurrentBlockWithZeros(pageSize);
 
      progress.Info(std::string("Writing level ")+std::to_string(1)+" ("+std::to_string(dataCount)+" entries)");
 
      N          lastId=0;
      FileOffset lastPos=0;
      uint32_t   currentPageSize=0;
 
      for (uint32_t d=0; d<dataCount; d++) {
        progress.SetProgress(d,dataCount);
 
        FileOffset readPos;
        T          data;
 
        readPos=scanner.GetPos();
 
        ReadData(*typeConfig,
                 scanner,
                 data);
 
        if (d>0) {
          if (data.GetId()<=lastId) {
            progress.Error("Current id "+std::to_string(data.GetId())+" <= last id "+std::to_string(lastId));
          }
          assert(data.GetId()>lastId);
          assert(readPos>lastPos);
        }
 
        if (currentPageSize>0) {
          std::array<char,10> b1;
          std::array<char,10> b2;
          N                   b1val=data.GetId()-lastId;
          FileOffset          b2val=readPos-lastPos;
          unsigned int        b1size;
          unsigned int        b2size;
 
 
          b1size=EncodeNumber(b1val,b1);
          b2size=EncodeNumber(b2val,b2);
 
          assert(b1size<=10);
          assert(b2size<=10);
 
          if (currentPageSize+b1size+b2size>pageSize) {
            // Next entry does not fit, fill rest of index page with zeros
            writer.FlushCurrentBlockWithZeros(pageSize);
 
            currentPageSize=0;
          }
          else {
            writer.Write(b1.data(),b1size);
            writer.Write(b2.data(),b2size);
 
            currentPageSize+=b1size+b2size;
          }
        }
 
        if (currentPageSize==0) {
          FileOffset writePos=writer.GetPos();
 
          startingIds.push_back(data.GetId());
          pageStarts.push_back(writePos);
 
          writer.WriteNumber(data.GetId());
          writer.WriteNumber(readPos);
 
          writePos=writer.GetPos();
          currentPageSize=writePos%pageSize;
        }
 
        lastId=data.GetId();
        lastPos=readPos;
      }
 
      writer.FlushCurrentBlockWithZeros(pageSize);
      indexPageCounts.push_back((uint32_t)pageStarts.size());
 
      while (pageStarts.size()>1) {
        std::vector<N>          si(startingIds);
        std::vector<FileOffset> po(pageStarts);
 
        startingIds.clear();
        pageStarts.clear();
 
        progress.Info(std::string("Writing level ")+std::to_string(indexPageCounts.size()+1)+" ("+std::to_string(si.size())+" entries)");
 
        size_t currentPageSize=0;
 
        for (size_t i=0; i<si.size(); i++) {
          if (currentPageSize>0) {
            std::array<char,10> b1;
            std::array<char,10> b2;
            N                   b1val=si[i]-si[i-1];
            FileOffset          b2val=po[i]-po[i-1];
            unsigned int        b1size;
            unsigned int        b2size;
 
            b1size=EncodeNumber(b1val,b1);
            b2size=EncodeNumber(b2val,b2);
 
            assert(b1size<=10);
            assert(b2size<=10);
 
            if (currentPageSize+b1size+b2size>pageSize) {
              // Fill rest of first index page with zeros
              writer.FlushCurrentBlockWithZeros(pageSize);
 
              currentPageSize=0;
            }
            else {
              writer.Write(b1.data(),b1size);
              writer.Write(b2.data(),b2size);
 
              currentPageSize+=b1size+b2size;
            }
          }
 
          if (currentPageSize==0) {
            FileOffset writePos;
 
            writePos=writer.GetPos();
 
            startingIds.push_back(si[i]);
            pageStarts.push_back(writePos);
 
            writer.WriteNumber(si[i]);
            writer.WriteNumber(po[i]);
 
            writePos=writer.GetPos();
            currentPageSize=writePos%pageSize;
          }
        }
 
        writer.FlushCurrentBlockWithZeros(pageSize);
        indexPageCounts.push_back((uint32_t)pageStarts.size());
      }
 
      // If we have data to index, we should have at least one root level index page
      if (dataCount>0) {
        assert(pageStarts.size()==1);
 
        FileOffset indexPageCountsPos;
 
        indexPageCountsPos=writer.GetPos();
 
        writer.SetPos(levelsOffset);
        writer.Write((uint32_t)indexPageCounts.size());
 
        writer.SetPos(lastLevelPageStartOffset);
        writer.WriteFileOffset(pageStarts[0]);
 
        writer.SetPos(indexPageCountsOffset);
        writer.WriteFileOffset(indexPageCountsPos);
 
        writer.SetPos(indexPageCountsPos);
      }
 
      progress.Info(std::string("Index for ")+std::to_string(dataCount)+" data elements will be stored in "+std::to_string(indexPageCounts.size())+ " levels");
      for (size_t level=0; level<indexPageCounts.size(); level++) {
        size_t levelIndex=indexPageCounts.size()-level-1;
 
        progress.Info(std::string("Page count for level ")+std::to_string(level)+" is "+std::to_string(indexPageCounts[levelIndex]));
        writer.WriteNumber(indexPageCounts[levelIndex]);
      }
 
      scanner.Close();
      writer.Close();
    }
    catch (IOException& e) {
      progress.Error(e.GetDescription());
 
      scanner.CloseFailsafe();
      writer.CloseFailsafe();
 
      return false;
    }
 
    return true;
  }
}
 
#endif