Scanner.h

Go to the documentation of this file.

/*
  This source is part of the libosmscout library
  Copyright (C) 2011  Tim Teulings
 
  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.
 
  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.
 
  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
*/
 
 
#if !defined(osmscout_ost_SCANNER_H)
#define osmscout_ost_SCANNER_H
 
#include <string>
#include <map>
#include <memory>
 
#include <osmscout/system/SystemTypes.h>
 
namespace osmscout {
namespace ost {
 
 
#define COCO_WCHAR_MAX 65535
 
// string handling, wide character
extern char* coco_string_create(const char* value);
extern char* coco_string_create(const char* value, int startIndex, int length);
extern void  coco_string_delete(char* &data);
 
class Token;
 
using TokenRef = std::shared_ptr<Token>;
 
class Token
{
public:
  int      kind;    // token kind
  int      pos;     // token position in the source text (starting at 0)
  int      charPos; // token position in characters in the source text (starting at 0)
  int      col;     // token column (starting at 1)
  int      line;    // token line (starting at 1)
  char*    val;     // token value
  TokenRef next;    // ML 2005-03-11 Peek tokens are kept in linked list
 
  Token();
  ~Token();
};
 
class Buffer
{
private:
  unsigned char *buf; // input buffer
  size_t bufLen;      // length of buffer
  int    bufPos;      // current position in buffer
 
public:
  static const int EoF = COCO_WCHAR_MAX + 1;
 
  Buffer(const unsigned char* buf, size_t len);
  virtual ~Buffer();
 
  int Read();
  int Peek();
  int GetPos();
  void SetPos(int value);
};
 
//-----------------------------------------------------------------------------------
// StartStates  -- maps characters to start states of tokens
//-----------------------------------------------------------------------------------
class StartStates
{
private:
  std::map<int,int> map;
 
public:
  StartStates() = default;
  virtual ~StartStates() = default;
 
  void set(int key, int val)
  {
    map[key]=val;
  }
 
  int state(int key) {
    std::map<int,int>::const_iterator iter=map.find(key);
 
    if (iter!=map.end()) {
      return iter->second;
    }
    else {
      return 0;
    }
  }
};
 
//-------------------------------------------------------------------------------------------
// KeywordMap  -- maps strings to integers (identifiers to keyword kinds)
//-------------------------------------------------------------------------------------------
class KeywordMap
{
private:
  std::map<std::string,int> map;
 
public:
  KeywordMap() = default;
  virtual ~KeywordMap() = default;
 
  void set(const char* key, int val)
  {
    map[std::string(key)]=val;
  }
 
  int get(const char* key, int defaultVal)
  {
    std::map<std::string,int>::const_iterator iter=map.find(std::string(key));
 
    if (iter!=map.end()) {
      return iter->second;
    }
    else {
      return defaultVal;
    }
  }
};
 
class Scanner
{
private:
  unsigned char EOL;
  int           eofSym;
  int           noSym;
  int           maxT;
  StartStates   start;
  KeywordMap    keywords;
 
  TokenRef      t;          // current token
  char          *tval;      // text of current token
  size_t        tvalLength; // length of text of current token
  size_t        tlen;       // length of current token
 
  TokenRef      tokens;     // list of tokens already peeked (first token is a dummy)
  TokenRef      pt;         // current peek token
 
  int           ch;         // current input character
 
  int           pos;        // byte position of current character
  int           charPos;    // position by unicode characters starting with 0
  int           line;       // line number of current character
  int           col;        // column number of current character
  int           oldEols;    // EOLs that appeared in a comment
 
  TokenRef CreateToken();
  void AppendVal(TokenRef& t);
 
  void Init();
  void NextCh();
  void AddCh();
    bool Comment0();
    bool Comment1();
 
  TokenRef NextToken();
 
public:
  Buffer *buffer;   // scanner buffer
 
  Scanner(const unsigned char* buf, size_t len);
  ~Scanner();
  TokenRef Scan();
  void SetScannerBehindT();
  TokenRef Peek();
  void ResetPeek();
 
}; // end Scanner
 
} // namespace
} // namespace
 
 
#endif