Package com.igormaznitsa.prologparser.tokenizer

Class Tokenizer

java.lang.Object
com.igormaznitsa.prologparser.tokenizer.Tokenizer
public final class Tokenizer extends Object
Internal tokenizer to extract next token from reader.

  • Field Details

    • strBuf

      private final StringBuilderEx strBuf

    • specCharBuf

      private final StringBuilderEx specCharBuf

    • insideCharBuffer

      private final StringBuilderEx insideCharBuffer

    • blockCommentsAllowed

      private final boolean blockCommentsAllowed

    • zeroSingleQuotationAllowed

      private final boolean zeroSingleQuotationAllowed

    • zeroQuotationAllowsWhitespaceChar

      private final boolean zeroQuotationAllowsWhitespaceChar

    • reader

      private final Reader reader

    • parser

      private final PrologParser parser

    • metaOperators

      private final Koi7CharOpMap metaOperators

    • lastPushedTerm

      private TokenizerResult lastPushedTerm

    • prevTokenLine

      private int prevTokenLine

    • prevTokenPos

      private int prevTokenPos

    • lastTokenLine

      private int lastTokenLine

    • lastTokenPos

      private int lastTokenPos

    • prevPos

      private int prevPos

    • prevLine

      private int prevLine

    • pos

      private int pos

    • line

      private int line

  • Constructor Details

  • Method Details

    • isCharAllowedForRadix

      public static boolean isCharAllowedForRadix(char chr, int radix)

    • getLastPushed

      TokenizerResult getLastPushed()

    • doReadChar

      private int doReadChar() throws IOException
      Throws:
      IOException

    • calcDiffAndPushResultBack

      public void calcDiffAndPushResultBack(String etalon, StringBuilderEx buffer)

    • push

      public void push(char ch)

    • close

      public void close(boolean closeReader) throws IOException
      Throws:
      IOException

    • hasOperatorStartsWith

      public boolean hasOperatorStartsWith(String operatorNameStartSubstring)

    • findOperatorForName

      public OpContainer findOperatorForName(String operatorName)

    • findOperatorForSingleChar

      public OpContainer findOperatorForSingleChar(char c)

    • push

      public void push(TokenizerResult object)

    • peek

      public TokenizerResult peek()

    • getLastTokenPos

      public int getLastTokenPos()

    • getLastTokenLine

      public int getLastTokenLine()

    • fixPosition

      public void fixPosition()

    • skipUntilBlockCommentEnd

      private void skipUntilBlockCommentEnd() throws IOException
      Throws:
      IOException

    • skipUntilNextString

      private void skipUntilNextString() throws IOException
      Throws:
      IOException

    • pop

      public TokenizerResult pop()

    • readNextToken

      public TokenizerResult readNextToken()
      Read next token
      Returns:
      next token or null if not found or stream ended

    • makeTermFromString

      public PrologTerm makeTermFromString(String str, int radix, Quotation quotingType, TokenizerState state)

    • getLine

      public int getLine()

    • getPos

      public int getPos()