Package gw.internal.gosu.parser

Class SourceCodeTokenizerInternal

java.lang.Object
gw.internal.gosu.parser.SourceCodeTokenizerInternal
public final class SourceCodeTokenizerInternal extends Object
This class is adapted from java.io.SourceCodeTokenizer. It adds the notion of operator to the mix. You can define your own operators or use the default set of operators, which are taken from the Java Language Spec. It also captures state information for use by our parser e.g., current token location, line number and column.

  • Field Details

    • CT_WHITESPACE

      protected static final int CT_WHITESPACE
      See Also:

    • CT_DIGIT

      protected static final int CT_DIGIT
      See Also:

    • CT_ALPHA

      protected static final int CT_ALPHA
      See Also:

    • CT_QUOTE

      protected static final int CT_QUOTE
      See Also:

    • CT_COMMENT

      protected static final int CT_COMMENT
      See Also:

    • CT_OPERATOR

      protected static final int CT_OPERATOR
      See Also:

    • CT_BITSHIFT_OPERATOR

      protected static final int CT_BITSHIFT_OPERATOR
      See Also:

    • CT_CHARQUOTE

      protected static final int CT_CHARQUOTE
      See Also:

    • _iType

      public int _iType

    • _keyword

      public Keyword _keyword

    • _strValue

      public String _strValue

    • _iInvalidCharPos

      public int _iInvalidCharPos

    • _bUnterminatedString

      public boolean _bUnterminatedString

    • _bUnterminatedComment

      public boolean _bUnterminatedComment

  • Constructor Details

    • SourceCodeTokenizerInternal

      public SourceCodeTokenizerInternal(CharSequence sourceCode)

    • SourceCodeTokenizerInternal

      public SourceCodeTokenizerInternal(Reader reader)

    • SourceCodeTokenizerInternal

      public SourceCodeTokenizerInternal(SourceCodeReader reader)

    • SourceCodeTokenizerInternal

      public SourceCodeTokenizerInternal(SourceCodeReader reader, ITokenizerInstructor instructor)

  • Method Details

    • copy

      public Token copy()

    • reset

      public void reset()

    • reset

      public void reset(Reader reader)

    • reset

      public void reset(SourceCodeReader reader)

    • getReader

      public SourceCodeReader getReader()

    • getSource

      public String getSource()

    • getInstructor

      public ITokenizerInstructor getInstructor()

    • setInstructor

      public void setInstructor(ITokenizerInstructor instructor)

    • isWhitespaceSignificant

      public boolean isWhitespaceSignificant()

    • setWhitespaceSignificant

      public void setWhitespaceSignificant(boolean bWhitespaceSignificant)

    • isCommentsSignificant

      public boolean isCommentsSignificant()

    • setCommentsSignificant

      public void setCommentsSignificant(boolean bCommentsSignificant)

    • getLineNumber

      public int getLineNumber()

    • getLineOffset

      public int getLineOffset()

    • incrementLineNumber

      protected void incrementLineNumber()

    • getTokenColumn

      public int getTokenColumn()

    • wordChars

      public void wordChars(int iLow, int iHigh)

    • whitespaceChars

      public void whitespaceChars(int iLow, int iHigh)

    • ordinaryChars

      public void ordinaryChars(int iLow, int iHigh)

    • ordinaryChar

      public void ordinaryChar(int ch)

    • getDefaultOperators

      public static String[] getDefaultOperators()

    • getDefaultBindingOperators

      public static List<String> getDefaultBindingOperators()

    • getBitshiftOperators

      public static String[] getBitshiftOperators()

    • operators

      public void operators(String[] astrOperators)

    • operatorChars

      public void operatorChars(int iLow, int iHigh)

    • bitshiftOperatorChars

      public void bitshiftOperatorChars(int iLow, int iHigh)

    • isOperator

      public boolean isOperator(String strOperator)

    • commentChar

      public void commentChar(int ch)

    • quoteChar

      public void quoteChar(int ch)

    • charQuoteChar

      public void charQuoteChar(int ch)

    • parseNumbers

      public void parseNumbers()

    • eolIsSignificant

      public void eolIsSignificant(boolean bFlag)

    • lowerCaseMode

      public void lowerCaseMode(boolean bLowerCaseMode)

    • isUnterminatedString

      public boolean isUnterminatedString()

    • isUnterminatedComment

      public boolean isUnterminatedComment()

    • setParseDotsAsOperators

      public void setParseDotsAsOperators(boolean parseDotsAsOperators)

    • isParseDotsAsOperators

      public boolean isParseDotsAsOperators()

    • popLastComment

      public DocCommentBlock popLastComment()

    • getTokenStart

      public int getTokenStart()

    • getTokenEnd

      public int getTokenEnd()

    • readWithInstructions

      protected int readWithInstructions(int c) throws IOException
      Throws:
      IOException

    • isAnalyzingSeparately

      public boolean isAnalyzingSeparately()

    • isAnalyzingDirective

      public boolean isAnalyzingDirective()

    • getTokens

      public final Stack<Token> getTokens()

    • nextToken

      public int nextToken() throws IOException
      Throws:
      IOException

    • toString

      public String toString()
      Overrides:
      toString in class Object

    • getType

      public int getType()

    • isEOF

      public boolean isEOF()

    • isNOTHING

      public boolean isNOTHING()

    • getEofToken

      public Token getEofToken()

    • isSupportsKeywords

      public boolean isSupportsKeywords()

    • setSupportsKeywords

      public void setSupportsKeywords(boolean supportsKeywords)