Package gw.internal.gosu.parser

Class SourceCodeTokenizerInternal

java.lang.Object
gw.internal.gosu.parser.SourceCodeTokenizerInternal
public final class SourceCodeTokenizerInternal extends Object
This class is adapted from java.io.SourceCodeTokenizer. It adds the notion of operator to the mix. You can define your own operators or use the default set of operators, which are taken from the Java Language Spec. It also captures state information for use by our parser e.g., current token location, line number and column.

  • Field Details

    • CT_WHITESPACE

      protected static final int CT_WHITESPACE
      See Also:

    • CT_DIGIT

      protected static final int CT_DIGIT
      See Also:

    • CT_ALPHA

      protected static final int CT_ALPHA
      See Also:

    • CT_QUOTE

      protected static final int CT_QUOTE
      See Also:

    • CT_COMMENT

      protected static final int CT_COMMENT
      See Also:

    • CT_OPERATOR

      protected static final int CT_OPERATOR
      See Also:

    • CT_BITSHIFT_OPERATOR

      protected static final int CT_BITSHIFT_OPERATOR
      See Also:

    • CT_CHARQUOTE

      protected static final int CT_CHARQUOTE
      See Also:

    • DEFAULT_OPERATORS

      private static final Set<String> DEFAULT_OPERATORS

    • BITSHIFT_OPERATORS

      private static final Set<String> BITSHIFT_OPERATORS

    • _reader

      private SourceCodeReader _reader

    • _peekc

      int _peekc

    • _bForceLower

      private boolean _bForceLower

    • _bEOLIsSignificantP

      private boolean _bEOLIsSignificantP

    • _ctype

      private int[] _ctype

    • _operators

      private Set<String> _operators

    • _iPos

      private int _iPos

    • _iLineNum

      private int _iLineNum

    • _iColumn

      private int _iColumn

    • _iTokenStart

      private int _iTokenStart

    • _iTokenColumn

      private int _iTokenColumn

    • _bWhitespaceSignificant

      private boolean _bWhitespaceSignificant

    • _bCommentsSignificant

      private boolean _bCommentsSignificant

    • _iType

      public int _iType

    • _keyword

      public Keyword _keyword

    • _strValue

      public String _strValue

    • _iInvalidCharPos

      public int _iInvalidCharPos

    • _bUnterminatedString

      public boolean _bUnterminatedString

    • _bUnterminatedComment

      public boolean _bUnterminatedComment

    • _instructor

      private ITokenizerInstructor _instructor

    • _lastComment

      private DocCommentBlock _lastComment

    • _bParseDotsAsOperators

      private boolean _bParseDotsAsOperators

    • _iLineOffset

      private int _iLineOffset

    • _tokens

      private Stack<Token> _tokens

    • _eof

      private Token _eof

    • _supportsKeywords

      private boolean _supportsKeywords

  • Constructor Details

    • SourceCodeTokenizerInternal

      SourceCodeTokenizerInternal(boolean initForCopy)

    • SourceCodeTokenizerInternal

      public SourceCodeTokenizerInternal(CharSequence sourceCode)

    • SourceCodeTokenizerInternal

      public SourceCodeTokenizerInternal(Reader reader)

    • SourceCodeTokenizerInternal

      public SourceCodeTokenizerInternal(SourceCodeReader reader)

    • SourceCodeTokenizerInternal

      public SourceCodeTokenizerInternal(SourceCodeReader reader, ITokenizerInstructor instructor)

  • Method Details

    • copy

      public Token copy()

    • reset

      public void reset()

    • reset

      public void reset(Reader reader)

    • reset

      public void reset(SourceCodeReader reader)

    • reset

      void reset(SourceCodeReader reader, boolean bResetReader)

    • getReader

      public SourceCodeReader getReader()

    • getSource

      public String getSource()

    • getInstructor

      public ITokenizerInstructor getInstructor()

    • setInstructor

      public void setInstructor(ITokenizerInstructor instructor)

    • isWhitespaceSignificant

      public boolean isWhitespaceSignificant()

    • setWhitespaceSignificant

      public void setWhitespaceSignificant(boolean bWhitespaceSignificant)

    • isCommentsSignificant

      public boolean isCommentsSignificant()

    • setCommentsSignificant

      public void setCommentsSignificant(boolean bCommentsSignificant)

    • getLineNumber

      public int getLineNumber()

    • getLineOffset

      public int getLineOffset()

    • incrementLineNumber

      protected void incrementLineNumber()

    • getTokenColumn

      public int getTokenColumn()

    • wordChars

      public void wordChars(int iLow, int iHigh)

    • whitespaceChars

      public void whitespaceChars(int iLow, int iHigh)

    • ordinaryChars

      public void ordinaryChars(int iLow, int iHigh)

    • ordinaryChar

      public void ordinaryChar(int ch)

    • getDefaultOperators

      public static String[] getDefaultOperators()

    • getDefaultBindingOperators

      public static List<String> getDefaultBindingOperators()

    • getBitshiftOperators

      public static String[] getBitshiftOperators()

    • operators

      public void operators(String[] astrOperators)

    • setOperatorChars

      private void setOperatorChars(Set<String> operators)

    • operatorChars

      public void operatorChars(int iLow, int iHigh)

    • setBitshiftOperatorChars

      private void setBitshiftOperatorChars(Set<String> operators)

    • bitshiftOperatorChars

      public void bitshiftOperatorChars(int iLow, int iHigh)

    • isOperator

      public boolean isOperator(String strOperator)

    • commentChar

      public void commentChar(int ch)

    • quoteChar

      public void quoteChar(int ch)

    • charQuoteChar

      public void charQuoteChar(int ch)

    • parseNumbers

      public void parseNumbers()

    • eolIsSignificant

      public void eolIsSignificant(boolean bFlag)

    • lowerCaseMode

      public void lowerCaseMode(boolean bLowerCaseMode)

    • isUnterminatedString

      public boolean isUnterminatedString()

    • isUnterminatedComment

      public boolean isUnterminatedComment()

    • setParseDotsAsOperators

      public void setParseDotsAsOperators(boolean parseDotsAsOperators)

    • isParseDotsAsOperators

      public boolean isParseDotsAsOperators()

    • popLastComment

      public DocCommentBlock popLastComment()

    • getTokenStart

      public int getTokenStart()

    • getTokenEnd

      public int getTokenEnd()

    • read

      private int read() throws IOException
      Throws:
      IOException

    • readOne

      private int readOne() throws IOException
      Throws:
      IOException

    • readWithInstructions

      protected int readWithInstructions(int c) throws IOException
      Throws:
      IOException

    • pushWhitespaceToken

      private void pushWhitespaceToken(StringBuilder sbWhitespace)

    • pushToken

      private void pushToken()

    • initEofToken

      private void initEofToken()

    • initToken

      Token initToken(Token token)

    • pushToken

      private void pushToken(Token token)

    • isAnalyzingSeparately

      public boolean isAnalyzingSeparately()

    • isAnalyzingDirective

      public boolean isAnalyzingDirective()

    • getTokens

      public final Stack<Token> getTokens()

    • rip

      void rip()

    • nextToken

      public int nextToken() throws IOException
      Throws:
      IOException

    • _nextTokenImpl

      private int _nextTokenImpl() throws IOException
      Throws:
      IOException

    • consumeBlockComment

      private void consumeBlockComment() throws IOException
      Throws:
      IOException

    • isReserved

      private boolean isReserved()

    • isAtIgnorePos

      private boolean isAtIgnorePos()

    • isHexDigit

      private boolean isHexDigit(int c)

    • isLeftOpenIntervalOp

      private boolean isLeftOpenIntervalOp(String strOpTest)

    • stopOnDot

      private boolean stopOnDot(int c)

    • toString

      public String toString()
      Overrides:
      toString in class Object

    • getType

      public int getType()

    • isEOF

      public boolean isEOF()

    • isNOTHING

      public boolean isNOTHING()

    • goToPosition

      void goToPosition(int iOffset) throws IOException
      Throws:
      IOException

    • setTokens

      void setTokens(Stack<Token> tokens)

    • getEofToken

      public Token getEofToken()

    • isSupportsKeywords

      public boolean isSupportsKeywords()

    • setSupportsKeywords

      public void setSupportsKeywords(boolean supportsKeywords)