Package org.htmlunit.corejs.javascript

Class TokenStream

  • class TokenStream
extends java.lang.Object
    This class implements the JavaScript scanner.

    It is based on the C source files jsscan.c and jsscan.h in the jsref package.

    See Also:
    Parser

    • Field Detail

      • REPORT_NUMBER_FORMAT_ERROR

        private static final int REPORT_NUMBER_FORMAT_ERROR
        See Also:
        Constant Field Values

      • rawString

        private java.lang.StringBuilder rawString

      • dirtyLine

        private boolean dirtyLine

      • regExpFlags

        java.lang.String regExpFlags

      • string

        private java.lang.String string

      • number

        private double number

      • bigInt

        private java.math.BigInteger bigInt

      • isBinary

        private boolean isBinary

      • isOldOctal

        private boolean isOldOctal

      • isOctal

        private boolean isOctal

      • isHex

        private boolean isHex

      • quoteChar

        private int quoteChar

      • stringBuffer

        private char[] stringBuffer

      • stringBufferTop

        private int stringBufferTop

      • ungetBuffer

        private final int[] ungetBuffer

      • ungetCursor

        private int ungetCursor

      • hitEOF

        private boolean hitEOF

      • lineStart

        private int lineStart

      • lineEndChar

        private int lineEndChar

      • lineno

        int lineno

      • sourceString

        private java.lang.String sourceString

      • sourceReader

        private java.io.Reader sourceReader

      • sourceBuffer

        private char[] sourceBuffer

      • sourceEnd

        private int sourceEnd

      • sourceCursor

        int sourceCursor

      • cursor

        int cursor

      • tokenBeg

        int tokenBeg

      • tokenEnd

        int tokenEnd

      • xmlIsAttribute

        private boolean xmlIsAttribute

      • xmlIsTagContent

        private boolean xmlIsTagContent

      • xmlOpenTagsCount

        private int xmlOpenTagsCount

      • parser

        private Parser parser

      • commentPrefix

        private java.lang.String commentPrefix

      • commentCursor

        private int commentCursor

    • Constructor Detail

      • TokenStream

        TokenStream​(Parser parser,
            java.io.Reader sourceReader,
            java.lang.String sourceString,
            int lineno)

    • Method Detail

      • tokenToString

        java.lang.String tokenToString​(int token)

      • isKeyword

        static boolean isKeyword​(java.lang.String s,
                         int version,
                         boolean isStrict)

      • stringToKeyword

        private static int stringToKeyword​(java.lang.String name,
                                   int version,
                                   boolean isStrict)

      • stringToKeywordForJS

        private static int stringToKeywordForJS​(java.lang.String name)
        JavaScript 1.8 and earlier

      • stringToKeywordForES

        private static int stringToKeywordForES​(java.lang.String name,
                                        boolean isStrict)
        ECMAScript 6.

      • isValidIdentifierName

        private static boolean isValidIdentifierName​(java.lang.String str)

      • getSourceString

        final java.lang.String getSourceString()

      • getLineno

        final int getLineno()

      • getString

        final java.lang.String getString()

      • getQuoteChar

        final char getQuoteChar()

      • getNumber

        final double getNumber()

      • getBigInt

        final java.math.BigInteger getBigInt()

      • isNumericBinary

        final boolean isNumericBinary()

      • isNumericOldOctal

        final boolean isNumericOldOctal()

      • isNumericOctal

        final boolean isNumericOctal()

      • isNumericHex

        final boolean isNumericHex()

      • eof

        final boolean eof()

      • getToken

        final int getToken()
            throws java.io.IOException
        Throws:
        java.io.IOException

      • readDigits

        private int readDigits​(int base,
                       int c)
                throws java.io.IOException
        Throws:
        java.io.IOException

      • isAlpha

        private static boolean isAlpha​(int c)

      • isDigit

        private static boolean isDigit​(int base,
                               int c)

      • isDualDigit

        private static boolean isDualDigit​(int c)

      • isOctalDigit

        private static boolean isOctalDigit​(int c)

      • isDigit

        private static boolean isDigit​(int c)

      • isHexDigit

        private static boolean isHexDigit​(int c)

      • isJSSpace

        private static boolean isJSSpace​(int c)

      • isJSFormatChar

        private static boolean isJSFormatChar​(int c)

      • readRegExp

        void readRegExp​(int startToken)
         throws java.io.IOException
        Parser calls the method when it gets / or /= in literal context.
        Throws:
        java.io.IOException

      • readAndClearRegExpFlags

        java.lang.String readAndClearRegExpFlags()

      • getRawString

        java.lang.String getRawString()

      • getTemplateLiteralChar

        private int getTemplateLiteralChar()
                            throws java.io.IOException
        Throws:
        java.io.IOException

      • ungetTemplateLiteralChar

        private void ungetTemplateLiteralChar​(int c)

      • matchTemplateLiteralChar

        private boolean matchTemplateLiteralChar​(int test)
                                  throws java.io.IOException
        Throws:
        java.io.IOException

      • peekTemplateLiteralChar

        private int peekTemplateLiteralChar()
                             throws java.io.IOException
        Throws:
        java.io.IOException

      • readTemplateLiteral

        int readTemplateLiteral​(boolean isTaggedLiteral)
                 throws java.io.IOException
        Throws:
        java.io.IOException

      • isXMLAttribute

        boolean isXMLAttribute()

      • getFirstXMLToken

        int getFirstXMLToken()
              throws java.io.IOException
        Throws:
        java.io.IOException

      • getNextXMLToken

        int getNextXMLToken()
             throws java.io.IOException
        Throws:
        java.io.IOException

      • readQuotedString

        private boolean readQuotedString​(int quote)
                          throws java.io.IOException
        Throws:
        java.io.IOException

      • readXmlComment

        private boolean readXmlComment()
                        throws java.io.IOException
        Throws:
        java.io.IOException

      • readCDATA

        private boolean readCDATA()
                   throws java.io.IOException
        Throws:
        java.io.IOException

      • readEntity

        private boolean readEntity()
                    throws java.io.IOException
        Throws:
        java.io.IOException

      • readPI

        private boolean readPI()
                throws java.io.IOException
        Throws:
        java.io.IOException

      • getStringFromBuffer

        private java.lang.String getStringFromBuffer()

      • addToString

        private void addToString​(int c)

      • canUngetChar

        private boolean canUngetChar()

      • ungetChar

        private void ungetChar​(int c)

      • matchChar

        private boolean matchChar​(int test)
                   throws java.io.IOException
        Throws:
        java.io.IOException

      • peekChar

        private int peekChar()
              throws java.io.IOException
        Throws:
        java.io.IOException

      • getChar

        private int getChar()
             throws java.io.IOException
        Throws:
        java.io.IOException

      • getChar

        private int getChar​(boolean skipFormattingChars)
             throws java.io.IOException
        Throws:
        java.io.IOException

      • getChar

        private int getChar​(boolean skipFormattingChars,
                    boolean ignoreLineEnd)
             throws java.io.IOException
        Throws:
        java.io.IOException

      • getCharIgnoreLineEnd

        private int getCharIgnoreLineEnd()
                          throws java.io.IOException
        Throws:
        java.io.IOException

      • getCharIgnoreLineEnd

        private int getCharIgnoreLineEnd​(boolean skipFormattingChars)
                          throws java.io.IOException
        Throws:
        java.io.IOException

      • ungetCharIgnoreLineEnd

        private void ungetCharIgnoreLineEnd​(int c)

      • skipLine

        private void skipLine()
               throws java.io.IOException
        Throws:
        java.io.IOException

      • getOffset

        final int getOffset()
        Returns the offset into the current line.

      • charAt

        private final int charAt​(int index)

      • substring

        private final java.lang.String substring​(int beginIndex,
                                         int endIndex)

      • getLine

        final java.lang.String getLine()

      • getLine

        final java.lang.String getLine​(int position,
                               int[] linep)

      • fillSourceBuffer

        private boolean fillSourceBuffer()
                          throws java.io.IOException
        Throws:
        java.io.IOException

      • getCursor

        public int getCursor()
        Return the current position of the scanner cursor.

      • getTokenBeg

        public int getTokenBeg()
        Return the absolute source offset of the last scanned token.

      • getTokenEnd

        public int getTokenEnd()
        Return the absolute source end-offset of the last scanned token.

      • getTokenLength

        public int getTokenLength()
        Return tokenEnd - tokenBeg

      • getCommentType

        public Token.CommentType getCommentType()
        Return the type of the last scanned comment.
        Returns:
        type of last scanned comment, or 0 if none have been scanned.

      • markCommentStart

        private void markCommentStart()

      • markCommentStart

        private void markCommentStart​(java.lang.String prefix)

      • isMarkingComment

        private boolean isMarkingComment()

      • getAndResetCurrentComment

        final java.lang.String getAndResetCurrentComment()

      • convertLastCharToHex

        private static java.lang.String convertLastCharToHex​(java.lang.String str)