Package org.jruby.lexer.yacc

Class RubyLexer

java.lang.Object

org.jruby.lexer.LexingCommon

org.jruby.lexer.yacc.RubyLexer

public class RubyLexer
extends LexingCommon

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static enum	RubyLexer.Keyword

Field Summary

Fields

Modifier and Type	Field	Description
private static final Map<ByteList,RubyLexer.Keyword>	byteList2Keyword
long	end
private final ByteList	FALSE
private StrTerm	lex_strterm
private static final HashMap<String,RubyLexer.Keyword>	map
private ByteList	numberBuffer
private ParserSupport	parserSupport
long	start
private final ByteList	TRUE
private IRubyWarnings	warnings

Fields inherited from class org.jruby.lexer.LexingCommon

__end__seen, AMPERSAND, AMPERSAND_AMPERSAND, AMPERSAND_DOT, ASCII8BIT_ENCODING, BACKSLASH, BACKTICK, BANG, BANG_EQ, BANG_TILDE, BEGIN_DOC_MARKER, braceNest, CALL, CARET, cmdArgumentState, CODING, COLON, COLON_COLON, COMMA, commandStart, conditionState, DOLLAR_BANG, DOLLAR_DOT, DOLLAR_UNDERSCORE, DOT, DOT_DOT, DOT_DOT_DOT, END_DOC_MARKER, END_MARKER, EOF, eofp, EQ, EQ_EQ, EQ_EQ_EQ, EQ_GT, EQ_TILDE, EXPR_ARG, EXPR_ARG_ANY, EXPR_BEG, EXPR_BEG_ANY, EXPR_CLASS, EXPR_CMDARG, EXPR_DOT, EXPR_END, EXPR_END_ANY, EXPR_ENDARG, EXPR_ENDFN, EXPR_FITEM, EXPR_FNAME, EXPR_LABEL, EXPR_LABELED, EXPR_MID, EXPR_VALUE, GT, GT_EQ, GT_GT, has_shebang, heredoc_end, heredoc_indent, heredoc_line_indent, inKwarg, last_cr_line, last_state, LBRACKET, LBRACKET_RBRACKET, LBRACKET_RBRACKET_EQ, LCURLY, lex_lastline, lex_nextline, lex_p, lex_pbeg, lex_pend, lex_state, lexb, line_count, line_offset, LT, LT_EQ, LT_EQ_RT, LT_LT, MINUS, MINUS_AT, MINUS_GT, OR, OR_OR, parenNest, PERCENT, PLUS, PLUS_AT, Q, QQ, QUESTION, RBRACKET, RCURLY, RPAREN, ruby_sourceline, SEMICOLON, SLASH, src, STAR, STAR_STAR, str_dquote, str_dsym, str_dword, STR_FUNC_ESCAPE, STR_FUNC_EXPAND, STR_FUNC_INDENT, STR_FUNC_LABEL, STR_FUNC_LIST, STR_FUNC_QWORDS, STR_FUNC_REGEXP, STR_FUNC_SYMBOL, STR_FUNC_TERM, str_label, str_regexp, str_squote, str_ssym, str_sword, str_xquote, SUFFIX_ALL, SUFFIX_I, SUFFIX_R, TAB_WIDTH, TILDE, token, tokenSeen, tokline, tokp, USASCII_ENCODING, UTF8_ENCODING, yaccValue

Constructor Summary

Constructors

Constructor	Description
RubyLexer(ParserSupport support, LexerSource source)	Deprecated.
RubyLexer(ParserSupport support, LexerSource source, IRubyWarnings warnings)

Method Summary

Modifier and Type	Method	Description
protected void	ambiguousOperator(String op, String syn)
private int	ampersand(boolean spaceSeen)
private boolean	arg_ambiguous()
protected int	asTruth(String name, ByteList value)
private int	at()
private int	backtick(boolean commandState)
private int	bang()
private int	caret()
private int	colon(boolean spaceSeen)
private int	comma(int c)
void	compile_error(String message)
void	compile_error(String message, long start, long end)
void	compile_error_pos(String message)
private int	considerComplex(int token, int suffix)
StrNode	createStr(ByteList buffer, int flags)
private int	doKeyword(int state)
private int	dollar()
private int	dot()
private int	doubleQuote(boolean commandState)
private int	getFloatToken(String number, int suffix)
private int	getIntegerToken(String value, int radix, int suffix)
static RubyLexer.Keyword	getKeyword(String str)
static RubyLexer.Keyword	getKeyword(ByteList str)
private int	getNumberToken(String number, boolean seen_e, boolean seen_point, int nondigit)
StrTerm	getStrTerm()
private int	greaterThan()
void	heredoc_dedent(Node root)
void	heredoc_restore(HeredocTerm here)
private int	hereDocumentIdentifier()
private int	identifier(int c, boolean commandState)
private int	identifierToken(int result, ByteList value)
private int	leftBracket(boolean spaceSeen)
private int	leftCurly()
private int	leftParen(boolean spaceSeen)
private int	lessThan(boolean spaceSeen)
private int	minus(boolean spaceSeen)
protected void	mismatchedRegexpEncodingError(org.jcodings.Encoding optionEncoding, org.jcodings.Encoding encoding)
private BignumNode	newBignumNode(String value, int radix)
private ComplexNode	newComplexNode(NumericNode number)
private FixnumNode	newFixnumNode(String value, int radix)
private RationalNode	newRationalNode(String value, int radix)
int	nextc()
int	nextToken()
private int	parseNumber(int c)	Parse a number from the input stream.
private int	parseQuote(int c)	What type/kind of quote are we dealing with?
protected RegexpOptions	parseRegexpFlags()
private int	percent(boolean spaceSeen)
private int	pipe()
private int	plus(boolean spaceSeen)
private String	prepareMessage(String message, ByteList line, long start, long end)
private void	printToken(int token)
private int	questionMark()
int	readEscape()
private void	readUTF8EscapeIntoBuffer(int codepoint, ByteList buffer, boolean stringLiteral, boolean[] encodingDetermined)
int	readUTFEscape(ByteList buffer, boolean stringLiteral, boolean[] encodingDetermined)
void	readUTFEscapeRegexpLiteral(ByteList buffer)
void	reset()
private int	rightBracket()
private int	rightCurly()
private int	rightParen()
private int	scanHex(int count, boolean strict, String errorMessage)	Read up to count hexadecimal digits.
private char	scanHexLiteral(ByteList buffer, int count, boolean strict, String errorMessage)	Read up to count hexadecimal digits and store those digits in a token numberBuffer.
protected void	setCompileOptionFlag(String name, ByteList value)
protected void	setEncoding(ByteList name)
void	setStrTerm(StrTerm strterm)
protected void	setTokenInfo(String name, ByteList value)
void	setWarnings(IRubyWarnings warnings)
private int	singleQuote(boolean commandState)
private int	slash(boolean spaceSeen)
private int	star(boolean spaceSeen)
private int	tilde()
int	tokenize_ident(int result)
private int	tokHex(int count, String errorMessage)
private void	updateStartPosition(int column)
private void	updateTokenPosition()
private int	yylex()	Returns the next token.
private int	yylex2()

Methods inherited from class org.jruby.lexer.LexingCommon

checkRegexpFragment, checkRegexpSyntax, column, comment_at_top, createAsEncodedString, createTokenByteList, createTokenByteList, createTokenString, createTokenString, dedent_string, flush, getBraceNest, getCmdArgumentState, getConditionState, getCurrentArg, getCurrentLine, getEncoding, getFile, getHeredocIndent, getHeredocLineIndent, getLeftParenBegin, getLineOffset, getRubySourceline, getState, getTokenCR, incrementParenNest, isAfterOperator, isARG, isASCII, isASCII, isBEG, isEND, isEndSeen, isGlobalCharPunct, isHexChar, isIdentifierChar, isLabelPossible, isLabelSuffix, isLexState, isLexStateAll, isNext_identchar, isOctChar, isSpace, isSpaceArg, lex_goto_eol, lineno, magicCommentEncoding, magicCommentMarker, newtok, numberLiteralSuffix, onMagicComment, optionsEncodingChar, p, parser_magic_comment, parser_prepare, parseRegexpFlags, peek, peek, peekVariableName, precise_mbclen, printState, pushback, resetStacks, scanOct, set_file_encoding, setBraceNest, setCurrentArg, setCurrentEncoding, setEncoding, setHeredocIndent, setHeredocLineIndent, setLeftParenBegin, setRegexpEncoding, setRubySourceline, setSource, setState, setValue, strncmp, tokAdd, tokadd_ident, tokadd_mbchar, tokadd_mbchar, tokaddmbc, tokCopy, token, update_heredoc_indent, validateFormalIdentifier, validateFormalIdentifier, value, warn_balanced, was_bol, whole_match_p

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

map

private static final HashMap<String,RubyLexer.Keyword> map

byteList2Keyword

private static final Map<ByteList,RubyLexer.Keyword> byteList2Keyword

parserSupport

private ParserSupport parserSupport

warnings

private IRubyWarnings warnings

lex_strterm

private StrTerm lex_strterm

start

public long start

end

public long end

TRUE

private final ByteList TRUE

FALSE

private final ByteList FALSE

numberBuffer

private ByteList numberBuffer

Constructor Details

RubyLexer

public RubyLexer(ParserSupport support,
 LexerSource source,
 IRubyWarnings warnings)

RubyLexer

@Deprecated
public RubyLexer(ParserSupport support,
 LexerSource source)

Deprecated.

Method Details

newBignumNode

private BignumNode newBignumNode(String value,
 int radix)

newFixnumNode

private FixnumNode newFixnumNode(String value,
 int radix)
                          throws NumberFormatException

Throws:

NumberFormatException

newRationalNode

private RationalNode newRationalNode(String value,
 int radix)
                              throws NumberFormatException

Throws:

NumberFormatException

newComplexNode

private ComplexNode newComplexNode(NumericNode number)

ambiguousOperator

protected void ambiguousOperator(String op,
 String syn)

Specified by:

ambiguousOperator in class LexingCommon

getKeyword

public static RubyLexer.Keyword getKeyword(ByteList str)

getKeyword

public static RubyLexer.Keyword getKeyword(String str)

tokenize_ident

public int tokenize_ident(int result)

Specified by:

tokenize_ident in class LexingCommon

reset

public void reset()

Overrides:

reset in class LexingCommon

nextc

public int nextc()

Specified by:

nextc in class LexingCommon

heredoc_dedent

public void heredoc_dedent(Node root)

compile_error

public void compile_error(String message,
 long start,
 long end)

compile_error

public void compile_error(String message)

Specified by:

compile_error in class LexingCommon

prepareMessage

private String prepareMessage(String message,
 ByteList line,
 long start,
 long end)

compile_error_pos

public void compile_error_pos(String message)

updateTokenPosition

private void updateTokenPosition()

updateStartPosition

private void updateStartPosition(int column)

heredoc_restore

public void heredoc_restore(HeredocTerm here)

nextToken

public int nextToken()
              throws IOException

Throws:

IOException

setCompileOptionFlag

protected void setCompileOptionFlag(String name,
 ByteList value)

Specified by:

setCompileOptionFlag in class LexingCommon

parseRegexpFlags

protected RegexpOptions parseRegexpFlags()
                                  throws IOException

Specified by:

parseRegexpFlags in class LexingCommon

Throws:

IOException

mismatchedRegexpEncodingError

protected void mismatchedRegexpEncodingError(org.jcodings.Encoding optionEncoding,
 org.jcodings.Encoding encoding)

Specified by:

mismatchedRegexpEncodingError in class LexingCommon

asTruth

protected int asTruth(String name,
 ByteList value)

setTokenInfo

protected void setTokenInfo(String name,
 ByteList value)

Specified by:

setTokenInfo in class LexingCommon

setEncoding

protected void setEncoding(ByteList name)

Specified by:

setEncoding in class LexingCommon

getStrTerm

public StrTerm getStrTerm()

setStrTerm

public void setStrTerm(StrTerm strterm)

setWarnings

public void setWarnings(IRubyWarnings warnings)

considerComplex

private int considerComplex(int token,
 int suffix)

getFloatToken

private int getFloatToken(String number,
 int suffix)

getIntegerToken

private int getIntegerToken(String value,
 int radix,
 int suffix)

createStr

public StrNode createStr(ByteList buffer,
 int flags)

parseQuote

private int parseQuote(int c)

What type/kind of quote are we dealing with?

Parameters:

c - first character the the quote construct

Returns:

a token that specifies the quote type

hereDocumentIdentifier

private int hereDocumentIdentifier()

arg_ambiguous

private boolean arg_ambiguous()

printToken

private void printToken(int token)

yylex2

private int yylex2()
            throws IOException

Throws:

IOException

yylex

private int yylex()
           throws IOException

Returns the next token. Also sets yyVal is needed.

Returns:

Description of the Returned Value

Throws:

IOException

identifierToken

private int identifierToken(int result,
 ByteList value)

ampersand

private int ampersand(boolean spaceSeen)

at

private int at()

backtick

private int backtick(boolean commandState)

bang

private int bang()

caret

private int caret()

colon

private int colon(boolean spaceSeen)

comma

private int comma(int c)

doKeyword

private int doKeyword(int state)

dollar

private int dollar()

dot

private int dot()

doubleQuote

private int doubleQuote(boolean commandState)

greaterThan

private int greaterThan()

identifier

private int identifier(int c,
 boolean commandState)

leftBracket

private int leftBracket(boolean spaceSeen)

leftCurly

private int leftCurly()

leftParen

private int leftParen(boolean spaceSeen)

lessThan

private int lessThan(boolean spaceSeen)

minus

private int minus(boolean spaceSeen)

percent

private int percent(boolean spaceSeen)

pipe

private int pipe()

plus

private int plus(boolean spaceSeen)

questionMark

private int questionMark()
                  throws IOException

Throws:

IOException

rightBracket

private int rightBracket()

rightCurly

private int rightCurly()

rightParen

private int rightParen()

singleQuote

private int singleQuote(boolean commandState)

slash

private int slash(boolean spaceSeen)

star

private int star(boolean spaceSeen)

tilde

private int tilde()

parseNumber

private int parseNumber(int c)

Parse a number from the input stream.

Parameters:

c - The first character of the number.

Returns:

An int constant which represents a token.

getNumberToken

private int getNumberToken(String number,
 boolean seen_e,
 boolean seen_point,
 int nondigit)

readUTFEscapeRegexpLiteral

public void readUTFEscapeRegexpLiteral(ByteList buffer)

readUTFEscape

public int readUTFEscape(ByteList buffer,
 boolean stringLiteral,
 boolean[] encodingDetermined)
                  throws IOException

Throws:

IOException

readUTF8EscapeIntoBuffer

private void readUTF8EscapeIntoBuffer(int codepoint,
 ByteList buffer,
 boolean stringLiteral,
 boolean[] encodingDetermined)
                               throws IOException

Throws:

IOException

readEscape

public int readEscape()
               throws IOException

Throws:

IOException

scanHexLiteral

private char scanHexLiteral(ByteList buffer,
 int count,
 boolean strict,
 String errorMessage)

Read up to count hexadecimal digits and store those digits in a token numberBuffer. If strict is provided then count number of hex digits must be present. If no digits can be read a syntax exception will be thrown. This will also return the codepoint as a value so codepoint ranges can be checked.

tokHex

private int tokHex(int count,
 String errorMessage)

scanHex

private int scanHex(int count,
 boolean strict,
 String errorMessage)

Read up to count hexadecimal digits. If strict is provided then count number of hex digits must be present. If no digits can be read a syntax exception will be thrown.