Package org.jruby.ext.ripper

Class RipperLexer

java.lang.Object

org.jruby.lexer.LexingCommon

org.jruby.ext.ripper.RipperLexer

public class RipperLexer
extends LexingCommon

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
static enum	RipperLexer.Keyword

Field Summary

Fields

Modifier and Type	Field	Description
private boolean	cr_seen
protected ByteList	delayed
private int	delayed_col
private int	delayed_line
(package private) String	identValue
private StrTerm	lex_strterm
private static final HashMap<String,RipperLexer.Keyword>	map
private ByteList	numberBuffer
private RipperParserBase	parser
(package private) static final int	STR_FUNC_INDENT

Fields inherited from class org.jruby.lexer.LexingCommon

__end__seen, AMPERSAND, AMPERSAND_AMPERSAND, AMPERSAND_DOT, ASCII8BIT_ENCODING, BACKSLASH, BACKTICK, BANG, BANG_EQ, BANG_TILDE, BEGIN_DOC_MARKER, braceNest, CALL, CARET, cmdArgumentState, CODING, COLON, COLON_COLON, COMMA, commandStart, conditionState, DOLLAR_BANG, DOLLAR_DOT, DOLLAR_UNDERSCORE, DOT, DOT_DOT, DOT_DOT_DOT, END_DOC_MARKER, END_MARKER, EOF, eofp, EQ, EQ_EQ, EQ_EQ_EQ, EQ_GT, EQ_TILDE, EXPR_ARG, EXPR_ARG_ANY, EXPR_BEG, EXPR_BEG_ANY, EXPR_CLASS, EXPR_CMDARG, EXPR_DOT, EXPR_END, EXPR_END_ANY, EXPR_ENDARG, EXPR_ENDFN, EXPR_FITEM, EXPR_FNAME, EXPR_LABEL, EXPR_LABELED, EXPR_MID, EXPR_VALUE, GT, GT_EQ, GT_GT, has_shebang, heredoc_end, heredoc_indent, heredoc_line_indent, inKwarg, last_cr_line, last_state, LBRACKET, LBRACKET_RBRACKET, LBRACKET_RBRACKET_EQ, LCURLY, lex_lastline, lex_nextline, lex_p, lex_pbeg, lex_pend, lex_state, lexb, line_count, line_offset, LT, LT_EQ, LT_EQ_RT, LT_LT, MINUS, MINUS_AT, MINUS_GT, OR, OR_OR, parenNest, PERCENT, PLUS, PLUS_AT, Q, QQ, QUESTION, RBRACKET, RCURLY, RPAREN, ruby_sourceline, SEMICOLON, SLASH, src, STAR, STAR_STAR, str_dquote, str_dsym, str_dword, STR_FUNC_ESCAPE, STR_FUNC_EXPAND, STR_FUNC_LABEL, STR_FUNC_LIST, STR_FUNC_QWORDS, STR_FUNC_REGEXP, STR_FUNC_SYMBOL, STR_FUNC_TERM, str_label, str_regexp, str_squote, str_ssym, str_sword, str_xquote, SUFFIX_ALL, SUFFIX_I, SUFFIX_R, TAB_WIDTH, TILDE, token, tokenSeen, tokline, tokp, USASCII_ENCODING, UTF8_ENCODING, yaccValue

Constructor Summary

Constructors

Constructor	Description
RipperLexer(RipperParserBase parser, LexerSource src)

Method Summary

Modifier and Type	Method	Description
void	addDelayedToken(int tok, int end)
protected void	ambiguousOperator(String op, String syn)
private int	ampersand(boolean spaceSeen)
private boolean	arg_ambiguous()
private int	at()
private int	backtick(boolean commandState)
private int	bang()
private int	caret()
private int	colon(boolean spaceSeen)
private int	comma(int c)
void	compile_error(String message)
private int	considerComplex(int token, int suffix)
private int	cr(int c)
IRubyObject	createStr(ByteList buffer, int flags)
void	dispatchDelayedToken(int token)
void	dispatchHeredocEnd()
void	dispatchIgnoredScanEvent(int token)
void	dispatchScanEvent(int token)
private int	doKeyword(int state)
private int	dollar()
private int	dot()
private int	doubleQuote(boolean commandState)
protected void	flush_string_content(org.jcodings.Encoding encoding)
private int	getFloatToken(String number, int suffix)
String	getIdent()
static RipperLexer.Keyword	getKeyword(String str)
private int	getNumberLiteral(String number, boolean seen_e, boolean seen_point, int nondigit)
Ruby	getRuntime()
StrTerm	getStrTerm()
private int	greaterThan()
boolean	hasScanEvent()
boolean	hasStarted()	Has lexing started yet?
void	heredoc_restore(HeredocTerm here)
private int	hereDocumentIdentifier()
private int	identifier(int c, boolean commandState)
private int	identifierToken(int last_state, int result, String value)
boolean	isVerbose()
private int	leftBracket(boolean spaceSeen)
private int	leftCurly()
private int	leftParen(boolean spaceSeen)
private int	lessThan(boolean spaceSeen)
private int	minus(boolean spaceSeen)
protected void	mismatchedRegexpEncodingError(org.jcodings.Encoding optionEncoding, org.jcodings.Encoding encoding)
int	nextc()
private boolean	nextLine()
int	nextToken()
protected boolean	onMagicComment(String name, ByteList value)
private int	parseNumber(int c)	Parse a number from the input stream.
private int	parseQuote(int c)	What type/kind of quote are we dealing with?
protected RegexpOptions	parseRegexpFlags()
private int	percent(boolean spaceSeen)
private int	pipe()
private int	plus(boolean spaceSeen)
private void	printToken(int token)
private int	questionMark()
int	readEscape()
private void	readUTF8EscapeIntoBuffer(int codepoint, ByteList buffer, boolean stringLiteral)
int	readUTFEscape(ByteList buffer, boolean stringLiteral, boolean symbolLiteral)
void	readUTFEscapeRegexpLiteral(ByteList buffer)
private int	rightBracket()
private int	rightCurly()
private int	rightParen()
private IRubyObject	scanEventValue(int token)
private int	scanHex(int count, boolean strict, String errorMessage)	Read up to count hexadecimal digits.
private char	scanHexLiteral(ByteList buffer, int count, boolean strict, String errorMessage)	Read up to count hexadecimal digits and store those digits in a token buffer.
protected void	setCompileOptionFlag(String name, ByteList value)
protected void	setEncoding(ByteList name)
private int	setIntegerLiteral(int suffix)
private int	setNumberLiteral(int type, int suffix)
void	setParser(RipperParserBase parserSupport)	Parse must pass its support object for some check at bottom of yylex().
void	setStrTerm(StrTerm strterm)
protected void	setTokenInfo(String name, ByteList value)
private int	singleQuote(boolean commandState)
private int	slash(boolean spaceSeen)
private int	star(boolean spaceSeen)
private int	tilde()
boolean	tokenAddMBC(int first_byte, ByteList buffer)
int	tokenize_ident(int result)
private String	tokenToEventId(int token)
void	warn(String message)
void	warning(String fmt)
void	warning(String fmt, String arg)
private int	yylex()	Returns the next token.
private int	yylex2()

Methods inherited from class org.jruby.lexer.LexingCommon

checkRegexpFragment, checkRegexpSyntax, column, comment_at_top, createAsEncodedString, createTokenByteList, createTokenByteList, createTokenString, createTokenString, dedent_string, flush, getBraceNest, getCmdArgumentState, getConditionState, getCurrentArg, getCurrentLine, getEncoding, getFile, getHeredocIndent, getHeredocLineIndent, getLeftParenBegin, getLineOffset, getRubySourceline, getState, getTokenCR, incrementParenNest, isAfterOperator, isARG, isASCII, isASCII, isBEG, isEND, isEndSeen, isGlobalCharPunct, isHexChar, isIdentifierChar, isLabelPossible, isLabelSuffix, isLexState, isLexStateAll, isNext_identchar, isOctChar, isSpace, isSpaceArg, lex_goto_eol, lineno, magicCommentEncoding, magicCommentMarker, newtok, numberLiteralSuffix, optionsEncodingChar, p, parser_magic_comment, parser_prepare, parseRegexpFlags, peek, peek, peekVariableName, precise_mbclen, printState, pushback, reset, resetStacks, scanOct, set_file_encoding, setBraceNest, setCurrentArg, setCurrentEncoding, setEncoding, setHeredocIndent, setHeredocLineIndent, setLeftParenBegin, setRegexpEncoding, setRubySourceline, setSource, setState, setValue, strncmp, tokAdd, tokadd_ident, tokadd_mbchar, tokadd_mbchar, tokaddmbc, tokCopy, token, update_heredoc_indent, validateFormalIdentifier, validateFormalIdentifier, value, warn_balanced, was_bol, whole_match_p

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

map

private static final HashMap<String,RipperLexer.Keyword> map

identValue

String identValue

parser

private RipperParserBase parser

lex_strterm

private StrTerm lex_strterm

STR_FUNC_INDENT

static final int STR_FUNC_INDENT

See Also:

• Constant Field Values

delayed

protected ByteList delayed

delayed_line

private int delayed_line

delayed_col

private int delayed_col

cr_seen

private boolean cr_seen

numberBuffer

private ByteList numberBuffer

Constructor Details

RipperLexer

public RipperLexer(RipperParserBase parser,
 LexerSource src)

Method Details

ambiguousOperator

protected void ambiguousOperator(String op,
 String syn)

Specified by:

ambiguousOperator in class LexingCommon

onMagicComment

protected boolean onMagicComment(String name,
 ByteList value)

Overrides:

onMagicComment in class LexingCommon

getFloatToken

private int getFloatToken(String number,
 int suffix)

considerComplex

private int considerComplex(int token,
 int suffix)

isVerbose

public boolean isVerbose()

warn

public void warn(String message)

warning

public void warning(String fmt)

warning

public void warning(String fmt,
 String arg)

getKeyword

public static RipperLexer.Keyword getKeyword(String str)

hasStarted

public boolean hasStarted()

Has lexing started yet?

flush_string_content

protected void flush_string_content(org.jcodings.Encoding encoding)

addDelayedToken

public void addDelayedToken(int tok,
 int end)

nextLine

private boolean nextLine()

cr

private int cr(int c)

nextc

public int nextc()

Specified by:

nextc in class LexingCommon

dispatchHeredocEnd

public void dispatchHeredocEnd()

compile_error

public void compile_error(String message)

Specified by:

compile_error in class LexingCommon

tokenize_ident

public int tokenize_ident(int result)

Specified by:

tokenize_ident in class LexingCommon

heredoc_restore

public void heredoc_restore(HeredocTerm here)

nextToken

public int nextToken()
              throws IOException

Throws:

IOException

getIdent

public String getIdent()

getRuntime

public Ruby getRuntime()

setParser

public void setParser(RipperParserBase parserSupport)

Parse must pass its support object for some check at bottom of yylex(). Ruby does it this way as well (i.e. a little parsing logic in the lexer).

Parameters:

parserSupport -

setCompileOptionFlag

protected void setCompileOptionFlag(String name,
 ByteList value)

Specified by:

setCompileOptionFlag in class LexingCommon

parseRegexpFlags

protected RegexpOptions parseRegexpFlags()
                                  throws IOException

Specified by:

parseRegexpFlags in class LexingCommon

Throws:

IOException

mismatchedRegexpEncodingError

protected void mismatchedRegexpEncodingError(org.jcodings.Encoding optionEncoding,
 org.jcodings.Encoding encoding)

Specified by:

mismatchedRegexpEncodingError in class LexingCommon

setTokenInfo

protected void setTokenInfo(String name,
 ByteList value)

Specified by:

setTokenInfo in class LexingCommon

setEncoding

protected void setEncoding(ByteList name)

Specified by:

setEncoding in class LexingCommon

getStrTerm

public StrTerm getStrTerm()

setStrTerm

public void setStrTerm(StrTerm strterm)

createStr

public IRubyObject createStr(ByteList buffer,
 int flags)

parseQuote

private int parseQuote(int c)
                throws IOException

What type/kind of quote are we dealing with?

Parameters:

c - first character the the quote construct

Returns:

a token that specifies the quote type

Throws:

IOException

hereDocumentIdentifier

private int hereDocumentIdentifier()
                            throws IOException

Throws:

IOException

arg_ambiguous

private boolean arg_ambiguous()

printToken

private void printToken(int token)

hasScanEvent

public boolean hasScanEvent()

dispatchDelayedToken

public void dispatchDelayedToken(int token)

dispatchIgnoredScanEvent

public void dispatchIgnoredScanEvent(int token)

dispatchScanEvent

public void dispatchScanEvent(int token)

scanEventValue

private IRubyObject scanEventValue(int token)

tokenToEventId

private String tokenToEventId(int token)

yylex2

private int yylex2()
            throws IOException

Throws:

IOException

yylex

private int yylex()
           throws IOException

Returns the next token. Also sets yyVal as needed.

Returns:

the next token

Throws:

IOException

identifierToken

private int identifierToken(int last_state,
 int result,
 String value)

ampersand

private int ampersand(boolean spaceSeen)
               throws IOException

Throws:

IOException

at

private int at()
        throws IOException

Throws:

IOException

backtick

private int backtick(boolean commandState)
              throws IOException

Throws:

IOException

bang

private int bang()
          throws IOException

Throws:

IOException

caret

private int caret()
           throws IOException

Throws:

IOException

colon

private int colon(boolean spaceSeen)
           throws IOException

Throws:

IOException

comma

private int comma(int c)
           throws IOException

Throws:

IOException

doKeyword

private int doKeyword(int state)

dollar

private int dollar()
            throws IOException

Throws:

IOException

dot

private int dot()
         throws IOException

Throws:

IOException

doubleQuote

private int doubleQuote(boolean commandState)
                 throws IOException

Throws:

IOException

greaterThan

private int greaterThan()
                 throws IOException

Throws:

IOException

identifier

private int identifier(int c,
 boolean commandState)
                throws IOException

Throws:

IOException

leftBracket

private int leftBracket(boolean spaceSeen)
                 throws IOException

Throws:

IOException

leftCurly

private int leftCurly()

leftParen

private int leftParen(boolean spaceSeen)
               throws IOException

Throws:

IOException

lessThan

private int lessThan(boolean spaceSeen)
              throws IOException

Throws:

IOException

minus

private int minus(boolean spaceSeen)
           throws IOException

Throws:

IOException

percent

private int percent(boolean spaceSeen)
             throws IOException

Throws:

IOException

pipe

private int pipe()
          throws IOException

Throws:

IOException

plus

private int plus(boolean spaceSeen)
          throws IOException

Throws:

IOException

questionMark

private int questionMark()
                  throws IOException

Throws:

IOException

rightBracket

private int rightBracket()

rightCurly

private int rightCurly()

rightParen

private int rightParen()

singleQuote

private int singleQuote(boolean commandState)
                 throws IOException

Throws:

IOException

slash

private int slash(boolean spaceSeen)
           throws IOException

Throws:

IOException

star

private int star(boolean spaceSeen)
          throws IOException

Throws:

IOException

tilde

private int tilde()
           throws IOException

Throws:

IOException

parseNumber

private int parseNumber(int c)
                 throws IOException

Parse a number from the input stream.

Parameters:

c - The first character of the number.

Returns:

A int constant which represents a token.

Throws:

IOException

getNumberLiteral

private int getNumberLiteral(String number,
 boolean seen_e,
 boolean seen_point,
 int nondigit)
                      throws IOException

Throws:

IOException

setNumberLiteral

private int setNumberLiteral(int type,
 int suffix)

setIntegerLiteral

private int setIntegerLiteral(int suffix)

readUTFEscapeRegexpLiteral

public void readUTFEscapeRegexpLiteral(ByteList buffer)
                                throws IOException

Throws:

IOException

tokenAddMBC

public boolean tokenAddMBC(int first_byte,
 ByteList buffer)

readUTFEscape

public int readUTFEscape(ByteList buffer,
 boolean stringLiteral,
 boolean symbolLiteral)
                  throws IOException

Throws:

IOException

readUTF8EscapeIntoBuffer

private void readUTF8EscapeIntoBuffer(int codepoint,
 ByteList buffer,
 boolean stringLiteral)

readEscape

public int readEscape()
               throws IOException

Throws:

IOException

scanHexLiteral

private char scanHexLiteral(ByteList buffer,
 int count,
 boolean strict,
 String errorMessage)
                     throws IOException

Read up to count hexadecimal digits and store those digits in a token buffer. If strict is provided then count number of hex digits must be present. If no digits can be read a syntax exception will be thrown. This will also return the codepoint as a value so codepoint ranges can be checked.

Throws:

IOException

scanHex

private int scanHex(int count,
 boolean strict,
 String errorMessage)
             throws IOException

Read up to count hexadecimal digits. If strict is provided then count number of hex digits must be present. If no digits can be read a syntax exception will be thrown.

Throws:

IOException