Package gw.internal.gosu.parser

Class SourceCodeTokenizer

java.lang.Object

gw.internal.gosu.parser.SourceCodeTokenizer

All Implemented Interfaces:

ISourceCodeTokenizer

public final class SourceCodeTokenizer
extends Object
implements ISourceCodeTokenizer

Field Summary

Fields

Modifier and Type	Field	Description
private SourceCodeTokenizerInternal	_internal
private Stack<ITokenizerOffsetMarker>	_offsetMarkers
private int	_state

Fields inherited from interface gw.lang.parser.ISourceCodeTokenizer

TT_COMMENT, TT_EOF, TT_EOL, TT_INTEGER, TT_KEYWORD, TT_NOTHING, TT_NUMBER, TT_OPERATOR, TT_WHITESPACE, TT_WORD

Constructor Summary

Constructors

Modifier	Constructor	Description
private	SourceCodeTokenizer()
private	SourceCodeTokenizer(boolean initForCopy)
	SourceCodeTokenizer(SourceCodeReader reader)
	SourceCodeTokenizer(SourceCodeReader reader, ITokenizerInstructor instructor)
	SourceCodeTokenizer(Reader reader)
	SourceCodeTokenizer(CharSequence sourceCode)

Method Summary

Modifier and Type	Method	Description
void	commentChar(int ch)
IToken	copy()
IToken	copyInto(IToken t)
int	countMatches(String s)
int	countMatches(String s, int tokenType)
void	eolIsSignificant(boolean bFlag)
final Token	getCurrentToken()
ITokenizerInstructor	getInstructor()
SourceCodeTokenizerInternal	getInternal()
int	getLineNumber()
int	getLineOffset()
ITokenizerOffsetMarker	getOffsetMarker()
Token	getPriorToken()
Token	getPriorToken(boolean bSkipWhitespace)
Token	getPriorToken(boolean bSkipWhitespace, boolean bSkipComments)
SourceCodeReader	getReader()
int	getRestoreState()
String	getSource()
int	getState()
String	getStringValue()
String	getTokenAsString()
Token	getTokenAt(int iTokenIndex)
IToken	getTokenAtPosition(int docPosition)
int	getTokenColumn()
int	getTokenEnd()
final Stack<Token>	getTokens()
int	getTokenStart()
int	getType()
(package private) void	goToPosition(int iOffset)
boolean	isAnalyzingDirective()
boolean	isAnalyzingSeparately()
boolean	isCommentsSignificant()
boolean	isEOF()
boolean	isNOTHING()
boolean	isOperator(String strOperator)
boolean	isParseDotsAsOperators()
boolean	isPositioned()
boolean	isSupportsKeywords()
boolean	isUnterminatedComment()
boolean	isUnterminatedString()
boolean	isWhitespaceSignificant()
SourceCodeTokenizer	lightweightRestore()
int	lookaheadType(int iTokens, boolean bSkipSpaces)
void	lowerCaseMode(boolean bLowerCaseMode)
int	mark()
private int	maybeSkipWhitespaceAndComments(int state, Stack<Token> tokens, int count)
void	nextToken()
void	operatorChars(int iLow, int iHigh)
void	operators(String[] astrOperators)
void	ordinaryChar(int ch)
void	ordinaryChars(int iLow, int iHigh)
void	parseNumbers()
DocCommentBlock	popLastComment()
void	popOffsetMarker(ITokenizerOffsetMarker offsetMarker)
void	pushOffsetMarker(ITokenizerOffsetMarker offsetMarker)
void	quoteChar(int ch)
void	reset()
void	reset(SourceCodeReader reader)
private void	reset(SourceCodeReader reader, boolean bResetReader)
void	reset(Reader reader)
void	resetButKeepTokens()
void	restoreToMark(int iMarkedOffset)
void	setCommentsSignificant(boolean bCommentsSignificant)
void	setInstructor(ITokenizerInstructor instructor)
void	setParseDotsAsOperators(boolean parseDotsAsOperators)
void	setSupportsKeywords(boolean supportsKeywords)
void	setWhitespaceSignificant(boolean bWhitespaceSignificant)
private int	skipSpaces(int iPos)
String	toString()
void	whitespaceChars(int iLow, int iHigh)
void	wordChars(int iLow, int iHigh)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Details

_state

private int _state

_internal

private SourceCodeTokenizerInternal _internal

_offsetMarkers

private Stack<ITokenizerOffsetMarker> _offsetMarkers

Constructor Details

SourceCodeTokenizer

private SourceCodeTokenizer()

SourceCodeTokenizer

private SourceCodeTokenizer(boolean initForCopy)

SourceCodeTokenizer

public SourceCodeTokenizer(CharSequence sourceCode)

SourceCodeTokenizer

public SourceCodeTokenizer(Reader reader)

SourceCodeTokenizer

public SourceCodeTokenizer(SourceCodeReader reader)

SourceCodeTokenizer

public SourceCodeTokenizer(SourceCodeReader reader,
 ITokenizerInstructor instructor)

Method Details

copy

public IToken copy()

Specified by:

copy in interface ISourceCodeTokenizer

copyInto

public IToken copyInto(IToken t)

Specified by:

copyInto in interface ISourceCodeTokenizer

getInternal

public SourceCodeTokenizerInternal getInternal()

lightweightRestore

public SourceCodeTokenizer lightweightRestore()

Specified by:

lightweightRestore in interface ISourceCodeTokenizer

Returns:

a very lightweight restored copy of this tokenizer, without deep cloning anything. This method should be used with care, as it does not clone internal state and cannot actually tokenize. It should be used only for offset information.

isPositioned

public boolean isPositioned()

Specified by:

isPositioned in interface ISourceCodeTokenizer

reset

public void reset()

Specified by:

reset in interface ISourceCodeTokenizer

reset

public void reset(Reader reader)

Specified by:

reset in interface ISourceCodeTokenizer

reset

public void reset(SourceCodeReader reader)

Specified by:

reset in interface ISourceCodeTokenizer

reset

private void reset(SourceCodeReader reader,
 boolean bResetReader)

getReader

public SourceCodeReader getReader()

Specified by:

getReader in interface ISourceCodeTokenizer

getSource

public String getSource()

Specified by:

getSource in interface ISourceCodeTokenizer

getInstructor

public ITokenizerInstructor getInstructor()

Specified by:

getInstructor in interface ISourceCodeTokenizer

setInstructor

public void setInstructor(ITokenizerInstructor instructor)

Specified by:

setInstructor in interface ISourceCodeTokenizer

isWhitespaceSignificant

public boolean isWhitespaceSignificant()

Specified by:

isWhitespaceSignificant in interface ISourceCodeTokenizer

setWhitespaceSignificant

public void setWhitespaceSignificant(boolean bWhitespaceSignificant)

Specified by:

setWhitespaceSignificant in interface ISourceCodeTokenizer

isCommentsSignificant

public boolean isCommentsSignificant()

Specified by:

isCommentsSignificant in interface ISourceCodeTokenizer

setCommentsSignificant

public void setCommentsSignificant(boolean bCommentsSignificant)

Specified by:

setCommentsSignificant in interface ISourceCodeTokenizer

getCurrentToken

public final Token getCurrentToken()

Specified by:

getCurrentToken in interface ISourceCodeTokenizer

getTokenAt

public Token getTokenAt(int iTokenIndex)

Specified by:

getTokenAt in interface ISourceCodeTokenizer

getTokenAtPosition

public IToken getTokenAtPosition(int docPosition)

Specified by:

getTokenAtPosition in interface ISourceCodeTokenizer

getLineNumber

public int getLineNumber()

Specified by:

getLineNumber in interface ISourceCodeTokenizer

getLineOffset

public int getLineOffset()

Specified by:

getLineOffset in interface ISourceCodeTokenizer

getTokenColumn

public int getTokenColumn()

Specified by:

getTokenColumn in interface ISourceCodeTokenizer

wordChars

public void wordChars(int iLow,
 int iHigh)

Specified by:

wordChars in interface ISourceCodeTokenizer

whitespaceChars

public void whitespaceChars(int iLow,
 int iHigh)

Specified by:

whitespaceChars in interface ISourceCodeTokenizer

ordinaryChars

public void ordinaryChars(int iLow,
 int iHigh)

Specified by:

ordinaryChars in interface ISourceCodeTokenizer

ordinaryChar

public void ordinaryChar(int ch)

Specified by:

ordinaryChar in interface ISourceCodeTokenizer

operators

public void operators(String[] astrOperators)

Specified by:

operators in interface ISourceCodeTokenizer

operatorChars

public void operatorChars(int iLow,
 int iHigh)

Specified by:

operatorChars in interface ISourceCodeTokenizer

isOperator

public boolean isOperator(String strOperator)

Specified by:

isOperator in interface ISourceCodeTokenizer

commentChar

public void commentChar(int ch)

Specified by:

commentChar in interface ISourceCodeTokenizer

quoteChar

public void quoteChar(int ch)

Specified by:

quoteChar in interface ISourceCodeTokenizer

parseNumbers

public void parseNumbers()

Specified by:

parseNumbers in interface ISourceCodeTokenizer

eolIsSignificant

public void eolIsSignificant(boolean bFlag)

Specified by:

eolIsSignificant in interface ISourceCodeTokenizer

lowerCaseMode

public void lowerCaseMode(boolean bLowerCaseMode)

Specified by:

lowerCaseMode in interface ISourceCodeTokenizer

isUnterminatedString

public boolean isUnterminatedString()

Specified by:

isUnterminatedString in interface ISourceCodeTokenizer

isUnterminatedComment

public boolean isUnterminatedComment()

Specified by:

isUnterminatedComment in interface ISourceCodeTokenizer

setParseDotsAsOperators

public void setParseDotsAsOperators(boolean parseDotsAsOperators)

Specified by:

setParseDotsAsOperators in interface ISourceCodeTokenizer

isParseDotsAsOperators

public boolean isParseDotsAsOperators()

Specified by:

isParseDotsAsOperators in interface ISourceCodeTokenizer

popLastComment

public DocCommentBlock popLastComment()

getTokenStart

public int getTokenStart()

Specified by:

getTokenStart in interface ISourceCodeTokenizer

getTokenEnd

public int getTokenEnd()

Specified by:

getTokenEnd in interface ISourceCodeTokenizer

getTokenAsString

public String getTokenAsString()

Specified by:

getTokenAsString in interface ISourceCodeTokenizer

mark

public int mark()

Specified by:

mark in interface ISourceCodeTokenizer

restoreToMark

public void restoreToMark(int iMarkedOffset)

Specified by:

restoreToMark in interface ISourceCodeTokenizer

getRestoreState

public int getRestoreState()

Specified by:

getRestoreState in interface ISourceCodeTokenizer

getTokens

public final Stack<Token> getTokens()

Specified by:

getTokens in interface ISourceCodeTokenizer

nextToken

public void nextToken()

Specified by:

nextToken in interface ISourceCodeTokenizer

maybeSkipWhitespaceAndComments

private int maybeSkipWhitespaceAndComments(int state,
 Stack<Token> tokens,
 int count)

getStringValue

public String getStringValue()

Specified by:

getStringValue in interface ISourceCodeTokenizer

toString

public String toString()

Overrides:

toString in class Object

getOffsetMarker

public ITokenizerOffsetMarker getOffsetMarker()

pushOffsetMarker

public void pushOffsetMarker(ITokenizerOffsetMarker offsetMarker)

Specified by:

pushOffsetMarker in interface ISourceCodeTokenizer

popOffsetMarker

public void popOffsetMarker(ITokenizerOffsetMarker offsetMarker)

Specified by:

popOffsetMarker in interface ISourceCodeTokenizer

getState

public int getState()

Specified by:

getState in interface ISourceCodeTokenizer

countMatches

public int countMatches(String s)

Specified by:

countMatches in interface ISourceCodeTokenizer

countMatches

public int countMatches(String s,
 int tokenType)

Specified by:

countMatches in interface ISourceCodeTokenizer

getType

public int getType()

Specified by:

getType in interface ISourceCodeTokenizer

isEOF

public boolean isEOF()

Specified by:

isEOF in interface ISourceCodeTokenizer

isNOTHING

public boolean isNOTHING()

goToPosition

void goToPosition(int iOffset)
           throws IOException

Throws:

IOException

resetButKeepTokens

public void resetButKeepTokens()

isAnalyzingSeparately

public boolean isAnalyzingSeparately()

Specified by:

isAnalyzingSeparately in interface ISourceCodeTokenizer

isAnalyzingDirective

public boolean isAnalyzingDirective()

Specified by:

isAnalyzingDirective in interface ISourceCodeTokenizer

isSupportsKeywords

public boolean isSupportsKeywords()

Specified by:

isSupportsKeywords in interface ISourceCodeTokenizer

setSupportsKeywords

public void setSupportsKeywords(boolean supportsKeywords)

Specified by:

setSupportsKeywords in interface ISourceCodeTokenizer

getPriorToken

public Token getPriorToken()

getPriorToken

public Token getPriorToken(boolean bSkipWhitespace)

getPriorToken

public Token getPriorToken(boolean bSkipWhitespace,
 boolean bSkipComments)

lookaheadType

public int lookaheadType(int iTokens,
 boolean bSkipSpaces)

skipSpaces

private int skipSpaces(int iPos)