cpp.tokenize Namespace Reference

Classes
class	Token

Functions
	_GetString (source, start, i)
	_GetChar (source, start, i)
	GetTokens (source)
	main (argv)

Variables
str	__author__ = 'nnorwitz@google.com (Neal Norwitz)'
str	_letters = 'abcdefghijklmnopqrstuvwxyz'
	VALID_IDENTIFIER_CHARS = set(_letters + _letters.upper() + '_0123456789$')
	HEX_DIGITS = set('0123456789abcdefABCDEF')
	INT_OR_FLOAT_DIGITS = set('01234567890eE-+')
	_STR_PREFIXES = set(('R', 'u8', 'u8R', 'u', 'uR', 'U', 'UR', 'L', 'LR'))
str	UNKNOWN = 'UNKNOWN'
str	SYNTAX = 'SYNTAX'
str	CONSTANT = 'CONSTANT'
str	NAME = 'NAME'
str	PREPROCESSOR = 'PREPROCESSOR'
	WHENCE_STREAM
	WHENCE_QUEUE

Detailed Description

Tokenize C++ source code.

Function Documentation

_GetChar()

cpp.tokenize._GetChar ( source, start, i )

protected

_GetString()

cpp.tokenize._GetString ( source, start, i )

protected

GetTokens()

cpp.tokenize.GetTokens

(

source

)

Returns a sequence of Tokens.

Args:
  source: string of C++ source code.

Yields:
  Token that represents the next token in the source.

main()

cpp.tokenize.main

(

argv

)

Driver mostly for testing purposes.

Variable Documentation

__author__

str cpp.tokenize.__author__ = 'nnorwitz@google.com (Neal Norwitz)'

private

_letters

str cpp.tokenize._letters = 'abcdefghijklmnopqrstuvwxyz'

protected

_STR_PREFIXES

cpp.tokenize._STR_PREFIXES = set(('R', 'u8', 'u8R', 'u', 'uR', 'U', 'UR', 'L', 'LR'))

protected

CONSTANT

str cpp.tokenize.CONSTANT = 'CONSTANT'

HEX_DIGITS

cpp.tokenize.HEX_DIGITS = set('0123456789abcdefABCDEF')

INT_OR_FLOAT_DIGITS

cpp.tokenize.INT_OR_FLOAT_DIGITS = set('01234567890eE-+')

NAME

str cpp.tokenize.NAME = 'NAME'

PREPROCESSOR

str cpp.tokenize.PREPROCESSOR = 'PREPROCESSOR'

SYNTAX

str cpp.tokenize.SYNTAX = 'SYNTAX'

UNKNOWN

str cpp.tokenize.UNKNOWN = 'UNKNOWN'

VALID_IDENTIFIER_CHARS

cpp.tokenize.VALID_IDENTIFIER_CHARS = set(_letters + _letters.upper() + '_0123456789$')

WHENCE_QUEUE

cpp.tokenize.WHENCE_QUEUE

WHENCE_STREAM

cpp.tokenize.WHENCE_STREAM