Tokenizer Class Reference

#include <Tokenizer.h>

Public Member Functions
	Tokenizer (TokenizerContext *context, const String &text)
	Tokenizer (TokenizerContext *context)
	Tokenizer (const String &text)
	Tokenizer ()
virtual	~Tokenizer ()
void	init ()
void	setContext (TokenizerContext *context)
TokenizerContext *	getContext ()
void	setText (const String &text)
void	setText (String &text)
void	setCaseSensitive (bool caseSensitive)
bool	isCaseSensitive ()
void	setGetCharLits (bool getCharLits)
bool	isGetCharLits ()
void	setGetStrings (bool getStrings)
bool	isGetStrings ()
void	setAllowSingleQuotes (bool allowSingleQuotes)
bool	isAllowSingleQuotes ()
void	setGetUnknown (bool getUnknown)
bool	isGetUnknown ()
void	setGetCRLF (bool getCRLF)
bool	isGetCRLF ()
void	setGetSpaces (bool getSpaces)
bool	isGetSpaces ()
void	setAllowNumCommas (bool allowNumCommas)
bool	isAllowNumCommas ()
void	resetScan ()
Integer *	findKeyword (String *keyword)
Integer *	findKeyword (String keyword)
bool	addKeyword (String *keyword, int code)
Integer *	findSingleOp (Char *singleOp)
Integer *	findSingleOp (Char singleOp)
bool	addSingleOp (Char *singleOp, int code)
Integer *	findMultiOp (String *multiOp)
Integer *	findMultiOp (String multiOp)
bool	addMultiOp (String *multiOp, int code)
bool	findIdentChar (Char *char1)
bool	findIdentChar (Char char1)
void	addIdentChars (const String &multiChars)
void	clearIdentChars ()
void	setIdentChars (const String &multiChars)
bool	hasNextChar (int index=1)
char	peekNextChar (int index=1)
char	peekCurrChar ()
bool	isEOS ()
bool	fetchToken (Token &token)
int	getCurrPosition ()
void	setIndentHere ()
void	setIndentPosition (int indentPos)
int	getIndentPosition ()
bool	hasIndented (int indentPos)
bool	hasOutdented (int indentPos)
void	resetIndent ()
Token *	getCurrToken ()
String	getTokenLineSegment ()
bool	isError ()
ErrorContext *	getErrorContext ()
void	setErrorContext (ErrorContext *errorContext)
String	getErrorLineSegment ()
int	getErrorCount ()
TokenizerState *	saveState ()
Token *	restoreState (TokenizerState *state)
void	deleteState (TokenizerState *state)
void	restartFromToken (Token &token)
void	error (String msg)

Constructor & Destructor Documentation

Tokenizer() [1/4]

Tokenizer::Tokenizer	(	TokenizerContext *	context,
		const String &	text
	)

Tokenizer() [2/4]

Tokenizer::Tokenizer ( TokenizerContext *  context )

explicit

Tokenizer() [3/4]

Tokenizer::Tokenizer ( const String &  text )

explicit

Tokenizer() [4/4]

Tokenizer::Tokenizer

(

)

~Tokenizer()

Tokenizer::~Tokenizer ( )

virtual

Member Function Documentation

addIdentChars()

void Tokenizer::addIdentChars

(

const String &

multiChars

)

addKeyword()

bool Tokenizer::addKeyword	(	String *	keyword,
		int	code
	)

addMultiOp()

bool Tokenizer::addMultiOp	(	String *	multiOp,
		int	code
	)

addSingleOp()

bool Tokenizer::addSingleOp	(	Char *	singleOp,
		int	code
	)

clearIdentChars()

void Tokenizer::clearIdentChars

(

)

deleteState()

void Tokenizer::deleteState

(

TokenizerState *

state

)

error()

void Tokenizer::error

(

String

msg

)

fetchToken()

bool Tokenizer::fetchToken

(

Token &

token

)

findIdentChar() [1/2]

bool Tokenizer::findIdentChar

(

Char *

char1

)

findIdentChar() [2/2]

bool Tokenizer::findIdentChar

(

Char

char1

)

findKeyword() [1/2]

Integer * Tokenizer::findKeyword

(

String *

keyword

)

findKeyword() [2/2]

Integer * Tokenizer::findKeyword

(

String

keyword

)

findMultiOp() [1/2]

Integer * Tokenizer::findMultiOp

(

String *

multiOp

)

findMultiOp() [2/2]

Integer * Tokenizer::findMultiOp

(

String

multiOp

)

findSingleOp() [1/2]

Integer * Tokenizer::findSingleOp

(

Char *

singleOp

)

findSingleOp() [2/2]

Integer * Tokenizer::findSingleOp

(

Char

singleOp

)

getContext()

TokenizerContext * Tokenizer::getContext

(

)

getCurrPosition()

int Tokenizer::getCurrPosition

(

)

getCurrToken()

Token * Tokenizer::getCurrToken

(

)

getErrorContext()

ErrorContext * Tokenizer::getErrorContext

(

)

getErrorCount()

int Tokenizer::getErrorCount

(

)

getErrorLineSegment()

String Tokenizer::getErrorLineSegment

(

)

getIndentPosition()

int Tokenizer::getIndentPosition

(

)

getTokenLineSegment()

String Tokenizer::getTokenLineSegment

(

)

hasIndented()

bool Tokenizer::hasIndented

(

int

indentPos

)

hasNextChar()

bool Tokenizer::hasNextChar

(

int

index = 1

)

hasOutdented()

bool Tokenizer::hasOutdented

(

int

indentPos

)

init()

void Tokenizer::init

(

)

isAllowNumCommas()

bool Tokenizer::isAllowNumCommas

(

)

isAllowSingleQuotes()

bool Tokenizer::isAllowSingleQuotes

(

)

isCaseSensitive()

bool Tokenizer::isCaseSensitive

(

)

isEOS()

bool Tokenizer::isEOS

(

)

isError()

bool Tokenizer::isError

(

)

isGetCharLits()

bool Tokenizer::isGetCharLits

(

)

isGetCRLF()

bool Tokenizer::isGetCRLF

(

)

isGetSpaces()

bool Tokenizer::isGetSpaces

(

)

isGetStrings()

bool Tokenizer::isGetStrings

(

)

isGetUnknown()

bool Tokenizer::isGetUnknown

(

)

peekCurrChar()

char Tokenizer::peekCurrChar

(

)

peekNextChar()

char Tokenizer::peekNextChar

(

int

index = 1

)

resetIndent()

void Tokenizer::resetIndent

(

)

resetScan()

void Tokenizer::resetScan

(

)

restartFromToken()

void Tokenizer::restartFromToken

(

Token &

token

)

restoreState()

Token * Tokenizer::restoreState

(

TokenizerState *

state

)

saveState()

TokenizerState * Tokenizer::saveState

(

)

setAllowNumCommas()

void Tokenizer::setAllowNumCommas

(

bool

allowNumCommas

)

setAllowSingleQuotes()

void Tokenizer::setAllowSingleQuotes

(

bool

allowSingleQuotes

)

setCaseSensitive()

void Tokenizer::setCaseSensitive

(

bool

caseSensitive

)

setContext()

void Tokenizer::setContext

(

TokenizerContext *

context

)

setErrorContext()

void Tokenizer::setErrorContext

(

ErrorContext *

errorContext

)

setGetCharLits()

void Tokenizer::setGetCharLits

(

bool

getCharLits

)

setGetCRLF()

void Tokenizer::setGetCRLF

(

bool

getCRLF

)

setGetSpaces()

void Tokenizer::setGetSpaces

(

bool

getSpaces

)

setGetStrings()

void Tokenizer::setGetStrings

(

bool

getStrings

)

setGetUnknown()

void Tokenizer::setGetUnknown

(

bool

getUnknown

)

setIdentChars()

void Tokenizer::setIdentChars

(

const String &

multiChars

)

setIndentHere()

void Tokenizer::setIndentHere

(

)

setIndentPosition()

void Tokenizer::setIndentPosition

(

int

indentPos

)

setText() [1/2]

void Tokenizer::setText

(

const String &

text

)

setText() [2/2]

void Tokenizer::setText

(

String &

text

)

---

The documentation for this class was generated from the following files:

• /github/workspace/src/Utilities/Parser/Tokenizer.h
• /github/workspace/src/Utilities/Parser/Tokenizer.cpp