Tokenizer

class Tokenizer

The base class for tokenizers, which convert a source into a sequence of tokens.

DEFINITION

class Tokenizer;

destructor ~Tokenizervirtual

DEFINITION

virtual ~Tokenizer

(

);

method collect_until

DEFINITION

std::vector< Token > collect_until

(

Token::Type type

);

PARAMETER	DESCRIPTION
`type`

RETURNS	DESCRIPTION
`std::vector< Token >`

method collect_until

DEFINITION

std::vector< Token > collect_until

(

const TokenGroup & types

);

PARAMETER	DESCRIPTION
`types`

RETURNS	DESCRIPTION
`std::vector< Token >`

method current_locationconstvirtual

DEFINITION

virtual Location current_location

(

) const

= 0;

RETURNS	DESCRIPTION
`Location`

method endedconst

DEFINITION

bool ended

(

) const;

RETURNS	DESCRIPTION
`bool`

method expect

DEFINITION

void expect

(

const Token & token

);

PARAMETER	DESCRIPTION
`token`

method expect

DEFINITION

Token expect

(

Token::Type type

);

PARAMETER	DESCRIPTION
`type`

RETURNS	DESCRIPTION
`Token`

method expect

DEFINITION

Token expect	(	Token::Type type,
		const TokenGroup & synchronize	);

PARAMETER	DESCRIPTION
`type`
`synchronize`

RETURNS	DESCRIPTION
`Token`

method expect

DEFINITION

Token expect	(	const TokenGroup & types,
		const TokenGroup & synchronize	);

PARAMETER	DESCRIPTION
`types`
`synchronize`

RETURNS	DESCRIPTION
`Token`

method expect_literals

DEFINITION

void expect_literals	(	const std::vector< Token::Type > & types,
		const TokenGroup & synchronize	);

PARAMETER	DESCRIPTION
`types`
`synchronize`

method next

DEFINITION

Token next

(

);

RETURNS	DESCRIPTION
`Token`

method peek

DEFINITION

Token peek

(

);

RETURNS	DESCRIPTION
`Token`

method skip

DEFINITION

void skip

(

Token::Type type

);

PARAMETER	DESCRIPTION
`type`

method skip

DEFINITION

void skip

(

const TokenGroup & types

);

PARAMETER	DESCRIPTION
`types`

method skip_until

DEFINITION

void skip_until	(	const Token & token,
		bool including_terminator = false	);

PARAMETER	DESCRIPTION
`token`
`including_terminator`

method skip_until

DEFINITION

void skip_until	(	const TokenGroup & types,
		bool including_terminator = false	);

PARAMETER	DESCRIPTION
`types`
`including_terminator`

method sub_endedconstvirtual

DEFINITION

virtual bool sub_ended

(

) const

= 0;

RETURNS	DESCRIPTION
`bool`

method sub_nextvirtual

DEFINITION

virtual Token sub_next

(

)

= 0;

RETURNS	DESCRIPTION
`Token`

method unget

DEFINITION

void unget

(

const Token & token

);

PARAMETER	DESCRIPTION
`token`