FileTokenizer

class FileTokenizer

This class gets tokens from a file.

DEFINITION

class FileTokenizer;

constructor FileTokenizer

DEFINITION

FileTokenizer	(	const SearchPath & search_path = SearchPath::empty_path,
		const Target * target = {},
		bool use_preprocessor = true,
		const std::unordered_set< Symbol > & defines = {}	);

PARAMETER	DESCRIPTION
`search_path`
`target`
`use_preprocessor`
`defines`

method add_literal

DEFINITION

void add_literal	(	Token::Type match,
		const std::string & name,
		const std::string & suffix_characters = ""	);

PARAMETER	DESCRIPTION
`match`
`name`
`suffix_characters`

method add_literal

DEFINITION

void add_literal

(

const Token & token

);

PARAMETER	DESCRIPTION
`token`

method add_punctuations

DEFINITION

void add_punctuations

(

const std::unordered_set< std::string > & names

);

PARAMETER	DESCRIPTION
`names`

method current_fileconst

DEFINITION

Symbol current_file

(

) const;

RETURNS	DESCRIPTION
`Symbol`

method current_locationconstvirtual

DEFINITION

virtual Location current_location

(

) const;

RETURNS	DESCRIPTION
`Location`

method define

DEFINITION

void define

(

const std::unordered_set< Symbol > & defines

);

PARAMETER	DESCRIPTION
`defines`

method define

DEFINITION

void define

(

Symbol name

);

PARAMETER	DESCRIPTION
`name`

method find_file

DEFINITION

Symbol find_file

(

Symbol file_name

);

PARAMETER	DESCRIPTION
`file_name`

RETURNS	DESCRIPTION
`Symbol`

method push

DEFINITION

void push

(

Symbol filename

);

PARAMETER	DESCRIPTION
`filename`

method set_target

DEFINITION

void set_target

(

const Target * new_target

);

PARAMETER	DESCRIPTION
`new_target`

method sub_endedconstvirtual

DEFINITION

virtual bool sub_ended

(

) const;

RETURNS	DESCRIPTION
`bool`

method sub_nextvirtual

DEFINITION

virtual Token sub_next

(

);

RETURNS	DESCRIPTION
`Token`

method undefine

DEFINITION

void undefine

(

Symbol name

);

PARAMETER	DESCRIPTION
`name`

member defines

DEFINITION

std::unordered_set< Symbol > defines;