FileTokenizer

INCLUDE FILE

#include "FileTokenizer.h"

class FileTokenizer

This class gets tokens from a file.

DEFINITION

            class FileTokenizer;
        

class MatcherNodeprivate

DEFINITION

            class MatcherNode;
        

method add

DEFINITION

void add	(	const char * string,
		Token::Type type,
		const std::unordered_set< char > & suffix_characters = {},
		bool match_in_word = false	);

PARAMETER	DESCRIPTION
`string`
`type`
`suffix_characters`
`match_in_word`

method conflictsconst private

DEFINITION

bool conflicts

(

const std::unordered_set< char > & new_suffix

) const;

PARAMETER	DESCRIPTION
`new_suffix`

RETURNS	DESCRIPTION
`bool`

method match

DEFINITION

std::optional< Token::Type > match	(	FileSource & source,
		std::string & name	);

PARAMETER	DESCRIPTION
`source`
`name`

RETURNS	DESCRIPTION
`std::optional< Token::Type >`

member match_in_word

DEFINITION

bool match_in_word;

member match_type

DEFINITION

std::optional< Token::Type > match_type;

member next

DEFINITION

std::unordered_map< char, std::unique_ptr< MatcherNode > > next;

member suffix_characters

DEFINITION

std::unordered_set< char > suffix_characters;

class PreStateprivate

DEFINITION

            class PreState;
        

constructor PreState

DEFINITION

PreState

(

bool condition

);

PARAMETER	DESCRIPTION
`condition`

method operator boolconst

DEFINITION

operator bool

(

) const;

method process_else

DEFINITION

void process_else

(

);

method process_else_if

DEFINITION

void process_else_if

(

bool condition

);

PARAMETER	DESCRIPTION
`condition`

method skipping_restconst

DEFINITION

bool skipping_rest

(

) const;

RETURNS	DESCRIPTION
`bool`

member else_seenprivate

DEFINITION

bool else_seen;

member processingprivate

DEFINITION

bool processing;

member skip_restprivate

DEFINITION

bool skip_rest;

class PreprocessorDirectiveprivate

DEFINITION

            class PreprocessorDirective;
        

constructor PreprocessorDirective

DEFINITION

PreprocessorDirective	(	std::optional< size_t > min_arguments,
		std::optional< size_t > max_arguments,
		std::vector< TokenGroup > argument_type,
		void(FileTokenizer::*)(const Token &, const std::vector< Token > &) handler	);

PARAMETER	DESCRIPTION
`min_arguments`
`max_arguments`
`argument_type`
`handler`

method operator()const

DEFINITION

void operator()	(	FileTokenizer & tokenizer,
		const Token & directive,
		const std::vector< Token > & arguments	) const;

PARAMETER	DESCRIPTION
`tokenizer`
`directive`
`arguments`

member argument_typeprivate

DEFINITION

std::vector< TokenGroup > argument_type;

member handlerprivate

DEFINITION

void(FileTokenizer::* handler;

member max_argumentsprivate

DEFINITION

std::optional< size_t > max_arguments;

member min_argumentsprivate

DEFINITION

std::optional< size_t > min_arguments;

class Sourceprivate

DEFINITION

            class Source;
        

constructor Source

DEFINITION

Source

(

Symbol filename

);

PARAMETER	DESCRIPTION
`filename`

member pre_states

DEFINITION

std::vector< PreState > pre_states;

constructor FileTokenizer

DEFINITION

FileTokenizer	(	const SearchPath & search_path = SearchPath::empty_path,
		const Target * target = {},
		bool use_preprocessor = true,
		const std::unordered_set< Symbol > & defines = {}	);

PARAMETER	DESCRIPTION
`search_path`
`target`
`use_preprocessor`
`defines`

method add_literal

DEFINITION

void add_literal	(	Token::Type match,
		const std::string & name,
		const std::string & suffix_characters = ""	);

PARAMETER	DESCRIPTION
`match`
`name`
`suffix_characters`

method add_literal

DEFINITION

void add_literal

(

const Token & token

);

PARAMETER	DESCRIPTION
`token`

method add_punctuations

DEFINITION

void add_punctuations

(

const std::unordered_set< std::string > & names

);

PARAMETER	DESCRIPTION
`names`

method convert_digitstatic private

DEFINITION

static int convert_digit

(

int c

);

PARAMETER	DESCRIPTION
`c`

RETURNS	DESCRIPTION
`int`

method current_fileconst

DEFINITION

Symbol current_file

(

) const;

RETURNS	DESCRIPTION
`Symbol`

method current_locationconst virtual

DEFINITION

virtual Location current_location

(

) const;

RETURNS	DESCRIPTION
`Location`

method define

DEFINITION

void define

(

const std::unordered_set< Symbol > & defines

);

PARAMETER	DESCRIPTION
`defines`

method define

DEFINITION

void define

(

Symbol name

);

PARAMETER	DESCRIPTION
`name`

method find_file

DEFINITION

Symbol find_file

(

Symbol file_name

);

PARAMETER	DESCRIPTION
`file_name`

RETURNS	DESCRIPTION
`Symbol`

method get_definesconst

DEFINITION

const std::unordered_set< Symbol > & get_defines

(

) const;

RETURNS	DESCRIPTION
`const std::unordered_set< Symbol > &`

method is_identifierstatic private

DEFINITION

static bool is_identifier

(

const std::string & s

);

PARAMETER	DESCRIPTION
`s`

RETURNS	DESCRIPTION
`bool`

method is_identifier_continuationstatic private

DEFINITION

static bool is_identifier_continuation

(

int c

);

PARAMETER	DESCRIPTION
`c`

RETURNS	DESCRIPTION
`bool`

method is_identifier_startstatic private

DEFINITION

static bool is_identifier_start

(

int c

);

PARAMETER	DESCRIPTION
`c`

RETURNS	DESCRIPTION
`bool`

method next_rawprivate

DEFINITION

Token next_raw

(

);

RETURNS	DESCRIPTION
`Token`

method parse_charprivate

DEFINITION

Token parse_char

(

Location location

);

PARAMETER	DESCRIPTION
`location`

RETURNS	DESCRIPTION
`Token`

method parse_hexprivate

DEFINITION

Token parse_hex

(

Location location

);

PARAMETER	DESCRIPTION
`location`

RETURNS	DESCRIPTION
`Token`

method parse_nameprivate

DEFINITION

Token parse_name	(	Token::Type type,
		Location location	);

PARAMETER	DESCRIPTION
`type`
`location`

RETURNS	DESCRIPTION
`Token`

method parse_numberprivate

DEFINITION

Token parse_number	(	unsigned int base,
		Location location	);

PARAMETER	DESCRIPTION
`base`
`location`

RETURNS	DESCRIPTION
`Token`

method parse_stringprivate

DEFINITION

Token parse_string

(

Location location

);

PARAMETER	DESCRIPTION
`location`

RETURNS	DESCRIPTION
`Token`

method parse_string_literalprivate

DEFINITION

std::string parse_string_literal	(	Location location,
		int terminator	);

PARAMETER	DESCRIPTION
`location`
`terminator`

RETURNS	DESCRIPTION
`std::string`

method pre_is_processingconst private

DEFINITION

bool pre_is_processing

(

) const;

RETURNS	DESCRIPTION
`bool`

method preprocessprivate

DEFINITION

void preprocess	(	const Token & directive,
		const std::vector< Token > & tokens	);

PARAMETER	DESCRIPTION
`directive`
`tokens`

method preprocess_defineprivate

DEFINITION

void preprocess_define	(	const Token & directive,
		const std::vector< Token > & tokens	);

PARAMETER	DESCRIPTION
`directive`
`tokens`

method preprocess_includeprivate

DEFINITION

void preprocess_include	(	const Token & directive,
		const std::vector< Token > & tokens	);

PARAMETER	DESCRIPTION
`directive`
`tokens`

method preprocess_pre_elseprivate

DEFINITION

void preprocess_pre_else	(	const Token & directive,
		const std::vector< Token > & tokens	);

PARAMETER	DESCRIPTION
`directive`
`tokens`

method preprocess_pre_endprivate

DEFINITION

void preprocess_pre_end	(	const Token & directive,
		const std::vector< Token > & tokens	);

PARAMETER	DESCRIPTION
`directive`
`tokens`

method preprocess_pre_ifprivate

DEFINITION

void preprocess_pre_if	(	const Token & directive,
		const std::vector< Token > & tokens	);

PARAMETER	DESCRIPTION
`directive`
`tokens`

method push

DEFINITION

void push

(

Symbol filename

);

PARAMETER	DESCRIPTION
`filename`

method set_target

DEFINITION

void set_target

(

const Target * new_target

);

PARAMETER	DESCRIPTION
`new_target`

method sub_endedconst virtual protected

DEFINITION

virtual bool sub_ended

(

) const;

RETURNS	DESCRIPTION
`bool`

method sub_nextvirtual protected

DEFINITION

virtual Token sub_next

(

);

RETURNS	DESCRIPTION
`Token`

method undefine

DEFINITION

void undefine

(

Symbol name

);

PARAMETER	DESCRIPTION
`name`

member current_sourceprivate

DEFINITION

Source * current_source;

member eof_locationprivate

DEFINITION

Location eof_location;

member last_was_newlineprivate

DEFINITION

bool last_was_newline;

member matcherprivate

DEFINITION

MatcherNode matcher;

member preprocessor_directivesstatic private

DEFINITION

static const std::unordered_map< Token, PreprocessorDirective > preprocessor_directives;

member preprocessor_scopeprivate

DEFINITION

std::shared_ptr< Scope > preprocessor_scope;

member search_pathprivate

DEFINITION

const SearchPath & search_path;

member sourcesprivate

DEFINITION

std::vector< Source > sources;

member targetprivate

DEFINITION

const Target * target;

member token_definestatic private

DEFINITION

static const Token token_define;

member token_includestatic private

DEFINITION

static const Token token_include;

member token_pre_elsestatic private

DEFINITION

static const Token token_pre_else;

member token_pre_else_ifstatic private

DEFINITION

static const Token token_pre_else_if;

member token_pre_endstatic private

DEFINITION

static const Token token_pre_end;

member token_pre_ifstatic private

DEFINITION

static const Token token_pre_if;

member token_undefinestatic private

DEFINITION

static const Token token_undefine;

member use_preprocessorprivate

DEFINITION

bool use_preprocessor;

class MatcherNodeprivate

DEFINITION

            class MatcherNode;
        

method add

DEFINITION

void add	(	const char * string,
		Token::Type type,
		const std::unordered_set< char > & suffix_characters = {},
		bool match_in_word = false	);

PARAMETER	DESCRIPTION
`string`
`type`
`suffix_characters`
`match_in_word`

method conflictsconst private

DEFINITION

bool conflicts

(

const std::unordered_set< char > & new_suffix

) const;

PARAMETER	DESCRIPTION
`new_suffix`

RETURNS	DESCRIPTION
`bool`

method match

DEFINITION

std::optional< Token::Type > match	(	FileSource & source,
		std::string & name	);

PARAMETER	DESCRIPTION
`source`
`name`

RETURNS	DESCRIPTION
`std::optional< Token::Type >`

member match_in_word

DEFINITION

bool match_in_word;

member match_type

DEFINITION

std::optional< Token::Type > match_type;

member next

DEFINITION

std::unordered_map< char, std::unique_ptr< MatcherNode > > next;

member suffix_characters

DEFINITION

std::unordered_set< char > suffix_characters;

class PreStateprivate

DEFINITION

            class PreState;
        

constructor PreState

DEFINITION

PreState

(

bool condition

);

PARAMETER	DESCRIPTION
`condition`

method operator boolconst

DEFINITION

operator bool

(

) const;

method process_else

DEFINITION

void process_else

(

);

method process_else_if

DEFINITION

void process_else_if

(

bool condition

);

PARAMETER	DESCRIPTION
`condition`

method skipping_restconst

DEFINITION

bool skipping_rest

(

) const;

RETURNS	DESCRIPTION
`bool`

member else_seenprivate

DEFINITION

bool else_seen;

member processingprivate

DEFINITION

bool processing;

member skip_restprivate

DEFINITION

bool skip_rest;

class PreprocessorDirectiveprivate

DEFINITION

            class PreprocessorDirective;
        

constructor PreprocessorDirective

DEFINITION

PreprocessorDirective	(	std::optional< size_t > min_arguments,
		std::optional< size_t > max_arguments,
		std::vector< TokenGroup > argument_type,
		void(FileTokenizer::*)(const Token &, const std::vector< Token > &) handler	);

PARAMETER	DESCRIPTION
`min_arguments`
`max_arguments`
`argument_type`
`handler`

method operator()const

DEFINITION

void operator()	(	FileTokenizer & tokenizer,
		const Token & directive,
		const std::vector< Token > & arguments	) const;

PARAMETER	DESCRIPTION
`tokenizer`
`directive`
`arguments`

member argument_typeprivate

DEFINITION

std::vector< TokenGroup > argument_type;

member handlerprivate

DEFINITION

void(FileTokenizer::* handler;

member max_argumentsprivate

DEFINITION

std::optional< size_t > max_arguments;

member min_argumentsprivate

DEFINITION

std::optional< size_t > min_arguments;

class Sourceprivate

DEFINITION

            class Source;
        

constructor Source

DEFINITION

Source

(

Symbol filename

);

PARAMETER	DESCRIPTION
`filename`

member pre_states

DEFINITION

std::vector< PreState > pre_states;