| /*============================================================================= |
| Boost.Wave: A Standard compliant C++ preprocessor library |
| http://www.boost.org/ |
| |
| Copyright (c) 2001-2010 Hartmut Kaiser. Distributed under the Boost |
| Software License, Version 1.0. (See accompanying file |
| LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) |
| =============================================================================*/ |
| |
| #if !defined(BOOST_WAVE_LEXERTL_LEXER_HPP_INCLUDED) |
| #define BOOST_WAVE_LEXERTL_LEXER_HPP_INCLUDED |
| |
| #include <fstream> |
| |
| #include <boost/iterator/iterator_traits.hpp> |
| |
| #include <boost/wave/wave_config.hpp> |
| #include <boost/wave/language_support.hpp> |
| #include <boost/wave/token_ids.hpp> |
| #include <boost/wave/util/time_conversion_helper.hpp> |
| |
| #include <boost/wave/cpplexer/validate_universal_char.hpp> |
| #include <boost/wave/cpplexer/convert_trigraphs.hpp> |
| #include <boost/wave/cpplexer/cpplexer_exceptions.hpp> |
| #if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0 |
| #include <boost/wave/cpplexer/detect_include_guards.hpp> |
| #endif |
| |
| #include "wave_lexertl_config.hpp" |
| #include "../lexertl_iterator.hpp" |
| |
| #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES != 0 |
| #include "wave_lexertl_tables.hpp" |
| #else |
| #include <boost/spirit/home/support/detail/lexer/generator.hpp> |
| #include <boost/spirit/home/support/detail/lexer/rules.hpp> |
| #include <boost/spirit/home/support/detail/lexer/state_machine.hpp> |
| #include <boost/spirit/home/support/detail/lexer/consts.hpp> |
| //#include "lexertl/examples/serialise.hpp> |
| // #if BOOST_WAVE_LEXERTL_GENERATE_CPP_CODE != 0 |
| // #include "lexertl/examples/cpp_code.hpp" |
| // #endif |
| #endif |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| namespace boost { namespace wave { namespace cpplexer { namespace lexertl |
| { |
| |
| #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| /////////////////////////////////////////////////////////////////////////////// |
| // The following numbers are the array sizes of the token regex's which we |
| // need to specify to make the CW compiler happy (at least up to V9.5). |
| #if BOOST_WAVE_SUPPORT_MS_EXTENSIONS != 0 |
| #define INIT_DATA_SIZE 176 |
| #else |
| #define INIT_DATA_SIZE 159 |
| #endif |
| #define INIT_DATA_CPP_SIZE 15 |
| #define INIT_DATA_PP_NUMBER_SIZE 2 |
| #define INIT_MACRO_DATA_SIZE 27 |
| #endif // #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| |
| // this is just a hack to have a unique token id not otherwise used by Wave |
| #define T_ANYCTRL T_LAST_TOKEN_ID |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| namespace lexer |
| { |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| // this is the wrapper for the lexertl lexer library |
| template <typename Iterator, typename Position> |
| class lexertl |
| { |
| private: |
| typedef BOOST_WAVE_STRINGTYPE string_type; |
| typedef typename boost::detail::iterator_traits<Iterator>::value_type |
| char_type; |
| |
| public: |
| wave::token_id next_token(Iterator &first, Iterator const &last, |
| string_type& token_value); |
| |
| #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES != 0 |
| lexertl() {} |
| void init_dfa(wave::language_support lang, Position const& pos, |
| bool force_reinit = false) {} |
| bool is_initialized() const { return true; } |
| #else |
| lexertl() : has_compiled_dfa_(false) {} |
| bool init_dfa(wave::language_support lang, Position const& pos, |
| bool force_reinit = false); |
| bool is_initialized() const { return has_compiled_dfa_; } |
| |
| // get time of last compilation |
| static std::time_t get_compilation_time() |
| { return compilation_time.get_time(); } |
| |
| bool load (std::istream& instrm); |
| bool save (std::ostream& outstrm); |
| |
| private: |
| boost::lexer::state_machine state_machine_; |
| bool has_compiled_dfa_; |
| |
| // initialization data (regular expressions for the token definitions) |
| struct lexer_macro_data { |
| char_type const *name; // macro name |
| char_type const *macro; // associated macro definition |
| }; |
| static lexer_macro_data const init_macro_data[INIT_MACRO_DATA_SIZE]; // macro patterns |
| |
| struct lexer_data { |
| token_id tokenid; // token data |
| char_type const *tokenregex; // associated token to match |
| }; |
| static lexer_data const init_data[INIT_DATA_SIZE]; // common patterns |
| static lexer_data const init_data_cpp[INIT_DATA_CPP_SIZE]; // C++ only patterns |
| static lexer_data const init_data_pp_number[INIT_DATA_PP_NUMBER_SIZE]; // pp-number only patterns |
| |
| // helper for calculation of the time of last compilation |
| static boost::wave::util::time_conversion_helper compilation_time; |
| #endif // #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| }; |
| |
| #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| /////////////////////////////////////////////////////////////////////////////// |
| // get time of last compilation of this file |
| template <typename IteratorT, typename PositionT> |
| boost::wave::util::time_conversion_helper |
| lexertl<IteratorT, PositionT>::compilation_time(__DATE__ " " __TIME__); |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| // token regex definitions |
| |
| // helper for initializing token data and macro definitions |
| #define Q(c) "\\" c |
| #define TRI(c) "{TRI}" c |
| #define OR "|" |
| #define MACRO_DATA(name, macro) { name, macro } |
| #define TOKEN_DATA(id, regex) { id, regex } |
| |
| // lexertl macro definitions |
| template <typename Iterator, typename Position> |
| typename lexertl<Iterator, Position>::lexer_macro_data const |
| lexertl<Iterator, Position>::init_macro_data[INIT_MACRO_DATA_SIZE] = |
| { |
| MACRO_DATA("ANY", "[\t\v\f\r\n\\040-\\377]"), |
| MACRO_DATA("ANYCTRL", "[\\000-\\037]"), |
| MACRO_DATA("TRI", "\\?\\?"), |
| MACRO_DATA("BLANK", "[ \t\v\f]"), |
| MACRO_DATA("CCOMMENT", "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/"), |
| MACRO_DATA("PPSPACE", "(" "{BLANK}" OR "{CCOMMENT}" ")*"), |
| MACRO_DATA("OCTALDIGIT", "[0-7]"), |
| MACRO_DATA("DIGIT", "[0-9]"), |
| MACRO_DATA("HEXDIGIT", "[0-9a-fA-F]"), |
| MACRO_DATA("OPTSIGN", "[-+]?"), |
| MACRO_DATA("EXPSTART", "[eE][-+]"), |
| MACRO_DATA("EXPONENT", "([eE]{OPTSIGN}{DIGIT}+)"), |
| MACRO_DATA("NONDIGIT", "[a-zA-Z_]"), |
| MACRO_DATA("INTEGER", "(" "(0x|0X){HEXDIGIT}+" OR "0{OCTALDIGIT}*" OR "[1-9]{DIGIT}*" ")"), |
| MACRO_DATA("INTEGER_SUFFIX", "(" "[uU][lL]?" OR "[lL][uU]?" ")"), |
| #if BOOST_WAVE_SUPPORT_MS_EXTENSIONS != 0 |
| MACRO_DATA("LONGINTEGER_SUFFIX", "([uU]([lL][lL])|([lL][lL])[uU]?|i64)"), |
| #else |
| MACRO_DATA("LONGINTEGER_SUFFIX", "([uU]([lL][lL])|([lL][lL])[uU]?)"), |
| #endif |
| MACRO_DATA("FLOAT_SUFFIX", "(" "[fF][lL]?" OR "[lL][fF]?" ")"), |
| MACRO_DATA("CHAR_SPEC", "L?"), |
| MACRO_DATA("BACKSLASH", "(" Q("\\") OR TRI(Q("/")) ")"), |
| MACRO_DATA("ESCAPESEQ", "{BACKSLASH}([abfnrtv?'\"]|{BACKSLASH}|x{HEXDIGIT}+|{OCTALDIGIT}{1,3})"), |
| MACRO_DATA("HEXQUAD", "{HEXDIGIT}{4}"), |
| MACRO_DATA("UNIVERSALCHAR", "{BACKSLASH}(u{HEXQUAD}|U{HEXQUAD}{2})"), |
| MACRO_DATA("POUNDDEF", "(" "#" OR TRI("=") OR Q("%:") ")"), |
| MACRO_DATA("NEWLINEDEF", "(" "\\n" OR "\\r" OR "\\r\\n" ")"), |
| #if BOOST_WAVE_SUPPORT_INCLUDE_NEXT != 0 |
| MACRO_DATA("INCLUDEDEF", "(include|include_next)"), |
| #else |
| MACRO_DATA("INCLUDEDEF", "include"), |
| #endif |
| MACRO_DATA("PP_NUMBERDEF", "\\.?{DIGIT}({DIGIT}|{NONDIGIT}|{EXPSTART}|\\.)*"), |
| MACRO_DATA(NULL, NULL) // should be the last entry |
| }; |
| |
| // common C++/C99 token definitions |
| template <typename Iterator, typename Position> |
| typename lexertl<Iterator, Position>::lexer_data const |
| lexertl<Iterator, Position>::init_data[INIT_DATA_SIZE] = |
| { |
| TOKEN_DATA(T_AND, "&"), |
| TOKEN_DATA(T_ANDAND, "&&"), |
| TOKEN_DATA(T_ASSIGN, "="), |
| TOKEN_DATA(T_ANDASSIGN, "&="), |
| TOKEN_DATA(T_OR, Q("|")), |
| TOKEN_DATA(T_OR_TRIGRAPH, "{TRI}!"), |
| TOKEN_DATA(T_ORASSIGN, Q("|=")), |
| TOKEN_DATA(T_ORASSIGN_TRIGRAPH, "{TRI}!="), |
| TOKEN_DATA(T_XOR, Q("^")), |
| TOKEN_DATA(T_XOR_TRIGRAPH, "{TRI}'"), |
| TOKEN_DATA(T_XORASSIGN, Q("^=")), |
| TOKEN_DATA(T_XORASSIGN_TRIGRAPH, "{TRI}'="), |
| TOKEN_DATA(T_COMMA, ","), |
| TOKEN_DATA(T_COLON, ":"), |
| TOKEN_DATA(T_DIVIDEASSIGN, Q("/=")), |
| TOKEN_DATA(T_DIVIDE, Q("/")), |
| TOKEN_DATA(T_DOT, Q(".")), |
| TOKEN_DATA(T_ELLIPSIS, Q(".") "{3}"), |
| TOKEN_DATA(T_EQUAL, "=="), |
| TOKEN_DATA(T_GREATER, ">"), |
| TOKEN_DATA(T_GREATEREQUAL, ">="), |
| TOKEN_DATA(T_LEFTBRACE, Q("{")), |
| TOKEN_DATA(T_LEFTBRACE_ALT, "<" Q("%")), |
| TOKEN_DATA(T_LEFTBRACE_TRIGRAPH, "{TRI}<"), |
| TOKEN_DATA(T_LESS, "<"), |
| TOKEN_DATA(T_LESSEQUAL, "<="), |
| TOKEN_DATA(T_LEFTPAREN, Q("(")), |
| TOKEN_DATA(T_LEFTBRACKET, Q("[")), |
| TOKEN_DATA(T_LEFTBRACKET_ALT, "<:"), |
| TOKEN_DATA(T_LEFTBRACKET_TRIGRAPH, "{TRI}" Q("(")), |
| TOKEN_DATA(T_MINUS, Q("-")), |
| TOKEN_DATA(T_MINUSASSIGN, Q("-=")), |
| TOKEN_DATA(T_MINUSMINUS, Q("-") "{2}"), |
| TOKEN_DATA(T_PERCENT, Q("%")), |
| TOKEN_DATA(T_PERCENTASSIGN, Q("%=")), |
| TOKEN_DATA(T_NOT, "!"), |
| TOKEN_DATA(T_NOTEQUAL, "!="), |
| TOKEN_DATA(T_OROR, Q("|") "{2}"), |
| TOKEN_DATA(T_OROR_TRIGRAPH, "{TRI}!\\||\\|{TRI}!|{TRI}!{TRI}!"), |
| TOKEN_DATA(T_PLUS, Q("+")), |
| TOKEN_DATA(T_PLUSASSIGN, Q("+=")), |
| TOKEN_DATA(T_PLUSPLUS, Q("+") "{2}"), |
| TOKEN_DATA(T_ARROW, Q("->")), |
| TOKEN_DATA(T_QUESTION_MARK, Q("?")), |
| TOKEN_DATA(T_RIGHTBRACE, Q("}")), |
| TOKEN_DATA(T_RIGHTBRACE_ALT, Q("%>")), |
| TOKEN_DATA(T_RIGHTBRACE_TRIGRAPH, "{TRI}>"), |
| TOKEN_DATA(T_RIGHTPAREN, Q(")")), |
| TOKEN_DATA(T_RIGHTBRACKET, Q("]")), |
| TOKEN_DATA(T_RIGHTBRACKET_ALT, ":>"), |
| TOKEN_DATA(T_RIGHTBRACKET_TRIGRAPH, "{TRI}" Q(")")), |
| TOKEN_DATA(T_SEMICOLON, ";"), |
| TOKEN_DATA(T_SHIFTLEFT, "<<"), |
| TOKEN_DATA(T_SHIFTLEFTASSIGN, "<<="), |
| TOKEN_DATA(T_SHIFTRIGHT, ">>"), |
| TOKEN_DATA(T_SHIFTRIGHTASSIGN, ">>="), |
| TOKEN_DATA(T_STAR, Q("*")), |
| TOKEN_DATA(T_COMPL, Q("~")), |
| TOKEN_DATA(T_COMPL_TRIGRAPH, "{TRI}-"), |
| TOKEN_DATA(T_STARASSIGN, Q("*=")), |
| TOKEN_DATA(T_ASM, "asm"), |
| TOKEN_DATA(T_AUTO, "auto"), |
| TOKEN_DATA(T_BOOL, "bool"), |
| TOKEN_DATA(T_FALSE, "false"), |
| TOKEN_DATA(T_TRUE, "true"), |
| TOKEN_DATA(T_BREAK, "break"), |
| TOKEN_DATA(T_CASE, "case"), |
| TOKEN_DATA(T_CATCH, "catch"), |
| TOKEN_DATA(T_CHAR, "char"), |
| TOKEN_DATA(T_CLASS, "class"), |
| TOKEN_DATA(T_CONST, "const"), |
| TOKEN_DATA(T_CONSTCAST, "const_cast"), |
| TOKEN_DATA(T_CONTINUE, "continue"), |
| TOKEN_DATA(T_DEFAULT, "default"), |
| TOKEN_DATA(T_DELETE, "delete"), |
| TOKEN_DATA(T_DO, "do"), |
| TOKEN_DATA(T_DOUBLE, "double"), |
| TOKEN_DATA(T_DYNAMICCAST, "dynamic_cast"), |
| TOKEN_DATA(T_ELSE, "else"), |
| TOKEN_DATA(T_ENUM, "enum"), |
| TOKEN_DATA(T_EXPLICIT, "explicit"), |
| TOKEN_DATA(T_EXPORT, "export"), |
| TOKEN_DATA(T_EXTERN, "extern"), |
| TOKEN_DATA(T_FLOAT, "float"), |
| TOKEN_DATA(T_FOR, "for"), |
| TOKEN_DATA(T_FRIEND, "friend"), |
| TOKEN_DATA(T_GOTO, "goto"), |
| TOKEN_DATA(T_IF, "if"), |
| TOKEN_DATA(T_INLINE, "inline"), |
| TOKEN_DATA(T_INT, "int"), |
| TOKEN_DATA(T_LONG, "long"), |
| TOKEN_DATA(T_MUTABLE, "mutable"), |
| TOKEN_DATA(T_NAMESPACE, "namespace"), |
| TOKEN_DATA(T_NEW, "new"), |
| TOKEN_DATA(T_OPERATOR, "operator"), |
| TOKEN_DATA(T_PRIVATE, "private"), |
| TOKEN_DATA(T_PROTECTED, "protected"), |
| TOKEN_DATA(T_PUBLIC, "public"), |
| TOKEN_DATA(T_REGISTER, "register"), |
| TOKEN_DATA(T_REINTERPRETCAST, "reinterpret_cast"), |
| TOKEN_DATA(T_RETURN, "return"), |
| TOKEN_DATA(T_SHORT, "short"), |
| TOKEN_DATA(T_SIGNED, "signed"), |
| TOKEN_DATA(T_SIZEOF, "sizeof"), |
| TOKEN_DATA(T_STATIC, "static"), |
| TOKEN_DATA(T_STATICCAST, "static_cast"), |
| TOKEN_DATA(T_STRUCT, "struct"), |
| TOKEN_DATA(T_SWITCH, "switch"), |
| TOKEN_DATA(T_TEMPLATE, "template"), |
| TOKEN_DATA(T_THIS, "this"), |
| TOKEN_DATA(T_THROW, "throw"), |
| TOKEN_DATA(T_TRY, "try"), |
| TOKEN_DATA(T_TYPEDEF, "typedef"), |
| TOKEN_DATA(T_TYPEID, "typeid"), |
| TOKEN_DATA(T_TYPENAME, "typename"), |
| TOKEN_DATA(T_UNION, "union"), |
| TOKEN_DATA(T_UNSIGNED, "unsigned"), |
| TOKEN_DATA(T_USING, "using"), |
| TOKEN_DATA(T_VIRTUAL, "virtual"), |
| TOKEN_DATA(T_VOID, "void"), |
| TOKEN_DATA(T_VOLATILE, "volatile"), |
| TOKEN_DATA(T_WCHART, "wchar_t"), |
| TOKEN_DATA(T_WHILE, "while"), |
| TOKEN_DATA(T_PP_DEFINE, "{POUNDDEF}{PPSPACE}define"), |
| TOKEN_DATA(T_PP_IF, "{POUNDDEF}{PPSPACE}if"), |
| TOKEN_DATA(T_PP_IFDEF, "{POUNDDEF}{PPSPACE}ifdef"), |
| TOKEN_DATA(T_PP_IFNDEF, "{POUNDDEF}{PPSPACE}ifndef"), |
| TOKEN_DATA(T_PP_ELSE, "{POUNDDEF}{PPSPACE}else"), |
| TOKEN_DATA(T_PP_ELIF, "{POUNDDEF}{PPSPACE}elif"), |
| TOKEN_DATA(T_PP_ENDIF, "{POUNDDEF}{PPSPACE}endif"), |
| TOKEN_DATA(T_PP_ERROR, "{POUNDDEF}{PPSPACE}error"), |
| TOKEN_DATA(T_PP_QHEADER, "{POUNDDEF}{PPSPACE}{INCLUDEDEF}{PPSPACE}" Q("\"") "[^\\n\\r\"]+" Q("\"")), |
| TOKEN_DATA(T_PP_HHEADER, "{POUNDDEF}{PPSPACE}{INCLUDEDEF}{PPSPACE}" "<" "[^\\n\\r>]+" ">"), |
| TOKEN_DATA(T_PP_INCLUDE, "{POUNDDEF}{PPSPACE}{INCLUDEDEF}{PPSPACE}"), |
| TOKEN_DATA(T_PP_LINE, "{POUNDDEF}{PPSPACE}line"), |
| TOKEN_DATA(T_PP_PRAGMA, "{POUNDDEF}{PPSPACE}pragma"), |
| TOKEN_DATA(T_PP_UNDEF, "{POUNDDEF}{PPSPACE}undef"), |
| TOKEN_DATA(T_PP_WARNING, "{POUNDDEF}{PPSPACE}warning"), |
| #if BOOST_WAVE_SUPPORT_MS_EXTENSIONS != 0 |
| TOKEN_DATA(T_MSEXT_INT8, "__int8"), |
| TOKEN_DATA(T_MSEXT_INT16, "__int16"), |
| TOKEN_DATA(T_MSEXT_INT32, "__int32"), |
| TOKEN_DATA(T_MSEXT_INT64, "__int64"), |
| TOKEN_DATA(T_MSEXT_BASED, "_?" "_based"), |
| TOKEN_DATA(T_MSEXT_DECLSPEC, "_?" "_declspec"), |
| TOKEN_DATA(T_MSEXT_CDECL, "_?" "_cdecl"), |
| TOKEN_DATA(T_MSEXT_FASTCALL, "_?" "_fastcall"), |
| TOKEN_DATA(T_MSEXT_STDCALL, "_?" "_stdcall"), |
| TOKEN_DATA(T_MSEXT_TRY , "__try"), |
| TOKEN_DATA(T_MSEXT_EXCEPT, "__except"), |
| TOKEN_DATA(T_MSEXT_FINALLY, "__finally"), |
| TOKEN_DATA(T_MSEXT_LEAVE, "__leave"), |
| TOKEN_DATA(T_MSEXT_INLINE, "_?" "_inline"), |
| TOKEN_DATA(T_MSEXT_ASM, "_?" "_asm"), |
| TOKEN_DATA(T_MSEXT_PP_REGION, "{POUNDDEF}{PPSPACE}region"), |
| TOKEN_DATA(T_MSEXT_PP_ENDREGION, "{POUNDDEF}{PPSPACE}endregion"), |
| #endif // BOOST_WAVE_SUPPORT_MS_EXTENSIONS != 0 |
| TOKEN_DATA(T_LONGINTLIT, "{INTEGER}{LONGINTEGER_SUFFIX}"), |
| TOKEN_DATA(T_INTLIT, "{INTEGER}{INTEGER_SUFFIX}?"), |
| TOKEN_DATA(T_FLOATLIT, |
| "(" "{DIGIT}*" Q(".") "{DIGIT}+" OR "{DIGIT}+" Q(".") "){EXPONENT}?{FLOAT_SUFFIX}?" OR |
| "{DIGIT}+{EXPONENT}{FLOAT_SUFFIX}?"), |
| #if BOOST_WAVE_USE_STRICT_LEXER != 0 |
| TOKEN_DATA(T_IDENTIFIER, |
| "(" "{NONDIGIT}" OR "{UNIVERSALCHAR}" ")" |
| "(" "{NONDIGIT}" OR "{DIGIT}" OR "{UNIVERSALCHAR}" ")*"), |
| #else |
| TOKEN_DATA(T_IDENTIFIER, |
| "(" "{NONDIGIT}" OR Q("$") OR "{UNIVERSALCHAR}" ")" |
| "(" "{NONDIGIT}" OR Q("$") OR "{DIGIT}" OR "{UNIVERSALCHAR}" ")*"), |
| #endif |
| TOKEN_DATA(T_CCOMMENT, "{CCOMMENT}"), |
| TOKEN_DATA(T_CPPCOMMENT, Q("/") Q("/[^\\n\\r]*") "{NEWLINEDEF}" ), |
| TOKEN_DATA(T_CHARLIT, |
| "{CHAR_SPEC}" "'" "({ESCAPESEQ}|[^\\n\\r']|{UNIVERSALCHAR})+" "'"), |
| TOKEN_DATA(T_STRINGLIT, |
| "{CHAR_SPEC}" Q("\"") "({ESCAPESEQ}|[^\\n\\r\"]|{UNIVERSALCHAR})*" Q("\"")), |
| TOKEN_DATA(T_SPACE, "{BLANK}+"), |
| TOKEN_DATA(T_CONTLINE, Q("\\") "\\n"), |
| TOKEN_DATA(T_NEWLINE, "{NEWLINEDEF}"), |
| TOKEN_DATA(T_POUND_POUND, "##"), |
| TOKEN_DATA(T_POUND_POUND_ALT, Q("%:") Q("%:")), |
| TOKEN_DATA(T_POUND_POUND_TRIGRAPH, "({TRI}=){2}"), |
| TOKEN_DATA(T_POUND, "#"), |
| TOKEN_DATA(T_POUND_ALT, Q("%:")), |
| TOKEN_DATA(T_POUND_TRIGRAPH, "{TRI}="), |
| TOKEN_DATA(T_ANY_TRIGRAPH, "{TRI}\\/"), |
| TOKEN_DATA(T_ANY, "{ANY}"), |
| TOKEN_DATA(T_ANYCTRL, "{ANYCTRL}"), // this should be the last recognized token |
| { token_id(0) } // this should be the last entry |
| }; |
| |
| // C++ only token definitions |
| template <typename Iterator, typename Position> |
| typename lexertl<Iterator, Position>::lexer_data const |
| lexertl<Iterator, Position>::init_data_cpp[INIT_DATA_CPP_SIZE] = |
| { |
| TOKEN_DATA(T_AND_ALT, "bitand"), |
| TOKEN_DATA(T_ANDASSIGN_ALT, "and_eq"), |
| TOKEN_DATA(T_ANDAND_ALT, "and"), |
| TOKEN_DATA(T_OR_ALT, "bitor"), |
| TOKEN_DATA(T_ORASSIGN_ALT, "or_eq"), |
| TOKEN_DATA(T_OROR_ALT, "or"), |
| TOKEN_DATA(T_XORASSIGN_ALT, "xor_eq"), |
| TOKEN_DATA(T_XOR_ALT, "xor"), |
| TOKEN_DATA(T_NOTEQUAL_ALT, "not_eq"), |
| TOKEN_DATA(T_NOT_ALT, "not"), |
| TOKEN_DATA(T_COMPL_ALT, "compl"), |
| #if BOOST_WAVE_SUPPORT_IMPORT_KEYWORD != 0 |
| TOKEN_DATA(T_IMPORT, "import"), |
| #endif |
| TOKEN_DATA(T_ARROWSTAR, Q("->") Q("*")), |
| TOKEN_DATA(T_DOTSTAR, Q(".") Q("*")), |
| TOKEN_DATA(T_COLON_COLON, "::"), |
| { token_id(0) } // this should be the last entry |
| }; |
| |
| // pp-number specific token definitions |
| template <typename Iterator, typename Position> |
| typename lexertl<Iterator, Position>::lexer_data const |
| lexertl<Iterator, Position>::init_data_pp_number[INIT_DATA_PP_NUMBER_SIZE] = |
| { |
| TOKEN_DATA(T_PP_NUMBER, "{PP_NUMBERDEF}"), |
| { token_id(0) } // this should be the last entry |
| }; |
| |
| #undef MACRO_DATA |
| #undef TOKEN_DATA |
| #undef OR |
| #undef TRI |
| #undef Q |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| // initialize lexertl lexer from C++ token regex's |
| template <typename Iterator, typename Position> |
| inline bool |
| lexertl<Iterator, Position>::init_dfa(wave::language_support lang, |
| Position const& pos, bool force_reinit) |
| { |
| if (has_compiled_dfa_) |
| return true; |
| |
| std::ifstream dfa_in("wave_lexertl_lexer.dfa", std::ios::in|std::ios::binary); |
| |
| if (force_reinit || !dfa_in.is_open() || !load (dfa_in)) |
| { |
| dfa_in.close(); |
| |
| state_machine_.clear(); |
| |
| // register macro definitions |
| boost::lexer::rules rules; |
| for (int k = 0; NULL != init_macro_data[k].name; ++k) { |
| rules.add_macro(init_macro_data[k].name, init_macro_data[k].macro); |
| } |
| |
| // if pp-numbers should be preferred, insert the corresponding rule first |
| if (wave::need_prefer_pp_numbers(lang)) { |
| for (int j = 0; 0 != init_data_pp_number[j].tokenid; ++j) { |
| rules.add(init_data_pp_number[j].tokenregex, |
| init_data_pp_number[j].tokenid); |
| } |
| } |
| |
| // if in C99 mode, some of the keywords are not valid |
| if (!wave::need_c99(lang)) { |
| for (int j = 0; 0 != init_data_cpp[j].tokenid; ++j) { |
| rules.add(init_data_cpp[j].tokenregex, |
| init_data_cpp[j].tokenid); |
| } |
| } |
| |
| for (int i = 0; 0 != init_data[i].tokenid; ++i) { |
| rules.add(init_data[i].tokenregex, init_data[i].tokenid); |
| } |
| |
| // generate minimized DFA |
| try { |
| boost::lexer::generator::build (rules, state_machine_); |
| boost::lexer::generator::minimise (state_machine_); |
| } |
| catch (std::runtime_error const& e) { |
| string_type msg("lexertl initialization error: "); |
| msg += e.what(); |
| BOOST_WAVE_LEXER_THROW(wave::cpplexer::lexing_exception, |
| unexpected_error, msg.c_str(), |
| pos.get_line(), pos.get_column(), pos.get_file().c_str()); |
| return false; |
| } |
| |
| std::ofstream dfa_out ("wave_lexertl_lexer.dfa", |
| std::ios::out|std::ios::binary|std::ios::trunc); |
| |
| if (dfa_out.is_open()) |
| save (dfa_out); |
| } |
| |
| has_compiled_dfa_ = true; |
| return true; |
| } |
| #endif // BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| // return next token from the input stream |
| template <typename Iterator, typename Position> |
| inline wave::token_id |
| lexertl<Iterator, Position>::next_token(Iterator &first, Iterator const &last, |
| string_type& token_value) |
| { |
| #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| size_t const* const lookup = &state_machine_.data()._lookup[0]->front (); |
| size_t const dfa_alphabet = state_machine_.data()._dfa_alphabet[0]; |
| |
| size_t const* dfa = &state_machine_.data()._dfa[0]->front(); |
| size_t const* ptr = dfa + dfa_alphabet + boost::lexer::dfa_offset; |
| #else |
| const std::size_t *ptr = dfa + dfa_offset; |
| #endif // BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| |
| Iterator curr = first; |
| Iterator end_token = first; |
| bool end_state = (*ptr != 0); |
| size_t id = *(ptr + 1); |
| |
| while (curr != last) { |
| size_t const state = ptr[lookup[int(*curr)]]; |
| if (0 == state) |
| break; |
| ++curr; |
| |
| #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| ptr = &dfa[state * (dfa_alphabet + boost::lexer::dfa_offset)]; |
| #else |
| ptr = &dfa[state * dfa_offset]; |
| #endif // BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| |
| if (0 != *ptr) { |
| end_state = true; |
| id = *(ptr + 1); |
| end_token = curr; |
| } |
| } |
| |
| if (end_state) { |
| if (T_ANY == id) { |
| id = TOKEN_FROM_ID(*first, UnknownTokenType); |
| } |
| |
| // return longest match |
| string_type str(first, end_token); |
| token_value.swap(str); |
| first = end_token; |
| return wave::token_id(id); |
| } |
| return T_EOF; |
| } |
| |
| #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| /////////////////////////////////////////////////////////////////////////////// |
| // load the DFA tables to/from a stream |
| template <typename Iterator, typename Position> |
| inline bool |
| lexertl<Iterator, Position>::load (std::istream& instrm) |
| { |
| // #if !defined(BOOST_WAVE_LEXERTL_GENERATE_CPP_CODE) |
| // std::size_t version = 0; |
| // boost::lexer::serialise::load_as_binary(instrm, state_machine_, version); |
| // if (version != (std::size_t)get_compilation_time()) |
| // return false; // too new for us |
| // return instrm.good(); |
| // #else |
| return false; // always create the dfa when generating the C++ code |
| // #endif |
| } |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| // save the DFA tables to/from a stream |
| template <typename Iterator, typename Position> |
| inline bool |
| lexertl<Iterator, Position>::save (std::ostream& outstrm) |
| { |
| // #if defined(BOOST_WAVE_LEXERTL_GENERATE_CPP_CODE) |
| // cpp_code::generate(state_machine_, outstrm); |
| // #else |
| // boost::lexer::serialise::save_as_binary(state_machine_, outstrm, |
| // (std::size_t)get_compilation_time()); |
| // #endif |
| return outstrm.good(); |
| } |
| #endif // #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0 |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| } // namespace lexer |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| template <typename Iterator, typename Position = wave::util::file_position_type> |
| class lexertl_functor |
| : public lexertl_input_interface<wave::cpplexer::lex_token<Position> > |
| { |
| public: |
| typedef wave::util::position_iterator<Iterator, Position> iterator_type; |
| typedef typename boost::detail::iterator_traits<Iterator>::value_type |
| char_type; |
| typedef BOOST_WAVE_STRINGTYPE string_type; |
| typedef wave::cpplexer::lex_token<Position> token_type; |
| |
| lexertl_functor(Iterator const &first_, Iterator const &last_, |
| Position const &pos_, wave::language_support language) |
| : first(first_, last_, pos_), language(language), at_eof(false) |
| { |
| lexer_.init_dfa(language, pos_); |
| } |
| ~lexertl_functor() {} |
| |
| // get the next token from the input stream |
| token_type& get(token_type& result) |
| { |
| if (lexer_.is_initialized() && !at_eof) { |
| do { |
| // generate and return the next token |
| string_type token_val; |
| Position pos = first.get_position(); // begin of token position |
| wave::token_id id = lexer_.next_token(first, last, token_val); |
| |
| if (T_CONTLINE != id) { |
| // The cast should avoid spurious warnings about missing case labels |
| // for the other token ids's. |
| switch (static_cast<unsigned int>(id)) { |
| case T_IDENTIFIER: |
| // test identifier characters for validity (throws if |
| // invalid chars found) |
| if (!wave::need_no_character_validation(language)) { |
| using wave::cpplexer::impl::validate_identifier_name; |
| validate_identifier_name(token_val, |
| pos.get_line(), pos.get_column(), pos.get_file()); |
| } |
| break; |
| |
| case T_STRINGLIT: |
| case T_CHARLIT: |
| // test literal characters for validity (throws if invalid |
| // chars found) |
| if (wave::need_convert_trigraphs(language)) { |
| using wave::cpplexer::impl::convert_trigraphs; |
| token_val = convert_trigraphs(token_val); |
| } |
| if (!wave::need_no_character_validation(language)) { |
| using wave::cpplexer::impl::validate_literal; |
| validate_literal(token_val, |
| pos.get_line(), pos.get_column(), pos.get_file()); |
| } |
| break; |
| |
| case T_LONGINTLIT: // supported in C99 and long_long mode |
| if (!wave::need_long_long(language)) { |
| // syntax error: not allowed in C++ mode |
| BOOST_WAVE_LEXER_THROW( |
| wave::cpplexer::lexing_exception, |
| invalid_long_long_literal, token_val.c_str(), |
| pos.get_line(), pos.get_column(), |
| pos.get_file().c_str()); |
| } |
| break; |
| |
| #if BOOST_WAVE_SUPPORT_INCLUDE_NEXT != 0 |
| case T_PP_HHEADER: |
| case T_PP_QHEADER: |
| case T_PP_INCLUDE: |
| // convert to the corresponding ..._next token, if appropriate |
| { |
| // Skip '#' and whitespace and see whether we find an |
| // 'include_next' here. |
| typename string_type::size_type start = token_val.find("include"); |
| if (0 == token_val.compare(start, 12, "include_next", 12)) |
| id = token_id(id | AltTokenType); |
| } |
| break; |
| #endif // BOOST_WAVE_SUPPORT_INCLUDE_NEXT != 0 |
| |
| case T_EOF: |
| // T_EOF is returned as a valid token, the next call will |
| // return T_EOI, i.e. the actual end of input |
| at_eof = true; |
| token_val.clear(); |
| break; |
| |
| case T_OR_TRIGRAPH: |
| case T_XOR_TRIGRAPH: |
| case T_LEFTBRACE_TRIGRAPH: |
| case T_RIGHTBRACE_TRIGRAPH: |
| case T_LEFTBRACKET_TRIGRAPH: |
| case T_RIGHTBRACKET_TRIGRAPH: |
| case T_COMPL_TRIGRAPH: |
| case T_POUND_TRIGRAPH: |
| case T_ANY_TRIGRAPH: |
| if (wave::need_convert_trigraphs(language)) |
| { |
| using wave::cpplexer::impl::convert_trigraph; |
| token_val = convert_trigraph(token_val); |
| } |
| break; |
| |
| case T_ANYCTRL: |
| // matched some unexpected character |
| { |
| // 21 is the max required size for a 64 bit integer |
| // represented as a string |
| char buffer[22]; |
| string_type msg("invalid character in input stream: '0x"); |
| |
| // for some systems sprintf is in namespace std |
| using namespace std; |
| sprintf(buffer, "%02x'", token_val[0]); |
| msg += buffer; |
| BOOST_WAVE_LEXER_THROW( |
| wave::cpplexer::lexing_exception, |
| generic_lexing_error, |
| msg.c_str(), pos.get_line(), pos.get_column(), |
| pos.get_file().c_str()); |
| } |
| break; |
| } |
| |
| result = token_type(id, token_val, pos); |
| #if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0 |
| return guards.detect_guard(result); |
| #else |
| return result; |
| #endif |
| } |
| } while (true); // skip the T_CONTLINE token |
| } |
| return result = token_type(); // return T_EOI |
| } |
| |
| void set_position(Position const &pos) |
| { |
| // set position has to change the file name and line number only |
| first.get_position().set_file(pos.get_file()); |
| first.get_position().set_line(pos.get_line()); |
| } |
| |
| #if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0 |
| bool has_include_guards(std::string& guard_name) const |
| { return guards.detected(guard_name); } |
| #endif |
| |
| private: |
| iterator_type first; |
| iterator_type last; |
| |
| wave::language_support language; |
| bool at_eof; |
| #if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0 |
| include_guards<token_type> guards; |
| #endif |
| |
| static lexer::lexertl<iterator_type, Position> lexer_; |
| }; |
| |
| template <typename Iterator, typename Position> |
| lexer::lexertl< |
| typename lexertl_functor<Iterator, Position>::iterator_type, Position> |
| lexertl_functor<Iterator, Position>::lexer_; |
| |
| #undef INIT_DATA_SIZE |
| #undef INIT_DATA_CPP_SIZE |
| #undef INIT_DATA_PP_NUMBER_SIZE |
| #undef INIT_MACRO_DATA_SIZE |
| #undef T_ANYCTRL |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| // |
| // The new_lexer_gen<>::new_lexer function (declared in lexertl_interface.hpp) |
| // should be defined inline, if the lex_functor shouldn't be instantiated |
| // separately from the lex_iterator. |
| // |
| // Separate (explicit) instantiation helps to reduce compilation time. |
| // |
| /////////////////////////////////////////////////////////////////////////////// |
| |
| #if BOOST_WAVE_SEPARATE_LEXER_INSTANTIATION != 0 |
| #define BOOST_WAVE_FLEX_NEW_LEXER_INLINE |
| #else |
| #define BOOST_WAVE_FLEX_NEW_LEXER_INLINE inline |
| #endif |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| // |
| // The 'new_lexer' function allows the opaque generation of a new lexer object. |
| // It is coupled to the iterator type to allow to decouple the lexer/iterator |
| // configurations at compile time. |
| // |
| // This function is declared inside the xlex_interface.hpp file, which is |
| // referenced by the source file calling the lexer and the source file, which |
| // instantiates the lex_functor. But is is defined here, so it will be |
| // instantiated only while compiling the source file, which instantiates the |
| // lex_functor. While the xlex_interface.hpp file may be included everywhere, |
| // this file (xlex_lexer.hpp) should be included only once. This allows |
| // to decouple the lexer interface from the lexer implementation and reduces |
| // compilation time. |
| // |
| /////////////////////////////////////////////////////////////////////////////// |
| |
| template <typename Iterator, typename Position> |
| BOOST_WAVE_FLEX_NEW_LEXER_INLINE |
| wave::cpplexer::lex_input_interface<wave::cpplexer::lex_token<Position> > * |
| new_lexer_gen<Iterator, Position>::new_lexer(Iterator const &first, |
| Iterator const &last, Position const &pos, wave::language_support language) |
| { |
| return new lexertl_functor<Iterator, Position>(first, last, pos, language); |
| } |
| |
| #undef BOOST_WAVE_FLEX_NEW_LEXER_INLINE |
| |
| /////////////////////////////////////////////////////////////////////////////// |
| }}}} // namespace boost::wave::cpplexer::lexertl |
| |
| #endif // !defined(BOOST_WAVE_LEXERTL_LEXER_HPP_INCLUDED) |
| |
