51 #include <unordered_map> 53 #include "../clib/clib.h" 54 #include "../clib/logfacility.h" 55 #include "../clib/passert.h" 56 #include "../clib/rawtypes.h" 57 #include "../clib/strutil.h" 58 #include "../clib/unittest.h" 67 #include <format/format.h> 89 "Unterminated String Literal",
90 "Invalid escape sequence in String",
94 "Illegal Construction",
96 "Token not legal here",
97 "Procedure calls not allowed here",
98 "Unexpected Semicolon",
105 "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 106 "abcdefghijklmnopqrstuvwxyz" 116 {1, 1, 0, 1, 1, 0, 0, 0},
117 {1, 0, 1, 0, 0, 1, 1, 1},
118 {0, 1, 0, 1, 1, 0, 0, 0},
119 {0, 1, 0, 0, 1, 0, 0, 0},
120 {0, 1, 0, 1, 1, 0, 0, 0},
121 {1, 0, 1, 0, 0, 1, 0, 1},
122 {0, 1, 0, 0, 1, 0, 1, 0},
123 {1, 0, 1, 0, 0, 1, 1, 1},
243 int n_operators =
sizeof binary_operators /
sizeof binary_operators[0];
254 int n_unary =
sizeof unary_operators /
sizeof unary_operators[0];
485 static auto cache = []() -> std::unordered_map<std::string, ObjMember*> {
486 std::unordered_map<std::string, ObjMember*> m;
489 m[object_members[i].
code] = &object_members[i];
493 std::string temp( token );
494 std::transform( temp.begin(), temp.end(), temp.begin(),
495 [](
char c ) {
return static_cast<char>(::tolower( c ) ); } );
496 auto member = cache.find( temp );
497 if ( member != cache.end() )
498 return member->second;
503 if (
id >= n_objmembers )
506 return &( object_members[id] );
669 static auto cache = []() -> std::unordered_map<std::string, ObjMethod*> {
670 std::unordered_map<std::string, ObjMethod*> m;
673 m[object_methods[i].
code] = &object_methods[i];
677 std::string temp( token );
678 std::transform( temp.begin(), temp.end(), temp.begin(),
679 [](
char c ) {
return static_cast<char>(::tolower( c ) ); } );
680 auto method = cache.find( temp );
681 if ( method != cache.end() )
682 return method->second;
687 if (
id >= n_objmethods )
690 return &( object_methods[id] );
697 if ( object_methods[i].
id != i )
699 INFO_PRINT <<
"ERROR: Object Method definition of " << object_methods[i].
code 700 <<
" has an invalid index!\n";
702 auto c =
reinterpret_cast<unsigned char*
>( object_methods[i].
code );
705 if ( *c != tolower( *c ) )
707 INFO_PRINT <<
"ERROR: Object Method definition of " << object_methods[i].
code 708 <<
" is not lowercase!\n";
716 if ( object_members[i].
id != i )
718 INFO_PRINT <<
"ERROR: Object Member definition of " << object_members[i].
code 719 <<
" has an invalid index!\n";
721 auto c =
reinterpret_cast<unsigned char*
>( object_members[i].
code );
724 if ( *c != tolower( *c ) )
726 INFO_PRINT <<
"ERROR: Object Member definition of " << object_members[i].
code 727 <<
" is not lowercase!\n";
741 int lenbuf = buf[1] ? 2 : 1;
744 *pTotalMatchOperator = NULL;
749 for (
int i = 0; i < n_ops; ++i, ++op )
751 if ( op->
code[0] == buf[0] )
753 if ( op->
code[1] ==
'\0' )
755 ( *pTotalMatchOperator ) = op;
769 for (
int i = 0; i < n_ops; ++i, ++op )
771 if ( op->
code[0] == buf[0] && op->
code[1] == buf[1] )
773 ( *pTotalMatchOperator ) = op;
911 unsigned n_reserved =
sizeof reserved_words /
sizeof reserved_words[0];
913 typedef std::map<std::string, ReservedWord*, Clib::ci_cmp_pred>
ReservedWords;
917 static std::once_flag flag;
918 std::call_once( flag, []() {
921 reservedWordsByName[reserved_words[i].
word] = &reserved_words[i];
928 os <<
"Reserved:" << std::endl;
931 os << reserved_words[i].
word << ( reserved_words[i].
deprecated ?
" (deprecated)" :
"" )
935 os <<
"Binary:" << std::endl;
938 os << binary_operators[i].
code << ( binary_operators[i].
deprecated ?
" (deprecated)" :
"" )
942 os <<
"Unary:" << std::endl;
943 for (
int i = 0; i <
n_unary; ++i )
945 os << unary_operators[i].
code << ( unary_operators[i].
deprecated ?
" (deprecated)" :
"" )
949 os <<
"Methodlist:" << std::endl;
952 os << object_methods[i].
id <<
" " << object_methods[i].
code << std::endl;
955 os <<
"Memberlist:" << std::endl;
958 os << object_members[i].
id <<
" " << object_members[i].
code << std::endl;
963 void matchReservedWords(
char *buf,
967 int lenbuf = strlen(buf);
968 assert(nPartial && nTotal);
973 if (strnicmp(reserved_words[i].word, buf, lenbuf)==0)
975 if (stricmp(reserved_words[i].word, buf)==0)
989 int Parser::tryReservedWord(
Token& tok,
char *t,
char **s)
993 int thisMatchPartial, thisMatchTotal;
994 int lastMatchTotal = 0;
999 opbuf[bufp++] = *t++;
1001 matchReservedWords(opbuf, &thisMatchPartial, &thisMatchTotal);
1002 if (!thisMatchPartial) {
1003 switch(lastMatchTotal) {
1009 opbuf[bufp-1] =
'\0';
1022 lastMatchTotal = thisMatchTotal;
1026 if (thisMatchTotal == 1) {
1052 int thisMatchPartial;
1066 opbuf[bufp++] = *t++;
1068 matchOperators( opList, n_ops, opbuf, &thisMatchPartial, &pMatch );
1069 if ( !thisMatchPartial )
1075 else if ( !pLastMatch )
1091 pLastMatch = pMatch;
1096 if ( pMatch == NULL )
1097 pMatch = pLastMatch;
1101 *s += strlen( pMatch->
code );
1104 tok.
id = pMatch->
id;
1128 res =
tryOperator( tok, ctx.
s, &ctx.
s, binary_operators, n_operators, opbuf );
1145 res =
tryOperator( tok, ctx.
s, &ctx.
s, unary_operators, n_unary, opbuf );
1159 if ( isdigit( ctx.
s[0] ) || ctx.
s[0] ==
'.' )
1161 char *endptr, *endptr2;
1162 int l = strtol( ctx.
s, &endptr, 0 );
1163 double d = strtod( ctx.
s, &endptr2 );
1169 if ( !( ctx.
s[0] ==
'0' && ctx.
s[1] && ( ctx.
s[1] ==
'x' || ctx.
s[1] ==
'X' ) ) )
1172 for (
const char* i = ctx.
s; i < endptr2; i++ )
1174 if ( *i ==
'd' || *i ==
'D' )
1178 size_t safelen = i - ctx.
s + 1;
1179 std::unique_ptr<char[]> safeptr(
new char[safelen]() );
1180 strncpy( safeptr.get(), ctx.
s, safelen - 1 );
1181 d = strtod( safeptr.get(), &endptr2 );
1182 size_t newlen = endptr2 - safeptr.get();
1183 endptr2 =
const_cast<char*
>( ctx.
s + newlen );
1190 if ( endptr >= endptr2 )
1217 if ( ctx.
s[0] ==
'\"' )
1219 const char* end = &ctx.
s[1];
1221 bool escnext =
false;
1224 memset( hexstr, 0, 3 );
1235 "Bug in the compiler. Please report this on the forums." );
1243 else if ( *end ==
't' )
1245 else if ( *end ==
'x' )
1253 hexstr[2 - hexnext] = *end;
1257 char ord =
static_cast<char>( strtol( hexstr, &endptr, 16 ) );
1258 if ( *endptr !=
'\0' )
1270 else if ( *end ==
'\"' )
1293 else if ( isalpha( ctx.
s[0] ) || ctx.
s[0] ==
'_' )
1295 const char* end = ctx.
s;
1296 while ( *end && !isspace( *end ) && strchr( ident_allowed, *end ) )
1301 if ( end[0] ==
':' && end[1] ==
':' )
1304 while ( *end && !isspace( *end ) && strchr( ident_allowed, *end ) )
1310 int len =
static_cast<int>( end - ctx.
s + 1 );
1327 if ( stricmp( buf, binary_operators[i].code ) == 0 )
1330 tok.
id = binary_operators[i].
id;
1331 tok.
type = binary_operators[i].
type;
1351 tok.
setStr( binary_operators[i].code );
1365 for (
int i = 0; i <
n_unary; i++ )
1367 if ( stricmp( buf, unary_operators[i].code ) == 0 )
1370 tok.
id = unary_operators[i].
id;
1371 tok.
type = unary_operators[i].
type;
1373 tok.
setStr( unary_operators[i].code );
1395 auto itr = reservedWordsByName.find( buf );
1396 if ( itr != reservedWordsByName.end() )
1435 while ( !ex.
CA.empty() )
1437 delete ex.
CA.front();
1442 while ( !ex.
TX.empty() )
1494 if ( ctx.
s[0] ==
';' )
1543 if ( ctx.
s[0] ==
':' )
1552 INFO_PRINT <<
"Your syntax frightens and confuses me.\n";
1565 return getToken( tctx, token, expr );
1595 INFO_PRINT <<
"isOkay(" << this_type <<
"," << last_type <<
")\n";
1609 return allowed_table[last_type][this_type];
1658 if (t[0] ==
':' && t[1] ==
':')
1664 if (res2 < 0)
return res2;
1671 if ( ctx.
s[0] ==
':' && ( ctx.
s[1] ==
'\0' || isspace( ctx.
s[1] ) ) )
1688 _tmp <<
"parseToken( " << *token <<
")\n";
1690 std::queue<Token*> ca( expr.
CA );
1691 while ( !ca.empty() )
1693 Token* tk = ca.front();
1699 std::stack<Token*> tx( expr.
TX );
1700 while ( !tx.empty() )
1702 Token* tk = tx.top();
1712 switch ( token->
type )
1717 expr.
CA.push( token );
1720 switch ( last->
type )
1724 delete expr.
TX.top();
1728 expr.
CA.push( expr.
TX.top() );
1732 expr.
CA.push( expr.
TX.top() );
1752 switch ( last->
type )
1759 expr.
TX.push( token );
1768 if ( this_prec > last_prec )
1770 expr.
TX.push( token );
1775 expr.
CA.push( expr.
TX.top() );
1786 expr.
TX.push( token );
1789 switch ( last->
type )
1796 expr.
CA.push( expr.
TX.top() );
1801 delete expr.
TX.top();
1805 INFO_PRINT <<
"Unmatched ')' in expression. (Trying to match against a '" << *last <<
"')\n" 1808 ERROR_PRINT <<
"parseToken(): Not sure what to do.\n" 1809 <<
"Token: " << *token <<
"\n" 1810 <<
"Last: " << *last <<
"\n";
1811 throw std::runtime_error(
"Error in parseToken() (1)" );
1819 switch ( last->
type )
1826 expr.
CA.push( expr.
TX.top() );
1836 if ( ptkn->
lval != 1 )
1838 expr.
CA.push( ptkn );
1841 delete expr.
TX.top();
1845 INFO_PRINT <<
"Unmatched ']' in expression. (Trying to match against a '" << *last <<
"')\n" 1847 ERROR_PRINT <<
"parseToken(): Not sure what to do.\n" 1848 <<
"Token: " << *token <<
"\n" 1849 <<
"Last: " << *last <<
"\n";
1850 throw std::runtime_error(
"Error in parseToken() (2)" );
1872 expr.
CA.push( expr.
TX.top() );
1883 INFO_PRINT <<
"Don't know what to do with '" << *token <<
"' in SmartParser::parseToken\n" 1932 if ( !pexpr->
TX.empty() )
1943 if ( isFunc( token, &modfunc_ ) )
1945 if ( isUserFunc( token, &userfunc_ ) )
1968 switch ( mfUse->
nargs )
2013 for ( nargs = 0; nargs < mfUse->
nargs; nargs++ )
2031 if ( nargs != mfUse->
nargs )
2058 Token tk_name, tk_paren;
2097 strtok( buf,
"\r\n" );
2102 int leftbracket_count = 0;
2103 while ( !done && ( res == 0 ) )
2105 const char* t = ctx.
s;
2112 if ( !isLegal( token ) )
2137 if ( leftbracket_count == 0 )
2147 if ( to_term_allowed && ( token.
id ==
RSV_TO ) )
2153 if ( rightbrace_term_allowed && token.
id ==
TOK_RBRACE )
2177 res =
getToken( ctx, token, &expr );
2183 ++leftbracket_count;
2185 --leftbracket_count;
2187 if ( !isOkay( token, last_type ) )
2192 if ( !isOkay( token, last_type ) )
2194 if ( auto_term_allowed )
2203 INFO_PRINT <<
"Token '" << token <<
"' cannot follow token '" << last_token <<
"'\n";
2206 INFO_PRINT <<
"Function " << last_token <<
"() is not defined.\n";
2216 <<
"Warning: Using { } is inappropriate; please define array, struct or dictionary.\n";
2218 throw std::runtime_error(
"Warnings treated as errors." );
2222 last_type = token.
type;
2229 if ( auto_term_allowed )
2245 res = getUserArgs( expr, ctx );
2249 ptok2 =
new Token( token );
2259 userfunc_ = modfunc_->uf;
2260 res = getUserArgs( expr, ctx,
false );
2263 INFO_PRINT <<
"Error getting arguments for function " << token.
tokval() <<
"\n" 2267 ptok2 =
new Token( token );
2305 res = getArrayElements( expr, ctx );
2311 res = getNewArrayElements( expr, ctx );
2315 INFO_PRINT <<
"Error getting elements for array\n";
2322 res = getNewArrayElements( expr, ctx );
2325 INFO_PRINT <<
"Error getting elements for array\n";
2333 expr.
CA.push( error_tkn );
2334 res = getStructMembers( expr, ctx );
2337 INFO_PRINT <<
"Error reading members for error\n";
2345 expr.
CA.push( struct_tkn );
2346 res = getStructMembers( expr, ctx );
2349 INFO_PRINT <<
"Error reading members for struct\n";
2354 auto dict_tkn =
new Token( token );
2357 expr.
CA.push( dict_tkn );
2358 res = getDictionaryMembers( expr, ctx );
2361 INFO_PRINT <<
"Error reading members for dictionary\n";
2366 auto ref_tkn =
new Token( token );
2367 res = getFunctionPArgument( expr, ctx, ref_tkn );
2370 INFO_PRINT <<
"Error reading function reference argument\n";
2372 expr.
CA.push( ref_tkn );
2374 else if ( token.
id ==
TOK_MEMBER && callingMethod( ctx ) )
2380 ptok2 =
new Token( token );
2390 std::string methodName( token.
tokval() );
2393 res = getMethodArguments( expr, ctx, nargs );
2404 ptok2->
lval = nargs;
2405 ptok2->
dval = objmeth->
id;
2411 ptok2->
lval = nargs;
2412 ptok2->
copyStr( methodName.c_str() );
2420 ptok2 =
new Token( token );
2444 if ( !comma_term_allowed && !
quiet )
virtual int tryBinaryOperator(Token &tok, CompilerContext &ctx)
const unsigned EXPR_FLAG_DICTKEY_TERM_ALLOWED
virtual int recognize(Token &tok, const char *buf, const char **s)
const unsigned EXPR_FLAG_AUTO_TERM_ALLOWED
static void write_words(std::ostream &os)
virtual bool recognize_reserved_word(Token &tok, const char *buf)
virtual int getToken(CompilerContext &ctx, Token &token, Expression *expr=NULL) POL_OVERRIDE
int getArgs(Expression &expr, CompilerContext &ctx)
ReservedWord reserved_words[]
virtual int tryOperator(Token &tok, const char *buf, const char **s, Operator *opList, int n_ops, char *opbuf)
const char * tokval() const
void reinit(Expression &ex)
ObjMember * getKnownObjMember(const char *token)
int IP(Expression &expr, char *s)
Operator binary_operators[]
std::map< std::string, ReservedWord *, Clib::ci_cmp_pred > ReservedWords
const unsigned EXPR_FLAG_SINGLE_ELEMENT
virtual int recognize_binary(Token &tok, const char *buf, const char **s)
char * stracpy(char *dest, const char *src, size_t maxlen)
CompilerConfig compilercfg
ObjMethod * getKnownObjMethod(const char *token)
const unsigned EXPR_FLAG_RIGHTPAREN_TERM_ALLOWED
virtual int recognize_unary(Token &tok, const char *buf)
int IIP(Expression &expr, CompilerContext &ctx, unsigned expr_flags)
void copyStr(const char *s)
const unsigned EXPR_FLAG_TO_TERM_ALLOWED
virtual int parseToken(CompilerContext &ctx, Expression &expr, Token *) POL_OVERRIDE
Clib::UnitTest testparserdefinitions_obj(testparserdefinitions)
void matchOperators(Operator *oplist, int n_ops, char *buf, int *nPartial, Operator **pTotalMatchOperator)
virtual int tryNumeric(Token &tok, CompilerContext &ctx)
virtual int getToken(CompilerContext &ctx, Token &token, Expression *expr=NULL)
ObjMember * getObjMember(int id)
ObjMethod * getObjMethod(int id)
bool OptimizeObjectMembers
virtual int tryLiteral(Token &tok, CompilerContext &ctx)
void testparserdefinitions()
#define passert_always_r(exp, reason)
virtual int tryUnaryOperator(Token &tok, CompilerContext &ctx)
const char * ParseErrorStr[PERR_NUM_ERRORS]
const unsigned EXPR_FLAG_RIGHTBRACE_TERM_ALLOWED
ReservedWords reservedWordsByName
virtual int peekToken(const CompilerContext &ctx, Token &token, Expression *expr=NULL)
virtual int tryLiteral(Token &tok, CompilerContext &ctx) POL_OVERRIDE
virtual int parseToken(CompilerContext &ctx, Expression &expr, Token *token)=0
void setStr(const char *s)
const unsigned EXPR_FLAG_COMMA_TERM_ALLOWED
void mklower(std::string &str)
Precedence find_precedence(Token &token)
bool callingMethod(CompilerContext &ctx)
ObjMethod object_methods[]
virtual int isOkay(const Token &token, BTokenType last_type)
const unsigned EXPR_FLAG_ENDENUM_TERM_ALLOWED
ObjMember object_members[]
static void init_tables()
Operator unary_operators[]
const unsigned EXPR_FLAG_SEMICOLON_TERM_ALLOWED