mirrors/php-src
0
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2024-11-24 10:24:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Regenerate tokenizer data

Browse Source 
Also fix the tokenizer_data_gen.sh script after 64bit changes.
This commit is contained in:
Nikita Popov 2014-10-05 21:59:46 +02:00
parent 292421d3a1
commit a86fd31c8c
2 changed files with 36 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
ext/tokenizer/tokenizer_data.c
View File

@ -51,6 +51,7 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
REGISTER_LONG_CONSTANT("T_MUL_EQUAL", T_MUL_EQUAL, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_MINUS_EQUAL", T_MINUS_EQUAL, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_PLUS_EQUAL", T_PLUS_EQUAL, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_COALESCE", T_COALESCE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_BOOLEAN_OR", T_BOOLEAN_OR, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_BOOLEAN_AND", T_BOOLEAN_AND, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_IS_NOT_IDENTICAL", T_IS_NOT_IDENTICAL, CONST_CS | CONST_PERSISTENT);
@ -74,22 +75,29 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
REGISTER_LONG_CONSTANT("T_POW", T_POW, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_CLONE", T_CLONE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_NEW", T_NEW, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_EXIT", T_EXIT, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_IF", T_IF, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ELSEIF", T_ELSEIF, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ELSE", T_ELSE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ENDIF", T_ENDIF, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_STRING_VARNAME", T_STRING_VARNAME, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_VARIABLE", T_VARIABLE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_NUM_STRING", T_NUM_STRING, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_INLINE_HTML", T_INLINE_HTML, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_CHARACTER", T_CHARACTER, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_BAD_CHARACTER", T_BAD_CHARACTER, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ENCAPSED_AND_WHITESPACE", T_ENCAPSED_AND_WHITESPACE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_CONSTANT_ENCAPSED_STRING", T_CONSTANT_ENCAPSED_STRING, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_STRING_VARNAME", T_STRING_VARNAME, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_NUM_STRING", T_NUM_STRING, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_EXIT", T_EXIT, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_IF", T_IF, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_CHARACTER", T_CHARACTER, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_BAD_CHARACTER", T_BAD_CHARACTER, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ECHO", T_ECHO, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DO", T_DO, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_WHILE", T_WHILE, CONST_CS | CONST_PERSISTENT);
@ -118,12 +126,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
REGISTER_LONG_CONSTANT("T_USE", T_USE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_INSTEADOF", T_INSTEADOF, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_GLOBAL", T_GLOBAL, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_VAR", T_VAR, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_UNSET", T_UNSET, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ISSET", T_ISSET, CONST_CS | CONST_PERSISTENT);
@ -135,16 +137,16 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
REGISTER_LONG_CONSTANT("T_EXTENDS", T_EXTENDS, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_IMPLEMENTS", T_IMPLEMENTS, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_OBJECT_OPERATOR", T_OBJECT_OPERATOR, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_LIST", T_LIST, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ARRAY", T_ARRAY, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_CALLABLE", T_CALLABLE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_LINE", T_LINE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_FILE", T_FILE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DIR", T_DIR, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_CLASS_C", T_CLASS_C, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_TRAIT_C", T_TRAIT_C, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_METHOD_C", T_METHOD_C, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_FUNC_C", T_FUNC_C, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_LINE", T_LINE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_FILE", T_FILE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_COMMENT", T_COMMENT, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DOC_COMMENT", T_DOC_COMMENT, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_OPEN_TAG", T_OPEN_TAG, CONST_CS | CONST_PERSISTENT);
@ -158,7 +160,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
REGISTER_LONG_CONSTANT("T_PAAMAYIM_NEKUDOTAYIM", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_NAMESPACE", T_NAMESPACE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_NS_C", T_NS_C, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DIR", T_DIR, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_NS_SEPARATOR", T_NS_SEPARATOR, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_ELLIPSIS", T_ELLIPSIS, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);
@ -190,6 +191,7 @@ char *get_token_type_name(int token_type)
case T_MUL_EQUAL: return "T_MUL_EQUAL";
case T_MINUS_EQUAL: return "T_MINUS_EQUAL";
case T_PLUS_EQUAL: return "T_PLUS_EQUAL";
case T_COALESCE: return "T_COALESCE";
case T_BOOLEAN_OR: return "T_BOOLEAN_OR";
case T_BOOLEAN_AND: return "T_BOOLEAN_AND";
case T_IS_NOT_IDENTICAL: return "T_IS_NOT_IDENTICAL";
@ -213,22 +215,29 @@ char *get_token_type_name(int token_type)
case T_POW: return "T_POW";
case T_CLONE: return "T_CLONE";
case T_NEW: return "T_NEW";
case T_EXIT: return "T_EXIT";
case T_IF: return "T_IF";
case T_ELSEIF: return "T_ELSEIF";
case T_ELSE: return "T_ELSE";
case T_ENDIF: return "T_ENDIF";
case T_PUBLIC: return "T_PUBLIC";
case T_PROTECTED: return "T_PROTECTED";
case T_PRIVATE: return "T_PRIVATE";
case T_FINAL: return "T_FINAL";
case T_ABSTRACT: return "T_ABSTRACT";
case T_STATIC: return "T_STATIC";
case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
case T_LNUMBER: return "T_LNUMBER";
case T_DNUMBER: return "T_DNUMBER";
case T_STRING: return "T_STRING";
case T_STRING_VARNAME: return "T_STRING_VARNAME";
case T_VARIABLE: return "T_VARIABLE";
case T_NUM_STRING: return "T_NUM_STRING";
case T_INLINE_HTML: return "T_INLINE_HTML";
case T_CHARACTER: return "T_CHARACTER";
case T_BAD_CHARACTER: return "T_BAD_CHARACTER";
case T_ENCAPSED_AND_WHITESPACE: return "T_ENCAPSED_AND_WHITESPACE";
case T_CONSTANT_ENCAPSED_STRING: return "T_CONSTANT_ENCAPSED_STRING";
case T_STRING_VARNAME: return "T_STRING_VARNAME";
case T_NUM_STRING: return "T_NUM_STRING";
case T_EXIT: return "T_EXIT";
case T_IF: return "T_IF";
case T_CHARACTER: return "T_CHARACTER";
case T_BAD_CHARACTER: return "T_BAD_CHARACTER";
case T_ECHO: return "T_ECHO";
case T_DO: return "T_DO";
case T_WHILE: return "T_WHILE";
@ -257,12 +266,6 @@ char *get_token_type_name(int token_type)
case T_USE: return "T_USE";
case T_INSTEADOF: return "T_INSTEADOF";
case T_GLOBAL: return "T_GLOBAL";
case T_PUBLIC: return "T_PUBLIC";
case T_PROTECTED: return "T_PROTECTED";
case T_PRIVATE: return "T_PRIVATE";
case T_FINAL: return "T_FINAL";
case T_ABSTRACT: return "T_ABSTRACT";
case T_STATIC: return "T_STATIC";
case T_VAR: return "T_VAR";
case T_UNSET: return "T_UNSET";
case T_ISSET: return "T_ISSET";
@ -274,16 +277,16 @@ char *get_token_type_name(int token_type)
case T_EXTENDS: return "T_EXTENDS";
case T_IMPLEMENTS: return "T_IMPLEMENTS";
case T_OBJECT_OPERATOR: return "T_OBJECT_OPERATOR";
case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
case T_LIST: return "T_LIST";
case T_ARRAY: return "T_ARRAY";
case T_CALLABLE: return "T_CALLABLE";
case T_LINE: return "T_LINE";
case T_FILE: return "T_FILE";
case T_DIR: return "T_DIR";
case T_CLASS_C: return "T_CLASS_C";
case T_TRAIT_C: return "T_TRAIT_C";
case T_METHOD_C: return "T_METHOD_C";
case T_FUNC_C: return "T_FUNC_C";
case T_LINE: return "T_LINE";
case T_FILE: return "T_FILE";
case T_COMMENT: return "T_COMMENT";
case T_DOC_COMMENT: return "T_DOC_COMMENT";
case T_OPEN_TAG: return "T_OPEN_TAG";
@ -297,7 +300,6 @@ char *get_token_type_name(int token_type)
case T_PAAMAYIM_NEKUDOTAYIM: return "T_DOUBLE_COLON";
case T_NAMESPACE: return "T_NAMESPACE";
case T_NS_C: return "T_NS_C";
case T_DIR: return "T_DIR";
case T_NS_SEPARATOR: return "T_NS_SEPARATOR";
case T_ELLIPSIS: return "T_ELLIPSIS";

4
ext/tokenizer/tokenizer_data_gen.sh
View File

@ -45,8 +45,8 @@ echo '/*
echo 'void tokenizer_register_constants(INIT_FUNC_ARGS) {' >> $OUTFILE
$AWK '/^#define T_/ { print " REGISTER_INT_CONSTANT(\"" $2 "\", " $2 ", CONST_CS | CONST_PERSISTENT);" }' < $INFILE >> $OUTFILE
echo ' REGISTER_INT_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);' >> $OUTFILE
$AWK '/^#define T_/ { print " REGISTER_LONG_CONSTANT(\"" $2 "\", " $2 ", CONST_CS | CONST_PERSISTENT);" }' < $INFILE >> $OUTFILE
echo ' REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);' >> $OUTFILE
echo '}' >> $OUTFILE