mirror of
https://github.com/python/cpython.git
synced 2024-12-06 16:25:31 +08:00
495da29225
This adds a `feature_version` flag to `ast.parse()` (documented) and `compile()` (hidden) that allow tweaking the parser to support older versions of the grammar. In particular if `feature_version` is 5 or 6, the hacks for the `async` and `await` keyword from PEP 492 are reinstated. (For 7 or higher, these are unconditionally treated as keywords, but they are still special tokens rather than `NAME` tokens that the parser driver recognizes.) https://bugs.python.org/issue35975
244 lines
4.5 KiB
C
Generated
244 lines
4.5 KiB
C
Generated
/* Auto-generated by Tools/scripts/generate_token.py */
|
|
|
|
#include "Python.h"
|
|
#include "token.h"
|
|
|
|
/* Token names */
|
|
|
|
const char * const _PyParser_TokenNames[] = {
|
|
"ENDMARKER",
|
|
"NAME",
|
|
"NUMBER",
|
|
"STRING",
|
|
"NEWLINE",
|
|
"INDENT",
|
|
"DEDENT",
|
|
"LPAR",
|
|
"RPAR",
|
|
"LSQB",
|
|
"RSQB",
|
|
"COLON",
|
|
"COMMA",
|
|
"SEMI",
|
|
"PLUS",
|
|
"MINUS",
|
|
"STAR",
|
|
"SLASH",
|
|
"VBAR",
|
|
"AMPER",
|
|
"LESS",
|
|
"GREATER",
|
|
"EQUAL",
|
|
"DOT",
|
|
"PERCENT",
|
|
"LBRACE",
|
|
"RBRACE",
|
|
"EQEQUAL",
|
|
"NOTEQUAL",
|
|
"LESSEQUAL",
|
|
"GREATEREQUAL",
|
|
"TILDE",
|
|
"CIRCUMFLEX",
|
|
"LEFTSHIFT",
|
|
"RIGHTSHIFT",
|
|
"DOUBLESTAR",
|
|
"PLUSEQUAL",
|
|
"MINEQUAL",
|
|
"STAREQUAL",
|
|
"SLASHEQUAL",
|
|
"PERCENTEQUAL",
|
|
"AMPEREQUAL",
|
|
"VBAREQUAL",
|
|
"CIRCUMFLEXEQUAL",
|
|
"LEFTSHIFTEQUAL",
|
|
"RIGHTSHIFTEQUAL",
|
|
"DOUBLESTAREQUAL",
|
|
"DOUBLESLASH",
|
|
"DOUBLESLASHEQUAL",
|
|
"AT",
|
|
"ATEQUAL",
|
|
"RARROW",
|
|
"ELLIPSIS",
|
|
"COLONEQUAL",
|
|
"OP",
|
|
"AWAIT",
|
|
"ASYNC",
|
|
"TYPE_IGNORE",
|
|
"TYPE_COMMENT",
|
|
"<ERRORTOKEN>",
|
|
"<COMMENT>",
|
|
"<NL>",
|
|
"<ENCODING>",
|
|
"<N_TOKENS>",
|
|
};
|
|
|
|
/* Return the token corresponding to a single character */
|
|
|
|
int
|
|
PyToken_OneChar(int c1)
|
|
{
|
|
switch (c1) {
|
|
case '%': return PERCENT;
|
|
case '&': return AMPER;
|
|
case '(': return LPAR;
|
|
case ')': return RPAR;
|
|
case '*': return STAR;
|
|
case '+': return PLUS;
|
|
case ',': return COMMA;
|
|
case '-': return MINUS;
|
|
case '.': return DOT;
|
|
case '/': return SLASH;
|
|
case ':': return COLON;
|
|
case ';': return SEMI;
|
|
case '<': return LESS;
|
|
case '=': return EQUAL;
|
|
case '>': return GREATER;
|
|
case '@': return AT;
|
|
case '[': return LSQB;
|
|
case ']': return RSQB;
|
|
case '^': return CIRCUMFLEX;
|
|
case '{': return LBRACE;
|
|
case '|': return VBAR;
|
|
case '}': return RBRACE;
|
|
case '~': return TILDE;
|
|
}
|
|
return OP;
|
|
}
|
|
|
|
int
|
|
PyToken_TwoChars(int c1, int c2)
|
|
{
|
|
switch (c1) {
|
|
case '!':
|
|
switch (c2) {
|
|
case '=': return NOTEQUAL;
|
|
}
|
|
break;
|
|
case '%':
|
|
switch (c2) {
|
|
case '=': return PERCENTEQUAL;
|
|
}
|
|
break;
|
|
case '&':
|
|
switch (c2) {
|
|
case '=': return AMPEREQUAL;
|
|
}
|
|
break;
|
|
case '*':
|
|
switch (c2) {
|
|
case '*': return DOUBLESTAR;
|
|
case '=': return STAREQUAL;
|
|
}
|
|
break;
|
|
case '+':
|
|
switch (c2) {
|
|
case '=': return PLUSEQUAL;
|
|
}
|
|
break;
|
|
case '-':
|
|
switch (c2) {
|
|
case '=': return MINEQUAL;
|
|
case '>': return RARROW;
|
|
}
|
|
break;
|
|
case '/':
|
|
switch (c2) {
|
|
case '/': return DOUBLESLASH;
|
|
case '=': return SLASHEQUAL;
|
|
}
|
|
break;
|
|
case ':':
|
|
switch (c2) {
|
|
case '=': return COLONEQUAL;
|
|
}
|
|
break;
|
|
case '<':
|
|
switch (c2) {
|
|
case '<': return LEFTSHIFT;
|
|
case '=': return LESSEQUAL;
|
|
case '>': return NOTEQUAL;
|
|
}
|
|
break;
|
|
case '=':
|
|
switch (c2) {
|
|
case '=': return EQEQUAL;
|
|
}
|
|
break;
|
|
case '>':
|
|
switch (c2) {
|
|
case '=': return GREATEREQUAL;
|
|
case '>': return RIGHTSHIFT;
|
|
}
|
|
break;
|
|
case '@':
|
|
switch (c2) {
|
|
case '=': return ATEQUAL;
|
|
}
|
|
break;
|
|
case '^':
|
|
switch (c2) {
|
|
case '=': return CIRCUMFLEXEQUAL;
|
|
}
|
|
break;
|
|
case '|':
|
|
switch (c2) {
|
|
case '=': return VBAREQUAL;
|
|
}
|
|
break;
|
|
}
|
|
return OP;
|
|
}
|
|
|
|
int
|
|
PyToken_ThreeChars(int c1, int c2, int c3)
|
|
{
|
|
switch (c1) {
|
|
case '*':
|
|
switch (c2) {
|
|
case '*':
|
|
switch (c3) {
|
|
case '=': return DOUBLESTAREQUAL;
|
|
}
|
|
break;
|
|
}
|
|
break;
|
|
case '.':
|
|
switch (c2) {
|
|
case '.':
|
|
switch (c3) {
|
|
case '.': return ELLIPSIS;
|
|
}
|
|
break;
|
|
}
|
|
break;
|
|
case '/':
|
|
switch (c2) {
|
|
case '/':
|
|
switch (c3) {
|
|
case '=': return DOUBLESLASHEQUAL;
|
|
}
|
|
break;
|
|
}
|
|
break;
|
|
case '<':
|
|
switch (c2) {
|
|
case '<':
|
|
switch (c3) {
|
|
case '=': return LEFTSHIFTEQUAL;
|
|
}
|
|
break;
|
|
}
|
|
break;
|
|
case '>':
|
|
switch (c2) {
|
|
case '>':
|
|
switch (c3) {
|
|
case '=': return RIGHTSHIFTEQUAL;
|
|
}
|
|
break;
|
|
}
|
|
break;
|
|
}
|
|
return OP;
|
|
}
|