mirror of
https://github.com/python/cpython.git
synced 2024-11-28 12:31:14 +08:00
b31c7f732a
* test_grammar.py, testall.out: added test for funny things in string literals * token.py, symbol.py: definitions used with built-in parser module. * tokenize.py: added double-quote recognition
51 lines
679 B
Python
Executable File
51 lines
679 B
Python
Executable File
# Tokens (from "token.h")
|
|
|
|
ENDMARKER = 0
|
|
NAME = 1
|
|
NUMBER = 2
|
|
STRING = 3
|
|
NEWLINE = 4
|
|
INDENT = 5
|
|
DEDENT = 6
|
|
LPAR = 7
|
|
RPAR = 8
|
|
LSQB = 9
|
|
RSQB = 10
|
|
COLON = 11
|
|
COMMA = 12
|
|
SEMI = 13
|
|
PLUS = 14
|
|
MINUS = 15
|
|
STAR = 16
|
|
SLASH = 17
|
|
VBAR = 18
|
|
AMPER = 19
|
|
LESS = 20
|
|
GREATER = 21
|
|
EQUAL = 22
|
|
DOT = 23
|
|
PERCENT = 24
|
|
BACKQUOTE = 25
|
|
LBRACE = 26
|
|
RBRACE = 27
|
|
EQEQUAL = 28
|
|
NOTEQUAL = 29
|
|
LESSEQUAL = 30
|
|
GREATEREQUAL = 31
|
|
TILDE = 32
|
|
CIRCUMFLEX = 33
|
|
LEFTSHIFT = 34
|
|
RIGHTSHIFT = 35
|
|
OP = 36
|
|
ERRORTOKEN = 37
|
|
|
|
names = dir()
|
|
tok_name = {}
|
|
for name in names:
|
|
number = eval(name)
|
|
tok_name[number] = name
|
|
|
|
N_TOKENS = 38 # Number of tokens including ERRORTOKEN
|
|
|
|
NT_OFFSET = 256 # Start of non-terminal symbols
|