mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-28 18:59:04 +08:00
Update token from v3.14.2
This commit is contained in:
1 parent
726a053ca2
commit
9c8aef3c05
1 file changed
+11
-8
Vendored
+11
-8
@@ -66,12 +66,15 @@ SOFT_KEYWORD = 58
|
||||
FSTRING_START = 59
|
||||
FSTRING_MIDDLE = 60
|
||||
FSTRING_END = 61
|
||||
COMMENT = 62
|
||||
NL = 63
|
||||
TSTRING_START = 62
|
||||
TSTRING_MIDDLE = 63
|
||||
TSTRING_END = 64
|
||||
COMMENT = 65
|
||||
NL = 66
|
||||
# These aren't used by the C tokenizer but are needed for tokenize.py
|
||||
ERRORTOKEN = 64
|
||||
ENCODING = 65
|
||||
N_TOKENS = 66
|
||||
ERRORTOKEN = 67
|
||||
ENCODING = 68
|
||||
N_TOKENS = 69
|
||||
# Special definitions for cooperation with parser
|
||||
NT_OFFSET = 256
|
||||
|
||||
@@ -131,11 +134,11 @@ EXACT_TOKEN_TYPES = {
|
||||
'~': TILDE,
|
||||
}
|
||||
|
||||
def ISTERMINAL(x):
|
||||
def ISTERMINAL(x: int) -> bool:
|
||||
return x < NT_OFFSET
|
||||
|
||||
def ISNONTERMINAL(x):
|
||||
def ISNONTERMINAL(x: int) -> bool:
|
||||
return x >= NT_OFFSET
|
||||
|
||||
def ISEOF(x):
|
||||
def ISEOF(x: int) -> bool:
|
||||
return x == ENDMARKER
|
||||
Reference in new issue
Block a user