Update token from v3.14.2

This commit is contained in:
CPython Developers authored and Jeong, YunWon committed 2026-01-22 08:02:43 +09:00
1 parent 726a053ca2
commit 9c8aef3c05
1 file changed
+11 -8
+11 -8
View File
@@ -66,12 +66,15 @@ SOFT_KEYWORD = 58
FSTRING_START = 59
FSTRING_MIDDLE = 60
FSTRING_END = 61
COMMENT = 62
NL = 63
TSTRING_START = 62
TSTRING_MIDDLE = 63
TSTRING_END = 64
COMMENT = 65
NL = 66
# These aren't used by the C tokenizer but are needed for tokenize.py
ERRORTOKEN = 64
ENCODING = 65
N_TOKENS = 66
ERRORTOKEN = 67
ENCODING = 68
N_TOKENS = 69
# Special definitions for cooperation with parser
NT_OFFSET = 256
@@ -131,11 +134,11 @@ EXACT_TOKEN_TYPES = {
'~': TILDE,
}
def ISTERMINAL(x):
def ISTERMINAL(x: int) -> bool:
return x < NT_OFFSET
def ISNONTERMINAL(x):
def ISNONTERMINAL(x: int) -> bool:
return x >= NT_OFFSET
def ISEOF(x):
def ISEOF(x: int) -> bool:
return x == ENDMARKER