From 9c8aef3c05ea2f1eda30291989eb94a5ce6c893c Mon Sep 17 00:00:00 2001 From: CPython Developers <> Date: Thu, 22 Jan 2026 01:00:15 +0900 Subject: [PATCH] Update token from v3.14.2 --- Lib/token.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Lib/token.py b/Lib/token.py index 54d7cdcca..f61723cc0 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -66,12 +66,15 @@ SOFT_KEYWORD = 58 FSTRING_START = 59 FSTRING_MIDDLE = 60 FSTRING_END = 61 -COMMENT = 62 -NL = 63 +TSTRING_START = 62 +TSTRING_MIDDLE = 63 +TSTRING_END = 64 +COMMENT = 65 +NL = 66 # These aren't used by the C tokenizer but are needed for tokenize.py -ERRORTOKEN = 64 -ENCODING = 65 -N_TOKENS = 66 +ERRORTOKEN = 67 +ENCODING = 68 +N_TOKENS = 69 # Special definitions for cooperation with parser NT_OFFSET = 256 @@ -131,11 +134,11 @@ EXACT_TOKEN_TYPES = { '~': TILDE, } -def ISTERMINAL(x): +def ISTERMINAL(x: int) -> bool: return x < NT_OFFSET -def ISNONTERMINAL(x): +def ISNONTERMINAL(x: int) -> bool: return x >= NT_OFFSET -def ISEOF(x): +def ISEOF(x: int) -> bool: return x == ENDMARKER