mirrors/cpython
0
0 Fork 0
mirror of https://github.com/python/cpython.git synced 2026-10-28 10:25:05 +08:00
Code Activity

gh-159465: Improve incomplete input detection (#159466)

Browse Source 
* gh-159465: Improve incomplete input detection

* gh-159465: Track newline effects and preserve parser layout

* gh-159465: Share incomplete-input error cleanup
This commit is contained in:
Pablo Galindo Salgado authored and GitHub committed 2026-10-18 11:54:19 +01:00
1 parent c5cbe312ca
commit 6af655301a
25 files changed
+395 -115
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Include/errcode.h
+1
View File
@@ -38,6 +38,7 @@ extern "C" {
#define E_BADSINGLE 27 /* Ill-formed single statement input */
#define E_INTERACT_STOP 28 /* Interactive mode stopped tokenization */
#define E_COLUMNOVERFLOW 29 /* Column offset overflow */
#define E_INCOMPLETE 30 /* An unfinished token in partial input */
#ifdef __cplusplus
}
Include/internal/pycore_parser.h
+5 -1
View File
@@ -27,13 +27,17 @@ _Py_DECLARE_STR(empty, "")
}, \
}
/* The caller must initialize *incomplete to NULL if supplied. A returned AST
can also return an owned exception there for a missing line terminator.
Validate the AST before reporting that exception. */
extern struct _mod* _PyParser_ASTFromString(
const char *str,
PyObject* filename,
int mode,
PyCompilerFlags *flags,
PyArena *arena,
PyObject *module);
PyObject *module,
PyObject **incomplete);
extern struct _mod* _PyParser_ASTFromFile(
FILE *fp,
Lib/codeop.py
+3 -8
View File
@@ -62,15 +62,10 @@ def _maybe_compile(compiler, source, filename, symbol, flags):
warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
try:
compiler(source, filename, symbol, flags=flags)
except _IncompleteInputError:
return None
except SyntaxError: # Let other compile() errors propagate.
try:
compiler(source + "\n", filename, symbol, flags=flags)
return None
except _IncompleteInputError:
return None
except SyntaxError:
pass
# fallthrough
pass
return compiler(source, filename, symbol, incomplete_input=False)
Lib/test/test_codeop.py
+93
View File
@@ -124,6 +124,99 @@ class CodeopTests(unittest.TestCase):
self.assertEqual(cm.exception.args, (
'incomplete input', ('<input>', 2, 9, text, 2, -1)))
@subTests('compiler', COMPILERS)
@subTests('newline', ('\n', '\r', '\r\n'))
def test_unterminated_trailing_line(self, compiler, newline):
for head, symbol in (('1', 'eval'), ('if 1: pass', 'single'),
('if 1:\n pass', 'single')):
for tail in (' ', '\t', '\f ', '\n '):
source = head + newline + tail
with self.subTest(source=source, symbol=symbol):
if compiler in RAW_COMPILERS:
with self.assertRaises(_IncompleteInputError):
compiler(source, '<input>', symbol)
else:
self.assertIsNone(compiler(source, '<input>', symbol))
self.assertValid(source + '\n', symbol, compiler=compiler)
for tail in ('\f', ' \f', '\n\f'):
with self.subTest(head=head, tail=tail, symbol=symbol):
self.assertValid(head + newline + tail, symbol,
compiler=compiler)
for tail in ('# comment', ' # comment'):
self.assertValid('if 1: pass' + newline + tail, compiler=compiler)
self.assertValid('1' + newline + tail, 'eval', compiler=compiler)
for head in ('1', 'if 1: pass', 'if 1:\n pass'):
for tail in (' ', '\t', '\f', '# comment'):
self.assertValid(head + newline + tail, 'exec',
compiler=compiler)
@subTests('symbol', ('single', 'exec', 'eval'))
def test_unfinished_token_in_invalid_position(self, symbol):
for source in ('a b', '(a b', '1 """', '(1 """',
'f"{lambda:\\', 't"{lambda:\\'):
for compiler in COMPILERS:
with self.subTest(source=source, compiler=compiler):
with self.assertRaises(SyntaxError) as cm:
compiler(source, '<input>', symbol)
self.assertNotIsInstance(cm.exception, _IncompleteInputError)
@subTests('symbol', ('single', 'exec', 'eval'))
def test_invalid_partial_string_diagnostic(self, symbol):
with self.assertRaisesRegex(SyntaxError, 'invalid syntax'):
Compile()('a"""', '<input>', symbol)
@subTests('symbol', ('single', 'exec', 'eval'))
@subTests('prefix', ('', 'r', 'b', 'f', 't', 'rf', 'rt'))
def test_string_continuation(self, symbol, prefix):
source = prefix + '"first\\'
compile(source + '\nsecond"\n', '<input>', symbol)
self.assertIsNone(compile_command(source, symbol=symbol))
with self.assertRaises(_IncompleteInputError):
Compile()(source, '<input>', symbol)
@subTests('symbol', ('single', 'exec', 'eval'))
@subTests('newline', ('\n', '\r', '\r\n'))
def test_continued_final_line(self, symbol, newline):
source = '1\\' + newline
compile(source + '+ 2\n', '<input>', symbol)
self.assertIsNone(compile_command(source, symbol=symbol))
with self.assertRaises(_IncompleteInputError):
Compile()(source, '<input>', symbol)
with self.assertRaises(SyntaxError) as cm:
Compile()('1 + \\' + newline, '<input>', symbol)
# Appending LF to CR completes the CRLF line terminator.
if newline != '\r':
self.assertNotIsInstance(cm.exception, _IncompleteInputError)
@subTests('symbol', ('single', 'exec'))
def test_eof_before_nested_body(self, symbol):
for source in ('if True:\n if True:',
'if True:\n def f():',
'match 1:\n case 1:'):
for tail in ('', '\n', '\n\n'):
with self.subTest(source=source, tail=tail):
compile(source + tail + '\n pass\n', '<input>', symbol)
self.assertIsNone(compile_command(source + tail, symbol=symbol))
with self.assertRaises(_IncompleteInputError):
Compile()(source + tail, '<input>', symbol)
def test_incomplete_suite_semantic_errors(self):
cases = (
('if True: break', "'break' outside loop"),
('if True: return 1', "'return' outside function"),
('def f(x, x):\n pass', 'duplicate parameter'),
)
for source, message in cases:
for compiler in COMPILERS:
with self.subTest(source=source, compiler=compiler):
with self.assertRaises(SyntaxError) as cm:
compiler(source, '<input>', 'single')
self.assertNotIsInstance(cm.exception, _IncompleteInputError)
if compiler in RAW_COMPILERS:
self.assertIn(message, str(cm.exception))
@subTests('compiler', COMPILERS)
def test_incomplete(self, compiler):
ai = functools.partial(self.assertIncomplete, compiler=compiler)
Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst
+2
View File
@@ -0,0 +1,2 @@
Improve incomplete input detection in :mod:`codeop` without an extra parser
pass, and preserve syntax and compiler errors.
Modules/_testinternalcapi/tokenizer.c
+29
View File
@@ -95,6 +95,35 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
goto error;
}
_PyTok_SourceClear(&source);
if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) == 0,
"cannot append empty terminated line") < 0 ||
check(_PyTok_SourceAppendLineWithNewline(&source, tail, 4) == 1,
"wrong terminated line offset") < 0 ||
check(source.len == 6 && source.nlines == 2 &&
memcmp(source.bytes, "\ntail\n", 7) == 0,
"wrong terminated source contents") < 0 ||
check_system_error(_PyTok_SourceAppendLineWithNewline(
&source, terminated_line, 2) < 0,
"accepted newline in line payload") < 0) {
goto error;
}
source.base_offset = PY_SSIZE_T_MAX - source.len;
if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) < 0 &&
PyErr_ExceptionMatches(PyExc_MemoryError),
"accepted overflowing appended newline") < 0) {
goto error;
}
PyErr_Clear();
source.base_offset = 0;
source.nlines = INT_MAX - 1;
if (check(_PyTok_SourceAppendLineWithNewline(&source, "", 0) < 0 &&
PyErr_ExceptionMatches(PyExc_OverflowError),
"accepted overflowing newline count") < 0) {
goto error;
}
PyErr_Clear();
_PyTok_SourceClear(&source);
Py_RETURN_NONE;
Parser/action_helpers.c
+10 -10
View File
@@ -1497,11 +1497,11 @@ _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b
}
expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
Py_ssize_t bsize;
char* bstr;
if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) {
return NULL;
if (tok->bytes == NULL) {
return _PyPegen_raise_incomplete_input(p);
}
Py_ssize_t bsize = PyBytes_GET_SIZE(tok->bytes);
const char *bstr = PyBytes_AS_STRING(tok->bytes);
int is_raw = tok->is_raw;
@@ -1519,10 +1519,10 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
}
expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
const char* bstr = PyBytes_AsString(tok->bytes);
if (bstr == NULL) {
return NULL;
if (tok->bytes == NULL) {
return _PyPegen_raise_incomplete_input(p);
}
const char *bstr = PyBytes_AS_STRING(tok->bytes);
PyObject* str = PyUnicode_FromString(bstr);
if (str == NULL) {
return NULL;
@@ -1537,10 +1537,10 @@ expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
}
expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok) {
const char* the_str = PyBytes_AsString(tok->bytes);
if (the_str == NULL) {
return NULL;
if (tok->bytes == NULL) {
return _PyPegen_raise_incomplete_input(p);
}
const char *the_str = PyBytes_AS_STRING(tok->bytes);
PyObject *s = _PyPegen_parse_string(p, tok);
if (s == NULL) {
_Pypegen_raise_decode_error(p);
Parser/lexer/layout.c
+12
View File
@@ -21,6 +21,9 @@ _PyLexer_ContinueLine(struct tok_state *tok)
return -1;
}
c = tok_nextc(tok);
if (tok->implicit_newline == _PyTokenizer_APPEND_NEWLINE) {
tok->needs_newline |= _PyTokenizer_NEWLINE_CONTINUATION;
}
if (c == EOF) {
tok->done = E_EOF;
tok->cur = tok->inp;
@@ -114,6 +117,11 @@ _PyLexer_BeginLine(struct tok_state *tok)
}
}
tok_backup(tok, c);
if (c == '\n' && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
tok->level == 0 && (cont_line_col || col)) {
// Without the newline, a whitespace-only line affects indentation.
tok->needs_newline |= _PyTokenizer_NEWLINE_INDENT;
}
if (c == '#' || c == '\n' || c == '\r') {
int interactive = _PyTok_ReaderIsInteractive(tok);
/* Lines with only whitespace and/or comments
@@ -171,6 +179,10 @@ _PyLexer_IndentationToken(struct tok_state *tok, struct token *token)
int
_PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline)
{
if (!blankline && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE) {
// The statement's terminating newline was not in the input.
tok->needs_newline |= _PyTokenizer_NEWLINE_STATEMENT;
}
tok->layout.at_bol = 1;
if (blankline || tok->level > 0) {
if (!tok->tok_extra_tokens) {
Parser/lexer/lexer.c
+6 -1
View File
@@ -235,6 +235,11 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
return type;
}
}
if (tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
tok->layout.depth > 0) {
// Ending this comment lets the next line dedent the suite.
tok->needs_newline |= _PyTokenizer_NEWLINE_STATEMENT;
}
if (tok->tok_extra_tokens) {
tok_backup(tok, c); /* don't eat the newline or EOF */
p_start = comment;
@@ -516,7 +521,7 @@ _PyTokenizer_Get(struct tok_state *tok, struct token *token)
Py_UNREACHABLE();
}
}
if (tok_failed(tok)) {
if (tok_failed(tok) && tok->done != E_INCOMPLETE) {
result = ERRORTOKEN;
}
token->type = result;
Parser/lexer/state.h
+2
View File
@@ -105,6 +105,8 @@ struct tok_state {
int debug;
#endif
int barry_as_bdfl;
unsigned char allow_incomplete_input;
unsigned char needs_newline;
};
static inline ftstring_state *
Parser/lexer/string.c
+17
View File
@@ -21,6 +21,15 @@ string_error_token(struct tok_state *tok, struct token *token,
return type;
}
static int
incomplete_string_token(struct tok_state *tok, struct token *token,
int type, _PyTok_Off start, _PyTok_Loc location)
{
string_error_token(tok, token, start, location);
tok->done = E_INCOMPLETE;
return type;
}
int
_PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
_PyTok_Off start, _PyTok_Off end)
@@ -337,6 +346,10 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
if (tok_failed(tok)) {
return MAKE_TOKEN(ERRORTOKEN);
}
if (c == EOF && tok->allow_incomplete_input) {
return incomplete_string_token(tok, token, STRING,
tok->start, tok->start_loc);
}
int end_lineno = tok->lineno;
_PyTok_Loc location = tok->start_loc;
const char *line = _PyTok_SourcePointer(
@@ -450,6 +463,10 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
if (tok_failed(tok)) {
return MAKE_TOKEN(ERRORTOKEN);
}
if (c == EOF && tok->allow_incomplete_input) {
return incomplete_string_token(tok, token,
FTSTRING_MIDDLE(current), current->start, current->start_loc);
}
if (in_format_spec && c == '\n') {
return MAKE_TOKEN(_PyTokenizer_syntaxerror(
Parser/peg_api.c
+3 -3
View File
@@ -5,14 +5,14 @@
mod_ty
_PyParser_ASTFromString(const char *str, PyObject* filename, int mode,
PyCompilerFlags *flags, PyArena *arena,
PyObject *module)
PyObject *module, PyObject **incomplete)
{
if (PySys_Audit("compile", "yO", str, filename) < 0) {
return NULL;
}
mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags,
arena, module);
arena, module, incomplete);
return result;
}
@@ -39,4 +39,4 @@ _PyParser_InteractiveASTFromFile(FILE *fp, PyObject *filename_ob, const char *en
}
return _PyPegen_run_parser_from_file_pointer(fp, mode, filename_ob, enc, ps1, ps2,
flags, errcode, interactive_src, arena);
}
}
Parser/pegen.c
+61 -30
View File
@@ -247,23 +247,26 @@ static int
initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) {
assert(parser_token != NULL);
Py_ssize_t length;
const char *text = _PyToken_TextView(p->tok, new_token, &length);
parser_token->type = token_type == NAME
? _get_keyword_or_name_type(p, text, length) : token_type;
if (token_needs_text(parser_token->type)) {
parser_token->bytes = PyBytes_FromStringAndSize(text, length);
if (parser_token->bytes == NULL) {
return -1;
parser_token->type = token_type;
parser_token->bytes = NULL;
if (token_needs_text(token_type) && new_token->span.start >= 0) {
Py_ssize_t length;
const char *text = _PyToken_TextView(p->tok, new_token, &length);
if (token_type == NAME) {
parser_token->type = _get_keyword_or_name_type(p, text, length);
}
if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) {
Py_DECREF(parser_token->bytes);
return -1;
// A name reclassified as a keyword does not need its text.
if (parser_token->type == token_type) {
parser_token->bytes = PyBytes_FromStringAndSize(text, length);
if (parser_token->bytes == NULL) {
return -1;
}
if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) {
Py_DECREF(parser_token->bytes);
return -1;
}
}
}
else {
parser_token->bytes = NULL;
}
parser_token->metadata = NULL;
if (new_token->metadata != NULL) {
@@ -287,7 +290,7 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
p->fill += 1;
if (token_type == ERRORTOKEN && _PyTokenizer_GetInfo(p->tok).status == E_DECODE) {
if (token_type == ERRORTOKEN && _PyTokenizer_GetStatus(p->tok) == E_DECODE) {
return _Pypegen_raise_decode_error(p);
}
@@ -352,6 +355,13 @@ _PyPegen_fill_token(Parser *p)
}
int type = new_token.type;
// EOF can finish an ordinary token. Only a boundary token means that
// the grammar requested input beyond the last physical line.
if (type == ENDMARKER ||
(type == DEDENT && (p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) &&
_PyTokenizer_GetStatus(p->tok) == E_EOF)) {
p->input_exhausted = 1;
}
// If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) {
@@ -953,6 +963,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
}
assert(tok != NULL);
_PyTokenizer_SetOptions(tok, 0, (flags & PyPARSE_TYPE_COMMENTS) > 0);
_PyTokenizer_AllowIncompleteInput(tok, flags & PyPARSE_ALLOW_INCOMPLETE_INPUT);
p->tok = tok;
p->keywords = NULL;
p->n_keyword_lists = -1;
@@ -998,6 +1009,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
#endif
p->level = 0;
p->call_invalid_rules = 0;
p->input_exhausted = 0;
p->last_stmt_location.lineno = 0;
p->last_stmt_location.col_offset = 0;
p->last_stmt_location.end_lineno = 0;
@@ -1035,16 +1047,12 @@ reset_parser_state_for_error_pass(Parser *p)
p->tokens[i]->memo_mask = 0;
}
p->mark = 0;
p->flags &= ~PyPARSE_ALLOW_INCOMPLETE_INPUT;
_PyTokenizer_AllowIncompleteInput(p->tok, 0);
p->call_invalid_rules = 1;
_PyTokenizer_StopInteractive(p->tok);
}
static inline int
_is_end_of_source(Parser *p) {
int err = _PyTokenizer_GetInfo(p->tok).status;
return err == E_EOF || err == E_EOFS || err == E_EOLS;
}
static void
_PyPegen_set_syntax_error_metadata(Parser *p) {
PyObject *exc = PyErr_GetRaisedException();
@@ -1095,13 +1103,15 @@ _PyPegen_run_parser(Parser *p)
return NULL;
}
if (res == NULL) {
if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) {
PyErr_Clear();
return _PyPegen_raise_error(p, PyExc_IncompleteInputError, 0, "incomplete input");
}
if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) {
if (PyErr_Occurred() &&
(PyErr_ExceptionMatches(PyExc_IncompleteInputError) ||
!PyErr_ExceptionMatches(PyExc_SyntaxError))) {
return NULL;
}
if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) &&
p->input_exhausted && !PyErr_Occurred()) {
return _PyPegen_raise_incomplete_input(p);
}
// Make a second parser pass. In this pass we activate heavier and slower checks
// to produce better error messages and more complete diagnostics. Extra "invalid_*"
// rules will be active during parsing.
@@ -1194,16 +1204,21 @@ error:
mod_ty
_PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob,
PyCompilerFlags *flags, PyArena *arena, PyObject *module)
PyCompilerFlags *flags, PyArena *arena, PyObject *module,
PyObject **incomplete)
{
int exec_input = start_rule == Py_file_input;
_PyTokenizer_Newline newline_mode = start_rule == Py_file_input
? _PyTokenizer_IMPLY_NEWLINE : _PyTokenizer_KEEP_NEWLINE;
int parser_flags = compute_parser_flags(flags);
if (parser_flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) {
newline_mode = _PyTokenizer_APPEND_NEWLINE;
}
struct tok_state *tok;
if (flags != NULL && flags->cf_flags & PyCF_IGNORE_COOKIE) {
tok = _PyTokenizer_FromUTF8(str, exec_input, 0);
tok = _PyTokenizer_FromUTF8(str, newline_mode, 0);
} else {
tok = _PyTokenizer_FromString(str, exec_input, 0);
tok = _PyTokenizer_FromString(str, newline_mode, 0);
}
if (tok == NULL) {
if (PyErr_Occurred()) {
@@ -1231,6 +1246,22 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
}
result = _PyPegen_run_parser(p);
if (result != NULL && (parser_flags & PyPARSE_ALLOW_INCOMPLETE_INPUT)) {
int needs_newline = _PyTokenizer_GetNewlineRequirements(tok);
if ((needs_newline & _PyTokenizer_NEWLINE_CONTINUATION) ||
(start_rule != Py_file_input &&
(needs_newline & _PyTokenizer_NEWLINE_INDENT)) ||
(start_rule == Py_single_input && p->input_exhausted &&
(needs_newline & _PyTokenizer_NEWLINE_STATEMENT))) {
_PyPegen_raise_incomplete_input(p);
if (incomplete != NULL && PyErr_ExceptionMatches(PyExc_IncompleteInputError)) {
*incomplete = PyErr_GetRaisedException();
}
else {
result = NULL;
}
}
}
_PyPegen_Parser_Free(p);
error:
Parser/pegen.h
+6 -2
View File
@@ -39,6 +39,8 @@ typedef struct _memo {
typedef struct {
int type;
// Text required by grammar actions or helpers; otherwise NULL.
// An unfinished string token also has NULL text. Its grammar action
// requests more input only if the token is valid at that position.
PyObject *bytes;
int level;
int is_raw;
@@ -86,7 +88,8 @@ typedef struct {
int n_keyword_lists;
int start_rule;
int *errcode;
int parsing_started;
unsigned char parsing_started;
unsigned char input_exhausted;
PyObject* normalize;
int starting_lineno;
int starting_col_offset;
@@ -210,6 +213,7 @@ typedef enum {
// Error handling functions and APIs
int _Pypegen_raise_decode_error(Parser *p);
int _Pypegen_tokenizer_error(Parser *p);
void *_PyPegen_raise_incomplete_input(Parser *p);
void *_PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...);
void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
Py_ssize_t lineno, Py_ssize_t col_offset,
@@ -422,7 +426,7 @@ mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char
const char *, const char *, PyCompilerFlags *, int *, PyObject **,
PyArena *);
void *_PyPegen_run_parser(Parser *);
mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *, PyObject *);
mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *, PyObject *, PyObject **);
asdl_stmt_seq *_PyPegen_interactive_exit(Parser *);
// Generated function in parse.c - function definition in python.gram
Parser/pegen_errors.c
+26
View File
@@ -9,6 +9,16 @@
// TOKENIZER ERRORS
void *
_PyPegen_raise_incomplete_input(Parser *p)
{
if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && !PyErr_Occurred()) {
return _PyPegen_raise_error(p, PyExc_IncompleteInputError, 0,
"incomplete input");
}
return NULL;
}
static inline void
raise_unclosed_parentheses_error(Parser *p) {
_PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
@@ -32,7 +42,17 @@ _Pypegen_tokenizer_error(Parser *p)
Py_ssize_t col_offset = -1;
p->error_indicator = 1;
_PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) &&
(info.status == E_EOF || info.status == E_INCOMPLETE))
{
_PyPegen_raise_incomplete_input(p);
return -1;
}
switch (info.status) {
case E_INCOMPLETE:
// The diagnostic pass rejected this unfinished token. Let the
// parser report the syntax error at its original failure point.
return -1;
case E_TOKEN:
msg = "invalid token";
break;
@@ -298,6 +318,12 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
Py_ssize_t size;
const char *line = _PyTokenizer_SpanView(
p->tok, text_span, &size);
if (errtype == PyExc_IncompleteInputError && size > 0 &&
(info.implicit_newline == _PyTokenizer_APPEND_NEWLINE ||
(info.implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
p->start_rule != Py_file_input))) {
size--;
}
error_line = PyUnicode_DecodeUTF8(line, size, "replace");
}
else if (!info.is_file) {
Parser/tokenizer/api.c
+19
View File
@@ -7,11 +7,24 @@
#include "reader_internal.h"
#include "../lexer/state.h"
int
_PyTokenizer_GetStatus(const struct tok_state *tok)
{
return tok->done;
}
int
_PyTokenizer_GetNewlineRequirements(const struct tok_state *tok)
{
return tok->needs_newline;
}
_PyTokenizer_Info
_PyTokenizer_GetInfo(const struct tok_state *tok)
{
_PyTokenizer_Info info = {
.status = tok->done,
.implicit_newline = tok->implicit_newline,
.diagnostic = tok->diagnostic,
.location = {tok->lineno, tok->line_start < 0
? -1 : (int)(tok->cur - tok->line_start)},
@@ -103,6 +116,12 @@ _PyTokenizer_ImplyDedents(struct tok_state *tok)
_PyLexer_ImplyDedents(tok);
}
void
_PyTokenizer_AllowIncompleteInput(struct tok_state *tok, int allow)
{
tok->allow_incomplete_input = allow != 0;
}
int
_PyTokenizer_HasTrailingStatement(const struct tok_state *tok)
{
Parser/tokenizer/decoder.c
+21 -32
View File
@@ -14,7 +14,6 @@
#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1))
#define UTF8_BOM "\xef\xbb\xbf"
#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1))
#define NORMALIZED_LINE_INITIAL_CAPACITY 256
char *
_PyTok_CopyBytes(const char *data, Py_ssize_t len)
@@ -284,9 +283,6 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len,
int preserve_crlf, int add_final_newline)
{
Py_ssize_t pos = 0;
// SourceAppendLine copies the bytes, so reuse this buffer for each line.
char *normalized = NULL;
Py_ssize_t capacity = 0;
while (pos < len) {
Py_ssize_t raw_line_len;
if (preserve_crlf) {
@@ -304,52 +300,45 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len,
int add_newline = add_final_newline &&
pos + raw_line_len == len && !terminated;
const char *line = data + pos;
Py_ssize_t line_len = raw_line_len;
// raw_line_length stops at the first CR or LF, so any CR is in
// the line terminator.
Py_ssize_t terminator_len = 0;
if (!preserve_crlf) {
if (line[line_len - 1] == '\r') {
if (line[raw_line_len - 1] == '\r') {
terminator_len = 1;
}
else if (line_len > 1 && line[line_len - 2] == '\r') {
else if (raw_line_len > 1 && line[raw_line_len - 2] == '\r') {
terminator_len = 2;
}
}
_PyTok_Off appended;
if (terminator_len || add_newline) {
Py_ssize_t payload_len = line_len - terminator_len;
if (payload_len > PY_SSIZE_T_MAX - 2) {
PyErr_NoMemory();
tok->done = E_NOMEM;
goto error;
}
Py_ssize_t needed = payload_len + 2;
if (_PyTok_ReserveBuffer(&normalized, &capacity, needed,
NORMALIZED_LINE_INITIAL_CAPACITY) < 0) {
tok->done = E_NOMEM;
goto error;
}
memcpy(normalized, line, payload_len);
normalized[payload_len] = '\n';
line_len = payload_len + 1;
normalized[line_len] = '\0';
line = normalized;
appended = _PyTok_SourceAppendLineWithNewline(
&tok->source, line, raw_line_len - terminator_len);
}
else {
appended = _PyTok_SourceAppendLine(&tok->source, line, raw_line_len);
}
_PyTok_Off appended = _PyTok_SourceAppendLine(
&tok->source, line, line_len);
if (appended < 0) {
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
? E_NOMEM : E_ERROR;
goto error;
return -1;
}
tok->reader->prepared_final_newline_is_implicit = add_newline;
tok->reader->prepared_final_newline = add_newline;
pos += raw_line_len;
}
PyMem_Free(normalized);
// Partial input includes the newline submitted by the next Enter key.
// A final CR pairs with that newline instead of starting a blank line.
if (add_final_newline == _PyTokenizer_APPEND_NEWLINE &&
!tok->reader->prepared_final_newline &&
(len == 0 || data[len - 1] != '\r'))
{
if (_PyTok_SourceAppendLine(&tok->source, "\n", 1) < 0) {
return -1;
}
tok->reader->prepared_final_newline = _PyTokenizer_APPEND_NEWLINE;
}
return 0;
error:
PyMem_Free(normalized);
return -1;
}
int
Parser/tokenizer/reader.c
+2 -2
View File
@@ -180,8 +180,8 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
chunk->data = (char *)_PyTok_SourceSpanView(
&tok->source, line, &chunk->len);
chunk->ownership = _PYTOK_CHUNK_BORROWED;
chunk->implicit_newline = line.end == tail.end &&
tok->reader->prepared_final_newline_is_implicit;
chunk->implicit_newline = line.end == tail.end
? tok->reader->prepared_final_newline : 0;
return _PYTOK_READ_LINE;
}
Parser/tokenizer/reader_internal.h
+1 -1
View File
@@ -58,7 +58,7 @@ typedef struct _PyTok_Reader {
Py_ssize_t decoded_len;
Py_ssize_t decoded_cap;
_PyTok_ReaderKind kind;
unsigned char prepared_final_newline_is_implicit;
unsigned char prepared_final_newline; // _PyTokenizer_Newline
unsigned char decoded_tail_is_implicit;
unsigned char file_initialized;
unsigned char file_eof;
Parser/tokenizer/source.c
+28 -11
View File
@@ -71,51 +71,68 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed)
static int
validate_line(const _PyTok_SourceText *source, const char *bytes,
Py_ssize_t len)
Py_ssize_t len, int append_newline)
{
if (len <= 0 || bytes == NULL ||
if (len < 0 || (len == 0 && !append_newline) || bytes == NULL ||
(source->nlines > 0 &&
(source->len == 0 || source->bytes[source->len - 1] != '\n'))) {
PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line");
return -1;
}
const char *newline = memchr(bytes, '\n', len);
if (newline != NULL && newline != bytes + len - 1) {
if (newline != NULL && (append_newline || newline != bytes + len - 1)) {
PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line");
return -1;
}
if (source->nlines == INT_MAX ||
(source->nlines == INT_MAX - 1 && newline != NULL)) {
(source->nlines == INT_MAX - 1 && (newline != NULL || append_newline))) {
PyErr_SetString(PyExc_OverflowError, "too many tokenizer source lines");
return -1;
}
return 0;
}
_PyTok_Off
_PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
Py_ssize_t len)
static _PyTok_Off
append_line(_PyTok_SourceText *source, const char *bytes,
Py_ssize_t len, int append_newline)
{
if (validate_line(source, bytes, len) < 0) {
if (validate_line(source, bytes, len, append_newline) < 0) {
return -1;
}
if (source->len > PY_SSIZE_T_MAX - len - 1 ||
source->base_offset > PY_SSIZE_T_MAX - source->len - len) {
if (source->len > PY_SSIZE_T_MAX - len - append_newline - 1 ||
source->base_offset > PY_SSIZE_T_MAX - source->len - len - append_newline) {
PyErr_NoMemory();
return -1;
}
if (reserve_bytes(source, source->len + len + 1) < 0) {
if (reserve_bytes(source, source->len + len + append_newline + 1) < 0) {
return -1;
}
_PyTok_Off start = source->len;
memcpy(source->bytes + start, bytes, len);
source->len += len;
if (append_newline) {
source->bytes[source->len++] = '\n';
}
source->bytes[source->len] = '\0';
source->nlines++;
return source->base_offset + start;
}
_PyTok_Off
_PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
Py_ssize_t len)
{
return append_line(source, bytes, len, 0);
}
_PyTok_Off
_PyTok_SourceAppendLineWithNewline(_PyTok_SourceText *source, const char *bytes,
Py_ssize_t len)
{
return append_line(source, bytes, len, 1);
}
const char *
_PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno,
Py_ssize_t *len)
Parser/tokenizer/source.h
+4
View File
@@ -70,6 +70,10 @@ PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *);
final line. The input must not point into source storage. */
PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
_PyTok_SourceText *source, const char *bytes, Py_ssize_t len);
/* Append a line payload followed by '\n'. The payload may be empty and must
contain no newline. Other SourceAppendLine constraints still apply. */
PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLineWithNewline(
_PyTok_SourceText *source, const char *bytes, Py_ssize_t len);
/* Return borrowed bytes excluding '\n', writing the byte length to *len.
Line numbers are 1-based and clamp to the first or final line; a trailing
'\n' adds an empty final line. The view need not be NUL-terminated.
Parser/tokenizer/tokenizer.h
+20 -1
View File
@@ -6,6 +6,19 @@
struct tok_state;
typedef enum {
_PyTokenizer_KEEP_NEWLINE, // Preserve the original line terminator.
_PyTokenizer_IMPLY_NEWLINE, // Add a missing line terminator.
_PyTokenizer_APPEND_NEWLINE, // Also append a blank line to terminated input.
} _PyTokenizer_Newline;
/* Reasons that the appended newline can change the token stream. */
enum {
_PyTokenizer_NEWLINE_STATEMENT = 1,
_PyTokenizer_NEWLINE_INDENT = 2,
_PyTokenizer_NEWLINE_CONTINUATION = 4,
};
/* Initialize before use. metadata owns a reference released by _PyToken_Free;
a consumer taking that reference must set metadata to NULL. */
struct token {
@@ -37,6 +50,7 @@ typedef struct {
typedef struct {
int status;
unsigned char implicit_newline; // _PyTokenizer_Newline
_PyTokenizer_Diagnostic diagnostic;
_PyTok_Loc location;
_PyTok_Off cursor;
@@ -54,7 +68,9 @@ typedef struct {
} _PyTokenizer_Info;
/* Get replaces the initialized token, releasing its previous metadata.
Errors are returned as ERRORTOKEN, with or without a Python exception. */
Errors are returned as ERRORTOKEN, with or without a Python exception.
In incomplete-input mode, an unfinished string retains its token type
with an absent span and E_INCOMPLETE status. No Python exception is set. */
void _PyTokenizer_Get(struct tok_state *, struct token *);
void _PyTokenizer_Free(struct tok_state *);
int _PyTokenizer_Traverse(struct tok_state *, visitproc, void *);
@@ -69,6 +85,8 @@ _PyToken_Free(struct token *token)
/* Views and borrowed snapshot references remain valid until the tokenizer is
mutated or freed. Source spans may be discarded when reading more input. */
_PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state *);
int _PyTokenizer_GetStatus(const struct tok_state *);
int _PyTokenizer_GetNewlineRequirements(const struct tok_state *);
/* An absent token span has a nonnull empty text view. */
const char *_PyToken_TextView(
const struct tok_state *, const struct token *, Py_ssize_t *);
@@ -91,6 +109,7 @@ void _PyTokenizer_SetContext(
struct tok_state *tok, PyObject *filename, PyObject *module);
void _PyTokenizer_SetOptions(
struct tok_state *tok, int extra_tokens, int type_comments);
void _PyTokenizer_AllowIncompleteInput(struct tok_state *, int);
void _PyTokenizer_ImplyDedents(struct tok_state *);
int _PyTokenizer_HasTrailingStatement(const struct tok_state *);
int _PyTokenizer_IsInteractive(const struct tok_state *);
Python/pythonrun.c
+22 -11
View File
@@ -1285,7 +1285,7 @@ _PyRun_String(const char *str, PyObject* name, int start,
goto done;
}
mod = _PyParser_ASTFromString(str, name, start, flags, arena, module);
mod = _PyParser_ASTFromString(str, name, start, flags, arena, module, NULL);
Py_XDECREF(module);
if (mod != NULL) {
@@ -1569,32 +1569,43 @@ _Py_CompileString(const char *str, PyObject *filename, int start,
return NULL;
}
PyCodeObject *co;
mod_ty mod;
PyArena *arena = _PyArena_New();
if (arena == NULL)
return NULL;
mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module);
PyObject *result = NULL;
PyObject *incomplete = NULL;
mod_ty mod = _PyParser_ASTFromString(str, filename, start, flags, arena,
module, &incomplete);
if (mod == NULL) {
_PyArena_Free(arena);
return NULL;
goto done;
}
if (flags && (flags->cf_flags & PyCF_ONLY_AST)) {
int syntax_check_only = ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */
if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena,
syntax_check_only, module) < 0)
{
_PyArena_Free(arena);
return NULL;
goto done;
}
PyObject *result = PyAST_mod2obj(mod);
if (incomplete != NULL) {
goto done;
}
result = PyAST_mod2obj(mod);
_PyArena_Free(arena);
return result;
}
co = _PyAST_Compile(mod, filename, flags, optimize, arena, module);
result = (PyObject *)_PyAST_Compile(mod, filename, flags, optimize, arena, module);
done:
_PyArena_Free(arena);
return (PyObject *)co;
if (incomplete != NULL && !PyErr_Occurred()) {
Py_CLEAR(result);
PyErr_SetRaisedException(incomplete);
}
else {
Py_XDECREF(incomplete);
}
return result;
}
PyObject *
Python/symtable.c
+1 -1
View File
@@ -3271,7 +3271,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
if (arena == NULL)
return NULL;
mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module);
mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module, NULL);
if (mod == NULL) {
_PyArena_Free(arena);
return NULL;
Tools/peg_generator/peg_extension/peg_extension.c
+1 -1
View File
@@ -93,7 +93,7 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds)
PyCompilerFlags flags = _PyCompilerFlags_INIT;
mod_ty res = _PyPegen_run_parser_from_string(the_string, Py_file_input, filename_ob,
&flags, arena, NULL);
&flags, arena, NULL, NULL);
if (res == NULL) {
goto error;
}