mirrors/cpython
0
0 Fork 0
mirror of https://github.com/python/cpython.git synced 2026-10-28 10:25:05 +08:00
Code Activity

gh-153569: Simplify tokenizer source views and reader ownership (#158849)

Browse Source 
Move source access and logical spans into the tokenizer source API, and keep input and encoding ownership in the reader. Export the incremental decoder helper for parser generator extensions.
This commit is contained in:
Pablo Galindo Salgado authored and GitHub committed 2026-10-13 13:01:17 +00:00
1 parent 50082b8d89
commit f892b11f22
14 files changed
+201 -225
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Include/internal/pycore_codecs.h
+3 -4
View File
@@ -59,10 +59,9 @@ extern PyObject* _PyCodec_DecodeText(
const char *encoding,
const char *errors);
/* These two aren't actually text encoding specific, but _io.TextIOWrapper
* is the only current API consumer.
*/
extern PyObject* _PyCodecInfo_GetIncrementalDecoder(
/* Helpers to create incremental codecs from a codec info object. */
// Export for test_peg_generator.
PyAPI_FUNC(PyObject*) _PyCodecInfo_GetIncrementalDecoder(
PyObject *codec_info,
const char *errors);
Parser/lexer/lexer.c
+36 -56
View File
@@ -13,12 +13,6 @@
tokenizing. */
static const char* type_comment_prefix = "# type: ";
static inline int
contains_null_bytes(const char* str, size_t size)
{
return memchr(str, 0, size) != NULL;
}
int
_PyLexer_refill(struct tok_state *tok)
{
@@ -29,8 +23,9 @@ _PyLexer_refill(struct tok_state *tok)
#if defined(Py_DEBUG)
if (tok->debug) {
fprintf(stderr, "line[%d] = ", tok->lineno);
_PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok, tok->cur),
tok->inp - tok->cur);
_PyTokenizer_print_escape(
stderr, _PyTok_SourcePointer(&tok->source, tok->cur),
tok->inp - tok->cur);
fprintf(stderr, " tok->done = %d\n", tok->done);
}
#endif
@@ -39,8 +34,8 @@ _PyLexer_refill(struct tok_state *tok)
return 0;
}
tok->line_start = tok->cur;
if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start),
tok->inp - tok->line_start)) {
_PyTok_Span line = {tok->line_start, tok->inp};
if (_PyTok_SourceFindByte(&tok->source, line, '\0') >= 0) {
_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
tok->cur = tok->inp;
return 0;
@@ -56,7 +51,7 @@ _PyLexer_backup(struct tok_state *tok, int c)
if (--tok->cur < tok->buf_offset) {
Py_FatalError("tokenizer beginning of buffer");
}
if ((int)(unsigned char)*_PyLexer_BufferPointer(tok, tok->cur) != Py_CHARMASK(c)) {
if (_PyTok_SourceByte(&tok->source, tok->cur) != Py_CHARMASK(c)) {
Py_FatalError("tok_backup: wrong character");
}
}
@@ -74,7 +69,8 @@ verify_identifier(struct tok_state *tok)
PyObject *s;
if (tok_failed(tok))
return 0;
s = PyUnicode_DecodeUTF8(_PyLexer_BufferPointer(tok, tok->start), tok->cur - tok->start, NULL);
s = PyUnicode_DecodeUTF8(_PyTok_SourcePointer(&tok->source, tok->start),
tok->cur - tok->start, NULL);
if (s == NULL) {
if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
tok->done = E_DECODE;
@@ -105,13 +101,13 @@ verify_identifier(struct tok_state *tok)
Py_DECREF(s);
if (Py_UNICODE_ISPRINTABLE(ch)) {
_PyTokenizer_syntaxerror_at(
tok, _PyLexer_BufferPointer(tok, tok->line_start),
tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
error_cursor - tok->line_start, tok->lineno, -1, -1,
"invalid character '%c' (U+%04X)", ch, ch);
}
else {
_PyTokenizer_syntaxerror_at(
tok, _PyLexer_BufferPointer(tok, tok->line_start),
tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
error_cursor - tok->line_start, tok->lineno, -1, -1,
"invalid non-printable character U+%04X", ch);
}
@@ -172,10 +168,6 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
/* Skip comment, unless it's a type comment */
if (c == '#') {
const char* p = NULL;
const char *prefix, *type_start;
int current_starting_col_offset;
while (c != EOF && c != '\n' && c != '\r') {
c = tok_nextc(tok);
}
@@ -192,23 +184,20 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
}
}
if (tok->tok_extra_tokens) {
p = _PyLexer_BufferPointer(tok, tok->start);
}
_PyTok_Off comment = tok->start;
if (tok->type_comments) {
p = _PyLexer_BufferPointer(tok, tok->start);
current_starting_col_offset = tok->start_loc.byte_col;
prefix = type_comment_prefix;
while (*prefix && p < _PyLexer_BufferPointer(tok, tok->cur)) {
const char *prefix = type_comment_prefix;
while (*prefix && comment < tok->cur) {
if (*prefix == ' ') {
while (*p == ' ' || *p == '\t') {
p++;
current_starting_col_offset++;
while (comment < tok->cur) {
int ch = _PyTok_SourceByte(&tok->source, comment);
if (ch != ' ' && ch != '\t') {
break;
}
comment++;
}
} else if (*prefix == *p) {
p++;
current_starting_col_offset++;
} else if (*prefix == _PyTok_SourceByte(&tok->source, comment)) {
comment++;
} else {
break;
}
@@ -218,35 +207,26 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
/* This is a type comment if we matched all of type_comment_prefix. */
if (!*prefix) {
int is_type_ignore = 1;
// +6 in order to skip the word 'ignore'
const char *ignore_end = p + 6;
const int ignore_end_col_offset = current_starting_col_offset + 6;
tok_backup(tok, c); /* don't eat the newline or EOF */
type_start = p;
/* A TYPE_IGNORE is "type: ignore" followed by the end of the token
* or anything ASCII and non-alphanumeric. */
is_type_ignore = (
_PyLexer_BufferPointer(tok, tok->cur) >= ignore_end && memcmp(p, "ignore", 6) == 0
&& !(_PyLexer_BufferPointer(tok, tok->cur) > ignore_end
&& ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
int is_type_ignore = tok->cur - comment >= 6 &&
memcmp(_PyTok_SourcePointer(&tok->source, comment),
"ignore", 6) == 0;
if (is_type_ignore && comment + 6 < tok->cur) {
int ch = _PyTok_SourceByte(&tok->source, comment + 6);
is_type_ignore = ch < 128 && !Py_ISALNUM(ch);
}
int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
int start_col_offset = is_type_ignore
? ignore_end_col_offset : current_starting_col_offset;
p_start = comment + (is_type_ignore ? 6 : 0);
p_end = tok->cur;
if (is_type_ignore) {
p_start = _PyLexer_BufferOffset(tok, ignore_end);
/* If this type ignore is the only thing on the line, consume the newline also. */
if (blankline) {
tok_nextc(tok);
tok->layout.at_bol = 1;
}
} else {
p_start = _PyLexer_BufferOffset(tok, type_start);
int start_col_offset = tok->start_loc.byte_col +
(int)(p_start - tok->start);
/* Consume the newline after a standalone type ignore. */
if (is_type_ignore && blankline) {
tok_nextc(tok);
tok->layout.at_bol = 1;
}
_PyLexer_token_setup(tok, token, type, p_start, p_end);
token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
@@ -257,7 +237,7 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
}
if (tok->tok_extra_tokens) {
tok_backup(tok, c); /* don't eat the newline or EOF */
p_start = _PyLexer_BufferOffset(tok, p);
p_start = comment;
p_end = tok->cur;
tok->layout.comment_newline = blankline;
return MAKE_TOKEN(COMMENT);
Parser/lexer/lexer_internal.h
+1 -4
View File
@@ -40,14 +40,11 @@ tok_nextc(struct tok_state *tok)
return EOF;
}
}
assert(tok->cur >= tok->source.base_offset);
assert(tok->cur - tok->source.base_offset < tok->source.len);
if (tok->cur - tok->line_start >= INT_MAX) {
tok->done = E_COLUMNOVERFLOW;
return EOF;
}
return Py_CHARMASK(
tok->source.bytes[tok->cur++ - tok->source.base_offset]);
return _PyTok_SourceByte(&tok->source, tok->cur++);
}
/* Return -1 on error, otherwise whether the line is blank. */
Parser/lexer/state.c
-3
View File
@@ -58,9 +58,6 @@ _PyLexer_PopFTString(struct tok_state *tok)
void
_PyTokenizer_Free(struct tok_state *tok)
{
if (tok->encoding != NULL) {
PyMem_Free(tok->encoding);
}
Py_XDECREF(tok->filename);
Py_XDECREF(tok->module);
_PyTok_ReaderFree(tok);
Parser/lexer/state.h
-31
View File
@@ -80,7 +80,6 @@ struct tok_state {
_PyTok_SourceText source;
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
/* NB If done != E_OK, cur must be == inp!!! */
FILE *fp; /* Rest of input; NULL if tokenizing a string */
lexer_layout_state layout;
int lineno; /* Current line number */
_PyTok_Loc start_loc;
@@ -92,9 +91,6 @@ struct tok_state {
int parencolstack[MAXLEVEL];
PyObject *filename;
PyObject *module;
/* Stuff for PEP 0263 */
char *encoding; /* Source encoding. */
struct _PyTok_Reader *reader;
int type_comments; /* Whether to look for type comments */
@@ -135,33 +131,6 @@ _PyLexer_FTStringBracketDepth(const struct tok_state *tok,
return tok->level - state->paren_level;
}
static inline _PyTok_Off
_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
{
const char *base = _PyTok_SourceData(&tok->source);
assert(position >= base && position <= base + tok->source.len);
return tok->source.base_offset + (position - base);
}
static inline const char *
_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
{
assert(offset >= tok->source.base_offset);
assert(offset - tok->source.base_offset <= tok->source.len);
return _PyTok_SourceData(&tok->source) + (offset - tok->source.base_offset);
}
static inline const char *
_PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span,
Py_ssize_t *length)
{
assert(length != NULL);
assert(_PyTok_SpanIsValid(span));
*length = span.end - span.start;
(void)_PyLexer_BufferPointer(tok, span.end);
return _PyLexer_BufferPointer(tok, span.start);
}
static inline int
_PyLexer_ByteColumn(const struct tok_state *tok)
{
Parser/lexer/string.c
+21 -19
View File
@@ -76,13 +76,11 @@ finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
if (!(state->debug_expr || tstring_interpolation) || token->metadata) {
return 0;
}
Py_ssize_t expr_len;
const char *expr = _PyLexer_BufferSpanView(
tok, state->expr_span, &expr_len);
tokenizer_comments *comments = state->comments;
PyObject *res;
if (comments != NULL && comments->count > 0) {
Py_ssize_t stripped_size = expr_len;
Py_ssize_t stripped_size =
state->expr_span.end - state->expr_span.start;
Py_ssize_t comment_count = 0;
for (Py_ssize_t i = 0; i < comments->count; i++) {
_PyTok_Span comment = comments->spans[i];
@@ -101,26 +99,28 @@ finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
PyErr_NoMemory();
return -1;
}
_PyTok_Off copied_to = state->expr_span.start;
_PyTok_Span kept = {state->expr_span.start, state->expr_span.start};
Py_ssize_t stripped_len = 0;
for (Py_ssize_t i = 0; i < comment_count; i++) {
_PyTok_Span comment = comments->spans[i];
Py_ssize_t length = comment.start - copied_to;
memcpy(stripped + stripped_len,
expr + copied_to - state->expr_span.start,
(size_t)length);
kept.end = comments->spans[i].start;
Py_ssize_t length;
const char *text = _PyTok_SourceSpanView(
&tok->source, kept, &length);
memcpy(stripped + stripped_len, text, (size_t)length);
stripped_len += length;
copied_to = comment.end;
kept.start = comments->spans[i].end;
}
Py_ssize_t length = state->expr_span.end - copied_to;
memcpy(stripped + stripped_len,
expr + copied_to - state->expr_span.start,
(size_t)length);
stripped_len += length;
res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL);
kept.end = state->expr_span.end;
Py_ssize_t length;
const char *text = _PyTok_SourceSpanView(&tok->source, kept, &length);
memcpy(stripped + stripped_len, text, (size_t)length);
res = PyUnicode_DecodeUTF8(stripped, stripped_size, NULL);
PyMem_Free(stripped);
}
else {
Py_ssize_t expr_len;
const char *expr = _PyTok_SourceSpanView(
&tok->source, state->expr_span, &expr_len);
res = PyUnicode_DecodeUTF8(expr, expr_len, NULL);
}
@@ -339,7 +339,8 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
}
int end_lineno = tok->lineno;
_PyTok_Loc location = tok->start_loc;
const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col;
const char *line = _PyTok_SourcePointer(
&tok->source, tok->start - location.byte_col);
Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
const ftstring_state *state = _PyLexer_CurrentFTString(tok);
@@ -460,7 +461,8 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
int end_lineno = tok->lineno;
_PyTok_Loc location = current->start_loc;
const char *line = _PyLexer_BufferPointer(tok, current->start) - location.byte_col;
const char *line = _PyTok_SourcePointer(
&tok->source, current->start - location.byte_col);
Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
if (quote_size == 3) {
Parser/tokenizer/api.c
+27 -26
View File
@@ -4,6 +4,7 @@
#include "tokenizer.h"
#include "reader.h"
#include "reader_internal.h"
#include "../lexer/state.h"
_PyTokenizer_Info
@@ -21,10 +22,10 @@ _PyTokenizer_GetInfo(const struct tok_state *tok)
.delimiter_loc = {-1, -1},
.in_formatted_string = tok->ftstring_depth != 0,
.is_interactive = _PyTok_ReaderIsInteractive(tok),
.is_file = tok->fp != NULL && tok->fp != stdin,
.is_file = tok->reader->fp != NULL && tok->reader->fp != stdin,
.filename = tok->filename,
.module = tok->module,
.encoding = tok->encoding,
.encoding = tok->reader->encoding,
};
if (tok->level > 0) {
int level = tok->level - 1;
@@ -45,14 +46,14 @@ _PyToken_TextView(const struct tok_state *tok, const struct token *token,
*length = 0;
return "";
}
return _PyLexer_BufferSpanView(tok, token->span, length);
return _PyTok_SourceSpanView(&tok->source, token->span, length);
}
const char *
_PyTokenizer_SpanView(const struct tok_state *tok, _PyTok_Span span,
Py_ssize_t *length)
{
return _PyLexer_BufferSpanView(tok, span, length);
return _PyTok_SourceSpanView(&tok->source, span, length);
}
void
@@ -60,19 +61,13 @@ _PyToken_GetView(const struct tok_state *tok, const struct token *token,
_PyToken_View *view)
{
assert(view != NULL);
assert((token->span.start == -1 && token->span.end == -1) ||
_PyTok_SpanIsValid(token->span));
if (token->span.start >= 0) {
(void)_PyLexer_BufferPointer(tok, token->span.end);
}
view->text = token->span.start < 0
? NULL : _PyLexer_BufferPointer(tok, token->span.start);
view->length = token->span.end - token->span.start;
view->end_line = _PyLexer_BufferPointer(tok, tok->line_start);
view->line = ISSTRINGLIT(token->type)
? view->text - token->start_loc.byte_col : view->end_line;
view->line_length = tok->inp - tok->line_start +
(view->end_line - view->line);
view->text = _PyToken_TextView(tok, token, &view->length);
view->end_line_start = tok->line_start;
view->line_span = (_PyTok_Span){
ISSTRINGLIT(token->type)
? token->span.start - token->start_loc.byte_col : tok->line_start,
tok->inp,
};
view->implicit_newline = tok->implicit_newline;
view->at_eof = tok->done == E_EOF;
}
@@ -111,22 +106,28 @@ _PyTokenizer_ImplyDedents(struct tok_state *tok)
int
_PyTokenizer_HasTrailingStatement(const struct tok_state *tok)
{
const char *cur = _PyLexer_BufferPointer(tok, tok->cur);
char c = *cur;
for (;;) {
while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
c = *++cur;
}
if (!c) {
_PyTok_Off cur = tok->cur;
_PyTok_Off end = tok->source.base_offset + tok->source.len;
while (cur < end) {
int c = _PyTok_SourceByte(&tok->source, cur++);
if (c == '\0') {
return 0;
}
if (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
continue;
}
if (c != '#') {
return 1;
}
while (c && c != '\n') {
c = *++cur;
while (cur < end) {
c = _PyTok_SourceByte(&tok->source, cur);
if (c == '\0' || c == '\n') {
break;
}
cur++;
}
}
return 0;
}
int
Parser/tokenizer/decoder.c
+12 -15
View File
@@ -117,8 +117,8 @@ _PyTok_SetEncoding(struct tok_state *tok, const char *encoding)
tok->done = E_NOMEM;
return -1;
}
PyMem_Free(tok->encoding);
tok->encoding = copy;
PyMem_Free(tok->reader->encoding);
tok->reader->encoding = copy;
return 0;
}
@@ -246,8 +246,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
PyMem_Free(cookie);
return _PYTOK_ENCODING_ERROR;
}
PyMem_Free(tok->encoding);
tok->encoding = cookie;
PyMem_Free(tok->reader->encoding);
tok->reader->encoding = cookie;
return _PYTOK_ENCODING_DONE;
}
@@ -393,9 +393,10 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only,
.len = raw_len,
.ownership = _PYTOK_CHUNK_BORROWED,
};
if (tok->encoding != NULL && strcmp(tok->encoding, "utf-8") != 0) {
const char *encoding = tok->reader->encoding;
if (encoding != NULL && strcmp(encoding, "utf-8") != 0) {
if (_PyTok_DecodeOnce(
tok, &decoded, tok->encoding, NULL) < 0) {
tok, &decoded, encoding, NULL) < 0) {
return -1;
}
}
@@ -407,7 +408,7 @@ _PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only,
return -1;
}
if (!utf8_only &&
(tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
(encoding == NULL || strcmp(encoding, "utf-8") == 0) &&
!_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) {
return -1;
}
@@ -418,22 +419,18 @@ int
_PyTok_StartDecoder(struct tok_state *tok, const char *errors)
{
_PyTok_Reader *reader = tok->reader;
if (tok->encoding == NULL || reader->decoder != NULL) {
if (reader->encoding == NULL || reader->decoder != NULL) {
return 0;
}
if (reader->kind == _PYTOK_READER_FILE &&
strcmp(tok->encoding, "utf-8") == 0) {
strcmp(reader->encoding, "utf-8") == 0) {
return 0;
}
PyObject *codec = _PyCodec_LookupTextEncoding(tok->encoding, NULL);
PyObject *codec = _PyCodec_LookupTextEncoding(reader->encoding, NULL);
if (codec != NULL) {
PyObject *factory = PyObject_GetAttrString(codec, "incrementaldecoder");
reader->decoder = _PyCodecInfo_GetIncrementalDecoder(codec, errors);
Py_DECREF(codec);
if (factory != NULL) {
reader->decoder = PyObject_CallFunction(factory, "s", errors);
Py_DECREF(factory);
}
}
if (reader->decoder == NULL) {
tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
Parser/tokenizer/helpers.c
+7 -6
View File
@@ -91,9 +91,9 @@ _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...)
// These errors are cleaned on startup. Todo: Fix it.
va_list vargs;
va_start(vargs, format);
int ret = _syntaxerror_range(tok, _PyLexer_BufferPointer(tok, tok->line_start),
tok->cur - tok->line_start, tok->lineno,
format, -1, -1, vargs);
int ret = _syntaxerror_range(
tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
tok->cur - tok->line_start, tok->lineno, format, -1, -1, vargs);
va_end(vargs);
return ret;
}
@@ -105,9 +105,10 @@ _PyTokenizer_syntaxerror_known_range(struct tok_state *tok,
{
va_list vargs;
va_start(vargs, format);
int ret = _syntaxerror_range(tok, _PyLexer_BufferPointer(tok, tok->line_start),
tok->cur - tok->line_start, tok->lineno,
format, col_offset, end_col_offset, vargs);
int ret = _syntaxerror_range(
tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
tok->cur - tok->line_start, tok->lineno,
format, col_offset, end_col_offset, vargs);
va_end(vargs);
return ret;
}
Parser/tokenizer/reader.c
+29 -28
View File
@@ -34,6 +34,7 @@ _PyTok_ReaderFree(struct tok_state *tok)
i < (int)Py_ARRAY_LENGTH(reader->prefetched_lines); i++) {
_PyTok_ChunkClear(&reader->prefetched_lines[i]);
}
PyMem_Free(reader->encoding);
PyMem_Free(reader->file_buffer);
PyMem_Free(reader->decoded);
PyMem_Free(reader);
@@ -173,15 +174,13 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
if (tok->lineno >= tok->source.nlines) {
return _PYTOK_READ_EOF;
}
const char *start = _PyLexer_BufferPointer(tok, tok->inp);
const char *newline = memchr(
start, '\n', tok->source.bytes + tok->source.len - start);
_PyTok_Off end = newline != NULL
? newline - tok->source.bytes + 1 : tok->source.len;
chunk->data = (char *)start;
chunk->len = tok->source.bytes + end - start;
_PyTok_Span tail = {tok->inp, tok->source.base_offset + tok->source.len};
_PyTok_Off newline = _PyTok_SourceFindByte(&tok->source, tail, '\n');
_PyTok_Span line = {tail.start, newline >= 0 ? newline + 1 : tail.end};
chunk->data = (char *)_PyTok_SourceSpanView(
&tok->source, line, &chunk->len);
chunk->ownership = _PYTOK_CHUNK_BORROWED;
chunk->implicit_newline = end == tok->source.len &&
chunk->implicit_newline = line.end == tail.end &&
tok->reader->prepared_final_newline_is_implicit;
return _PYTOK_READ_LINE;
}
@@ -203,7 +202,7 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk)
int available = (int)Py_MIN(reader->file_buffer_cap - len, INT_MAX);
size_t read = 0;
char *result = _Py_UniversalNewlineFgetsWithSize(
reader->file_buffer + len, available, tok->fp, NULL, &read);
reader->file_buffer + len, available, reader->fp, NULL, &read);
if (result == NULL) {
if (len == 0) {
return _PYTOK_READ_EOF;
@@ -228,7 +227,7 @@ initialize_file(struct tok_state *tok)
{
_PyTok_Reader *reader = tok->reader;
reader->file_initialized = 1;
if (tok->encoding != NULL) {
if (reader->encoding != NULL) {
return _PyTok_StartDecoder(tok, "strict");
}
@@ -411,7 +410,7 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk)
}
_PyTok_Chunk input = {0};
if (tok->encoding != NULL) {
if (reader->encoding != NULL) {
if (!PyBytes_Check(raw)) {
PyErr_SetString(PyExc_TypeError,
"readline() returned a non-bytes object");
@@ -434,7 +433,7 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk)
input.ownership = _PYTOK_CHUNK_PYOBJECT;
int decoded;
if (reader->decoder == NULL &&
strcmp(tok->encoding, "utf-8") == 0 &&
strcmp(reader->encoding, "utf-8") == 0 &&
chunk_is_line(&input)) {
decoded = _PyTok_DecodeOnce(
tok, &input, "utf-8", "replace");
@@ -510,7 +509,7 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
return _PYTOK_READ_STOPPED;
}
char *input = PyOS_Readline(
tok->fp != NULL ? tok->fp : stdin, stdout, reader->prompt);
reader->fp != NULL ? reader->fp : stdin, stdout, reader->prompt);
if (reader->nextprompt != NULL) {
reader->prompt = reader->nextprompt;
}
@@ -527,9 +526,9 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk)
.len = len,
.ownership = _PYTOK_CHUNK_PYMEM,
};
if (tok->encoding != NULL &&
if (reader->encoding != NULL &&
_PyTok_DecodeOnce(
tok, &decoded, tok->encoding, NULL) < 0) {
tok, &decoded, reader->encoding, NULL) < 0) {
_PyTok_ChunkClear(&decoded);
return _PYTOK_READ_ERROR;
}
@@ -605,10 +604,11 @@ int
_PyTok_ReaderUnderflow(struct tok_state *tok)
{
assert(tok->cur >= tok->buf_offset && tok->cur <= tok->inp);
_PyTok_ReaderKind kind = tok->reader->kind;
_PyTok_Reader *reader = tok->reader;
_PyTok_ReaderKind kind = reader->kind;
int prepared = kind == _PYTOK_READER_PREPARED;
int streaming = reader_is_streaming(kind);
int reset_buffer = !prepared && tok->start < 0 &&
int reset_buffer = tok->start < 0 &&
_PyLexer_CurrentFTString(tok) == NULL;
_PyTok_Chunk chunk;
@@ -661,23 +661,24 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
tok->cur = source_start;
tok->buf_offset = source_start;
tok->line_start = tok->buf_offset;
tok->start = -1;
}
tok->inp = source_start + chunk.len;
}
if (prepared) {
if (tok->start < 0 && _PyLexer_CurrentFTString(tok) == NULL) {
tok->buf_offset = tok->source.base_offset +
(chunk.data - tok->source.bytes);
else {
if (reset_buffer) {
tok->buf_offset = tok->inp;
}
tok->inp = _PyLexer_BufferOffset(tok, chunk.data) + chunk.len;
tok->inp += chunk.len;
}
tok->implicit_newline = chunk.implicit_newline;
tok->lineno++;
if (kind == _PYTOK_READER_FILE &&
(tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
!_PyTokenizer_ensure_utf8(_PyLexer_BufferPointer(tok, tok->cur), tok, tok->lineno)) {
(reader->encoding == NULL ||
strcmp(reader->encoding, "utf-8") == 0) &&
!_PyTokenizer_ensure_utf8(
_PyTok_SourcePointer(&tok->source, tok->cur),
tok, tok->lineno)) {
_PyTok_ChunkClear(&chunk);
return 0;
}
@@ -779,7 +780,7 @@ _PyTokenizer_FromFile(FILE *fp, const char *encoding,
_PyTokenizer_Free(tok);
return NULL;
}
tok->fp = fp;
tok->reader->fp = fp;
tok->reader->prompt = ps1;
tok->reader->nextprompt = ps2;
return tok;
@@ -840,8 +841,8 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
tok->filename = Py_NewRef(filename != NULL ? filename : &_Py_STR(anon_string));
char *encoding = NULL;
if (initialize_file(tok) == 0) {
encoding = tok->encoding;
tok->encoding = NULL;
encoding = tok->reader->encoding;
tok->reader->encoding = NULL;
}
fclose(fp);
_PyTokenizer_Free(tok);
Parser/tokenizer/reader_internal.h
+2
View File
@@ -41,6 +41,8 @@ typedef struct {
} _PyTok_Chunk;
typedef struct _PyTok_Reader {
FILE *fp; // Borrowed input stream; NULL for string and readline input.
char *encoding; // Owned source encoding.
PyObject *readline;
PyObject *decoder;
const char *prompt;
Parser/tokenizer/source.h
+40
View File
@@ -19,6 +19,46 @@ _PyTok_SourceData(const _PyTok_SourceText *source)
return source->bytes != NULL ? source->bytes : "";
}
/* Convert positions within the retained source window. Pointers and views
are borrowed; append, discard, and clear invalidate them. */
static inline const char *
_PyTok_SourcePointer(const _PyTok_SourceText *source, _PyTok_Off offset)
{
assert(offset >= source->base_offset);
assert(offset - source->base_offset <= source->len);
return _PyTok_SourceData(source) + (offset - source->base_offset);
}
static inline unsigned char
_PyTok_SourceByte(const _PyTok_SourceText *source, _PyTok_Off offset)
{
assert(offset >= source->base_offset);
assert(offset - source->base_offset < source->len);
return (unsigned char)source->bytes[offset - source->base_offset];
}
static inline const char *
_PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
Py_ssize_t *length)
{
assert(length != NULL);
assert(_PyTok_SpanIsValid(span));
assert(span.end - source->base_offset <= source->len);
*length = span.end - span.start;
return _PyTok_SourcePointer(source, span.start);
}
/* Return the first matching offset within span, or -1 if absent. */
static inline _PyTok_Off
_PyTok_SourceFindByte(const _PyTok_SourceText *source, _PyTok_Span span,
int byte)
{
Py_ssize_t length;
const char *data = _PyTok_SourceSpanView(source, span, &length);
const char *found = memchr(data, byte, length);
return found != NULL ? span.start + (found - data) : -1;
}
PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
/* Clear invalidates all spans and views for the source. */
PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
Parser/tokenizer/tokenizer.h
+5 -5
View File
@@ -21,9 +21,8 @@ struct token {
typedef struct {
const char *text;
Py_ssize_t length;
const char *line;
Py_ssize_t line_length;
const char *end_line;
_PyTok_Span line_span;
_PyTok_Off end_line_start;
int implicit_newline;
int at_eof;
} _PyToken_View;
@@ -73,8 +72,9 @@ _PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state *);
/* An absent token span has a nonnull empty text view. */
const char *_PyToken_TextView(
const struct tok_state *, const struct token *, Py_ssize_t *);
/* Use the token from the most recent Get. text is NULL for an absent span;
line includes the token's complete physical line range. */
/* Use the token from the most recent Get. An absent span has nonnull empty text.
line_span covers the token's complete physical line range. end_line_start
is the logical offset of its final physical line. */
void _PyToken_GetView(
const struct tok_state *tok, const struct token *token,
_PyToken_View *view);
Python/Python-tokenize.c
+18 -28
View File
@@ -158,12 +158,16 @@ exit:
static PyObject *
_get_current_line(tokenizeriterobject *it, int current_lineno,
const char *line_start, Py_ssize_t size, int *line_changed)
const _PyToken_View *view, int *line_changed)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
if (current_lineno != it->last_lineno) {
// Line has changed since last token, so we fetch the new line and cache it
// in the iter object.
Py_ssize_t size;
const char *line_start = _PyTokenizer_SpanView(
it->tok, view->line_span, &size);
if (size > 0 && view->implicit_newline) {
size--;
}
Py_XDECREF(it->last_line);
it->last_line = PyUnicode_DecodeUTF8(line_start, size, "replace");
it->byte_col_offset_diff = 0;
@@ -180,14 +184,13 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token,
Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
const char *token_start = view->text;
const char *token_end = token_start == NULL
? NULL : token_start + view->length;
_PyTok_Off token_start = token->span.start;
_PyTok_Off token_end = token->span.end;
Py_ssize_t lineno = token->start_loc.lineno;
Py_ssize_t end_lineno = token->end_loc.lineno;
Py_ssize_t byte_offset = -1;
if (token_start != NULL && token_start >= view->line) {
byte_offset = token_start - view->line;
if (token_start >= view->line_span.start) {
byte_offset = token_start - view->line_span.start;
if (line_changed) {
*col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
if (*col_offset < 0) {
@@ -200,8 +203,8 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token,
}
}
if (token_end != NULL && token_end >= view->end_line) {
Py_ssize_t end_byte_offset = token_end - view->end_line;
if (token_end >= view->end_line_start) {
Py_ssize_t end_byte_offset = token_end - view->end_line_start;
if (lineno == end_lineno) {
// Avoid rescanning the prefix of a very long line.
Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);
@@ -213,7 +216,8 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token,
}
else {
*end_col_offset = _PyPegen_byte_offset_to_character_offset_line(
line, view->end_line - view->line, token_end - view->line);
line, view->end_line_start - view->line_span.start,
token_end - view->line_span.start);
if (*end_col_offset < 0) {
return -1;
}
@@ -251,15 +255,7 @@ tokenizeriter_next(PyObject *op)
}
_PyToken_View view;
_PyToken_GetView(it->tok, &token, &view);
const char *token_start = view.text;
PyObject *str;
if (token.span.start < 0) {
assert(token.span.start == -1 && token.span.end == -1);
str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
}
else {
str = PyUnicode_FromStringAndSize(token_start, view.length);
}
PyObject *str = PyUnicode_FromStringAndSize(view.text, view.length);
if (str == NULL) {
goto exit;
}
@@ -271,13 +267,8 @@ tokenizeriter_next(PyObject *op)
if (it->extra_tokens && is_trailing_token) {
line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
} else {
Py_ssize_t size = view.line_length;
if (size >= 1 && view.implicit_newline) {
size -= 1;
}
line = _get_current_line(
it, token.end_loc.lineno, view.line, size, &line_changed);
it, token.end_loc.lineno, &view, &line_changed);
}
if (line == NULL) {
Py_DECREF(str);
@@ -307,8 +298,7 @@ tokenizeriter_next(PyObject *op)
else if (type == NEWLINE) {
if (!view.implicit_newline) {
Py_DECREF(str);
assert(token_start != NULL);
if (token_start[0] == '\r') {
if (view.text[0] == '\r') {
str = PyUnicode_FromString("\r\n");
} else {
str = PyUnicode_FromString("\n");