mirror of
https://github.com/python/cpython.git
synced 2026-10-28 18:35:21 +08:00
Pablo Galindo Salgado
6af655301a
* gh-159465: Improve incomplete input detection * gh-159465: Track newline effects and preserve parser layout * gh-159465: Share incomplete-input error cleanup
85 lines
3.2 KiB
C
85 lines
3.2 KiB
C
#ifndef Py_TOKENIZER_SOURCE_H
|
|
#define Py_TOKENIZER_SOURCE_H
|
|
|
|
#include "Python.h"
|
|
|
|
#include "types.h"
|
|
|
|
typedef struct {
|
|
char *bytes;
|
|
_PyTok_Off base_offset;
|
|
_PyTok_Off len;
|
|
_PyTok_Off cap;
|
|
int nlines;
|
|
} _PyTok_SourceText;
|
|
|
|
static inline const char *
|
|
_PyTok_SourceData(const _PyTok_SourceText *source)
|
|
{
|
|
return source->bytes != NULL ? source->bytes : "";
|
|
}
|
|
|
|
/* Convert positions within the retained source window. Pointers and views
|
|
are borrowed; append, discard, and clear invalidate them. */
|
|
static inline const char *
|
|
_PyTok_SourcePointer(const _PyTok_SourceText *source, _PyTok_Off offset)
|
|
{
|
|
assert(offset >= source->base_offset);
|
|
assert(offset - source->base_offset <= source->len);
|
|
return _PyTok_SourceData(source) + (offset - source->base_offset);
|
|
}
|
|
|
|
static inline unsigned char
|
|
_PyTok_SourceByte(const _PyTok_SourceText *source, _PyTok_Off offset)
|
|
{
|
|
assert(offset >= source->base_offset);
|
|
assert(offset - source->base_offset < source->len);
|
|
return (unsigned char)source->bytes[offset - source->base_offset];
|
|
}
|
|
|
|
static inline const char *
|
|
_PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
|
|
Py_ssize_t *length)
|
|
{
|
|
assert(length != NULL);
|
|
assert(_PyTok_SpanIsValid(span));
|
|
assert(span.end - source->base_offset <= source->len);
|
|
*length = span.end - span.start;
|
|
return _PyTok_SourcePointer(source, span.start);
|
|
}
|
|
|
|
/* Return the first matching offset within span, or -1 if absent. */
|
|
static inline _PyTok_Off
|
|
_PyTok_SourceFindByte(const _PyTok_SourceText *source, _PyTok_Span span,
|
|
int byte)
|
|
{
|
|
Py_ssize_t length;
|
|
const char *data = _PyTok_SourceSpanView(source, span, &length);
|
|
const char *found = memchr(data, byte, length);
|
|
return found != NULL ? span.start + (found - data) : -1;
|
|
}
|
|
|
|
PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *);
|
|
/* Clear invalidates all spans and views for the source. */
|
|
PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
|
|
/* Discard the retained window and invalidate its spans and views.
|
|
Keep its allocation and advance the logical base to the end of the window. */
|
|
PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *);
|
|
/* Append one nonempty logical line and return its start offset. The input may
|
|
contain one newline, as its final byte. An unterminated line must be the
|
|
final line. The input must not point into source storage. */
|
|
PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
|
|
_PyTok_SourceText *source, const char *bytes, Py_ssize_t len);
|
|
/* Append a line payload followed by '\n'. The payload may be empty and must
|
|
contain no newline. Other SourceAppendLine constraints still apply. */
|
|
PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLineWithNewline(
|
|
_PyTok_SourceText *source, const char *bytes, Py_ssize_t len);
|
|
/* Return borrowed bytes excluding '\n', writing the byte length to *len.
|
|
Line numbers are 1-based and clamp to the first or final line; a trailing
|
|
'\n' adds an empty final line. The view need not be NUL-terminated.
|
|
This does not set an exception. Append, discard, and clear invalidate the view. */
|
|
PyAPI_FUNC(const char *) _PyTok_SourceLineView(
|
|
const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len);
|
|
|
|
#endif
|