From 44ef2d1caace7e77285079fb9601c6b5bad4ec4d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 30 Aug 2026 13:36:50 +0100 Subject: [PATCH 1/6] gh-153569: bound tokenizer input storage File and readline tokenizers keep every byte they read because the lexer owns buffer growth and repairs its pointers after reallocations. Long inputs therefore grow tokenizer memory with the entire source. Let the reader reuse a bounded input window when no token or formatted string needs older bytes. Track the absolute offset of that window and save pointer offsets only when backing storage moves. --- Lib/test/test_tokenize.py | 25 +++++ Parser/lexer/buffer.c | 82 +++++++--------- Parser/lexer/buffer.h | 18 +++- Parser/lexer/state.c | 1 - Parser/lexer/state.h | 4 +- Parser/lexer/string.c | 8 +- Parser/tokenizer/reader.c | 148 +++++++++++++++++++++-------- Parser/tokenizer/reader_internal.h | 2 + 8 files changed, 190 insertions(+), 98 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index c471f857660ec90..571de34051f7006 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2427,6 +2427,31 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self): (token.ENDMARKER, "", (1, 0), (1, 0), ""), ) + def test_fstring_offsets_survive_buffer_reallocation(self): + padding = " " * 9000 + expression_line = ")=:>{2}}\n" + physical_lines = [ + 'f"""\n', + "{(\n", + padding + "1\n", + expression_line, + '"""\n', + ] + source = "".join(physical_lines) + chunks = iter([ + "".join(physical_lines[:2]), + "".join(physical_lines[2:4]), + physical_lines[4], + "", + ]) + + expected = self._get_tokens(source, extra_tokens=True) + tokens = list(tokenize._generate_tokens_from_c_tokenizer( + chunks.__next__, + extra_tokens=True, + )) + self.assertEqual(tokens, expected) + def test_extra_tokens_relaxes_lexer_errors(self): cases = [ ( diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c index cd6885a7d01040a..9c39544ca7c4790 100644 --- a/Parser/lexer/buffer.c +++ b/Parser/lexer/buffer.c @@ -1,62 +1,46 @@ #include "Python.h" -#include "errcode.h" - +#include "buffer.h" #include "state.h" -/* Traverse and remember all f-string buffers, in order to be able to restore - them after reallocating tok->buf */ void -_PyLexer_remember_fstring_buffers(struct tok_state *tok) +_PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base, + _PyLexer_BufferPointers *pointers) { - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); + pointers->buf_from_base = tok->buf - base; + pointers->cur_from_buf = tok->cur - tok->buf; + pointers->inp_from_buf = tok->inp - tok->buf; + pointers->start_from_buf = tok->start == NULL + ? -1 : tok->start - tok->buf; + pointers->line_start_from_buf = tok->line_start == NULL + ? -1 : tok->line_start - tok->buf; + pointers->multi_line_start_from_buf = tok->multi_line_start == NULL + ? -1 : tok->multi_line_start - tok->buf; + for (int index = tok->tok_mode_stack_index; index > 0; --index) { + tokenizer_mode *mode = &tok->tok_mode_stack[index]; mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; - mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf; + mode->multi_line_start_offset = mode->multi_line_start == NULL + ? -1 : mode->multi_line_start - tok->buf; } } -/* Traverse and restore all f-string buffers after reallocating tok->buf */ void -_PyLexer_restore_fstring_buffers(struct tok_state *tok) -{ - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); - mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset; - mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset; - } -} - -int -_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) +_PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base, + const _PyLexer_BufferPointers *pointers) { - Py_ssize_t cur = tok->cur - tok->buf; - Py_ssize_t oldsize = tok->inp - tok->buf; - Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1); - if (newsize > tok->end - tok->buf) { - char *newbuf = tok->buf; - Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf; - Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf; - Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf; - _PyLexer_remember_fstring_buffers(tok); - newbuf = (char *)PyMem_Realloc(newbuf, newsize); - if (newbuf == NULL) { - tok->done = E_NOMEM; - return 0; - } - tok->buf = newbuf; - tok->cur = tok->buf + cur; - tok->inp = tok->buf + oldsize; - tok->end = tok->buf + newsize; - tok->start = start < 0 ? NULL : tok->buf + start; - tok->line_start = line_start < 0 ? NULL : tok->buf + line_start; - tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start; - _PyLexer_restore_fstring_buffers(tok); + tok->buf = base + pointers->buf_from_base; + tok->cur = tok->buf + pointers->cur_from_buf; + tok->inp = tok->buf + pointers->inp_from_buf; + tok->start = pointers->start_from_buf < 0 + ? NULL : tok->buf + pointers->start_from_buf; + tok->line_start = pointers->line_start_from_buf < 0 + ? NULL : tok->buf + pointers->line_start_from_buf; + tok->multi_line_start = pointers->multi_line_start_from_buf < 0 + ? NULL : tok->buf + pointers->multi_line_start_from_buf; + for (int index = tok->tok_mode_stack_index; index > 0; --index) { + tokenizer_mode *mode = &tok->tok_mode_stack[index]; + mode->start = mode->start_offset < 0 + ? NULL : tok->buf + mode->start_offset; + mode->multi_line_start = mode->multi_line_start_offset < 0 + ? NULL : tok->buf + mode->multi_line_start_offset; } - return 1; } diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h index bb218162ff48453..285da124226d50e 100644 --- a/Parser/lexer/buffer.h +++ b/Parser/lexer/buffer.h @@ -3,8 +3,20 @@ #include "pyport.h" -void _PyLexer_remember_fstring_buffers(struct tok_state *tok); -void _PyLexer_restore_fstring_buffers(struct tok_state *tok); -int _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size); +struct tok_state; + +typedef struct { + Py_ssize_t buf_from_base; + Py_ssize_t cur_from_buf; + Py_ssize_t inp_from_buf; + Py_ssize_t start_from_buf; + Py_ssize_t line_start_from_buf; + Py_ssize_t multi_line_start_from_buf; +} _PyLexer_BufferPointers; + +void _PyLexer_SaveBufferPointers( + struct tok_state *, const char *, _PyLexer_BufferPointers *); +void _PyLexer_RestoreBufferPointers( + struct tok_state *, char *, const _PyLexer_BufferPointers *); #endif diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 2a6408bef927a36..e9829c60fafa129 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -26,7 +26,6 @@ _PyTokenizer_tok_new(void) tok->interactive_src_start = NULL; tok->interactive_src_end = NULL; tok->start = NULL; - tok->end = NULL; tok->done = E_OK; tok->fp = NULL; tok->tabsize = TABSIZE; diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 0824785195491ee..55ddc015e21a447 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -67,15 +67,15 @@ typedef struct _tokenizer_mode { /* Tokenizer state */ struct tok_state { - /* Input state; buf <= cur <= inp <= end */ + /* Input state; buf <= cur <= inp */ /* NB an entire line is held in the buffer */ char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ + _PyTok_Off buf_offset; /* Logical offset of buf[0]. */ int fp_interactive; /* If the file descriptor is interactive */ char *interactive_src_start; /* The start of the source parsed so far in interactive mode */ char *interactive_src_end; /* The end of the source parsed so far in interactive mode */ - const char *end; /* End of input buffer if buf != NULL */ const char *start; /* Start of current token if not NULL */ int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ /* NB If done != E_OK, cur must be == inp!!! */ diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index d67c48f7f678eda..fc0299c5c7c592f 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -125,7 +125,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) { assert(tok->cur != NULL); - Py_ssize_t size = strlen(tok->cur); + Py_ssize_t size = cur == 0 + ? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur); tokenizer_mode *tok_mode = TOK_GET_MODE(tok); switch (cur) { @@ -142,7 +143,8 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) goto error; } tok_mode->last_expr_buffer = new_buffer; - strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size); + memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, + tok->cur, size); tok_mode->last_expr_size += size; break; case '{': @@ -155,7 +157,7 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) } tok_mode->last_expr_size = size; tok_mode->last_expr_end = -1; - strncpy(tok_mode->last_expr_buffer, tok->cur, size); + memcpy(tok_mode->last_expr_buffer, tok->cur, size); break; case '}': case '!': diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 82b824f56374fcf..68c8da2186ede96 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -13,6 +13,12 @@ # include #endif +static inline int +reader_is_streaming(_PyTok_ReaderKind kind) +{ + return kind == _PYTOK_READER_FILE || kind == _PYTOK_READER_READLINE; +} + void _PyTok_ReaderFree(struct tok_state *tok) { @@ -28,10 +34,10 @@ _PyTok_ReaderFree(struct tok_state *tok) } PyMem_Free(reader->file_buffer); PyMem_Free(reader->decoded); - if (reader->kind != _PYTOK_READER_PREPARED) { + if (reader_is_streaming(reader->kind)) { PyMem_Free(tok->buf); - tok->buf = NULL; } + tok->buf = NULL; PyMem_Free(reader); tok->reader = NULL; } @@ -60,6 +66,26 @@ reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed) return 0; } +static int +reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed) +{ + _PyTok_Reader *reader = tok->reader; + if (needed <= reader->input_buffer_cap) { + return 0; + } + assert(tok->buf != NULL); + assert(tok->cur >= tok->buf && tok->cur <= tok->inp); + assert(tok->inp - tok->buf <= reader->input_buffer_cap); + _PyLexer_BufferPointers pointers; + _PyLexer_SaveBufferPointers(tok, tok->buf, &pointers); + if (reserve_buffer( + &tok->buf, &reader->input_buffer_cap, needed) < 0) { + return -1; + } + _PyLexer_RestoreBufferPointers(tok, tok->buf, &pointers); + return 0; +} + static int append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len) { @@ -529,19 +555,31 @@ reader_next(struct tok_state *tok, _PyTok_Chunk *chunk) Py_UNREACHABLE(); } +static void +reset_streaming_buffer(struct tok_state *tok) +{ + assert(tok->buf != NULL); + assert(tok->cur >= tok->buf && tok->cur <= tok->inp); + Py_ssize_t consumed = tok->inp - tok->buf; + assert(tok->buf_offset <= PY_SSIZE_T_MAX - consumed); + tok->buf_offset += consumed; + tok->cur = tok->inp = tok->buf; +} + int _PyTok_ReaderUnderflow(struct tok_state *tok) { - int prepared = tok->reader->kind == _PYTOK_READER_PREPARED; + _PyTok_ReaderKind kind = tok->reader->kind; + int prepared = kind == _PYTOK_READER_PREPARED; + int streaming = reader_is_streaming(kind); int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok); - if (reset_buffer && tok->reader->kind != _PYTOK_READER_INTERACTIVE) { - tok->cur = tok->inp = tok->buf; - } - _PyTok_Chunk chunk; _PyTok_ReadResult result = reader_next(tok, &chunk); if (result != _PYTOK_READ_LINE) { + if (reset_buffer && streaming) { + reset_streaming_buffer(tok); + } if (result == _PYTOK_READ_EOF) { tok->done = E_EOF; } @@ -558,34 +596,68 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) ? E_NOMEM : E_ERROR; } } - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && + if (kind == _PYTOK_READER_INTERACTIVE && result != _PYTOK_READ_STOPPED) { PySys_WriteStderr("\n"); } return 0; } - Py_ssize_t copy_len = chunk.len; - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && + Py_ssize_t scan_len = chunk.len; + if (kind == _PYTOK_READER_INTERACTIVE && chunk.implicit_newline) { - copy_len--; + scan_len--; } - if (reset_buffer && tok->reader->kind == _PYTOK_READER_INTERACTIVE) { - tok->cur = tok->inp = tok->buf; - } - if (!prepared && !_PyLexer_tok_reserve_buf(tok, copy_len + 1)) { - _PyTok_ChunkClear(&chunk); - tok->input_error = 1; - return 0; + if (streaming) { + if (reset_buffer) { + reset_streaming_buffer(tok); + } + Py_ssize_t used = tok->inp - tok->buf; + int overflow = scan_len > PY_SSIZE_T_MAX - used - 1 || + tok->buf_offset > PY_SSIZE_T_MAX - used - scan_len; + if (overflow) { + PyErr_NoMemory(); + } + if (overflow || reserve_input_buffer(tok, used + scan_len + 1) < 0) { + _PyTok_ChunkClear(&chunk); + tok->done = E_NOMEM; + tok->input_error = 1; + return 0; + } + memcpy(tok->inp, chunk.data, (size_t)scan_len); + tok->inp += scan_len; + *tok->inp = '\0'; } - if (tok->reader->kind == _PYTOK_READER_INTERACTIVE && - _PyTok_SourceAppendLine(&tok->source, chunk.data, chunk.len, - chunk.implicit_newline) < 0) { - _PyTok_ChunkClear(&chunk); - tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) - ? E_NOMEM : E_ERROR; - tok->input_error = 1; - return 0; + else if (!prepared) { + int source_will_grow = + chunk.len > tok->source.cap - tok->source.len - 1; + _PyLexer_BufferPointers pointers; + if (!reset_buffer && source_will_grow) { + _PyLexer_SaveBufferPointers( + tok, tok->source.bytes, &pointers); + } + _PyTok_Off source_start = _PyTok_SourceAppendLine( + &tok->source, chunk.data, chunk.len, + chunk.implicit_newline); + if (source_start < 0) { + _PyTok_ChunkClear(&chunk); + tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) + ? E_NOMEM : E_ERROR; + tok->input_error = 1; + return 0; + } + if (reset_buffer) { + tok->buf = tok->cur = tok->source.bytes + source_start; + tok->buf_offset = source_start; + tok->line_start = tok->buf; + tok->start = NULL; + tok->multi_line_start = NULL; + } + else if (source_will_grow) { + _PyLexer_RestoreBufferPointers( + tok, tok->source.bytes, &pointers); + } + tok->inp = tok->source.bytes + source_start + scan_len; } if (tok->fp_interactive) { tok->interactive_src_start = tok->source.bytes; @@ -594,14 +666,10 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) if (prepared) { if (tok->start == NULL) { tok->buf = tok->cur; + tok->buf_offset = chunk.data - tok->source.bytes; } tok->inp = chunk.data + chunk.len; } - else { - memcpy(tok->inp, chunk.data, (size_t)copy_len); - tok->inp += copy_len; - *tok->inp = '\0'; - } tok->implicit_newline = chunk.implicit_newline; if (!prepared && tok->tok_mode_stack_index && @@ -611,7 +679,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) return 0; } ADVANCE_LINENO(); - if (tok->reader->kind == _PYTOK_READER_FILE && + if (kind == _PYTOK_READER_FILE && (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) { _PyTok_ChunkClear(&chunk); @@ -639,14 +707,15 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind) if (kind == _PYTOK_READER_PREPARED) { return tok; } - tok->buf = PyMem_Malloc(BUFSIZ); - if (tok->buf == NULL) { - PyErr_NoMemory(); - _PyTokenizer_Free(tok); - return NULL; + if (reader_is_streaming(kind)) { + if (reserve_buffer( + &tok->buf, &tok->reader->input_buffer_cap, BUFSIZ) < 0) { + _PyTokenizer_Free(tok); + return NULL; + } + tok->cur = tok->inp = tok->buf; + tok->buf[0] = '\0'; } - tok->cur = tok->inp = tok->buf; - tok->end = tok->buf + BUFSIZ; return tok; } @@ -664,7 +733,6 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input, return NULL; } tok->buf = tok->cur = tok->inp = tok->str; - tok->end = tok->buf; return tok; } diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 121d0f96f6698a2..49a6f04ec60af27 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -44,6 +44,8 @@ typedef struct _PyTok_Reader { PyObject *decoder; const char *nextprompt; + Py_ssize_t input_buffer_cap; + char *file_buffer; Py_ssize_t file_buffer_cap; _PyTok_Chunk prefetched_lines[2]; From f381082d3b34f3cc2747f5da998651c58079e116 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 30 Aug 2026 13:36:50 +0100 Subject: [PATCH 2/6] gh-153569: return tokenizer tokens as source spans Tokenizer results expose pointers into the active input buffer. That ties every consumer to the buffer lifetime and prevents the reader from reusing older storage. Return logical source spans with their start and end locations instead. Pegen and `_tokenize` materialize a short-lived view before requesting another token, and the tokenizer no longer keeps a second end pointer for the last token. --- Parser/lexer/lexer.c | 14 ++++++----- Parser/lexer/lexer.h | 21 ++++++++++++++++ Parser/lexer/state.c | 51 +++++++++++++++++++++------------------ Parser/lexer/state.h | 9 +++---- Parser/pegen.c | 41 +++++++++++++++++-------------- Parser/tokenizer/source.h | 3 ++- Python/Python-tokenize.c | 45 ++++++++++++++++++++-------------- 7 files changed, 113 insertions(+), 71 deletions(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index a96362c8961023a..f96b31b9d2f38a1 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -12,8 +12,6 @@ #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) -#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\ - _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end)) /* Spaces in this constant are treated as "zero or more spaces or tabs" when tokenizing. */ @@ -360,21 +358,25 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str && !(tok->cur > ignore_end && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0])))); + int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT; + int start_col_offset = is_type_ignore + ? ignore_end_col_offset : current_starting_col_offset; + p_end = tok->cur; if (is_type_ignore) { p_start = ignore_end; - p_end = tok->cur; /* If this type ignore is the only thing on the line, consume the newline also. */ if (blankline) { tok_nextc(tok); tok->atbol = 1; } - return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset); } else { p_start = type_start; - p_end = tok->cur; - return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset); } + _PyLexer_token_setup(tok, token, type, p_start, p_end); + token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset}; + token->end_loc = (_PyTok_Loc){tok->lineno, tok->col_offset}; + return type; } } if (tok->tok_extra_tokens) { diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index 1d97ac57b745b09..040935a7e689138 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -7,4 +7,25 @@ int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur); int _PyTokenizer_Get(struct tok_state *, struct token *); +/* The view points into the current input window. The next + _PyTokenizer_Get() call may discard it. */ +static inline const char * +_PyToken_TextView(const struct tok_state *tok, const struct token *token, + Py_ssize_t *length) +{ + assert(length != NULL); + if (token->span.start < 0) { + assert(token->span.start == -1 && token->span.end == -1); + *length = 0; + return ""; + } + assert(_PyTok_SpanIsValid(token->span)); + assert(tok->buf != NULL); + assert(tok->inp >= tok->buf); + assert(token->span.start >= tok->buf_offset); + assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf); + *length = token->span.end - token->span.start; + return tok->buf + (token->span.start - tok->buf_offset); +} + #endif diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index e9829c60fafa129..d82a7d0f296bac0 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -100,41 +100,46 @@ _PyToken_Free(struct token *token) { void _PyToken_Init(struct token *token) { +#ifdef Py_DEBUG + token->span = (_PyTok_Span){-1, -1}; + token->start_loc = (_PyTok_Loc){-1, -1}; + token->end_loc = (_PyTok_Loc){-1, -1}; +#endif token->metadata = NULL; } -int -_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, - int end_col_offset, const char *start, const char *end) +static inline _PyTok_Span +buffer_span(const struct tok_state *tok, const char *start, const char *end) { - token->level = tok->level; - token->lineno = token->end_lineno = tok->lineno; - token->col_offset = col_offset; - token->end_col_offset = end_col_offset; - token->start = start; - token->end = end; - return type; + if (start == NULL) { + assert(end == NULL); + return (_PyTok_Span){-1, -1}; + } + assert(end != NULL); + const char *base = tok->buf; + assert(base != NULL); + assert(tok->inp >= base); + Py_ssize_t start_offset = start - base; + Py_ssize_t end_offset = end - base; + assert(start_offset >= 0 && start_offset <= end_offset); + assert(end_offset <= tok->inp - base); + assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset); + return _PyTok_SpanFromBounds( + tok->buf_offset + start_offset, tok->buf_offset + end_offset); } int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end) { - assert((start == NULL && end == NULL) || (start != NULL && end != NULL)); token->level = tok->level; - if (ISSTRINGLIT(type)) { - token->lineno = tok->first_lineno; - } - else { - token->lineno = tok->lineno; - } - token->end_lineno = tok->lineno; - token->col_offset = token->end_col_offset = -1; - token->start = start; - token->end = end; + token->span = buffer_span(tok, start, end); + int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno; + token->start_loc = (_PyTok_Loc){lineno, -1}; + token->end_loc = (_PyTok_Loc){tok->lineno, -1}; if (start != NULL && end != NULL) { - token->col_offset = tok->starting_col_offset; - token->end_col_offset = tok->col_offset; + token->start_loc.byte_col = tok->starting_col_offset; + token->end_loc.byte_col = tok->col_offset; } return type; } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 55ddc015e21a447..496962fd0484f03 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -23,8 +23,9 @@ enum interactive_underflow_t { struct token { int level; - int lineno, col_offset, end_lineno, end_col_offset; - const char *start, *end; + _PyTok_Span span; + _PyTok_Loc start_loc; + _PyTok_Loc end_loc; PyObject *metadata; }; @@ -69,7 +70,7 @@ typedef struct _tokenizer_mode { struct tok_state { /* Input state; buf <= cur <= inp */ /* NB an entire line is held in the buffer */ - char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */ + char *buf; /* Owned for file/readline input; source-backed otherwise. */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ _PyTok_Off buf_offset; /* Logical offset of buf[0]. */ @@ -128,8 +129,6 @@ struct tok_state { #endif }; -int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, - int end_col_offset, const char *start, const char *end); int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); struct tok_state *_PyTokenizer_tok_new(void); diff --git a/Parser/pegen.c b/Parser/pegen.c index fcec810037e98d4..d86dd22444e6a7b 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -171,18 +171,17 @@ growable_comment_array_deallocate(growable_comment_array *arr) { } static int -_get_keyword_or_name_type(Parser *p, struct token *new_token) +_get_keyword_or_name_type(Parser *p, const char *text, Py_ssize_t length) { - Py_ssize_t name_len = new_token->end_col_offset - new_token->col_offset; - assert(name_len > 0); + assert(length > 0); - if (name_len >= p->n_keyword_lists || - p->keywords[name_len] == NULL || - p->keywords[name_len]->type == -1) { + if (length >= p->n_keyword_lists || + p->keywords[length] == NULL || + p->keywords[length]->type == -1) { return NAME; } - for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) { - if (strncmp(k->str, new_token->start, (size_t)name_len) == 0) { + for (KeywordToken *k = p->keywords[length]; k != NULL && k->type != -1; k++) { + if (memcmp(k->str, text, (size_t)length) == 0) { return k->type; } } @@ -193,8 +192,11 @@ static int initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) { assert(parser_token != NULL); - parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type; - parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start); + Py_ssize_t length; + const char *text = _PyToken_TextView(p->tok, new_token, &length); + parser_token->type = token_type == NAME + ? _get_keyword_or_name_type(p, text, length) : token_type; + parser_token->bytes = PyBytes_FromStringAndSize(text, length); if (parser_token->bytes == NULL) { return -1; } @@ -214,12 +216,14 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to } parser_token->level = new_token->level; - parser_token->lineno = new_token->lineno; - parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset - : new_token->col_offset; - parser_token->end_lineno = new_token->end_lineno; - parser_token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_col_offset - : new_token->end_col_offset; + parser_token->lineno = new_token->start_loc.lineno; + parser_token->col_offset = p->tok->lineno == p->starting_lineno + ? p->starting_col_offset + new_token->start_loc.byte_col + : new_token->start_loc.byte_col; + parser_token->end_lineno = new_token->end_loc.lineno; + parser_token->end_col_offset = p->tok->lineno == p->starting_lineno + ? p->starting_col_offset + new_token->end_loc.byte_col + : new_token->end_loc.byte_col; p->fill += 1; @@ -261,13 +265,14 @@ _PyPegen_fill_token(Parser *p) // Record and skip '# type: ignore' comments while (type == TYPE_IGNORE) { - Py_ssize_t len = new_token.end_col_offset - new_token.col_offset; + Py_ssize_t len; + const char *text = _PyToken_TextView(p->tok, &new_token, &len); char *tag = PyMem_Malloc((size_t)len + 1); if (tag == NULL) { PyErr_NoMemory(); goto error; } - strncpy(tag, new_token.start, (size_t)len); + memcpy(tag, text, (size_t)len); tag[len] = '\0'; // Ownership of tag passes to the growable array if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) { diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index b42ecda1b31aa50..363475ff9015e35 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -5,7 +5,8 @@ typedef Py_ssize_t _PyTok_Off; -/* Half-open byte offsets into a _PyTok_SourceText. */ +/* Spans use half-open logical byte offsets into decoded input. Their backing + storage may retain only the current input window. */ typedef struct { _PyTok_Off start; _PyTok_Off end; diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 762b7b3e4c8d71d..71f236b08d93c8f 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -203,14 +203,19 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si } static void -_get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_start, - PyObject *line, int line_changed, Py_ssize_t lineno, Py_ssize_t end_lineno, +_get_col_offsets(tokenizeriterobject *it, const struct token *token, + const char *token_start, const char *line_start, + PyObject *line, int line_changed, Py_ssize_t *col_offset, Py_ssize_t *end_col_offset) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); + const char *token_end = token_start == NULL + ? NULL : token_start + token->span.end - token->span.start; + Py_ssize_t lineno = token->start_loc.lineno; + Py_ssize_t end_lineno = token->end_loc.lineno; Py_ssize_t byte_offset = -1; - if (token.start != NULL && token.start >= line_start) { - byte_offset = token.start - line_start; + if (token_start != NULL && token_start >= line_start) { + byte_offset = token_start - line_start; if (line_changed) { *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); it->byte_col_offset_diff = byte_offset - *col_offset; @@ -220,15 +225,13 @@ _get_col_offsets(tokenizeriterobject *it, struct token token, const char *line_s } } - if (token.end != NULL && token.end >= it->tok->line_start) { - Py_ssize_t end_byte_offset = token.end - it->tok->line_start; + if (token_end != NULL && token_end >= it->tok->line_start) { + Py_ssize_t end_byte_offset = token_end - it->tok->line_start; if (lineno == end_lineno) { - // If the whole token is at the same line, we can just use the token.start - // buffer for figuring out the new column offset, since using line is not - // performant for very long lines. + // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); *end_col_offset = *col_offset + token_col_offset; - it->byte_col_offset_diff += token.end - token.start - token_col_offset; + it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset); @@ -263,12 +266,17 @@ tokenizeriter_next(PyObject *op) it->done = 1; goto exit; } - PyObject *str = NULL; - if (token.start == NULL || token.end == NULL) { + const char *token_start = NULL; + PyObject *str; + if (token.span.start < 0) { + assert(token.span.start == -1 && token.span.end == -1); str = Py_GetConstant(Py_CONSTANT_EMPTY_STR); } else { - str = PyUnicode_FromStringAndSize(token.start, token.end - token.start); + Py_ssize_t token_length; + token_start = _PyToken_TextView( + it->tok, &token, &token_length); + str = PyUnicode_FromStringAndSize(token_start, token_length); } if (str == NULL) { goto exit; @@ -297,12 +305,12 @@ tokenizeriter_next(PyObject *op) goto exit; } - Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno; - Py_ssize_t end_lineno = it->tok->lineno; + Py_ssize_t lineno = token.start_loc.lineno; + Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - _get_col_offsets(it, token, line_start, line, line_changed, - lineno, end_lineno, &col_offset, &end_col_offset); + _get_col_offsets(it, &token, token_start, line_start, line, line_changed, + &col_offset, &end_col_offset); if (it->tok->tok_extra_tokens) { if (is_trailing_token) { @@ -317,7 +325,8 @@ tokenizeriter_next(PyObject *op) else if (type == NEWLINE) { Py_DECREF(str); if (!it->tok->implicit_newline) { - if (it->tok->start[0] == '\r') { + assert(token_start != NULL); + if (token_start[0] == '\r') { str = PyUnicode_FromString("\r\n"); } else { str = PyUnicode_FromString("\n"); From b60cc049a7098266e574667eba32aa94a0b5bcf7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 5 Sep 2026 18:22:13 +0100 Subject: [PATCH 3/6] gh-153569: enforce tokenizer buffer view lifetimes --- Lib/test/test_repl.py | 16 ++++++++++++++ Lib/test/test_tokenize.py | 45 +++++++++++++++++++-------------------- Parser/tokenizer/reader.c | 19 +++++++++++++---- 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index ea0f72e0e8d4568..372c110783bce7a 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -198,6 +198,22 @@ def test_lexer_buffer_realloc_with_null_start(self): self.assertEqual(p.returncode, 0) self.assertIn(long_value, output) + @cpython_only + def test_multiline_fstring_source_reallocation(self): + long_line = " " * 9000 + "+ 2" + user_input = ( + 'value = f"""{(\n' + '1\n' + f'{long_line}\n' + ')}"""\n' + 'print(value)\n' + ) + p = spawn_repl() + p.stdin.write(user_input) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(">>> 3\n>>> ", output) + def test_close_stdin(self): user_input = dedent(''' import os diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 571de34051f7006..948b341a5dd72ec 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2428,29 +2428,28 @@ def test_stop_iteration_skips_encoded_readline_codec_lookup(self): ) def test_fstring_offsets_survive_buffer_reallocation(self): - padding = " " * 9000 - expression_line = ")=:>{2}}\n" - physical_lines = [ - 'f"""\n', - "{(\n", - padding + "1\n", - expression_line, - '"""\n', - ] - source = "".join(physical_lines) - chunks = iter([ - "".join(physical_lines[:2]), - "".join(physical_lines[2:4]), - physical_lines[4], - "", - ]) - - expected = self._get_tokens(source, extra_tokens=True) - tokens = list(tokenize._generate_tokens_from_c_tokenizer( - chunks.__next__, - extra_tokens=True, - )) - self.assertEqual(tokens, expected) + for prefix in ("f", "t"): + for extra_tokens in (False, True): + with self.subTest(prefix=prefix, extra_tokens=extra_tokens): + physical_lines = [ + prefix + '"""\n', + "{(\n", + " " * 9000 + "1\n", + ")=:>{2}}\n", + '"""\n', + ] + source = "".join(physical_lines) + chunks = iter([ + "".join(physical_lines[:2]), + "".join(physical_lines[2:4]), + physical_lines[4], + "", + ]) + expected = self._get_tokens( + source, extra_tokens=extra_tokens) + tokens = list(tokenize._generate_tokens_from_c_tokenizer( + chunks.__next__, extra_tokens=extra_tokens)) + self.assertEqual(tokens, expected) def test_extra_tokens_relaxes_lexer_errors(self): cases = [ diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 68c8da2186ede96..ccf1f7095ed740e 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -78,10 +78,23 @@ reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed) assert(tok->inp - tok->buf <= reader->input_buffer_cap); _PyLexer_BufferPointers pointers; _PyLexer_SaveBufferPointers(tok, tok->buf, &pointers); +#ifdef Py_DEBUG + char *buffer = NULL; + Py_ssize_t capacity = reader->input_buffer_cap; + if (reserve_buffer(&buffer, &capacity, needed) < 0) { + return -1; + } + memcpy(buffer, tok->buf, (size_t)(tok->inp - tok->buf) + 1); + memset(tok->buf, 0xDD, reader->input_buffer_cap); + PyMem_Free(tok->buf); + tok->buf = buffer; + reader->input_buffer_cap = capacity; +#else if (reserve_buffer( &tok->buf, &reader->input_buffer_cap, needed) < 0) { return -1; } +#endif _PyLexer_RestoreBufferPointers(tok, tok->buf, &pointers); return 0; } @@ -629,10 +642,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) *tok->inp = '\0'; } else if (!prepared) { - int source_will_grow = - chunk.len > tok->source.cap - tok->source.len - 1; _PyLexer_BufferPointers pointers; - if (!reset_buffer && source_will_grow) { + if (!reset_buffer) { _PyLexer_SaveBufferPointers( tok, tok->source.bytes, &pointers); } @@ -653,7 +664,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) tok->start = NULL; tok->multi_line_start = NULL; } - else if (source_will_grow) { + else { _PyLexer_RestoreBufferPointers( tok, tok->source.bytes, &pointers); } From 2bdef783ebcec30fa5905b57d68d79b3a6172e28 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 5 Sep 2026 19:25:17 +0100 Subject: [PATCH 4/6] gh-153569: Unify decoded tokenizer storage ownership --- Lib/test/test_capi/test_tokenizer.py | 3 + Modules/_testinternalcapi/tokenizer.c | 105 ++++++++++++++++++++++++++ Parser/tokenizer/cursor.c | 9 ++- Parser/tokenizer/cursor.h | 16 +++- Parser/tokenizer/reader.c | 83 ++++---------------- Parser/tokenizer/reader_internal.h | 2 - Parser/tokenizer/source.c | 47 +++++++++--- Parser/tokenizer/source.h | 22 ++++-- 8 files changed, 191 insertions(+), 96 deletions(-) diff --git a/Lib/test/test_capi/test_tokenizer.py b/Lib/test/test_capi/test_tokenizer.py index 2fe1fef241e90ae..eb04f6c0136022d 100644 --- a/Lib/test/test_capi/test_tokenizer.py +++ b/Lib/test/test_capi/test_tokenizer.py @@ -9,6 +9,9 @@ class TokenizerTests(unittest.TestCase): def test_source(self): _testinternalcapi.test_tokenizer_source() + def test_source_discard(self): + _testinternalcapi.test_tokenizer_source_discard() + def test_cursor(self): _testinternalcapi.test_tokenizer_cursor() diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 0b292410d3eb4ef..df481cb832a4363 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -195,6 +195,28 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } + _PyTok_SourceDiscard(&source); + if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4, + "wrong retained source offset") < 0 || + _PyTok_SourceLine(&source, 1, &line) < 0 || + check(line.start == 4 && line.end == 6, + "wrong retained source line") < 0 || + _PyTok_SourceLocation( + &source, 4, _PYTOK_AFFINITY_LEFT, &loc) < 0 || + check(loc.lineno == 1 && loc.byte_col == 0, + "wrong retained source location") < 0) { + goto error; + } + view = _PyTok_SourceSpanView( + &source, _PyTok_SpanFromBounds(4, 5), &view_len); + if (check(view != NULL && view_len == 1 && view[0] == 'a', + "wrong retained source span") < 0 || + check_system_error(_PyTok_SourceSpanView( + &source, _PyTok_SpanFromBounds(0, 1), &view_len) == NULL, + "accepted discarded source span") < 0) { + goto error; + } + _PyTok_SourceClear(&source); Py_RETURN_NONE; @@ -296,6 +318,88 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module), } #endif + _PyTok_Off base = source.len; + _PyTok_SourceDiscard(&source); + if (_PyTok_SourceAppendLine(&source, "ab\n", 3, 0) < 0 || + _PyTok_SourceAppendLine(&source, "cd", 2, 0) < 0) { + goto error; + } + _PyTok_CursorInit(&cursor, &source); + if (_PyTok_CursorSetLine(&cursor, 1) < 0 || + check(cursor.pos == base && _PyTok_CursorPeek(&cursor, 1) == 'b', + "wrong retained cursor line") < 0 || + _PyTok_CursorSetLine(&cursor, 2) < 0 || + check(_PyTok_CursorAdvance(&cursor) == 'c', + "wrong retained cursor byte") < 0 || + _PyTok_CursorSetOffset(&cursor, base + 5) < 0 || + check(cursor.lineno == 2 && _PyTok_CursorAdvance(&cursor) == EOF, + "wrong retained cursor EOF") < 0) { + goto error; + } + + _PyTok_SourceClear(&source); + Py_RETURN_NONE; + +error: + _PyTok_SourceClear(&source); + return NULL; +} + +static PyObject * +test_tokenizer_source_discard(PyObject *Py_UNUSED(module), + PyObject *Py_UNUSED(args)) +{ + _PyTok_SourceText source; + _PyTok_SourceInit(&source); + for (int i = 0; i < 260; i++) { + if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) { + goto error; + } + } + char *bytes = source.bytes; + _PyTok_Off capacity = source.cap; + _PyTok_SourceDiscard(&source); + if (check(source.base_offset == 520 && source.len == 0 && + source.nlines == 0 && source.bytes == bytes && + source.cap == capacity && source.bytes[0] == '\0', + "discard did not preserve source allocation") < 0) { + goto error; + } + for (int i = 0; i < 260; i++) { + if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i, + "wrong source offset after discard") < 0 || + check(!_PyTok_SourceLineIsImplicit(&source, i + 1), + "discard preserved implicit newline flag") < 0) { + goto error; + } + } + if (check(source.bytes == bytes && source.cap == capacity, + "discarded allocation was not reused") < 0) { + goto error; + } + _PyTok_SourceDiscard(&source); + if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040, + "wrong source offset after repeated discard") < 0) { + goto error; + } + _PyTok_SourceDiscard(&source); + if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044, + "cannot append after discarding unterminated line") < 0) { + goto error; + } + _PyTok_SourceDiscard(&source); + source.base_offset = PY_SSIZE_T_MAX - 1; + if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 && + PyErr_ExceptionMatches(PyExc_MemoryError), + "accepted overflowing logical source offset") < 0) { + goto error; + } + PyErr_Clear(); + if (check(source.len == 0 && source.nlines == 0 && + source.base_offset == PY_SSIZE_T_MAX - 1, + "overflow changed retained source") < 0) { + goto error; + } _PyTok_SourceClear(&source); Py_RETURN_NONE; @@ -307,6 +411,7 @@ test_tokenizer_cursor(PyObject *Py_UNUSED(module), static PyMethodDef test_methods[] = { {"test_tokenizer_source", test_tokenizer_source, METH_NOARGS}, {"test_tokenizer_cursor", test_tokenizer_cursor, METH_NOARGS}, + {"test_tokenizer_source_discard", test_tokenizer_source_discard, METH_NOARGS}, {NULL}, }; diff --git a/Parser/tokenizer/cursor.c b/Parser/tokenizer/cursor.c index 698a26a740fd249..523b99dedc6160a 100644 --- a/Parser/tokenizer/cursor.c +++ b/Parser/tokenizer/cursor.c @@ -23,7 +23,7 @@ _PyTok_CursorSetLine(_PyTok_Cursor *cursor, int lineno) if (lineno > 0 && cursor->lineno == lineno - 1 && lineno <= source->nlines) { _PyTok_Off start = cursor->line_end; - _PyTok_Off end = source->len; + _PyTok_Off end = source->base_offset + source->len; if (lineno < source->nlines) { end = _PyTok_SourceFindLineEnd(source, start); if (end < 0) { @@ -53,8 +53,9 @@ _PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset) int stays_on_line = cursor->lineno > 0 && offset >= cursor->line_start && offset < cursor->line_end; if (!stays_on_line && cursor->lineno > 0 && - offset == cursor->line_end && offset == source->len && - (offset == 0 || source->bytes[offset - 1] != '\n')) { + offset == cursor->line_end && + offset - source->base_offset == source->len && + (source->len == 0 || source->bytes[source->len - 1] != '\n')) { stays_on_line = 1; } if (stays_on_line) { @@ -68,7 +69,7 @@ _PyTok_CursorSetOffset(_PyTok_Cursor *cursor, _PyTok_Off offset) return -1; } _PyTok_Off start = offset - loc.byte_col; - _PyTok_Off end = source->len; + _PyTok_Off end = source->base_offset + source->len; if (loc.lineno < source->nlines) { end = _PyTok_SourceFindLineEnd(source, start); if (end < 0) { diff --git a/Parser/tokenizer/cursor.h b/Parser/tokenizer/cursor.h index d0fd9cf80b77b71..18e404251316f0c 100644 --- a/Parser/tokenizer/cursor.h +++ b/Parser/tokenizer/cursor.h @@ -21,8 +21,12 @@ PyAPI_FUNC(int) _PyTok_CursorSetOffset(_PyTok_Cursor *, _PyTok_Off); static inline void _PyTok_CursorInit(_PyTok_Cursor *cursor, const _PyTok_SourceText *source) { + _PyTok_Off base = source != NULL ? source->base_offset : 0; *cursor = (_PyTok_Cursor){ .source = source, + .pos = base, + .line_start = base, + .line_end = base, }; } @@ -35,14 +39,16 @@ _PyTok_CursorAdvance(_PyTok_Cursor *cursor) assert(cursor->source != NULL); assert(cursor->pos >= cursor->line_start); assert(cursor->pos <= cursor->line_end); - assert(cursor->line_end <= cursor->source->len); + assert(cursor->line_start >= cursor->source->base_offset); + assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len); if (cursor->pos >= cursor->line_end) { return EOF; } if (cursor->pos - cursor->line_start >= INT_MAX) { return EOF; } - return Py_CHARMASK(cursor->source->bytes[cursor->pos++]); + return Py_CHARMASK(cursor->source->bytes[ + cursor->pos++ - cursor->source->base_offset]); } /* Return the byte at a nonnegative distance within the current line, or EOF @@ -53,13 +59,15 @@ _PyTok_CursorPeek(const _PyTok_Cursor *cursor, int distance) assert(cursor->source != NULL); assert(cursor->pos >= cursor->line_start); assert(cursor->pos <= cursor->line_end); - assert(cursor->line_end <= cursor->source->len); + assert(cursor->line_start >= cursor->source->base_offset); + assert(cursor->line_end - cursor->source->base_offset <= cursor->source->len); assert(distance >= 0); if (distance < 0 || distance >= cursor->line_end - cursor->pos) { return EOF; } - return Py_CHARMASK(cursor->source->bytes[cursor->pos + distance]); + return Py_CHARMASK(cursor->source->bytes[ + cursor->pos - cursor->source->base_offset + distance]); } #endif diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index ccf1f7095ed740e..b9b4a4610874419 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -34,9 +34,6 @@ _PyTok_ReaderFree(struct tok_state *tok) } PyMem_Free(reader->file_buffer); PyMem_Free(reader->decoded); - if (reader_is_streaming(reader->kind)) { - PyMem_Free(tok->buf); - } tok->buf = NULL; PyMem_Free(reader); tok->reader = NULL; @@ -66,39 +63,6 @@ reserve_buffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed) return 0; } -static int -reserve_input_buffer(struct tok_state *tok, Py_ssize_t needed) -{ - _PyTok_Reader *reader = tok->reader; - if (needed <= reader->input_buffer_cap) { - return 0; - } - assert(tok->buf != NULL); - assert(tok->cur >= tok->buf && tok->cur <= tok->inp); - assert(tok->inp - tok->buf <= reader->input_buffer_cap); - _PyLexer_BufferPointers pointers; - _PyLexer_SaveBufferPointers(tok, tok->buf, &pointers); -#ifdef Py_DEBUG - char *buffer = NULL; - Py_ssize_t capacity = reader->input_buffer_cap; - if (reserve_buffer(&buffer, &capacity, needed) < 0) { - return -1; - } - memcpy(buffer, tok->buf, (size_t)(tok->inp - tok->buf) + 1); - memset(tok->buf, 0xDD, reader->input_buffer_cap); - PyMem_Free(tok->buf); - tok->buf = buffer; - reader->input_buffer_cap = capacity; -#else - if (reserve_buffer( - &tok->buf, &reader->input_buffer_cap, needed) < 0) { - return -1; - } -#endif - _PyLexer_RestoreBufferPointers(tok, tok->buf, &pointers); - return 0; -} - static int append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len) { @@ -573,10 +537,10 @@ reset_streaming_buffer(struct tok_state *tok) { assert(tok->buf != NULL); assert(tok->cur >= tok->buf && tok->cur <= tok->inp); - Py_ssize_t consumed = tok->inp - tok->buf; - assert(tok->buf_offset <= PY_SSIZE_T_MAX - consumed); - tok->buf_offset += consumed; - tok->cur = tok->inp = tok->buf; + _PyTok_SourceDiscard(&tok->source); + tok->buf_offset = tok->source.base_offset; + tok->buf = tok->cur = tok->inp = (char *)_PyTok_SourceData(&tok->source); + tok->line_start = tok->buf; } int @@ -621,27 +585,10 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) chunk.implicit_newline) { scan_len--; } - if (streaming) { - if (reset_buffer) { + if (!prepared) { + if (streaming && reset_buffer) { reset_streaming_buffer(tok); } - Py_ssize_t used = tok->inp - tok->buf; - int overflow = scan_len > PY_SSIZE_T_MAX - used - 1 || - tok->buf_offset > PY_SSIZE_T_MAX - used - scan_len; - if (overflow) { - PyErr_NoMemory(); - } - if (overflow || reserve_input_buffer(tok, used + scan_len + 1) < 0) { - _PyTok_ChunkClear(&chunk); - tok->done = E_NOMEM; - tok->input_error = 1; - return 0; - } - memcpy(tok->inp, chunk.data, (size_t)scan_len); - tok->inp += scan_len; - *tok->inp = '\0'; - } - else if (!prepared) { _PyLexer_BufferPointers pointers; if (!reset_buffer) { _PyLexer_SaveBufferPointers( @@ -658,7 +605,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) return 0; } if (reset_buffer) { - tok->buf = tok->cur = tok->source.bytes + source_start; + tok->buf = tok->cur = + tok->source.bytes + (source_start - tok->source.base_offset); tok->buf_offset = source_start; tok->line_start = tok->buf; tok->start = NULL; @@ -668,7 +616,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) _PyLexer_RestoreBufferPointers( tok, tok->source.bytes, &pointers); } - tok->inp = tok->source.bytes + source_start + scan_len; + tok->inp = tok->source.bytes + + (source_start - tok->source.base_offset) + scan_len; } if (tok->fp_interactive) { tok->interactive_src_start = tok->source.bytes; @@ -677,7 +626,8 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) if (prepared) { if (tok->start == NULL) { tok->buf = tok->cur; - tok->buf_offset = chunk.data - tok->source.bytes; + tok->buf_offset = tok->source.base_offset + + (chunk.data - tok->source.bytes); } tok->inp = chunk.data + chunk.len; } @@ -719,13 +669,8 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind) return tok; } if (reader_is_streaming(kind)) { - if (reserve_buffer( - &tok->buf, &tok->reader->input_buffer_cap, BUFSIZ) < 0) { - _PyTokenizer_Free(tok); - return NULL; - } - tok->cur = tok->inp = tok->buf; - tok->buf[0] = '\0'; + tok->buf = tok->cur = tok->inp = + (char *)_PyTok_SourceData(&tok->source); } return tok; } diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 49a6f04ec60af27..121d0f96f6698a2 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -44,8 +44,6 @@ typedef struct _PyTok_Reader { PyObject *decoder; const char *nextprompt; - Py_ssize_t input_buffer_cap; - char *file_buffer; Py_ssize_t file_buffer_cap; _PyTok_Chunk prefetched_lines[2]; diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index c0f7925e33f8b97..04c24b77e0d8256 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -19,6 +19,23 @@ _PyTok_SourceClear(_PyTok_SourceText *source) _PyTok_SourceInit(source); } +void +_PyTok_SourceDiscard(_PyTok_SourceText *source) +{ + assert(source->base_offset <= PY_SSIZE_T_MAX - source->len); + source->base_offset += source->len; + source->len = 0; + if (source->bytes != NULL) { + source->bytes[0] = '\0'; + } + if (source->implicit_lines != NULL) { + memset(source->implicit_lines, 0, + Py_MIN(((Py_ssize_t)source->nlines + 7) / 8, + source->implicit_cap)); + } + source->nlines = 0; +} + static int reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) { @@ -143,7 +160,8 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, if (validate_line(source, bytes, len, implicit_newline) < 0) { return -1; } - if (source->len > PY_SSIZE_T_MAX - len - 1) { + if (source->len > PY_SSIZE_T_MAX - len - 1 || + source->base_offset > PY_SSIZE_T_MAX - source->len - len) { PyErr_NoMemory(); return -1; } @@ -162,26 +180,28 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, source->len += len; source->bytes[source->len] = '\0'; if (checkpoint) { - source->line_checkpoints[checkpoint_count - 1] = start; + source->line_checkpoints[checkpoint_count - 1] = + source->base_offset + start; } if (implicit_newline) { source->implicit_lines[(nlines - 1) / 8] |= (unsigned char)(1U << ((nlines - 1) & 7)); } source->nlines = nlines; - return start; + return source->base_offset + start; } const char * _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span, Py_ssize_t *len) { - if (!_PyTok_SpanIsValid(span) || span.end > source->len || len == NULL) { + if (!_PyTok_SpanIsValid(span) || span.start < source->base_offset || + span.end - source->base_offset > source->len || len == NULL) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source span"); return NULL; } *len = span.end - span.start; - return source->bytes == NULL ? "" : source->bytes + span.start; + return _PyTok_SourceData(source) + (span.start - source->base_offset); } int @@ -220,8 +240,8 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, } if (lineno > source->nlines) { *line = (_PyTok_Line){ - .start = source->len, - .end = source->len, + .start = source->base_offset + source->len, + .end = source->base_offset + source->len, }; return 0; } @@ -236,7 +256,7 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, } current++; } - _PyTok_Off end = source->len; + _PyTok_Off end = source->base_offset + source->len; if (lineno < source->nlines) { end = _PyTok_SourceFindLineEnd(source, start); if (end < 0) { @@ -248,7 +268,8 @@ _PyTok_SourceLine(const _PyTok_SourceText *source, int lineno, .end = end, .implicit_newline = _PyTok_SourceLineIsImplicit(source, lineno), .contains_nul = memchr( - source->bytes + start, 0, end - start) != NULL, + source->bytes + (start - source->base_offset), + 0, end - start) != NULL, }; return 0; } @@ -257,21 +278,23 @@ int _PyTok_SourceLocation(const _PyTok_SourceText *source, _PyTok_Off offset, _PyTok_Affinity affinity, _PyTok_Loc *loc) { - if (offset < 0 || offset > source->len || loc == NULL || + if (offset < source->base_offset || + offset - source->base_offset > source->len || loc == NULL || (affinity != _PYTOK_AFFINITY_LEFT && affinity != _PYTOK_AFFINITY_RIGHT)) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source offset"); return -1; } if (source->nlines == 0 || - (offset == source->len && source_ends_in_newline(source) && + (offset - source->base_offset == source->len && + source_ends_in_newline(source) && affinity == _PYTOK_AFFINITY_RIGHT)) { *loc = (_PyTok_Loc){eof_lineno(source), 0}; return 0; } _PyTok_Off key = offset; - if (affinity == _PYTOK_AFFINITY_LEFT && key > 0) { + if (affinity == _PYTOK_AFFINITY_LEFT && key > source->base_offset) { key--; } int low = 0; diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 363475ff9015e35..7a2f46f73aff470 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -33,6 +33,7 @@ typedef struct { typedef struct { char *bytes; + _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; _PyTok_Off *line_checkpoints; @@ -42,9 +43,18 @@ typedef struct { Py_ssize_t implicit_cap; } _PyTok_SourceText; +static inline const char * +_PyTok_SourceData(const _PyTok_SourceText *source) +{ + return source->bytes != NULL ? source->bytes : ""; +} + PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *); /* Clear invalidates all cursors, spans, and views for the source. */ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); +/* Discard the retained window and invalidate its cursors, spans, and views. + Keep its allocation and advance the logical base to the end of the window. */ +PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may contain one newline, as its final byte. An unterminated line must be the final line. implicit_newline means that the final newline was synthesized. @@ -55,8 +65,8 @@ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( /* The returned view is invalidated by SourceAppendLine and SourceClear. */ PyAPI_FUNC(const char *) _PyTok_SourceSpanView( const _PyTok_SourceText *, _PyTok_Span, Py_ssize_t *); -/* Look up a 1-based line. Empty and newline-terminated sources have an empty - virtual line at EOF. */ +/* Look up a 1-based line in the retained window. Empty and newline-terminated + sources have an empty virtual line at EOF. */ PyAPI_FUNC(int) _PyTok_SourceLine( const _PyTok_SourceText *, int, _PyTok_Line *); /* Return false for invalid line numbers and the virtual EOF line. */ @@ -82,19 +92,21 @@ _PyTok_SpanIsValid(_PyTok_Span span) static inline _PyTok_Off _PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start) { - if (source->bytes == NULL || start < 0 || start >= source->len) { + if (source->bytes == NULL || start < source->base_offset || + start - source->base_offset >= source->len) { PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index"); return -1; } + _PyTok_Off relative_start = start - source->base_offset; const char *newline = memchr( - source->bytes + start, '\n', source->len - start); + source->bytes + relative_start, '\n', source->len - relative_start); if (newline == NULL) { PyErr_SetString(PyExc_SystemError, "corrupt tokenizer source line index"); return -1; } - return newline - source->bytes + 1; + return source->base_offset + (newline - source->bytes) + 1; } #endif From 27c3d732c3d375d9d133782038984b2da7647da7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 5 Sep 2026 19:30:23 +0100 Subject: [PATCH 5/6] gh-153569: Avoid overflow when sizing implicit line flags --- Parser/tokenizer/source.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index 04c24b77e0d8256..2f2aaf2589246d9 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -29,9 +29,8 @@ _PyTok_SourceDiscard(_PyTok_SourceText *source) source->bytes[0] = '\0'; } if (source->implicit_lines != NULL) { - memset(source->implicit_lines, 0, - Py_MIN(((Py_ssize_t)source->nlines + 7) / 8, - source->implicit_cap)); + Py_ssize_t used = source->nlines / 8 + (source->nlines % 8 != 0); + memset(source->implicit_lines, 0, Py_MIN(used, source->implicit_cap)); } source->nlines = 0; } @@ -106,7 +105,7 @@ reserve_checkpoints(_PyTok_SourceText *source, int needed) static int reserve_implicit_lines(_PyTok_SourceText *source, int nlines) { - Py_ssize_t needed = ((Py_ssize_t)nlines + 7) / 8; + Py_ssize_t needed = nlines / 8 + (nlines % 8 != 0); if (needed <= source->implicit_cap) { return 0; } From 6117ae337c2242a5a7767a34a81809f6ff6fb4c0 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 5 Sep 2026 20:24:37 +0100 Subject: [PATCH 6/6] Remove stale tokenizer buffer ownership comment --- Parser/lexer/state.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 496962fd0484f03..6d19e685bd7f81b 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -70,7 +70,7 @@ typedef struct _tokenizer_mode { struct tok_state { /* Input state; buf <= cur <= inp */ /* NB an entire line is held in the buffer */ - char *buf; /* Owned for file/readline input; source-backed otherwise. */ + char *buf; char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ _PyTok_Off buf_offset; /* Logical offset of buf[0]. */