Skip to content

gh-153569: hide tokenizer state behind token and source operations #156654

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
pablogsal wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

gh-153569: hide tokenizer state behind token and source operations #156654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8c56c04
gh-153569: hide tokenizer state behind token and source operations
pablogsal Sep 6, 2026
7c45b95
gh-153569: keep persistent scanner positions as source offsets
pablogsal Sep 6, 2026
569f343
gh-153569: include token types and owned metadata in tokenizer results
pablogsal Sep 6, 2026
b619385
gh-153569: keep character scanning inline with source offsets
pablogsal Sep 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Lib/test/test_fstring.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1876,6 +1876,9 @@ def __format__(self, format):
self.assertEqual(f"{UnchangedFormat():{r'\xFF'}}", '\\xFF')
self.assertEqual(rf"{UnchangedFormat():{r'\xFF'}}", '\\xFF')

self.assertEqual(rf"{UnchangedFormat():{f'\xFF'}}\n", 'ÿ\\n')
self.assertEqual(f"{UnchangedFormat():{rf'\xFF'}}\n", '\\xFF\n')

# Test continuation character in format specs
self.assertEqual(f"""{UnchangedFormat():{'a'\
'b'}}""", 'ab')
Expand Down
12 changes: 12 additions & 0 deletions Lib/test/test_tstring.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,14 @@ def test_raw_tstrings(self):
t = tr"{path}\Documents"
self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])

value = 42
t = rt"{value:{f'\xFF'}}\n"
self.assertTStringEqual(
t, ("", "\\n"), [(value, "value", None, 'ÿ')])
t = t"{value:{rf'\xFF'}}\n"
self.assertTStringEqual(
t, ("", "\n"), [(value, "value", None, '\\xFF')])

def test_template_concatenation(self):
# Test template + template
t1 = t"Hello, "
Expand Down Expand Up @@ -217,6 +225,10 @@ def test_syntax_errors(self):
("t'{x=!}'", "t-string: missing conversion character"),
("t'{x!z}'", "t-string: invalid conversion character 'z': "
"expected 's', 'r', or 'a'"),
("f\"{t'{x!z}'}\"", "t-string: invalid conversion character 'z': "
"expected 's', 'r', or 'a'"),
("t'{f\"{x!z}\"}'", "f-string: invalid conversion character 'z': "
"expected 's', 'r', or 'a'"),
("t'{lambda:1}'", "t-string: lambda expressions are not allowed "
"without parentheses"),
("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"),
Expand Down
6 changes: 3 additions & 3 deletions Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,13 +394,13 @@ PEGEN_OBJS= \
Parser/peg_api.o

TOKENIZER_OBJS= \
Parser/lexer/buffer.o \
Parser/lexer/lexer.o \
Parser/lexer/number.o \
Parser/lexer/state.o \
Parser/lexer/string.o \
Parser/tokenizer/cursor.o \
Parser/tokenizer/decoder.o \
Parser/tokenizer/api.o \
Parser/tokenizer/reader.o \
Parser/tokenizer/source.o \
Parser/tokenizer/helpers.o
Expand All @@ -411,14 +411,14 @@ PEGEN_HEADERS= \
$(srcdir)/Parser/string_parser.h

TOKENIZER_HEADERS= \
Parser/lexer/buffer.h \
Parser/lexer/lexer.h \
Parser/lexer/lexer_internal.h \
Parser/lexer/state.h \
Parser/tokenizer/cursor.h \
Parser/tokenizer/reader.h \
Parser/tokenizer/reader_internal.h \
Parser/tokenizer/source.h \
Parser/tokenizer/types.h \
Parser/tokenizer/tokenizer.h \
Parser/tokenizer/helpers.h

Expand Down Expand Up @@ -3463,7 +3463,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h

Expand Down
2 changes: 1 addition & 1 deletion PCbuild/_freeze_module.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,12 +181,12 @@
<ClCompile Include="..\Parser\action_helpers.c" />
<ClCompile Include="..\Parser\string_parser.c" />
<ClCompile Include="..\Parser\token.c" />
<ClCompile Include="..\Parser\lexer\buffer.c" />
<ClCompile Include="..\Parser\lexer\state.c" />
<ClCompile Include="..\Parser\lexer\lexer.c" />
<ClCompile Include="..\Parser\lexer\number.c" />
<ClCompile Include="..\Parser\lexer\string.c" />
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
<ClCompile Include="..\Parser\tokenizer\api.c" />
<ClCompile Include="..\Parser\tokenizer\reader.c" />
<ClCompile Include="..\Parser\tokenizer\source.c" />
<ClCompile Include="..\Parser\tokenizer\helpers.c" />
Expand Down
6 changes: 3 additions & 3 deletions PCbuild/_freeze_module.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,15 +469,15 @@
<ClCompile Include="..\Parser\lexer\string.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\lexer\buffer.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\lexer\state.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\decoder.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\api.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\reader.c">
<Filter>Source Files</Filter>
</ClCompile>
Expand Down
4 changes: 2 additions & 2 deletions PCbuild/pythoncore.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,10 +423,10 @@
<ClInclude Include="..\Parser\lexer\state.h" />
<ClInclude Include="..\Parser\lexer\lexer.h" />
<ClInclude Include="..\Parser\lexer\lexer_internal.h" />
<ClInclude Include="..\Parser\lexer\buffer.h" />
<ClInclude Include="..\Parser\tokenizer\cursor.h" />
<ClInclude Include="..\Parser\tokenizer\reader.h" />
<ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
<ClInclude Include="..\Parser\tokenizer\types.h" />
<ClInclude Include="..\Parser\tokenizer\source.h" />
<ClInclude Include="..\Parser\tokenizer\helpers.h" />
<ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
Expand Down Expand Up @@ -593,10 +593,10 @@
<ClCompile Include="..\Parser\lexer\lexer.c" />
<ClCompile Include="..\Parser\lexer\number.c" />
<ClCompile Include="..\Parser\lexer\string.c" />
<ClCompile Include="..\Parser\lexer\buffer.c" />
<ClCompile Include="..\Parser\tokenizer\cursor.c" />
<ClCompile Include="..\Parser\tokenizer\source.c" />
<ClCompile Include="..\Parser\tokenizer\decoder.c" />
<ClCompile Include="..\Parser\tokenizer\api.c" />
<ClCompile Include="..\Parser\tokenizer\reader.c" />
<ClCompile Include="..\Parser\tokenizer\helpers.c" />
<ClCompile Include="..\Parser\token.c" />
Expand Down
12 changes: 6 additions & 6 deletions PCbuild/pythoncore.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,9 +330,6 @@
<ClInclude Include="..\Parser\lexer\lexer_internal.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\lexer\buffer.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\tokenizer\cursor.h">
<Filter>Parser</Filter>
</ClInclude>
Expand All @@ -342,6 +339,9 @@
<ClInclude Include="..\Parser\tokenizer\reader_internal.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\tokenizer\types.h">
<Filter>Parser</Filter>
</ClInclude>
<ClInclude Include="..\Parser\tokenizer\source.h">
<Filter>Parser</Filter>
</ClInclude>
Expand Down Expand Up @@ -1361,9 +1361,6 @@
<ClCompile Include="..\Parser\lexer\state.c">
<Filter>Parser</Filter>
</ClCompile>
<ClCompile Include="..\Parser\lexer\buffer.c">
<Filter>Parser</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\cursor.c">
<Filter>Parser</Filter>
</ClCompile>
Expand All @@ -1373,6 +1370,9 @@
<ClCompile Include="..\Parser\tokenizer\decoder.c">
<Filter>Parser</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\api.c">
<Filter>Parser</Filter>
</ClCompile>
<ClCompile Include="..\Parser\tokenizer\reader.c">
<Filter>Parser</Filter>
</ClCompile>
Expand Down
54 changes: 35 additions & 19 deletions Parser/action_helpers.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1001,14 +1001,33 @@ result_token_with_metadata(Parser *p, void *result, PyObject *metadata)
return res;
}

static char
formatted_string_prefix(const Parser *p)
{
int nested = 0;
for (int i = p->mark - 1; i >= 0; i--) {
int type = p->tokens[i]->type;
if (type == FSTRING_END || type == TSTRING_END) {
nested++;
}
else if (type == FSTRING_START || type == TSTRING_START) {
if (nested == 0) {
return type == TSTRING_START ? 't' : 'f';
}
nested--;
}
}
Py_UNREACHABLE();
}

ResultTokenWithMetadata *
_PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
{
if (conv_token->lineno != conv->lineno || conv_token->end_col_offset != conv->col_offset) {
return RAISE_SYNTAX_ERROR_KNOWN_RANGE(
conv_token, conv,
"%c-string: conversion type must come right after the exclamation mark",
TOK_GET_STRING_PREFIX(p->tok)
formatted_string_prefix(p)
);
}

Expand All @@ -1017,7 +1036,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
!(first == 's' || first == 'r' || first == 'a')) {
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(conv,
"%c-string: invalid conversion character %R: expected 's', 'r', or 'a'",
TOK_GET_STRING_PREFIX(p->tok),
formatted_string_prefix(p),
conv->v.Name.id);
return NULL;
}
Expand Down Expand Up @@ -1344,7 +1363,8 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* tok
}

static asdl_expr_seq *
_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b, enum string_kind_t string_kind)
_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
Token *b, int is_tstring)
{
Py_ssize_t n_items = asdl_seq_LEN(raw_expressions);
Py_ssize_t total_items = n_items;
Expand All @@ -1370,15 +1390,13 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
for (Py_ssize_t i = 0; i < n_items; i++) {
expr_ty item = asdl_seq_GET(raw_expressions, i);

// This should correspond to a JoinedStr node of two elements
// created _PyPegen_formatted_value. This situation can only be the result of
// a (f|t)-string debug expression where the first element is a constant with the text and the second
// a formatted value with the expression.
/* Debug expressions arrive as JoinedStr(text, value); flatten them
into the surrounding string. */
if (item->kind == JoinedStr_kind) {
asdl_expr_seq *values = item->v.JoinedStr.values;
if (asdl_seq_LEN(values) != 2) {
PyErr_Format(PyExc_SystemError,
string_kind == TSTRING
is_tstring
? "unexpected TemplateStr node without debug data in t-string at line %d"
: "unexpected JoinedStr node without debug data in f-string at line %d",
item->lineno);
Expand All @@ -1390,7 +1408,9 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
asdl_seq_SET(seq, index++, first);

expr_ty second = asdl_seq_GET(values, 1);
assert((string_kind == TSTRING && second->kind == Interpolation_kind) || second->kind == FormattedValue_kind);
assert((is_tstring &&
second->kind == Interpolation_kind) ||
second->kind == FormattedValue_kind);
asdl_seq_SET(seq, index++, second);

continue;
Expand Down Expand Up @@ -1432,7 +1452,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
expr_ty
_PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b) {

asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, TSTRING);
asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, 1);
if (resized_exprs == NULL) {
return NULL;
}
Expand All @@ -1444,7 +1464,7 @@ _PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token
expr_ty
_PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) {

asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, FSTRING);
asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, 0);
if (resized_exprs == NULL) {
return NULL;
}
Expand All @@ -1460,12 +1480,7 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
return NULL;
}

// Check if we're inside a raw f-string for format spec decoding
int is_raw = 0;
if (INSIDE_FSTRING(p->tok)) {
tokenizer_mode *mode = TOK_GET_MODE(p->tok);
is_raw = mode->raw;
}
int is_raw = tok->is_raw;

PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);
if (str == NULL) {
Expand Down Expand Up @@ -2047,13 +2062,14 @@ _warn_relative_import_of_lazy(Parser *p, asdl_seq *dots, expr_ty module)
return -1;
}

_PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
int res = _PyErr_EmitSyntaxWarning(msg,
p->tok->filename,
info.filename,
module->lineno,
module->col_offset + 1,
module->end_lineno,
module->end_col_offset + 1,
p->tok->module);
info.module);
Py_DECREF(msg);
return res;
}
Expand Down
46 changes: 0 additions & 46 deletions Parser/lexer/buffer.c
View file
Open in desktop

This file was deleted.

22 changes: 0 additions & 22 deletions Parser/lexer/buffer.h
View file
Open in desktop

This file was deleted.

Loading
Loading