diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index c1ef1a73f05c20..4fff42ad873c5d 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -1876,6 +1876,9 @@ def __format__(self, format):
self.assertEqual(f"{UnchangedFormat():{r'\xFF'}}", '\\xFF')
self.assertEqual(rf"{UnchangedFormat():{r'\xFF'}}", '\\xFF')
+ self.assertEqual(rf"{UnchangedFormat():{f'\xFF'}}\n", 'ÿ\\n')
+ self.assertEqual(f"{UnchangedFormat():{rf'\xFF'}}\n", '\\xFF\n')
+
# Test continuation character in format specs
self.assertEqual(f"""{UnchangedFormat():{'a'\
'b'}}""", 'ab')
diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py
index 74653c77c55de1..eb51f3c242d07b 100644
--- a/Lib/test/test_tstring.py
+++ b/Lib/test/test_tstring.py
@@ -150,6 +150,14 @@ def test_raw_tstrings(self):
t = tr"{path}\Documents"
self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])
+ value = 42
+ t = rt"{value:{f'\xFF'}}\n"
+ self.assertTStringEqual(
+ t, ("", "\\n"), [(value, "value", None, 'ÿ')])
+ t = t"{value:{rf'\xFF'}}\n"
+ self.assertTStringEqual(
+ t, ("", "\n"), [(value, "value", None, '\\xFF')])
+
def test_template_concatenation(self):
# Test template + template
t1 = t"Hello, "
@@ -217,6 +225,10 @@ def test_syntax_errors(self):
("t'{x=!}'", "t-string: missing conversion character"),
("t'{x!z}'", "t-string: invalid conversion character 'z': "
"expected 's', 'r', or 'a'"),
+ ("f\"{t'{x!z}'}\"", "t-string: invalid conversion character 'z': "
+ "expected 's', 'r', or 'a'"),
+ ("t'{f\"{x!z}\"}'", "f-string: invalid conversion character 'z': "
+ "expected 's', 'r', or 'a'"),
("t'{lambda:1}'", "t-string: lambda expressions are not allowed "
"without parentheses"),
("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"),
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 78a486623181fa..166087f32dff18 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -394,13 +394,13 @@ PEGEN_OBJS= \
Parser/peg_api.o
TOKENIZER_OBJS= \
- Parser/lexer/buffer.o \
Parser/lexer/lexer.o \
Parser/lexer/number.o \
Parser/lexer/state.o \
Parser/lexer/string.o \
Parser/tokenizer/cursor.o \
Parser/tokenizer/decoder.o \
+ Parser/tokenizer/api.o \
Parser/tokenizer/reader.o \
Parser/tokenizer/source.o \
Parser/tokenizer/helpers.o
@@ -411,7 +411,6 @@ PEGEN_HEADERS= \
$(srcdir)/Parser/string_parser.h
TOKENIZER_HEADERS= \
- Parser/lexer/buffer.h \
Parser/lexer/lexer.h \
Parser/lexer/lexer_internal.h \
Parser/lexer/state.h \
@@ -419,6 +418,7 @@ TOKENIZER_HEADERS= \
Parser/tokenizer/reader.h \
Parser/tokenizer/reader_internal.h \
Parser/tokenizer/source.h \
+ Parser/tokenizer/types.h \
Parser/tokenizer/tokenizer.h \
Parser/tokenizer/helpers.h
@@ -3463,7 +3463,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
-MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
+MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index 469fd77cc8be9d..70c54e0e41efc6 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -181,12 +181,12 @@
-
+
diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters
index 976c99b7d24bdf..b0799b8dc9ecdd 100644
--- a/PCbuild/_freeze_module.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -469,15 +469,15 @@
Source Files
-
- Source Files
-
Source Files
Source Files
+
+ Source Files
+
Source Files
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index 79dfc9ccf39ec2..cd37db73930192 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -423,10 +423,10 @@
-
+
@@ -593,10 +593,10 @@
-
+
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index 765b4d46b12dd0..e4b9039eec13fc 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -330,9 +330,6 @@
Parser
-
- Parser
-
Parser
@@ -342,6 +339,9 @@
Parser
+
+ Parser
+
Parser
@@ -1361,9 +1361,6 @@
Parser
-
- Parser
-
Parser
@@ -1373,6 +1370,9 @@
Parser
+
+ Parser
+
Parser
diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c
index 8690dca8331b5c..3e4d463b36ab87 100644
--- a/Parser/action_helpers.c
+++ b/Parser/action_helpers.c
@@ -1001,6 +1001,25 @@ result_token_with_metadata(Parser *p, void *result, PyObject *metadata)
return res;
}
+static char
+formatted_string_prefix(const Parser *p)
+{
+ int nested = 0;
+ for (int i = p->mark - 1; i >= 0; i--) {
+ int type = p->tokens[i]->type;
+ if (type == FSTRING_END || type == TSTRING_END) {
+ nested++;
+ }
+ else if (type == FSTRING_START || type == TSTRING_START) {
+ if (nested == 0) {
+ return type == TSTRING_START ? 't' : 'f';
+ }
+ nested--;
+ }
+ }
+ Py_UNREACHABLE();
+}
+
ResultTokenWithMetadata *
_PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
{
@@ -1008,7 +1027,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
return RAISE_SYNTAX_ERROR_KNOWN_RANGE(
conv_token, conv,
"%c-string: conversion type must come right after the exclamation mark",
- TOK_GET_STRING_PREFIX(p->tok)
+ formatted_string_prefix(p)
);
}
@@ -1017,7 +1036,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
!(first == 's' || first == 'r' || first == 'a')) {
RAISE_SYNTAX_ERROR_KNOWN_LOCATION(conv,
"%c-string: invalid conversion character %R: expected 's', 'r', or 'a'",
- TOK_GET_STRING_PREFIX(p->tok),
+ formatted_string_prefix(p),
conv->v.Name.id);
return NULL;
}
@@ -1344,7 +1363,8 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* tok
}
static asdl_expr_seq *
-_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b, enum string_kind_t string_kind)
+_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
+ Token *b, int is_tstring)
{
Py_ssize_t n_items = asdl_seq_LEN(raw_expressions);
Py_ssize_t total_items = n_items;
@@ -1370,15 +1390,13 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
for (Py_ssize_t i = 0; i < n_items; i++) {
expr_ty item = asdl_seq_GET(raw_expressions, i);
- // This should correspond to a JoinedStr node of two elements
- // created _PyPegen_formatted_value. This situation can only be the result of
- // a (f|t)-string debug expression where the first element is a constant with the text and the second
- // a formatted value with the expression.
+ /* Debug expressions arrive as JoinedStr(text, value); flatten them
+ into the surrounding string. */
if (item->kind == JoinedStr_kind) {
asdl_expr_seq *values = item->v.JoinedStr.values;
if (asdl_seq_LEN(values) != 2) {
PyErr_Format(PyExc_SystemError,
- string_kind == TSTRING
+ is_tstring
? "unexpected TemplateStr node without debug data in t-string at line %d"
: "unexpected JoinedStr node without debug data in f-string at line %d",
item->lineno);
@@ -1390,7 +1408,9 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
asdl_seq_SET(seq, index++, first);
expr_ty second = asdl_seq_GET(values, 1);
- assert((string_kind == TSTRING && second->kind == Interpolation_kind) || second->kind == FormattedValue_kind);
+ assert((is_tstring &&
+ second->kind == Interpolation_kind) ||
+ second->kind == FormattedValue_kind);
asdl_seq_SET(seq, index++, second);
continue;
@@ -1432,7 +1452,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
expr_ty
_PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b) {
- asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, TSTRING);
+ asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, 1);
if (resized_exprs == NULL) {
return NULL;
}
@@ -1444,7 +1464,7 @@ _PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token
expr_ty
_PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) {
- asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, FSTRING);
+ asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, 0);
if (resized_exprs == NULL) {
return NULL;
}
@@ -1460,12 +1480,7 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
return NULL;
}
- // Check if we're inside a raw f-string for format spec decoding
- int is_raw = 0;
- if (INSIDE_FSTRING(p->tok)) {
- tokenizer_mode *mode = TOK_GET_MODE(p->tok);
- is_raw = mode->raw;
- }
+ int is_raw = tok->is_raw;
PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);
if (str == NULL) {
@@ -2047,13 +2062,14 @@ _warn_relative_import_of_lazy(Parser *p, asdl_seq *dots, expr_ty module)
return -1;
}
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
int res = _PyErr_EmitSyntaxWarning(msg,
- p->tok->filename,
+ info.filename,
module->lineno,
module->col_offset + 1,
module->end_lineno,
module->end_col_offset + 1,
- p->tok->module);
+ info.module);
Py_DECREF(msg);
return res;
}
diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c
deleted file mode 100644
index 9c39544ca7c479..00000000000000
--- a/Parser/lexer/buffer.c
+++ /dev/null
@@ -1,46 +0,0 @@
-#include "Python.h"
-#include "buffer.h"
-#include "state.h"
-
-void
-_PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base,
- _PyLexer_BufferPointers *pointers)
-{
- pointers->buf_from_base = tok->buf - base;
- pointers->cur_from_buf = tok->cur - tok->buf;
- pointers->inp_from_buf = tok->inp - tok->buf;
- pointers->start_from_buf = tok->start == NULL
- ? -1 : tok->start - tok->buf;
- pointers->line_start_from_buf = tok->line_start == NULL
- ? -1 : tok->line_start - tok->buf;
- pointers->multi_line_start_from_buf = tok->multi_line_start == NULL
- ? -1 : tok->multi_line_start - tok->buf;
- for (int index = tok->tok_mode_stack_index; index > 0; --index) {
- tokenizer_mode *mode = &tok->tok_mode_stack[index];
- mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
- mode->multi_line_start_offset = mode->multi_line_start == NULL
- ? -1 : mode->multi_line_start - tok->buf;
- }
-}
-
-void
-_PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base,
- const _PyLexer_BufferPointers *pointers)
-{
- tok->buf = base + pointers->buf_from_base;
- tok->cur = tok->buf + pointers->cur_from_buf;
- tok->inp = tok->buf + pointers->inp_from_buf;
- tok->start = pointers->start_from_buf < 0
- ? NULL : tok->buf + pointers->start_from_buf;
- tok->line_start = pointers->line_start_from_buf < 0
- ? NULL : tok->buf + pointers->line_start_from_buf;
- tok->multi_line_start = pointers->multi_line_start_from_buf < 0
- ? NULL : tok->buf + pointers->multi_line_start_from_buf;
- for (int index = tok->tok_mode_stack_index; index > 0; --index) {
- tokenizer_mode *mode = &tok->tok_mode_stack[index];
- mode->start = mode->start_offset < 0
- ? NULL : tok->buf + mode->start_offset;
- mode->multi_line_start = mode->multi_line_start_offset < 0
- ? NULL : tok->buf + mode->multi_line_start_offset;
- }
-}
diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h
deleted file mode 100644
index 285da124226d50..00000000000000
--- a/Parser/lexer/buffer.h
+++ /dev/null
@@ -1,22 +0,0 @@
-#ifndef _LEXER_BUFFER_H_
-#define _LEXER_BUFFER_H_
-
-#include "pyport.h"
-
-struct tok_state;
-
-typedef struct {
- Py_ssize_t buf_from_base;
- Py_ssize_t cur_from_buf;
- Py_ssize_t inp_from_buf;
- Py_ssize_t start_from_buf;
- Py_ssize_t line_start_from_buf;
- Py_ssize_t multi_line_start_from_buf;
-} _PyLexer_BufferPointers;
-
-void _PyLexer_SaveBufferPointers(
- struct tok_state *, const char *, _PyLexer_BufferPointers *);
-void _PyLexer_RestoreBufferPointers(
- struct tok_state *, char *, const _PyLexer_BufferPointers *);
-
-#endif
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index f96b31b9d2f38a..fa1d6e3bd7fc39 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -23,44 +23,33 @@ contains_null_bytes(const char* str, size_t size)
return memchr(str, 0, size) != NULL;
}
-/* Get next char, updating state; error code goes into tok->done */
int
-_PyLexer_nextc(struct tok_state *tok)
+_PyLexer_refill(struct tok_state *tok)
{
- int rc;
- for (;;) {
- if (tok->cur != tok->inp) {
- if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
- tok->done = E_COLUMNOVERFLOW;
- return EOF;
- }
- tok->col_offset++;
- return Py_CHARMASK(*tok->cur++); /* Fast path */
- }
- if (tok->done != E_OK) {
- return EOF;
- }
- rc = _PyTok_ReaderUnderflow(tok);
+ if (tok->done != E_OK) {
+ return 0;
+ }
+ int rc = _PyTok_ReaderUnderflow(tok);
#if defined(Py_DEBUG)
- if (tok->debug) {
- fprintf(stderr, "line[%d] = ", tok->lineno);
- _PyTokenizer_print_escape(stderr, tok->cur, tok->inp - tok->cur);
- fprintf(stderr, " tok->done = %d\n", tok->done);
- }
+ if (tok->debug) {
+ fprintf(stderr, "line[%d] = ", tok->lineno);
+ _PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok, tok->cur),
+ tok->inp - tok->cur);
+ fprintf(stderr, " tok->done = %d\n", tok->done);
+ }
#endif
- if (!rc) {
- tok->cur = tok->inp;
- return EOF;
- }
- tok->line_start = tok->cur;
-
- if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) {
- _PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
- tok->cur = tok->inp;
- return EOF;
- }
+ if (!rc) {
+ tok->cur = tok->inp;
+ return 0;
}
- Py_UNREACHABLE();
+ tok->line_start = tok->cur;
+ if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start),
+ tok->inp - tok->line_start)) {
+ _PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
+ tok->cur = tok->inp;
+ return 0;
+ }
+ return 1;
}
/* Back-up one character */
@@ -68,10 +57,10 @@ void
_PyLexer_backup(struct tok_state *tok, int c)
{
if (c != EOF) {
- if (--tok->cur < tok->buf) {
+ if (--tok->cur < tok->buf_offset) {
Py_FatalError("tokenizer beginning of buffer");
}
- if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
+ if ((int)(unsigned char)*_PyLexer_BufferPointer(tok, tok->cur) != Py_CHARMASK(c)) {
Py_FatalError("tok_backup: wrong character");
}
tok->col_offset--;
@@ -90,7 +79,7 @@ verify_identifier(struct tok_state *tok)
PyObject *s;
if (tok->input_error)
return 0;
- s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
+ s = PyUnicode_DecodeUTF8(_PyLexer_BufferPointer(tok, tok->start), tok->cur - tok->start, NULL);
if (s == NULL) {
if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
tok->done = E_DECODE;
@@ -115,7 +104,7 @@ verify_identifier(struct tok_state *tok)
tok->done = E_ERROR;
return 0;
}
- tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
+ tok->cur = tok->start + PyBytes_GET_SIZE(s);
}
Py_DECREF(s);
if (Py_UNICODE_ISPRINTABLE(ch)) {
@@ -161,10 +150,10 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
int c;
int blankline, nonascii;
- const char *p_start = NULL;
- const char *p_end = NULL;
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
nextline:
- tok->start = NULL;
+ tok->start = -1;
tok->starting_col_offset = -1;
blankline = 0;
@@ -283,7 +272,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
}
else {
if (tok->tok_extra_tokens) {
- p_start = tok->buf;
+ p_start = tok->buf_offset;
p_end = tok->cur;
}
tok->pendin--;
@@ -296,14 +285,14 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
tok_backup(tok, c);
again:
- tok->start = NULL;
+ tok->start = -1;
/* Skip spaces */
do {
c = tok_nextc(tok);
} while (c == ' ' || c == '\t' || c == '\014');
/* Set start of current token */
- tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
+ tok->start = tok->cur - 1;
tok->starting_col_offset = tok->col_offset - 1;
/* Skip comment, unless it's a type comment */
@@ -318,14 +307,14 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
}
if (tok->tok_extra_tokens) {
- p = tok->start;
+ p = _PyLexer_BufferPointer(tok, tok->start);
}
if (tok->type_comments) {
- p = tok->start;
+ p = _PyLexer_BufferPointer(tok, tok->start);
current_starting_col_offset = tok->starting_col_offset;
prefix = type_comment_prefix;
- while (*prefix && p < tok->cur) {
+ while (*prefix && p < _PyLexer_BufferPointer(tok, tok->cur)) {
if (*prefix == ' ') {
while (*p == ' ' || *p == '\t') {
p++;
@@ -354,8 +343,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
/* A TYPE_IGNORE is "type: ignore" followed by the end of the token
* or anything ASCII and non-alphanumeric. */
is_type_ignore = (
- tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
- && !(tok->cur > ignore_end
+ _PyLexer_BufferPointer(tok, tok->cur) >= ignore_end && memcmp(p, "ignore", 6) == 0
+ && !(_PyLexer_BufferPointer(tok, tok->cur) > ignore_end
&& ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
@@ -363,7 +352,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
? ignore_end_col_offset : current_starting_col_offset;
p_end = tok->cur;
if (is_type_ignore) {
- p_start = ignore_end;
+ p_start = _PyLexer_BufferOffset(tok, ignore_end);
/* If this type ignore is the only thing on the line, consume the newline also. */
if (blankline) {
@@ -371,7 +360,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
tok->atbol = 1;
}
} else {
- p_start = type_start;
+ p_start = _PyLexer_BufferOffset(tok, type_start);
}
_PyLexer_token_setup(tok, token, type, p_start, p_end);
token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
@@ -381,7 +370,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
}
if (tok->tok_extra_tokens) {
tok_backup(tok, c); /* don't eat the newline or EOF */
- p_start = p;
+ p_start = _PyLexer_BufferOffset(tok, p);
p_end = tok->cur;
tok->comment_newline = blankline;
return MAKE_TOKEN(COMMENT);
@@ -684,12 +673,13 @@ tok_get(struct tok_state *tok, struct token *token)
}
}
-int
+void
_PyTokenizer_Get(struct tok_state *tok, struct token *token)
{
+ _PyToken_Free(token);
int result = tok_get(tok, token);
if (tok->input_error) {
result = ERRORTOKEN;
}
- return result;
+ token->type = result;
}
diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h
index 040935a7e68913..5fce4ad8470789 100644
--- a/Parser/lexer/lexer.h
+++ b/Parser/lexer/lexer.h
@@ -5,27 +5,4 @@
int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
-int _PyTokenizer_Get(struct tok_state *, struct token *);
-
-/* The view points into the current input window. The next
- _PyTokenizer_Get() call may discard it. */
-static inline const char *
-_PyToken_TextView(const struct tok_state *tok, const struct token *token,
- Py_ssize_t *length)
-{
- assert(length != NULL);
- if (token->span.start < 0) {
- assert(token->span.start == -1 && token->span.end == -1);
- *length = 0;
- return "";
- }
- assert(_PyTok_SpanIsValid(token->span));
- assert(tok->buf != NULL);
- assert(tok->inp >= tok->buf);
- assert(token->span.start >= tok->buf_offset);
- assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf);
- *length = token->span.end - token->span.start;
- return tok->buf + (token->span.start - tok->buf_offset);
-}
-
#endif
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index c6d3b9045c7292..6a108990d9ae9d 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -1,6 +1,7 @@
#ifndef _PY_LEXER_INTERNAL_H_
#define _PY_LEXER_INTERNAL_H_
+#include "errcode.h"
#include "lexer.h"
#define is_potential_identifier_start(c) (\
@@ -41,10 +42,29 @@ TOK_NEXT_MODE(struct tok_state *tok)
#define FTSTRING_END(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_END : FSTRING_END)
#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f')
-#define tok_nextc _PyLexer_nextc
#define tok_backup _PyLexer_backup
-int _PyLexer_nextc(struct tok_state *);
+int _PyLexer_refill(struct tok_state *);
+
+static inline int
+tok_nextc(struct tok_state *tok)
+{
+ while (tok->cur == tok->inp) {
+ if (!_PyLexer_refill(tok)) {
+ return EOF;
+ }
+ }
+ assert(tok->cur >= tok->source.base_offset);
+ assert(tok->cur - tok->source.base_offset < tok->source.len);
+ if ((unsigned int)tok->col_offset >= (unsigned int)INT_MAX) {
+ tok->done = E_COLUMNOVERFLOW;
+ return EOF;
+ }
+ tok->col_offset++;
+ return Py_CHARMASK(
+ tok->source.bytes[tok->cur++ - tok->source.base_offset]);
+}
+
void _PyLexer_backup(struct tok_state *, int);
int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char);
int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
diff --git a/Parser/lexer/number.c b/Parser/lexer/number.c
index 8bca8cbb9adfe5..220952a69cd69e 100644
--- a/Parser/lexer/number.c
+++ b/Parser/lexer/number.c
@@ -113,8 +113,8 @@ int
_PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
int leading_dot)
{
- const char *p_start = NULL;
- const char *p_end = NULL;
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
if (leading_dot) {
goto fraction;
@@ -214,7 +214,7 @@ _PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
}
c = tok_nextc(tok);
}
- char* zeros_end = tok->cur;
+ _PyTok_Off zeros_end = tok->cur;
if (Py_ISDIGIT(c)) {
nonzero = 1;
c = tok_decimal_tail(tok);
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index d82a7d0f296bac..9a2f53920e72ba 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -21,11 +21,12 @@ _PyTokenizer_tok_new(void)
return NULL;
}
- tok->buf = tok->cur = tok->inp = NULL;
+ tok->cur = tok->inp = 0;
+ tok->line_start = tok->multi_line_start = -1;
tok->fp_interactive = 0;
tok->interactive_src_start = NULL;
tok->interactive_src_end = NULL;
- tok->start = NULL;
+ tok->start = -1;
tok->done = E_OK;
tok->fp = NULL;
tok->tabsize = TABSIZE;
@@ -93,51 +94,29 @@ _PyTokenizer_Free(struct tok_state *tok)
PyMem_Free(tok);
}
-void
-_PyToken_Free(struct token *token) {
- Py_XDECREF(token->metadata);
-}
-
void
_PyToken_Init(struct token *token) {
-#ifdef Py_DEBUG
- token->span = (_PyTok_Span){-1, -1};
- token->start_loc = (_PyTok_Loc){-1, -1};
- token->end_loc = (_PyTok_Loc){-1, -1};
-#endif
- token->metadata = NULL;
-}
-
-static inline _PyTok_Span
-buffer_span(const struct tok_state *tok, const char *start, const char *end)
-{
- if (start == NULL) {
- assert(end == NULL);
- return (_PyTok_Span){-1, -1};
- }
- assert(end != NULL);
- const char *base = tok->buf;
- assert(base != NULL);
- assert(tok->inp >= base);
- Py_ssize_t start_offset = start - base;
- Py_ssize_t end_offset = end - base;
- assert(start_offset >= 0 && start_offset <= end_offset);
- assert(end_offset <= tok->inp - base);
- assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset);
- return _PyTok_SpanFromBounds(
- tok->buf_offset + start_offset, tok->buf_offset + end_offset);
+ *token = (struct token){
+ .type = -1,
+ .span = {-1, -1},
+ .start_loc = {-1, -1},
+ .end_loc = {-1, -1},
+ };
}
int
-_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
+_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, _PyTok_Off start, _PyTok_Off end)
{
token->level = tok->level;
- token->span = buffer_span(tok, start, end);
+ token->is_raw = ISSTRINGLIT(type)
+ && tok->tok_mode_stack[tok->tok_mode_stack_index].raw;
+ assert((start == -1 && end == -1) || (start >= 0 && end >= start));
+ token->span = (_PyTok_Span){start, end};
int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
token->start_loc = (_PyTok_Loc){lineno, -1};
token->end_loc = (_PyTok_Loc){tok->lineno, -1};
- if (start != NULL && end != NULL) {
+ if (start >= 0 && end >= 0) {
token->start_loc.byte_col = tok->starting_col_offset;
token->end_loc.byte_col = tok->col_offset;
}
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 6d19e685bd7f81..fdf2e195edf11e 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -3,6 +3,7 @@
#include "object.h"
#include "../tokenizer/source.h"
+#include "../tokenizer/tokenizer.h"
#define MAXINDENT 100 /* Max indentation level */
#define MAXLEVEL 200 /* Max parentheses level */
@@ -21,13 +22,6 @@ enum interactive_underflow_t {
IUNDERFLOW_STOP,
};
-struct token {
- int level;
- _PyTok_Span span;
- _PyTok_Loc start_loc;
- _PyTok_Loc end_loc;
- PyObject *metadata;
-};
enum tokenizer_mode_kind_t {
TOK_REGULAR_MODE,
@@ -50,13 +44,10 @@ typedef struct _tokenizer_mode {
char quote;
int quote_size;
int raw;
- const char* start;
- const char* multi_line_start;
+ _PyTok_Off start;
+ _PyTok_Off multi_line_start;
int first_line;
- Py_ssize_t start_offset;
- Py_ssize_t multi_line_start_offset;
-
Py_ssize_t last_expr_size;
Py_ssize_t last_expr_end;
char* last_expr_buffer;
@@ -68,16 +59,15 @@ typedef struct _tokenizer_mode {
/* Tokenizer state */
struct tok_state {
- /* Input state; buf <= cur <= inp */
- /* NB an entire line is held in the buffer */
- char *buf;
- char *cur; /* Next character in buffer */
- char *inp; /* End of data in buffer */
- _PyTok_Off buf_offset; /* Logical offset of buf[0]. */
+ _PyTok_Off buf_offset;
+ _PyTok_Off cur;
+ _PyTok_Off inp;
+ _PyTok_Off start;
+ _PyTok_Off line_start;
+ _PyTok_SourceText source;
int fp_interactive; /* If the file descriptor is interactive */
char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
- const char *start; /* Start of current token if not NULL */
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
/* NB If done != E_OK, cur must be == inp!!! */
FILE *fp; /* Rest of input; NULL if tokenizing a string */
@@ -104,13 +94,9 @@ struct tok_state {
/* Stuff for PEP 0263 */
int input_error;
char *encoding; /* Source encoding. */
- const char* line_start; /* pointer to start of current line */
- const char* multi_line_start; /* pointer to start of first line of
- a single line or multi line string
- expression (cf. issue 16806) */
+ _PyTok_Off multi_line_start;
char* str; /* Source string being tokenized (if tokenizing from a string)*/
- _PyTok_SourceText source;
struct _PyTok_Reader *reader;
int type_comments; /* Whether to look for type comments */
@@ -129,12 +115,37 @@ struct tok_state {
#endif
};
-int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
+static inline _PyTok_Off
+_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
+{
+ const char *base = _PyTok_SourceData(&tok->source);
+ assert(position >= base && position <= base + tok->source.len);
+ return tok->source.base_offset + (position - base);
+}
+
+static inline const char *
+_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
+{
+ assert(offset >= tok->source.base_offset);
+ assert(offset - tok->source.base_offset <= tok->source.len);
+ return _PyTok_SourceData(&tok->source) + (offset - tok->source.base_offset);
+}
+
+static inline const char *
+_PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span,
+ Py_ssize_t *length)
+{
+ assert(length != NULL);
+ assert(_PyTok_SpanIsValid(span));
+ *length = span.end - span.start;
+ (void)_PyLexer_BufferPointer(tok, span.end);
+ return _PyLexer_BufferPointer(tok, span.start);
+}
+
+int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, _PyTok_Off start, _PyTok_Off end);
struct tok_state *_PyTokenizer_tok_new(void);
void _PyTokenizer_Free(struct tok_state *);
-void _PyToken_Free(struct token *);
-void _PyToken_Init(struct token *);
#endif
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index fc0299c5c7c592..cef2fa787c162f 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -123,10 +123,10 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
int
_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
{
- assert(tok->cur != NULL);
+ assert(tok->cur >= 0);
Py_ssize_t size = cur == 0
- ? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur);
+ ? tok->inp - tok->cur : (Py_ssize_t)strlen(_PyLexer_BufferPointer(tok, tok->cur));
tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
switch (cur) {
@@ -144,7 +144,7 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
}
tok_mode->last_expr_buffer = new_buffer;
memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size,
- tok->cur, size);
+ _PyLexer_BufferPointer(tok, tok->cur), size);
tok_mode->last_expr_size += size;
break;
case '{':
@@ -157,15 +157,15 @@ _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
}
tok_mode->last_expr_size = size;
tok_mode->last_expr_end = -1;
- memcpy(tok_mode->last_expr_buffer, tok->cur, size);
+ memcpy(tok_mode->last_expr_buffer, _PyLexer_BufferPointer(tok, tok->cur), size);
break;
case '}':
case '!':
- tok_mode->last_expr_end = strlen(tok->start);
+ tok_mode->last_expr_end = strlen(_PyLexer_BufferPointer(tok, tok->start));
break;
case ':':
if (tok_mode->last_expr_end == -1) {
- tok_mode->last_expr_end = strlen(tok->start);
+ tok_mode->last_expr_end = strlen(_PyLexer_BufferPointer(tok, tok->start));
}
break;
default:
@@ -225,8 +225,8 @@ _PyLexer_check_string_prefixes(struct tok_state *tok,
int
_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
{
- const char *p_start = NULL;
- const char *p_end = NULL;
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
int quote = c;
int quote_size = 1; /* 1 or 3 */
@@ -268,8 +268,6 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
the_current_tok->start = tok->start;
the_current_tok->multi_line_start = tok->line_start;
the_current_tok->first_line = tok->lineno;
- the_current_tok->start_offset = -1;
- the_current_tok->multi_line_start_offset = -1;
the_current_tok->last_expr_buffer = NULL;
the_current_tok->last_expr_size = 0;
the_current_tok->last_expr_end = -1;
@@ -277,20 +275,20 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
the_current_tok->in_debug = 0;
enum string_kind_t string_kind = FSTRING;
- switch (*tok->start) {
+ switch (*_PyLexer_BufferPointer(tok, tok->start)) {
case 'T':
case 't':
- the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
+ the_current_tok->raw = Py_TOLOWER(*(_PyLexer_BufferPointer(tok, tok->start) + 1)) == 'r';
string_kind = TSTRING;
break;
case 'F':
case 'f':
- the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
+ the_current_tok->raw = Py_TOLOWER(*(_PyLexer_BufferPointer(tok, tok->start) + 1)) == 'r';
break;
case 'R':
case 'r':
the_current_tok->raw = 1;
- if (Py_TOLOWER(*(tok->start + 1)) == 't') {
+ if (Py_TOLOWER(*(_PyLexer_BufferPointer(tok, tok->start) + 1)) == 't') {
string_kind = TSTRING;
}
break;
@@ -307,8 +305,8 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
int
_PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
{
- const char *p_start = NULL;
- const char *p_end = NULL;
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
int quote = c;
int quote_size = 1; /* 1 or 3 */
@@ -347,11 +345,11 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
break;
}
if (c == EOF || (quote_size == 1 && c == '\n')) {
- assert(tok->multi_line_start != NULL);
+ assert(tok->multi_line_start >= 0);
// shift the tok_state's location into
// the start of string, and report the error
// from the initial quote character
- tok->cur = (char *)tok->start;
+ tok->cur = tok->start;
tok->cur++;
tok->line_start = tok->multi_line_start;
int start = tok->lineno;
@@ -423,8 +421,8 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
int
_PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
{
- const char *p_start = NULL;
- const char *p_end = NULL;
+ _PyTok_Off p_start = -1;
+ _PyTok_Off p_end = -1;
int end_quote_size = 0;
int unicode_escape = 0;
@@ -516,11 +514,11 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
}
- assert(tok->multi_line_start != NULL);
+ assert(tok->multi_line_start >= 0);
// shift the tok_state's location into
// the start of string, and report the error
// from the initial quote character
- tok->cur = (char *)current_tok->start;
+ tok->cur = current_tok->start;
tok->cur++;
tok->line_start = current_tok->multi_line_start;
int start = tok->lineno;
diff --git a/Parser/pegen.c b/Parser/pegen.c
index d86dd22444e6a7..e709031ae78159 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -7,9 +7,7 @@
#include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal
#include
-#include "lexer/lexer.h"
#include "tokenizer/tokenizer.h"
-#include "tokenizer/helpers.h"
#include "pegen.h"
#define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two.
@@ -208,7 +206,6 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
parser_token->metadata = NULL;
if (new_token->metadata != NULL) {
if (_PyArena_AddPyObject(p->arena, new_token->metadata) < 0) {
- Py_DECREF(new_token->metadata);
return -1;
}
parser_token->metadata = new_token->metadata;
@@ -216,18 +213,19 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
}
parser_token->level = new_token->level;
+ parser_token->is_raw = new_token->is_raw;
parser_token->lineno = new_token->start_loc.lineno;
- parser_token->col_offset = p->tok->lineno == p->starting_lineno
+ parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno
? p->starting_col_offset + new_token->start_loc.byte_col
: new_token->start_loc.byte_col;
parser_token->end_lineno = new_token->end_loc.lineno;
- parser_token->end_col_offset = p->tok->lineno == p->starting_lineno
+ parser_token->end_col_offset = new_token->end_loc.lineno == p->starting_lineno
? p->starting_col_offset + new_token->end_loc.byte_col
: new_token->end_loc.byte_col;
p->fill += 1;
- if (token_type == ERRORTOKEN && p->tok->done == E_DECODE) {
+ if (token_type == ERRORTOKEN && _PyTokenizer_GetInfo(p->tok).status == E_DECODE) {
return _Pypegen_raise_decode_error(p);
}
@@ -261,10 +259,10 @@ _PyPegen_fill_token(Parser *p)
{
struct token new_token;
_PyToken_Init(&new_token);
- int type = _PyTokenizer_Get(p->tok, &new_token);
+ _PyTokenizer_Get(p->tok, &new_token);
// Record and skip '# type: ignore' comments
- while (type == TYPE_IGNORE) {
+ while (new_token.type == TYPE_IGNORE) {
Py_ssize_t len;
const char *text = _PyToken_TextView(p->tok, &new_token, &len);
char *tag = PyMem_Malloc((size_t)len + 1);
@@ -275,21 +273,22 @@ _PyPegen_fill_token(Parser *p)
memcpy(tag, text, (size_t)len);
tag[len] = '\0';
// Ownership of tag passes to the growable array
- if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
+ if (!growable_comment_array_add(&p->type_ignore_comments, new_token.end_loc.lineno, tag)) {
PyErr_NoMemory();
goto error;
}
- type = _PyTokenizer_Get(p->tok, &new_token);
+ _PyTokenizer_Get(p->tok, &new_token);
}
+ int type = new_token.type;
+
// If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) {
type = NEWLINE; /* Add an extra newline */
p->parsing_started = 0;
- if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
- p->tok->pendin = -p->tok->indent;
- p->tok->indent = 0;
+ if (!(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
+ _PyTokenizer_ImplyDedents(p->tok);
}
}
else {
@@ -302,7 +301,9 @@ _PyPegen_fill_token(Parser *p)
}
Token *t = p->tokens[p->fill];
- return initialize_token(p, t, &new_token, type);
+ int result = initialize_token(p, t, &new_token, type);
+ _PyToken_Free(&new_token);
+ return result;
error:
_PyToken_Free(&new_token);
return -1;
@@ -793,34 +794,7 @@ _PyPegen_number_token(Parser *p)
t->end_col_offset, p->arena);
}
-/* Check that the source for a single input statement really is a single
- statement by looking at what is left in the buffer after parsing.
- Trailing whitespace and comments are OK. */
-static int // bool
-bad_single_statement(Parser *p)
-{
- char *cur = p->tok->cur;
- char c = *cur;
-
- for (;;) {
- while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
- c = *++cur;
- }
-
- if (!c) {
- return 0;
- }
-
- if (c != '#') {
- return 1;
- }
- /* Suck up comment. */
- while (c && c != '\n') {
- c = *++cur;
- }
- }
-}
static int
compute_parser_flags(PyCompilerFlags *flags)
@@ -858,7 +832,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
return (Parser *) PyErr_NoMemory();
}
assert(tok != NULL);
- tok->type_comments = (flags & PyPARSE_TYPE_COMMENTS) > 0;
+ _PyTokenizer_SetOptions(tok, 0, (flags & PyPARSE_TYPE_COMMENTS) > 0);
p->tok = tok;
p->keywords = NULL;
p->n_keyword_lists = -1;
@@ -943,14 +917,12 @@ reset_parser_state_for_error_pass(Parser *p)
}
p->mark = 0;
p->call_invalid_rules = 1;
- // Don't try to get extra tokens in interactive mode when trying to
- // raise specialized errors in the second pass.
- p->tok->interactive_underflow = IUNDERFLOW_STOP;
+ _PyTokenizer_StopInteractive(p->tok);
}
static inline int
_is_end_of_source(Parser *p) {
- int err = p->tok->done;
+ int err = _PyTokenizer_GetInfo(p->tok).status;
return err == E_EOF || err == E_EOFS || err == E_EOLS;
}
@@ -961,19 +933,14 @@ _PyPegen_set_syntax_error_metadata(Parser *p) {
PyErr_SetRaisedException(exc);
return;
}
- const char *source = NULL;
- if (p->tok->str != NULL) {
- source = p->tok->str;
- }
- if (!source && p->tok->fp_interactive && p->tok->interactive_src_start) {
- source = p->tok->interactive_src_start;
- }
+ const char *source = _PyTokenizer_RetainedSource(p->tok);
+ const char *encoding = _PyTokenizer_GetInfo(p->tok).encoding;
PyObject* the_source = NULL;
if (source) {
- if (p->tok->encoding == NULL) {
+ if (encoding == NULL) {
the_source = PyUnicode_FromString(source);
} else {
- the_source = PyUnicode_Decode(source, strlen(source), p->tok->encoding, NULL);
+ the_source = PyUnicode_Decode(source, strlen(source), encoding, NULL);
}
}
if (!the_source) {
@@ -1034,8 +1001,7 @@ _PyPegen_run_parser(Parser *p)
return NULL;
}
- if (p->start_rule == Py_single_input && bad_single_statement(p)) {
- p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
+ if (p->start_rule == Py_single_input && _PyTokenizer_HasTrailingStatement(p->tok)) {
return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
}
@@ -1070,20 +1036,16 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
}
return NULL;
}
- if (!tok->fp || ps1 != NULL || ps2 != NULL ||
- PyUnicode_CompareWithASCIIString(filename_ob, "") == 0) {
- tok->fp_interactive = 1;
- }
- // This transfers the ownership to the tokenizer
- tok->filename = Py_NewRef(filename_ob);
// From here on we need to clean up even if there's an error
mod_ty result = NULL;
- tok->module = PyUnicode_FromString("__main__");
- if (tok->module == NULL) {
+ PyObject *module = PyUnicode_FromString("__main__");
+ if (module == NULL) {
goto error;
}
+ _PyTokenizer_SetContext(tok, filename_ob, module);
+ Py_DECREF(module);
int parser_flags = compute_parser_flags(flags);
Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
@@ -1095,8 +1057,9 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
result = _PyPegen_run_parser(p);
_PyPegen_Parser_Free(p);
- if (tok->fp_interactive && tok->interactive_src_start && result && interactive_src != NULL) {
- *interactive_src = PyUnicode_FromString(tok->interactive_src_start);
+ const char *source = _PyTokenizer_RetainedSource(tok);
+ if (source != NULL && result && interactive_src != NULL) {
+ *interactive_src = PyUnicode_FromString(source);
if (!*interactive_src || _PyArena_AddPyObject(arena, *interactive_src) < 0) {
Py_XDECREF(*interactive_src);
result = NULL;
@@ -1131,9 +1094,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
}
return NULL;
}
- // This transfers the ownership to the tokenizer
- tok->filename = Py_NewRef(filename_ob);
- tok->module = Py_XNewRef(module);
+ _PyTokenizer_SetContext(tok, filename_ob, module);
// We need to clear up from here on
mod_ty result = NULL;
diff --git a/Parser/pegen.h b/Parser/pegen.h
index 3cca698692bf6b..84804621d3ed67 100644
--- a/Parser/pegen.h
+++ b/Parser/pegen.h
@@ -5,7 +5,7 @@
#include
#include
-#include "lexer/state.h"
+#include "tokenizer/tokenizer.h"
#if 0
#define PyPARSE_YIELD_IS_KEYWORD 0x0001
@@ -26,8 +26,6 @@
#define CURRENT_POS (-5)
-#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index]))
-#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f')
typedef struct _memo {
int type;
@@ -40,6 +38,7 @@ typedef struct {
int type;
PyObject *bytes;
int level;
+ int is_raw;
int lineno, col_offset, end_lineno, end_col_offset;
Memo *memo;
// Filter over the rule types present in `memo` (bit `type & 63` is set
diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index b13e1c079220a9..7841f01b612915 100644
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -4,20 +4,20 @@
#include "pycore_pyerrors.h" // _PyErr_ProgramDecodedTextObject()
#include "pycore_runtime.h" // _Py_ID()
#include "pycore_tuple.h" // _PyTuple_FromPair
-#include "lexer/state.h"
-#include "lexer/lexer.h"
#include "pegen.h"
+#include "tokenizer/tokenizer.h"
// TOKENIZER ERRORS
static inline void
raise_unclosed_parentheses_error(Parser *p) {
- int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
- int error_col = p->tok->parencolstack[p->tok->level-1];
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ int error_lineno = info.delimiter_loc.lineno;
+ int error_col = info.delimiter_loc.byte_col;
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError,
error_lineno, error_col, error_lineno, -1,
"'%c' was never closed",
- p->tok->parenstack[p->tok->level-1]);
+ info.delimiter);
}
int
@@ -31,12 +31,13 @@ _Pypegen_tokenizer_error(Parser *p)
PyObject* errtype = PyExc_SyntaxError;
Py_ssize_t col_offset = -1;
p->error_indicator = 1;
- switch (p->tok->done) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ switch (info.status) {
case E_TOKEN:
msg = "invalid token";
break;
case E_EOF:
- if (p->tok->level) {
+ if (info.level) {
raise_unclosed_parentheses_error(p);
} else {
RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
@@ -62,7 +63,7 @@ _Pypegen_tokenizer_error(Parser *p)
msg = "too many levels of indentation";
break;
case E_LINECONT: {
- col_offset = p->tok->cur - p->tok->buf - 1;
+ col_offset = info.cursor - info.line_span.start - 1;
msg = "unexpected character after line continuation character";
break;
}
@@ -74,9 +75,9 @@ _Pypegen_tokenizer_error(Parser *p)
msg = "unknown parsing error";
}
- RAISE_ERROR_KNOWN_LOCATION(p, errtype, p->tok->lineno,
+ RAISE_ERROR_KNOWN_LOCATION(p, errtype, info.location.lineno,
col_offset >= 0 ? col_offset : 0,
- p->tok->lineno, -1, msg);
+ info.location.lineno, -1, msg);
return -1;
}
@@ -122,7 +123,7 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
// before the one that we had for the generic error.
// We don't want to tokenize to the end for interactive input
- if (p->tok->prompt != NULL) {
+ if (_PyTokenizer_IsInteractive(p->tok)) {
return 0;
}
@@ -137,14 +138,16 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
_PyToken_Init(&new_token);
for (;;) {
- switch (_PyTokenizer_Get(p->tok, &new_token)) {
- case ERRORTOKEN:
+ _PyTokenizer_Get(p->tok, &new_token);
+ switch (new_token.type) {
+ case ERRORTOKEN: {
if (PyErr_Occurred()) {
ret = -1;
goto exit;
}
- if (p->tok->level != 0) {
- int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ if (info.level != 0) {
+ int error_lineno = info.delimiter_loc.lineno;
if (current_err_line > error_lineno) {
raise_unclosed_parentheses_error(p);
ret = -1;
@@ -152,6 +155,7 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
}
}
break;
+ }
case ENDMARKER:
break;
default:
@@ -163,10 +167,9 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
exit:
_PyToken_Free(&new_token);
- // If we're in an f-string, we want the syntax error in the expression part
- // to propagate, so that tokenizer errors (like expecting '}') that happen afterwards
- // do not swallow it.
- if (PyErr_Occurred() && p->tok->tok_mode_stack_index <= 0) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ // Preserve expression errors over later formatted-string errors.
+ if (PyErr_Occurred() && !info.in_formatted_string) {
Py_XDECREF(value);
Py_XDECREF(type);
Py_XDECREF(traceback);
@@ -202,11 +205,12 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err
Py_ssize_t col_offset;
Py_ssize_t end_col_offset = -1;
if (t->col_offset == -1) {
- if (p->tok->cur == p->tok->buf) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ if (info.cursor == info.input_span.start) {
col_offset = 0;
} else {
- const char* start = p->tok->buf ? p->tok->line_start : p->tok->buf;
- col_offset = Py_SAFE_DOWNCAST(p->tok->cur - start, intptr_t, int);
+ col_offset = Py_SAFE_DOWNCAST(
+ info.cursor - info.line_span.start, intptr_t, int);
}
} else {
col_offset = t->col_offset + 1;
@@ -225,46 +229,15 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err
}
static PyObject *
-get_error_line_from_tokenizer_buffers(Parser *p, Py_ssize_t lineno)
+get_error_line_from_source(Parser *p, Py_ssize_t lineno)
{
- /* If the file descriptor is interactive, the source lines of the current
- * (multi-line) statement are stored in p->tok->interactive_src_start.
- * If not, we're parsing from a string, which means that the whole source
- * is stored in p->tok->str. */
- assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp != NULL);
-
- char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str;
- if (cur_line == NULL) {
- assert(p->tok->fp_interactive);
- // We can reach this point if the tokenizer buffers for interactive source have not been
- // initialized because we failed to decode the original source with the given locale.
+ if (_PyTokenizer_RetainedSource(p->tok) == NULL) {
return Py_GetConstant(Py_CONSTANT_EMPTY_STR);
}
-
Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno;
- const char* buf_end = p->tok->fp_interactive ? p->tok->interactive_src_end : p->tok->inp;
-
- if (buf_end < cur_line) {
- buf_end = cur_line + strlen(cur_line);
- }
-
- for (int i = 0; i < relative_lineno - 1; i++) {
- char *new_line = strchr(cur_line, '\n');
- // The assert is here for debug builds but the conditional that
- // follows is there so in release builds we do not crash at the cost
- // to report a potentially wrong line.
- assert(new_line != NULL && new_line + 1 < buf_end);
- if (new_line == NULL || new_line + 1 > buf_end) {
- break;
- }
- cur_line = new_line + 1;
- }
-
- char *next_newline;
- if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line
- next_newline = cur_line + strlen(cur_line);
- }
- return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
+ Py_ssize_t len;
+ const char *line = _PyTokenizer_LineView(p->tok, relative_lineno, &len);
+ return PyUnicode_DecodeUTF8(line, len, "replace");
}
void *
@@ -282,12 +255,15 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
PyObject *error_line = NULL;
PyObject *tmp = NULL;
p->error_indicator = 1;
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ _PyTok_Loc location = info.location;
+ _PyTok_Span text_span = info.line_span;
if (end_lineno == CURRENT_POS) {
- end_lineno = p->tok->lineno;
+ end_lineno = location.lineno;
}
if (end_col_offset == CURRENT_POS) {
- end_col_offset = p->tok->cur - p->tok->line_start;
+ end_col_offset = location.byte_col;
}
errstr = PyUnicode_FromFormatV(errmsg, va);
@@ -295,12 +271,12 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
goto error;
}
- if (p->tok->fp_interactive && p->tok->interactive_src_start != NULL) {
- error_line = get_error_line_from_tokenizer_buffers(p, lineno);
+ if (info.is_interactive && _PyTokenizer_RetainedSource(p->tok) != NULL) {
+ error_line = get_error_line_from_source(p, lineno);
}
else if (p->start_rule == Py_file_input) {
- error_line = _PyErr_ProgramDecodedTextObject(p->tok->filename,
- (int) lineno, p->tok->encoding);
+ error_line = _PyErr_ProgramDecodedTextObject(info.filename,
+ (int) lineno, info.encoding);
}
if (!error_line) {
@@ -311,14 +287,17 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
we're actually parsing from a file, which has an E_EOF SyntaxError and in that case
`PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which
does not physically exist */
- assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);
-
- if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) {
- Py_ssize_t size = p->tok->inp - p->tok->line_start;
- error_line = PyUnicode_DecodeUTF8(p->tok->line_start, size, "replace");
+ assert(!info.is_file || info.status == E_EOF);
+
+ if (location.lineno <= lineno &&
+ info.input_span.end > info.input_span.start) {
+ Py_ssize_t size;
+ const char *line = _PyTokenizer_SpanView(
+ p->tok, text_span, &size);
+ error_line = PyUnicode_DecodeUTF8(line, size, "replace");
}
- else if (p->tok->fp == NULL || p->tok->fp == stdin) {
- error_line = get_error_line_from_tokenizer_buffers(p, lineno);
+ else if (!info.is_file) {
+ error_line = get_error_line_from_source(p, lineno);
}
else {
error_line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
@@ -343,7 +322,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
}
}
- tmp = Py_BuildValue("(OnnNnn)", p->tok->filename, lineno, col_number, error_line, end_lineno, end_col_number);
+ tmp = Py_BuildValue("(OnnNnn)", info.filename, lineno, col_number, error_line, end_lineno, end_col_number);
if (!tmp) {
goto error;
}
@@ -366,11 +345,12 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
void
_Pypegen_set_syntax_error(Parser* p, Token* last_token) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
// Existing syntax error
if (PyErr_Occurred()) {
// Prioritize tokenizer errors to custom syntax errors raised
// on the second phase only if the errors come from the parser.
- int is_tok_ok = (p->tok->done == E_DONE || p->tok->done == E_OK);
+ int is_tok_ok = (info.status == E_DONE || info.status == E_OK);
if (is_tok_ok && PyErr_ExceptionMatches(PyExc_SyntaxError)) {
_PyPegen_tokenize_full_source_to_check_for_errors(p);
}
@@ -382,8 +362,8 @@ _Pypegen_set_syntax_error(Parser* p, Token* last_token) {
RAISE_SYNTAX_ERROR("error at start before reading any input");
}
// Parser encountered EOF (End of File) unexpectedtly
- if (last_token->type == ERRORTOKEN && p->tok->done == E_EOF) {
- if (p->tok->level) {
+ if (last_token->type == ERRORTOKEN && info.status == E_EOF) {
+ if (info.level) {
raise_unclosed_parentheses_error(p);
} else {
RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
diff --git a/Parser/string_parser.c b/Parser/string_parser.c
index 431d2703d88e2c..e57460cb1fa648 100644
--- a/Parser/string_parser.c
+++ b/Parser/string_parser.c
@@ -2,7 +2,6 @@
#include "pycore_bytesobject.h" // _PyBytes_DecodeEscape()
#include "pycore_unicodeobject.h" // _PyUnicode_DecodeUnicodeEscapeInternal()
-#include "lexer/state.h"
#include "pegen.h"
#include "string_parser.h"
@@ -87,8 +86,9 @@ warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_in
col_offset += quote_count;
}
- if (PyErr_WarnExplicitObject(category, msg, p->tok->filename,
- lineno, p->tok->module, NULL) < 0) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
+ if (PyErr_WarnExplicitObject(category, msg, info.filename,
+ lineno, info.module, NULL) < 0) {
if (PyErr_ExceptionMatches(category)) {
/* Replace the Syntax/DeprecationWarning exception with a SyntaxError
to get a more accurate error report */
diff --git a/Parser/tokenizer/api.c b/Parser/tokenizer/api.c
new file mode 100644
index 00000000000000..c69e7113c130b1
--- /dev/null
+++ b/Parser/tokenizer/api.c
@@ -0,0 +1,170 @@
+#include "Python.h"
+#include "errcode.h"
+#include "pycore_token.h"
+
+#include "tokenizer.h"
+#include "reader.h"
+#include "reader_internal.h"
+#include "../lexer/state.h"
+
+_PyTokenizer_Info
+_PyTokenizer_GetInfo(const struct tok_state *tok)
+{
+ _PyTokenizer_Info info = {
+ .status = tok->done,
+ .location = {tok->lineno, tok->line_start < 0
+ ? -1 : (int)(tok->cur - tok->line_start)},
+ .cursor = tok->cur,
+ .input_span = {tok->buf_offset, tok->inp},
+ .line_span = {tok->line_start, tok->inp},
+ .level = tok->level,
+ .delimiter_loc = {-1, -1},
+ .in_formatted_string = INSIDE_FSTRING(tok),
+ .is_interactive = tok->reader->kind == _PYTOK_READER_INTERACTIVE,
+ .is_file = tok->fp != NULL && tok->fp != stdin,
+ .filename = tok->filename,
+ .module = tok->module,
+ .encoding = tok->encoding,
+ };
+ if (tok->level > 0) {
+ int level = tok->level - 1;
+ info.delimiter = tok->parenstack[level];
+ info.delimiter_loc = (_PyTok_Loc){
+ tok->parenlinenostack[level], tok->parencolstack[level]};
+ }
+ return info;
+}
+
+const char *
+_PyToken_TextView(const struct tok_state *tok, const struct token *token,
+ Py_ssize_t *length)
+{
+ assert(length != NULL);
+ if (token->span.start < 0) {
+ assert(token->span.start == -1 && token->span.end == -1);
+ *length = 0;
+ return "";
+ }
+ return _PyLexer_BufferSpanView(tok, token->span, length);
+}
+
+const char *
+_PyTokenizer_SpanView(const struct tok_state *tok, _PyTok_Span span,
+ Py_ssize_t *length)
+{
+ return _PyLexer_BufferSpanView(tok, span, length);
+}
+
+void
+_PyToken_GetView(const struct tok_state *tok, const struct token *token,
+ _PyToken_View *view)
+{
+ assert(view != NULL);
+ assert((token->span.start == -1 && token->span.end == -1) ||
+ _PyTok_SpanIsValid(token->span));
+ if (token->span.start >= 0) {
+ (void)_PyLexer_BufferPointer(tok, token->span.end);
+ }
+ view->text = token->span.start < 0
+ ? NULL : _PyLexer_BufferPointer(tok, token->span.start);
+ view->length = token->span.end - token->span.start;
+ view->end_line = _PyLexer_BufferPointer(tok, tok->line_start);
+ view->line = ISSTRINGLIT(token->type)
+ ? view->text - token->start_loc.byte_col : view->end_line;
+ view->line_length = tok->inp - tok->line_start +
+ (view->end_line - view->line);
+ view->implicit_newline = tok->implicit_newline;
+ view->at_eof = tok->done == E_EOF;
+}
+
+const char *
+_PyTokenizer_LineView(const struct tok_state *tok, Py_ssize_t lineno,
+ Py_ssize_t *length)
+{
+ const char *line = _PyTokenizer_RetainedSource(tok);
+ if (line == NULL) {
+ line = _PyLexer_BufferPointer(tok, tok->buf_offset);
+ }
+ for (Py_ssize_t i = 1; i < lineno; i++) {
+ const char *next = strchr(line, '\n');
+ if (next == NULL) {
+ break;
+ }
+ line = next + 1;
+ }
+ const char *end = strchr(line, '\n');
+ *length = end != NULL ? end - line : (Py_ssize_t)strlen(line);
+ return line;
+}
+
+const char *
+_PyTokenizer_RetainedSource(const struct tok_state *tok)
+{
+ if (tok->reader->kind == _PYTOK_READER_PREPARED) {
+ return _PyTok_SourceData(&tok->source);
+ }
+ if (tok->reader->kind == _PYTOK_READER_INTERACTIVE) {
+ return tok->source.bytes;
+ }
+ return NULL;
+}
+
+void
+_PyTokenizer_SetContext(struct tok_state *tok, PyObject *filename,
+ PyObject *module)
+{
+ Py_XINCREF(filename);
+ Py_XINCREF(module);
+ Py_XSETREF(tok->filename, filename);
+ Py_XSETREF(tok->module, module);
+}
+
+void
+_PyTokenizer_SetOptions(struct tok_state *tok, int extra_tokens,
+ int type_comments)
+{
+ tok->tok_extra_tokens = extra_tokens;
+ tok->type_comments = type_comments;
+}
+
+void
+_PyTokenizer_ImplyDedents(struct tok_state *tok)
+{
+ if (tok->indent != 0) {
+ tok->pendin = -tok->indent;
+ tok->indent = 0;
+ }
+}
+
+int
+_PyTokenizer_HasTrailingStatement(const struct tok_state *tok)
+{
+ const char *cur = _PyLexer_BufferPointer(tok, tok->cur);
+ char c = *cur;
+ for (;;) {
+ while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
+ c = *++cur;
+ }
+ if (!c) {
+ return 0;
+ }
+ if (c != '#') {
+ return 1;
+ }
+ while (c && c != '\n') {
+ c = *++cur;
+ }
+ }
+}
+
+int
+_PyTokenizer_IsInteractive(const struct tok_state *tok)
+{
+ return tok->prompt != NULL;
+}
+
+void
+_PyTokenizer_StopInteractive(struct tok_state *tok)
+{
+ tok->interactive_underflow = IUNDERFLOW_STOP;
+}
diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c
index af17b8b63235f5..5b588572f1fb7e 100644
--- a/Parser/tokenizer/decoder.c
+++ b/Parser/tokenizer/decoder.c
@@ -238,22 +238,13 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
const _PyTok_Chunk *line = cookie_line == 2 ? second : first;
const char *line_data = line->data + (cookie_line == 1 ? 3 : 0);
Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0);
- const char *saved_line_start = tok->line_start;
- char *saved_cur = tok->cur;
- int saved_lineno = tok->lineno;
- tok->line_start = line_data;
- tok->cur = (char *)line_data;
- tok->lineno = cookie_line;
int end_col = (int)Py_MIN(line_len, INT_MAX);
if (end_col > 0 && (line_data[end_col - 1] == '\n' ||
line_data[end_col - 1] == '\r')) {
end_col--;
}
- _PyTokenizer_syntaxerror_known_range(
- tok, 0, end_col, "encoding problem: %s with BOM", cookie);
- tok->line_start = saved_line_start;
- tok->cur = saved_cur;
- tok->lineno = saved_lineno;
+ _PyTokenizer_syntaxerror_at(
+ tok, line_data, 0, cookie_line, 0, end_col, "encoding problem: %s with BOM", cookie);
PyMem_Free(cookie);
return _PYTOK_ENCODING_ERROR;
}
diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c
index e99be70ea5e980..c803b787d9dae6 100644
--- a/Parser/tokenizer/helpers.c
+++ b/Parser/tokenizer/helpers.c
@@ -24,7 +24,8 @@ byte_col_to_char_col(const char *line, int byte_col)
}
static int
-_syntaxerror_range(struct tok_state *tok, const char *format,
+_syntaxerror_range(struct tok_state *tok, const char *line,
+ Py_ssize_t cursor, int lineno, const char *format,
int col_offset, int end_col_offset,
va_list vargs)
{
@@ -40,7 +41,7 @@ _syntaxerror_range(struct tok_state *tok, const char *format,
goto error;
}
- errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start,
+ errtext = PyUnicode_DecodeUTF8(line, cursor,
"replace");
if (!errtext) {
goto error;
@@ -50,19 +51,19 @@ _syntaxerror_range(struct tok_state *tok, const char *format,
col_offset = (int)PyUnicode_GET_LENGTH(errtext);
}
else if (col_offset > 0) {
- col_offset = byte_col_to_char_col(tok->line_start, col_offset);
+ col_offset = byte_col_to_char_col(line, col_offset);
}
if (end_col_offset == -1) {
end_col_offset = col_offset;
}
else if (end_col_offset > 0) {
- end_col_offset = byte_col_to_char_col(tok->line_start, end_col_offset);
+ end_col_offset = byte_col_to_char_col(line, end_col_offset);
}
- Py_ssize_t line_len = strcspn(tok->line_start, "\n");
- if (line_len != tok->cur - tok->line_start) {
+ Py_ssize_t line_len = strcspn(line, "\n");
+ if (line_len != cursor) {
Py_DECREF(errtext);
- errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len,
+ errtext = PyUnicode_DecodeUTF8(line, line_len,
"replace");
}
if (!errtext) {
@@ -71,8 +72,8 @@ _syntaxerror_range(struct tok_state *tok, const char *format,
args = Py_BuildValue("(O(OiiNii))", errmsg,
tok->filename ? tok->filename : Py_None,
- tok->lineno, col_offset, errtext,
- tok->lineno, end_col_offset);
+ lineno, col_offset, errtext,
+ lineno, end_col_offset);
if (args) {
PyErr_SetObject(PyExc_SyntaxError, args);
Py_DECREF(args);
@@ -90,7 +91,9 @@ _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...)
// These errors are cleaned on startup. Todo: Fix it.
va_list vargs;
va_start(vargs, format);
- int ret = _syntaxerror_range(tok, format, -1, -1, vargs);
+ int ret = _syntaxerror_range(tok, _PyLexer_BufferPointer(tok, tok->line_start),
+ tok->cur - tok->line_start, tok->lineno,
+ format, -1, -1, vargs);
va_end(vargs);
return ret;
}
@@ -102,7 +105,23 @@ _PyTokenizer_syntaxerror_known_range(struct tok_state *tok,
{
va_list vargs;
va_start(vargs, format);
- int ret = _syntaxerror_range(tok, format, col_offset, end_col_offset, vargs);
+ int ret = _syntaxerror_range(tok, _PyLexer_BufferPointer(tok, tok->line_start),
+ tok->cur - tok->line_start, tok->lineno,
+ format, col_offset, end_col_offset, vargs);
+ va_end(vargs);
+ return ret;
+}
+
+int
+_PyTokenizer_syntaxerror_at(struct tok_state *tok, const char *line,
+ Py_ssize_t cursor, int lineno,
+ int col_offset, int end_col_offset,
+ const char *format, ...)
+{
+ va_list vargs;
+ va_start(vargs, format);
+ int ret = _syntaxerror_range(tok, line, cursor, lineno, format,
+ col_offset, end_col_offset, vargs);
va_end(vargs);
return ret;
}
@@ -326,8 +345,8 @@ _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
}
if (badchar) {
tok->lineno = lineno;
- tok->line_start = line_start;
- tok->cur = (char *)badchar;
+ tok->line_start = _PyLexer_BufferOffset(tok, line_start);
+ tok->cur = _PyLexer_BufferOffset(tok, badchar);
_PyTokenizer_syntaxerror_known_range(tok,
col_offset + 1, col_offset + 1,
"Non-UTF-8 code starting with '\\x%.2x'"
diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h
index 5edf5a3dfd2e0d..4e2e4e9525ec32 100644
--- a/Parser/tokenizer/helpers.h
+++ b/Parser/tokenizer/helpers.h
@@ -9,6 +9,8 @@
tok->lineno++; \
tok->col_offset = 0;
+int _PyTokenizer_syntaxerror_at(struct tok_state *, const char *,
+ Py_ssize_t, int, int, int, const char *, ...);
int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...);
int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...);
int _PyTokenizer_indenterror(struct tok_state *tok);
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index b9b4a461087441..1fd9d540df2e55 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -5,7 +5,6 @@
#include "helpers.h"
#include "reader.h"
#include "reader_internal.h"
-#include "../lexer/buffer.h"
#include "../lexer/lexer.h"
#include "../lexer/state.h"
@@ -34,7 +33,6 @@ _PyTok_ReaderFree(struct tok_state *tok)
}
PyMem_Free(reader->file_buffer);
PyMem_Free(reader->decoded);
- tok->buf = NULL;
PyMem_Free(reader);
tok->reader = NULL;
}
@@ -141,7 +139,7 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
if (lineno > tok->source.nlines) {
return _PYTOK_READ_EOF;
}
- const char *start = tok->inp;
+ const char *start = _PyLexer_BufferPointer(tok, tok->inp);
const char *newline = memchr(
start, '\n', tok->source.bytes + tok->source.len - start);
_PyTok_Off end = newline != NULL
@@ -535,12 +533,11 @@ reader_next(struct tok_state *tok, _PyTok_Chunk *chunk)
static void
reset_streaming_buffer(struct tok_state *tok)
{
- assert(tok->buf != NULL);
- assert(tok->cur >= tok->buf && tok->cur <= tok->inp);
+ assert(tok->cur >= tok->buf_offset && tok->cur <= tok->inp);
_PyTok_SourceDiscard(&tok->source);
tok->buf_offset = tok->source.base_offset;
- tok->buf = tok->cur = tok->inp = (char *)_PyTok_SourceData(&tok->source);
- tok->line_start = tok->buf;
+ tok->cur = tok->inp = tok->source.base_offset;
+ tok->line_start = tok->buf_offset;
}
int
@@ -549,7 +546,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
_PyTok_ReaderKind kind = tok->reader->kind;
int prepared = kind == _PYTOK_READER_PREPARED;
int streaming = reader_is_streaming(kind);
- int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok);
+ int reset_buffer = !prepared && tok->start < 0 && !INSIDE_FSTRING(tok);
_PyTok_Chunk chunk;
_PyTok_ReadResult result = reader_next(tok, &chunk);
@@ -589,11 +586,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
if (streaming && reset_buffer) {
reset_streaming_buffer(tok);
}
- _PyLexer_BufferPointers pointers;
- if (!reset_buffer) {
- _PyLexer_SaveBufferPointers(
- tok, tok->source.bytes, &pointers);
- }
_PyTok_Off source_start = _PyTok_SourceAppendLine(
&tok->source, chunk.data, chunk.len,
chunk.implicit_newline);
@@ -605,31 +597,24 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
return 0;
}
if (reset_buffer) {
- tok->buf = tok->cur =
- tok->source.bytes + (source_start - tok->source.base_offset);
+ tok->cur = source_start;
tok->buf_offset = source_start;
- tok->line_start = tok->buf;
- tok->start = NULL;
- tok->multi_line_start = NULL;
- }
- else {
- _PyLexer_RestoreBufferPointers(
- tok, tok->source.bytes, &pointers);
+ tok->line_start = tok->buf_offset;
+ tok->start = -1;
+ tok->multi_line_start = -1;
}
- tok->inp = tok->source.bytes +
- (source_start - tok->source.base_offset) + scan_len;
+ tok->inp = source_start + scan_len;
}
if (tok->fp_interactive) {
tok->interactive_src_start = tok->source.bytes;
tok->interactive_src_end = tok->source.bytes + tok->source.len;
}
if (prepared) {
- if (tok->start == NULL) {
- tok->buf = tok->cur;
+ if (tok->start < 0) {
tok->buf_offset = tok->source.base_offset +
(chunk.data - tok->source.bytes);
}
- tok->inp = chunk.data + chunk.len;
+ tok->inp = _PyLexer_BufferOffset(tok, chunk.data) + chunk.len;
}
tok->implicit_newline = chunk.implicit_newline;
@@ -642,7 +627,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
ADVANCE_LINENO();
if (kind == _PYTOK_READER_FILE &&
(tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
- !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) {
+ !_PyTokenizer_ensure_utf8(_PyLexer_BufferPointer(tok, tok->cur), tok, tok->lineno)) {
_PyTok_ChunkClear(&chunk);
tok->input_error = 1;
return 0;
@@ -669,8 +654,7 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind)
return tok;
}
if (reader_is_streaming(kind)) {
- tok->buf = tok->cur = tok->inp =
- (char *)_PyTok_SourceData(&tok->source);
+ tok->cur = tok->inp = tok->source.base_offset;
}
return tok;
}
@@ -688,7 +672,7 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input,
_PyTokenizer_Free(tok);
return NULL;
}
- tok->buf = tok->cur = tok->inp = tok->str;
+ tok->cur = tok->inp = tok->source.base_offset;
return tok;
}
diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h
index 7a2f46f73aff47..2f74ed8b1f4fab 100644
--- a/Parser/tokenizer/source.h
+++ b/Parser/tokenizer/source.h
@@ -3,20 +3,7 @@
#include "Python.h"
-typedef Py_ssize_t _PyTok_Off;
-
-/* Spans use half-open logical byte offsets into decoded input. Their backing
- storage may retain only the current input window. */
-typedef struct {
- _PyTok_Off start;
- _PyTok_Off end;
-} _PyTok_Span;
-
-/* Lines are 1-based and byte columns are 0-based. */
-typedef struct {
- int lineno;
- int byte_col;
-} _PyTok_Loc;
+#include "types.h"
typedef enum {
_PYTOK_AFFINITY_LEFT,
@@ -77,18 +64,6 @@ PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit(
PyAPI_FUNC(int) _PyTok_SourceLocation(
const _PyTok_SourceText *, _PyTok_Off, _PyTok_Affinity, _PyTok_Loc *);
-static inline _PyTok_Span
-_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end)
-{
- return (_PyTok_Span){start, end};
-}
-
-static inline int
-_PyTok_SpanIsValid(_PyTok_Span span)
-{
- return span.start >= 0 && span.end >= span.start;
-}
-
static inline _PyTok_Off
_PyTok_SourceFindLineEnd(const _PyTok_SourceText *source, _PyTok_Off start)
{
diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h
index d8c889115cfd73..e9229d12087162 100644
--- a/Parser/tokenizer/tokenizer.h
+++ b/Parser/tokenizer/tokenizer.h
@@ -2,12 +2,98 @@
#define Py_TOKENIZER_H
#include "Python.h"
+#include "types.h"
-struct tok_state *_PyTokenizer_FromString(const char *, int, int);
-struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int);
-struct tok_state *_PyTokenizer_FromReadline(PyObject *, const char *);
-struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
- const char *, const char *);
+struct tok_state;
+
+/* Initialize before use. metadata owns a reference released by _PyToken_Free;
+ a consumer taking that reference must set metadata to NULL. */
+struct token {
+ int type;
+ int level;
+ int is_raw;
+ _PyTok_Span span;
+ _PyTok_Loc start_loc;
+ _PyTok_Loc end_loc;
+ PyObject *metadata;
+};
+
+typedef struct {
+ const char *text;
+ Py_ssize_t length;
+ const char *line;
+ Py_ssize_t line_length;
+ const char *end_line;
+ int implicit_newline;
+ int at_eof;
+} _PyToken_View;
+
+typedef struct {
+ int status;
+ _PyTok_Loc location;
+ _PyTok_Off cursor;
+ _PyTok_Span input_span;
+ _PyTok_Span line_span;
+ int level;
+ char delimiter;
+ _PyTok_Loc delimiter_loc;
+ int in_formatted_string;
+ int is_interactive;
+ int is_file;
+ PyObject *filename;
+ PyObject *module;
+ const char *encoding;
+} _PyTokenizer_Info;
+
+/* Get replaces the initialized token, releasing its previous metadata.
+ Errors are returned as ERRORTOKEN, with or without a Python exception. */
+void _PyTokenizer_Get(struct tok_state *, struct token *);
+void _PyTokenizer_Free(struct tok_state *);
+void _PyTokenizer_raise_init_error(PyObject *filename);
+void _PyToken_Init(struct token *);
+static inline void
+_PyToken_Free(struct token *token)
+{
+ Py_CLEAR(token->metadata);
+}
+
+/* Views and borrowed snapshot references remain valid until the tokenizer is
+ mutated or freed. Source spans may be discarded when reading more input. */
+_PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state *);
+/* An absent token span has a nonnull empty text view. */
+const char *_PyToken_TextView(
+ const struct tok_state *, const struct token *, Py_ssize_t *);
+/* Use the token from the most recent Get. text is NULL for an absent span;
+ line includes the token's complete physical line range. */
+void _PyToken_GetView(
+ const struct tok_state *tok, const struct token *token,
+ _PyToken_View *view);
+const char *_PyTokenizer_SpanView(
+ const struct tok_state *, _PyTok_Span, Py_ssize_t *);
+/* For retained input: 1-based lines clamp to the first or final line.
+ The view excludes the newline and need not be NUL-terminated. */
+const char *_PyTokenizer_LineView(
+ const struct tok_state *, Py_ssize_t, Py_ssize_t *);
+/* Return NUL-terminated retained input, or NULL without setting an exception
+ for streaming input or interactive input before its first line. */
+const char *_PyTokenizer_RetainedSource(const struct tok_state *);
+void _PyTokenizer_SetContext(
+ struct tok_state *tok, PyObject *filename, PyObject *module);
+void _PyTokenizer_SetOptions(
+ struct tok_state *tok, int extra_tokens, int type_comments);
+void _PyTokenizer_ImplyDedents(struct tok_state *);
+int _PyTokenizer_HasTrailingStatement(const struct tok_state *);
+int _PyTokenizer_IsInteractive(const struct tok_state *);
+void _PyTokenizer_StopInteractive(struct tok_state *);
+
+struct tok_state *_PyTokenizer_FromString(
+ const char *input, int exec_input, int preserve_crlf);
+struct tok_state *_PyTokenizer_FromUTF8(
+ const char *input, int exec_input, int preserve_crlf);
+struct tok_state *_PyTokenizer_FromReadline(
+ PyObject *readline, const char *encoding);
+struct tok_state *_PyTokenizer_FromFile(
+ FILE *fp, const char *encoding, const char *ps1, const char *ps2);
/* Return the declared encoding in PyMem-allocated storage, or NULL.
An exception is set on error. */
char *_PyTokenizer_FindEncodingFilename(int, PyObject *);
diff --git a/Parser/tokenizer/types.h b/Parser/tokenizer/types.h
new file mode 100644
index 00000000000000..c12b6de449c700
--- /dev/null
+++ b/Parser/tokenizer/types.h
@@ -0,0 +1,33 @@
+#ifndef Py_TOKENIZER_TYPES_H
+#define Py_TOKENIZER_TYPES_H
+
+#include "Python.h"
+
+typedef Py_ssize_t _PyTok_Off;
+
+/* Spans use half-open logical byte offsets into decoded input. Their backing
+ storage may retain only the current input window. */
+typedef struct {
+ _PyTok_Off start;
+ _PyTok_Off end;
+} _PyTok_Span;
+
+/* Lines are 1-based and byte columns are 0-based. */
+typedef struct {
+ int lineno;
+ int byte_col;
+} _PyTok_Loc;
+
+static inline _PyTok_Span
+_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end)
+{
+ return (_PyTok_Span){start, end};
+}
+
+static inline int
+_PyTok_SpanIsValid(_PyTok_Span span)
+{
+ return span.start >= 0 && span.end >= span.start;
+}
+
+#endif
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 71f236b08d93c8..3802709732e29d 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -2,8 +2,6 @@
#include "errcode.h"
#include "internal/pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION
#include "internal/pycore_tuple.h" // _PyTuple_FromPair
-#include "../Parser/lexer/state.h"
-#include "../Parser/lexer/lexer.h"
#include "../Parser/tokenizer/tokenizer.h"
#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset()
@@ -34,6 +32,7 @@ typedef struct
{
PyObject_HEAD struct tok_state *tok;
int done;
+ int extra_tokens;
/* Needed to cache line for performance */
PyObject *last_line;
@@ -71,10 +70,10 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline,
Py_DECREF(filename);
return NULL;
}
- self->tok->filename = filename;
- if (extra_tokens) {
- self->tok->tok_extra_tokens = 1;
- }
+ _PyTokenizer_SetContext(self->tok, filename, NULL);
+ Py_DECREF(filename);
+ _PyTokenizer_SetOptions(self->tok, extra_tokens, 0);
+ self->extra_tokens = extra_tokens;
self->done = 0;
self->last_line = NULL;
@@ -96,14 +95,16 @@ _tokenizer_error(tokenizeriterobject *it)
const char *msg = NULL;
PyObject* errtype = PyExc_SyntaxError;
struct tok_state *tok = it->tok;
- switch (tok->done) {
+ _PyTokenizer_Info info = _PyTokenizer_GetInfo(tok);
+ switch (info.status) {
case E_TOKEN:
msg = "invalid token";
break;
case E_EOF:
PyErr_SetString(PyExc_SyntaxError, "unexpected EOF in multi-line statement");
- PyErr_SyntaxLocationObject(tok->filename, tok->lineno,
- tok->inp - tok->buf < 0 ? 0 : (int)(tok->inp - tok->buf));
+ PyErr_SyntaxLocationObject(
+ info.filename, info.location.lineno,
+ (int)Py_MAX(0, info.input_span.end - info.input_span.start));
return -1;
case E_DEDENT:
msg = "unindent does not match any outer indentation level";
@@ -139,21 +140,24 @@ _tokenizer_error(tokenizeriterobject *it)
PyObject* value = NULL;
int result = 0;
- Py_ssize_t size = tok->inp - tok->buf;
- assert(tok->buf[size-1] == '\n');
+ Py_ssize_t input_size;
+ const char *input = _PyTokenizer_SpanView(
+ tok, info.input_span, &input_size);
+ Py_ssize_t size = input_size;
+ assert(input[size-1] == '\n');
size -= 1; // Remove the newline character from the end of the line
- error_line = PyUnicode_DecodeUTF8(tok->buf, size, "replace");
+ error_line = PyUnicode_DecodeUTF8(input, size, "replace");
if (!error_line) {
result = -1;
goto exit;
}
- Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, tok->inp - tok->buf);
+ Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, input_size);
if (offset == -1) {
result = -1;
goto exit;
}
- tmp = Py_BuildValue("(OnnOOO)", tok->filename, tok->lineno, offset, error_line, Py_None, Py_None);
+ tmp = Py_BuildValue("(OnnOOO)", info.filename, info.location.lineno, offset, error_line, Py_None, Py_None);
if (!tmp) {
result = -1;
goto exit;
@@ -182,12 +186,12 @@ _tokenizer_error(tokenizeriterobject *it)
}
static PyObject *
-_get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t size,
- int *line_changed)
+_get_current_line(tokenizeriterobject *it, int current_lineno,
+ const char *line_start, Py_ssize_t size, int *line_changed)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
PyObject *line;
- if (it->tok->lineno != it->last_lineno) {
+ if (current_lineno != it->last_lineno) {
// Line has changed since last token, so we fetch the new line and cache it
// in the iter object.
Py_XDECREF(it->last_line);
@@ -205,7 +209,7 @@ _get_current_line(tokenizeriterobject *it, const char *line_start, Py_ssize_t si
static void
_get_col_offsets(tokenizeriterobject *it, const struct token *token,
const char *token_start, const char *line_start,
- PyObject *line, int line_changed,
+ const char *end_line_start, PyObject *line, int line_changed,
Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
@@ -225,8 +229,8 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token,
}
}
- if (token_end != NULL && token_end >= it->tok->line_start) {
- Py_ssize_t end_byte_offset = token_end - it->tok->line_start;
+ if (token_end != NULL && token_end >= end_line_start) {
+ Py_ssize_t end_byte_offset = token_end - end_line_start;
if (lineno == end_lineno) {
// Avoid rescanning the prefix of a very long line.
Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);
@@ -234,7 +238,7 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token,
it->byte_col_offset_diff += token_end - token_start - token_col_offset;
}
else {
- *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(it->tok->line_start, end_byte_offset);
+ *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(end_line_start, end_byte_offset);
it->byte_col_offset_diff += end_byte_offset - *end_col_offset;
}
}
@@ -253,7 +257,8 @@ tokenizeriter_next(PyObject *op)
struct token token;
_PyToken_Init(&token);
- int type = _PyTokenizer_Get(it->tok, &token);
+ _PyTokenizer_Get(it->tok, &token);
+ int type = token.type;
if (type == ERRORTOKEN) {
if(!PyErr_Occurred()) {
_tokenizer_error(it);
@@ -266,39 +271,38 @@ tokenizeriter_next(PyObject *op)
it->done = 1;
goto exit;
}
- const char *token_start = NULL;
+ _PyToken_View view;
+ _PyToken_GetView(it->tok, &token, &view);
+ const char *token_start = view.text;
PyObject *str;
if (token.span.start < 0) {
assert(token.span.start == -1 && token.span.end == -1);
str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
}
else {
- Py_ssize_t token_length;
- token_start = _PyToken_TextView(
- it->tok, &token, &token_length);
- str = PyUnicode_FromStringAndSize(token_start, token_length);
+ str = PyUnicode_FromStringAndSize(token_start, view.length);
}
if (str == NULL) {
goto exit;
}
int is_trailing_token = 0;
- if (type == ENDMARKER || (type == DEDENT && it->tok->done == E_EOF)) {
+ if (type == ENDMARKER || (type == DEDENT && view.at_eof)) {
is_trailing_token = 1;
}
- const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
PyObject* line = NULL;
int line_changed = 1;
- if (it->tok->tok_extra_tokens && is_trailing_token) {
+ if (it->extra_tokens && is_trailing_token) {
line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
} else {
- Py_ssize_t size = it->tok->inp - line_start;
- if (size >= 1 && it->tok->implicit_newline) {
+ Py_ssize_t size = view.line_length;
+ if (size >= 1 && view.implicit_newline) {
size -= 1;
}
- line = _get_current_line(it, line_start, size, &line_changed);
+ line = _get_current_line(
+ it, token.end_loc.lineno, view.line, size, &line_changed);
}
if (line == NULL) {
Py_DECREF(str);
@@ -309,10 +313,10 @@ tokenizeriter_next(PyObject *op)
Py_ssize_t end_lineno = token.end_loc.lineno;
Py_ssize_t col_offset = -1;
Py_ssize_t end_col_offset = -1;
- _get_col_offsets(it, &token, token_start, line_start, line, line_changed,
- &col_offset, &end_col_offset);
+ _get_col_offsets(it, &token, token_start, view.line, view.end_line, line,
+ line_changed, &col_offset, &end_col_offset);
- if (it->tok->tok_extra_tokens) {
+ if (it->extra_tokens) {
if (is_trailing_token) {
lineno = end_lineno = lineno + 1;
col_offset = end_col_offset = 0;
@@ -324,7 +328,7 @@ tokenizeriter_next(PyObject *op)
}
else if (type == NEWLINE) {
Py_DECREF(str);
- if (!it->tok->implicit_newline) {
+ if (!view.implicit_newline) {
assert(token_start != NULL);
if (token_start[0] == '\r') {
str = PyUnicode_FromString("\r\n");
@@ -335,7 +339,7 @@ tokenizeriter_next(PyObject *op)
end_col_offset++;
}
else if (type == NL) {
- if (it->tok->implicit_newline) {
+ if (view.implicit_newline) {
Py_DECREF(str);
str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
}
diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py
index bfd8e43c6912e8..1dc33520e5387d 100644
--- a/Tools/peg_generator/pegen/build.py
+++ b/Tools/peg_generator/pegen/build.py
@@ -128,8 +128,8 @@ def compile_c_extension(
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"),
- str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "buffer.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "decoder.c"),
+ str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "api.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "reader.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "pegen.c"),