From 55da4dd79fc9c8a429ed15104c6ff0dc413fbd19 Mon Sep 17 00:00:00 2001
From: Pablo Galindo Salgado <pablogsal@gmail.com>
Date: Sun, 30 Aug 2026 13:39:00 +0100
Subject: [PATCH 1/4] gh-153569: store formatted-string text as source spans

Formatted-string expressions and comments can outlive the buffer window where
scanning began. Pointer boundaries require buffer relocation to repair tokenizer
state.

Record logical source ranges instead. Retain the active input window while a
formatted string is open, give each mode ownership of its comment spans, and
materialize text through shared span views.
---
 Lib/test/test_fstring.py      |  14 ++
 Lib/test/test_tokenize.py     |  19 +++
 Lib/test/test_tstring.py      |  17 +++
 Parser/lexer/buffer.c         |  13 --
 Parser/lexer/lexer.c          |  26 +++-
 Parser/lexer/lexer.h          |  10 +-
 Parser/lexer/lexer_internal.h |   5 +-
 Parser/lexer/state.c          |  44 +-----
 Parser/lexer/state.h          |  65 +++++++--
 Parser/lexer/string.c         | 254 ++++++++++++++--------------------
 Parser/tokenizer/reader.c     |   8 +-
 11 files changed, 244 insertions(+), 231 deletions(-)

diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index c1ef1a73f05c204..ae443b269fe4dde 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -1679,6 +1679,20 @@ def __repr__(self):
 
         self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'')
         self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'')
+        self.assertEqual(f'{"""a" # inside"""=}',
+                         '"""a" # inside"""=\'a" # inside\'')
+        self.assertEqual(f"{'''a' # inside'''=}",
+                         "'''a' # inside'''=\"a' # inside\"")
+        self.assertEqual(f'{"""a""""#" # outside
+=}', '"""a""""#" \n=\'a#\'')
+
+        x, y = 1, 2
+        self.assertEqual(f'{x != y # outside
+=}', 'x != y \n=True')
+
+        d = {'a#b': 42}
+        self.assertEqual(f'''{f"{d["a#b"]}"=}''',
+                         'f"{d["a#b"]}"=\'42\'')
 
         self.assertEqual(f'{ # some comment goes here
   """hello"""=}',  ' \n  """hello"""=\'hello\'')
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 948b341a5dd72ec..c4584628b7fc1b4 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -2573,6 +2573,25 @@ def test_degraded_fstring_format_spec(self):
             ("f-string: single '}' is not allowed", (1, 11)),
         )
 
+    def test_carriage_return_after_debug_comment(self):
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                tokens = self._get_tokens(f"{prefix}'''{{x=# comment\r}}'''")
+                self.assertEqual(tokens[4].string, "# comment\r}")
+
+    def test_incomplete_formatted_string_comment_after_carriage_return(self):
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                for extra_tokens in (False, True):
+                    with self.assertRaises(tokenize.TokenError) as caught:
+                        self._get_tokens(
+                            f"{prefix}'{{#\r!", extra_tokens=extra_tokens
+                        )
+                    self.assertEqual(
+                        caught.exception.args,
+                        ("unexpected EOF in multi-line statement", (1, 7)),
+                    )
+
     def test_escaped_fstring_brace_has_a_position_gap(self):
         tokens = self._get_tokens('f"a{{"', extra_tokens=True)
         self.assertEqual(
diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py
index 74653c77c55de17..c90f18a4ced5296 100644
--- a/Lib/test/test_tstring.py
+++ b/Lib/test/test_tstring.py
@@ -287,5 +287,22 @@ def test_triple_quoted(self):
         )
         self.assertEqual(fstring(t), "\n        Hello,\n        Python\n        ")
 
+        t = t'{"""a" # inside"""}'
+        self.assertEqual(t.interpolations[0].expression,
+                         '"""a" # inside"""')
+
+        t = t'{"""a""""#" # outside
+}'
+        self.assertEqual(t.interpolations[0].expression, '"""a""""#"')
+
+        x, y = 1, 2
+        t = t'{x != y # outside
+}'
+        self.assertEqual(t.interpolations[0].expression, 'x != y')
+
+        d = {'a#b': 42}
+        t = t'''{f"{d["a#b"]}"}'''
+        self.assertEqual(t.interpolations[0].expression, 'f"{d["a#b"]}"')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c
index 9c39544ca7c4790..7e2330482ec85dc 100644
--- a/Parser/lexer/buffer.c
+++ b/Parser/lexer/buffer.c
@@ -15,12 +15,6 @@ _PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base,
         ? -1 : tok->line_start - tok->buf;
     pointers->multi_line_start_from_buf = tok->multi_line_start == NULL
         ? -1 : tok->multi_line_start - tok->buf;
-    for (int index = tok->tok_mode_stack_index; index > 0; --index) {
-        tokenizer_mode *mode = &tok->tok_mode_stack[index];
-        mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
-        mode->multi_line_start_offset = mode->multi_line_start == NULL
-            ? -1 : mode->multi_line_start - tok->buf;
-    }
 }
 
 void
@@ -36,11 +30,4 @@ _PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base,
         ? NULL : tok->buf + pointers->line_start_from_buf;
     tok->multi_line_start = pointers->multi_line_start_from_buf < 0
         ? NULL : tok->buf + pointers->multi_line_start_from_buf;
-    for (int index = tok->tok_mode_stack_index; index > 0; --index) {
-        tokenizer_mode *mode = &tok->tok_mode_stack[index];
-        mode->start = mode->start_offset < 0
-            ? NULL : tok->buf + mode->start_offset;
-        mode->multi_line_start = mode->multi_line_start_offset < 0
-            ? NULL : tok->buf + mode->multi_line_start_offset;
-    }
 }
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index f96b31b9d2f38a1..b4f94015c1545b4 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -317,6 +317,18 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             c = tok_nextc(tok);
         }
 
+        if (INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
+            const char *comment_end = tok->cur;
+            if (c == '\n' || c == '\r') {
+                comment_end--;
+            }
+            if (_PyLexer_record_ftstring_comment(
+                    tok, tok->start, comment_end) < 0) {
+                tok->done = E_NOMEM;
+                return MAKE_TOKEN(ERRORTOKEN);
+            }
+        }
+
         if (tok->tok_extra_tokens) {
             p = tok->start;
         }
@@ -544,10 +556,18 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
          int cursor_in_format_with_debug =
              cursor == 1 && (current_tok->in_debug || in_format_spec);
          int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
-        if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
-            return MAKE_TOKEN(ENDMARKER);
+        if (cursor_valid && c == '!') {
+            int c2 = tok_nextc(tok);
+            if (c2 == '=') {
+                cursor_valid = 0;
+            }
+            tok_backup(tok, c2);
+        }
+        if (cursor_valid) {
+            _PyLexer_update_ftstring_expr(tok, c);
         }
-        if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) {
+        if (cursor_valid && c != '{' &&
+                _PyLexer_set_ftstring_expr_metadata(tok, token)) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
 
diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h
index 040935a7e689138..63f6e628640107c 100644
--- a/Parser/lexer/lexer.h
+++ b/Parser/lexer/lexer.h
@@ -3,8 +3,6 @@
 
 #include "state.h"
 
-int _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur);
-
 int _PyTokenizer_Get(struct tok_state *, struct token *);
 
 /* The view points into the current input window. The next
@@ -19,13 +17,7 @@ _PyToken_TextView(const struct tok_state *tok, const struct token *token,
         *length = 0;
         return "";
     }
-    assert(_PyTok_SpanIsValid(token->span));
-    assert(tok->buf != NULL);
-    assert(tok->inp >= tok->buf);
-    assert(token->span.start >= tok->buf_offset);
-    assert(token->span.end - tok->buf_offset <= tok->inp - tok->buf);
-    *length = token->span.end - token->span.start;
-    return tok->buf + (token->span.start - tok->buf_offset);
+    return _PyLexer_BufferSpanView(tok, token->span, length);
 }
 
 #endif
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index c6d3b9045c72921..9dc70b1e6a42801 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -46,7 +46,10 @@ TOK_NEXT_MODE(struct tok_state *tok)
 
 int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
-int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char);
+void _PyLexer_update_ftstring_expr(struct tok_state *, char);
+int _PyLexer_record_ftstring_comment(
+    struct tok_state *, const char *, const char *);
+int _PyLexer_set_ftstring_expr_metadata(struct tok_state *, struct token *);
 int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
 int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index d82a7d0f296bac0..13ae159476d67f1 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -60,24 +60,6 @@ _PyTokenizer_tok_new(void)
     return tok;
 }
 
-static void
-free_fstring_expressions(struct tok_state *tok)
-{
-    int index;
-    tokenizer_mode *mode;
-
-    for (index = tok->tok_mode_stack_index; index >= 0; --index) {
-        mode = &(tok->tok_mode_stack[index]);
-        if (mode->last_expr_buffer != NULL) {
-            PyMem_Free(mode->last_expr_buffer);
-            mode->last_expr_buffer = NULL;
-            mode->last_expr_size = 0;
-            mode->last_expr_end = -1;
-            mode->in_format_spec = 0;
-        }
-    }
-}
-
 /* Free a tok_state structure */
 void
 _PyTokenizer_Free(struct tok_state *tok)
@@ -89,7 +71,9 @@ _PyTokenizer_Free(struct tok_state *tok)
     Py_XDECREF(tok->module);
     _PyTok_ReaderFree(tok);
     _PyTok_SourceClear(&tok->source);
-    free_fstring_expressions(tok);
+    for (int i = 0; i <= tok->tok_mode_stack_index; i++) {
+        PyMem_Free(tok->tok_mode_stack[i].comments);
+    }
     PyMem_Free(tok);
 }
 
@@ -108,31 +92,11 @@ _PyToken_Init(struct token *token) {
     token->metadata = NULL;
 }
 
-static inline _PyTok_Span
-buffer_span(const struct tok_state *tok, const char *start, const char *end)
-{
-    if (start == NULL) {
-        assert(end == NULL);
-        return (_PyTok_Span){-1, -1};
-    }
-    assert(end != NULL);
-    const char *base = tok->buf;
-    assert(base != NULL);
-    assert(tok->inp >= base);
-    Py_ssize_t start_offset = start - base;
-    Py_ssize_t end_offset = end - base;
-    assert(start_offset >= 0 && start_offset <= end_offset);
-    assert(end_offset <= tok->inp - base);
-    assert(tok->buf_offset <= PY_SSIZE_T_MAX - end_offset);
-    return _PyTok_SpanFromBounds(
-        tok->buf_offset + start_offset, tok->buf_offset + end_offset);
-}
-
 int
 _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
 {
     token->level = tok->level;
-    token->span = buffer_span(tok, start, end);
+    token->span = _PyLexer_BufferSpan(tok, start, end);
     int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
     token->start_loc = (_PyTok_Loc){lineno, -1};
     token->end_loc = (_PyTok_Loc){tok->lineno, -1};
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 6d19e685bd7f81b..144bffd1a170fc3 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -41,6 +41,12 @@ enum string_kind_t {
 
 #define MAX_EXPR_NESTING 3
 
+typedef struct _tokenizer_comments {
+    Py_ssize_t count;
+    Py_ssize_t capacity;
+    _PyTok_Span spans[];
+} tokenizer_comments;
+
 typedef struct _tokenizer_mode {
     enum tokenizer_mode_kind_t kind;
 
@@ -50,20 +56,16 @@ typedef struct _tokenizer_mode {
     char quote;
     int quote_size;
     int raw;
-    const char* start;
-    const char* multi_line_start;
+    _PyTok_Off start;
+    _PyTok_Off multi_line_start;
     int first_line;
 
-    Py_ssize_t start_offset;
-    Py_ssize_t multi_line_start_offset;
-
-    Py_ssize_t last_expr_size;
-    Py_ssize_t last_expr_end;
-    char* last_expr_buffer;
+    _PyTok_Span expr_span;
     int in_debug;
     int in_format_spec;
 
     enum string_kind_t string_kind;
+    tokenizer_comments *comments;
 } tokenizer_mode;
 
 /* Tokenizer state */
@@ -129,6 +131,53 @@ struct tok_state {
 #endif
 };
 
+static inline _PyTok_Off
+_PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
+{
+    assert(tok->buf != NULL);
+    assert(tok->inp >= tok->buf);
+    assert(position >= tok->buf && position <= tok->inp);
+    Py_ssize_t offset = position - tok->buf;
+    assert(tok->buf_offset <= PY_SSIZE_T_MAX - offset);
+    return tok->buf_offset + offset;
+}
+
+static inline char *
+_PyLexer_BufferPointer(const struct tok_state *tok, _PyTok_Off offset)
+{
+    assert(tok->buf != NULL);
+    assert(tok->inp >= tok->buf);
+    assert(offset >= tok->buf_offset);
+    assert(offset - tok->buf_offset <= tok->inp - tok->buf);
+    return tok->buf + (offset - tok->buf_offset);
+}
+
+static inline const char *
+_PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span,
+                        Py_ssize_t *length)
+{
+    assert(length != NULL);
+    assert(_PyTok_SpanIsValid(span));
+    *length = span.end - span.start;
+    (void)_PyLexer_BufferPointer(tok, span.end);
+    return _PyLexer_BufferPointer(tok, span.start);
+}
+
+static inline _PyTok_Span
+_PyLexer_BufferSpan(const struct tok_state *tok, const char *start,
+                    const char *end)
+{
+    if (start == NULL) {
+        assert(end == NULL);
+        return (_PyTok_Span){-1, -1};
+    }
+    assert(end != NULL);
+    assert(start <= end);
+    return _PyTok_SpanFromBounds(
+        _PyLexer_BufferOffset(tok, start),
+        _PyLexer_BufferOffset(tok, end));
+}
+
 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
 
 struct tok_state *_PyTokenizer_tok_new(void);
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index fc0299c5c7c592f..e2489c34bfe31e3 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -8,109 +8,100 @@
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
 
 int
-_PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
+_PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start,
+                                 const char *end)
+{
+    tokenizer_mode *mode = TOK_GET_MODE(tok);
+    if (mode->expr_span.end >= 0) {
+        return 0;
+    }
+    assert(mode->expr_span.start >= 0);
+    tokenizer_comments *comments = mode->comments;
+    if (comments == NULL || comments->count == comments->capacity) {
+        int create = comments == NULL;
+        Py_ssize_t max_capacity = (PY_SSIZE_T_MAX -
+            (Py_ssize_t)sizeof(*comments)) /
+            (Py_ssize_t)sizeof(*comments->spans);
+        if (comments != NULL && comments->capacity > max_capacity / 2) {
+            PyErr_NoMemory();
+            return -1;
+        }
+        Py_ssize_t capacity = comments == NULL ? 4 : comments->capacity * 2;
+        size_t size = sizeof(*comments) +
+            (size_t)capacity * sizeof(*comments->spans);
+        tokenizer_comments *resized = PyMem_Realloc(comments, size);
+        if (resized == NULL) {
+            PyErr_NoMemory();
+            return -1;
+        }
+        comments = resized;
+        if (create) {
+            comments->count = 0;
+        }
+        comments->capacity = capacity;
+        mode->comments = comments;
+    }
+    comments->spans[comments->count++] =
+        _PyLexer_BufferSpan(tok, start, end);
+    return 0;
+}
+
+int
+_PyLexer_set_ftstring_expr_metadata(struct tok_state *tok, struct token *token)
+{
     assert(token != NULL);
-    assert(c == '}' || c == ':' || c == '!');
     tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
 
     if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
         return 0;
     }
-    PyObject *res = NULL;
-
-    // Look for a # character outside of string literals
-    int hash_detected = 0;
-    int in_string = 0;
-    char quote_char = 0;
-
-    for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) {
-        char ch = tok_mode->last_expr_buffer[i];
-
-        // Skip escaped characters
-        if (ch == '\\') {
-            i++;
-            continue;
-        }
-
-        // Handle quotes
-        if (ch == '"' || ch == '\'') {
-            // The following if/else block works becase there is an off number
-            // of quotes in STRING tokens and the lexer only ever reaches this
-            // function with valid STRING tokens.
-            // For example: """hello"""
-            // First quote: in_string = 1
-            // Second quote: in_string = 0
-            // Third quote: in_string = 1
-            if (!in_string) {
-                in_string = 1;
-                quote_char = ch;
-            }
-            else if (ch == quote_char) {
-                in_string = 0;
+    Py_ssize_t expr_len;
+    const char *expr = _PyLexer_BufferSpanView(
+        tok, tok_mode->expr_span, &expr_len);
+    tokenizer_comments *comments = tok_mode->comments;
+    PyObject *res;
+    if (comments != NULL && comments->count > 0) {
+        Py_ssize_t stripped_size = expr_len;
+        _PyTok_Off previous_end = tok_mode->expr_span.start;
+        Py_ssize_t comment_count = 0;
+        for (Py_ssize_t i = 0; i < comments->count; i++) {
+            _PyTok_Span comment = comments->spans[i];
+            assert(_PyTok_SpanIsValid(comment));
+            assert(comment.start >= previous_end);
+            if (comment.start >= tok_mode->expr_span.end) {
+                break;
             }
-            continue;
-        }
-
-        // Check for # outside strings
-        if (ch == '#' && !in_string) {
-            hash_detected = 1;
-            break;
+            assert(comment.end <= tok_mode->expr_span.end);
+            stripped_size -= comment.end - comment.start;
+            previous_end = comment.end;
+            comment_count++;
         }
-    }
-    // If we found a # character in the expression, we need to handle comments
-    if (hash_detected) {
-        // Allocate buffer for processed result
-        char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char));
-        if (!result) {
+        char *stripped = PyMem_Malloc((size_t)stripped_size);
+        if (stripped == NULL) {
+            PyErr_NoMemory();
             return -1;
         }
-
-        Py_ssize_t i = 0;  // Input position
-        Py_ssize_t j = 0;  // Output position
-        in_string = 0;     // Whether we're in a string
-        quote_char = 0;    // Current string quote char
-
-        // Process each character
-        while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
-            char ch = tok_mode->last_expr_buffer[i];
-
-            // Handle string quotes
-            if (ch == '"' || ch == '\'') {
-                // See comment above to understand this part
-                if (!in_string) {
-                    in_string = 1;
-                    quote_char = ch;
-                } else if (ch == quote_char) {
-                    in_string = 0;
-                }
-                result[j++] = ch;
-            }
-            // Skip comments
-            else if (ch == '#' && !in_string) {
-                while (i < tok_mode->last_expr_size - tok_mode->last_expr_end &&
-                       tok_mode->last_expr_buffer[i] != '\n') {
-                    i++;
-                }
-                if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
-                    result[j++] = '\n';
-                }
-            }
-            // Copy other chars
-            else {
-                result[j++] = ch;
-            }
-            i++;
+        _PyTok_Off copied_to = tok_mode->expr_span.start;
+        Py_ssize_t stripped_len = 0;
+        for (Py_ssize_t i = 0; i < comment_count; i++) {
+            _PyTok_Span comment = comments->spans[i];
+            Py_ssize_t length = comment.start - copied_to;
+            memcpy(stripped + stripped_len,
+                   expr + copied_to - tok_mode->expr_span.start,
+                   (size_t)length);
+            stripped_len += length;
+            copied_to = comment.end;
         }
-
-        result[j] = '\0';  // Null-terminate the result string
-        res = PyUnicode_DecodeUTF8(result, j, NULL);
-        PyMem_Free(result);
-    } else {
-        res = PyUnicode_DecodeUTF8(
-            tok_mode->last_expr_buffer,
-            tok_mode->last_expr_size - tok_mode->last_expr_end,
-            NULL
-        );
+        Py_ssize_t length = tok_mode->expr_span.end - copied_to;
+        memcpy(stripped + stripped_len,
+               expr + copied_to - tok_mode->expr_span.start,
+               (size_t)length);
+        stripped_len += length;
+        res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL);
+        PyMem_Free(stripped);
+    }
+    else {
+        res = PyUnicode_DecodeUTF8(expr, expr_len, NULL);
     }
 
     if (!res) {
@@ -120,61 +111,33 @@ _PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
     return 0;
 }
 
-int
+void
 _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
 {
-    assert(tok->cur != NULL);
-
-    Py_ssize_t size = cur == 0
-        ? tok->inp - tok->cur : (Py_ssize_t)strlen(tok->cur);
     tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
 
     switch (cur) {
-       case 0:
-            if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) {
-                return 1;
-            }
-            char *new_buffer = PyMem_Realloc(
-                tok_mode->last_expr_buffer,
-                tok_mode->last_expr_size + size
-            );
-            if (new_buffer == NULL) {
-                PyMem_Free(tok_mode->last_expr_buffer);
-                goto error;
-            }
-            tok_mode->last_expr_buffer = new_buffer;
-            memcpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size,
-                   tok->cur, size);
-            tok_mode->last_expr_size += size;
-            break;
         case '{':
-            if (tok_mode->last_expr_buffer != NULL) {
-                PyMem_Free(tok_mode->last_expr_buffer);
+            tok_mode->expr_span = (_PyTok_Span){
+                _PyLexer_BufferOffset(tok, tok->cur), -1};
+            tokenizer_comments *comments = tok_mode->comments;
+            if (comments != NULL) {
+                comments->count = 0;
             }
-            tok_mode->last_expr_buffer = PyMem_Malloc(size);
-            if (tok_mode->last_expr_buffer == NULL) {
-                goto error;
-            }
-            tok_mode->last_expr_size = size;
-            tok_mode->last_expr_end = -1;
-            memcpy(tok_mode->last_expr_buffer, tok->cur, size);
             break;
         case '}':
         case '!':
-            tok_mode->last_expr_end = strlen(tok->start);
+            tok_mode->expr_span.end = _PyLexer_BufferOffset(tok, tok->start);
             break;
         case ':':
-            if (tok_mode->last_expr_end == -1) {
-               tok_mode->last_expr_end = strlen(tok->start);
+            if (tok_mode->expr_span.end < 0) {
+                tok_mode->expr_span.end =
+                    _PyLexer_BufferOffset(tok, tok->start);
             }
             break;
         default:
             Py_UNREACHABLE();
     }
-    return 1;
-error:
-    tok->done = E_NOMEM;
-    return 0;
 }
 
 int
@@ -265,16 +228,14 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
     the_current_tok->kind = TOK_FSTRING_MODE;
     the_current_tok->quote = quote;
     the_current_tok->quote_size = quote_size;
-    the_current_tok->start = tok->start;
-    the_current_tok->multi_line_start = tok->line_start;
+    the_current_tok->start = _PyLexer_BufferOffset(tok, tok->start);
+    the_current_tok->multi_line_start =
+        _PyLexer_BufferOffset(tok, tok->line_start);
     the_current_tok->first_line = tok->lineno;
-    the_current_tok->start_offset = -1;
-    the_current_tok->multi_line_start_offset = -1;
-    the_current_tok->last_expr_buffer = NULL;
-    the_current_tok->last_expr_size = 0;
-    the_current_tok->last_expr_end = -1;
+    the_current_tok->expr_span = (_PyTok_Span){-1, -1};
     the_current_tok->in_format_spec = 0;
     the_current_tok->in_debug = 0;
+    the_current_tok->comments = NULL;
 
     enum string_kind_t string_kind = FSTRING;
     switch (*tok->start) {
@@ -462,15 +423,10 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
         }
     }
 
-    if (current_tok->last_expr_buffer != NULL) {
-        PyMem_Free(current_tok->last_expr_buffer);
-        current_tok->last_expr_buffer = NULL;
-        current_tok->last_expr_size = 0;
-        current_tok->last_expr_end = -1;
-    }
-
     p_start = tok->start;
     p_end = tok->cur;
+    PyMem_Free(current_tok->comments);
+    current_tok->comments = NULL;
     tok->tok_mode_stack_index--;
     return MAKE_TOKEN(FTSTRING_END(current_tok));
 
@@ -520,9 +476,9 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
             // shift the tok_state's location into
             // the start of string, and report the error
             // from the initial quote character
-            tok->cur = (char *)current_tok->start;
-            tok->cur++;
-            tok->line_start = current_tok->multi_line_start;
+            tok->cur = _PyLexer_BufferPointer(tok, current_tok->start) + 1;
+            tok->line_start = _PyLexer_BufferPointer(
+                tok, current_tok->multi_line_start);
             int start = tok->lineno;
 
             tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
@@ -553,9 +509,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
         }
 
         if (c == '{') {
-            if (!_PyLexer_update_ftstring_expr(tok, c)) {
-                return MAKE_TOKEN(ENDMARKER);
-            }
+            _PyLexer_update_ftstring_expr(tok, c);
             int peek = tok_nextc(tok);
             if (peek != '{' || in_format_spec) {
                 tok_backup(tok, peek);
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index b9b4a4610874419..18c55ddb224ce7c 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -624,7 +624,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         tok->interactive_src_end = tok->source.bytes + tok->source.len;
     }
     if (prepared) {
-        if (tok->start == NULL) {
+        if (tok->start == NULL && !INSIDE_FSTRING(tok)) {
             tok->buf = tok->cur;
             tok->buf_offset = tok->source.base_offset +
                 (chunk.data - tok->source.bytes);
@@ -633,12 +633,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
     }
     tok->implicit_newline = chunk.implicit_newline;
 
-    if (!prepared && tok->tok_mode_stack_index &&
-            !_PyLexer_update_ftstring_expr(tok, 0)) {
-        _PyTok_ChunkClear(&chunk);
-        tok->input_error = 1;
-        return 0;
-    }
     ADVANCE_LINENO();
     if (kind == _PYTOK_READER_FILE &&
             (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&

From 4c8ba41af8620b469da49c744dce08fbcf4b7851 Mon Sep 17 00:00:00 2001
From: Pablo Galindo Salgado <pablogsal@gmail.com>
Date: Sun, 6 Sep 2026 22:20:03 +0100
Subject: [PATCH 2/4] gh-153569: derive tokenizer locations and failures from
 scanner state

---
 Lib/test/test_syntax.py       |  4 +++
 Parser/lexer/buffer.c         |  4 ---
 Parser/lexer/buffer.h         |  1 -
 Parser/lexer/lexer.c          | 25 +++++++++---------
 Parser/lexer/lexer_internal.h |  8 ++++++
 Parser/lexer/state.c          | 19 +++++---------
 Parser/lexer/state.h          | 20 +++++++-------
 Parser/lexer/string.c         | 49 ++++++++++++-----------------------
 Parser/pegen.c                |  1 -
 Parser/pegen_errors.c         |  2 +-
 Parser/tokenizer/helpers.h    |  4 ---
 Parser/tokenizer/reader.c     |  8 +++---
 Python/Python-tokenize.c      |  3 ++-
 13 files changed, 64 insertions(+), 84 deletions(-)

diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
index dbd518707c18266..f667707f2bb08b9 100644
--- a/Lib/test/test_syntax.py
+++ b/Lib/test/test_syntax.py
@@ -3393,6 +3393,10 @@ def test_invalid_line_continuation_error_position(self):
         self._check_error('\nfgdfgf\n1,\\#\n2\n',
                           "unexpected character after line continuation character",
                           lineno=3, offset=4)
+        for prefix in ("f", "t"):
+            self._check_error(f'{prefix}"""{{\n\\ x}}"""',
+                              "unexpected character after line continuation character",
+                              lineno=2, offset=2)
 
     def test_invalid_line_continuation_left_recursive(self):
         # Check bpo-42218: SyntaxErrors following left-recursive rules
diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c
index 7e2330482ec85dc..ead973b0e85234b 100644
--- a/Parser/lexer/buffer.c
+++ b/Parser/lexer/buffer.c
@@ -13,8 +13,6 @@ _PyLexer_SaveBufferPointers(struct tok_state *tok, const char *base,
         ? -1 : tok->start - tok->buf;
     pointers->line_start_from_buf = tok->line_start == NULL
         ? -1 : tok->line_start - tok->buf;
-    pointers->multi_line_start_from_buf = tok->multi_line_start == NULL
-        ? -1 : tok->multi_line_start - tok->buf;
 }
 
 void
@@ -28,6 +26,4 @@ _PyLexer_RestoreBufferPointers(struct tok_state *tok, char *base,
         ? NULL : tok->buf + pointers->start_from_buf;
     tok->line_start = pointers->line_start_from_buf < 0
         ? NULL : tok->buf + pointers->line_start_from_buf;
-    tok->multi_line_start = pointers->multi_line_start_from_buf < 0
-        ? NULL : tok->buf + pointers->multi_line_start_from_buf;
 }
diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h
index 285da124226d50e..af57732a69363f0 100644
--- a/Parser/lexer/buffer.h
+++ b/Parser/lexer/buffer.h
@@ -11,7 +11,6 @@ typedef struct {
     Py_ssize_t inp_from_buf;
     Py_ssize_t start_from_buf;
     Py_ssize_t line_start_from_buf;
-    Py_ssize_t multi_line_start_from_buf;
 } _PyLexer_BufferPointers;
 
 void _PyLexer_SaveBufferPointers(
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index b4f94015c1545b4..fbec2ec92f64103 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -7,7 +7,7 @@
 #include "../tokenizer/helpers.h"
 #include "../tokenizer/reader.h"
 
-/* Alternate tab spacing */
+#define TABSIZE 8
 #define ALTTABSIZE 1
 
 
@@ -30,11 +30,10 @@ _PyLexer_nextc(struct tok_state *tok)
     int rc;
     for (;;) {
         if (tok->cur != tok->inp) {
-            if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
+            if (tok->cur - tok->line_start >= INT_MAX) {
                 tok->done = E_COLUMNOVERFLOW;
                 return EOF;
             }
-            tok->col_offset++;
             return Py_CHARMASK(*tok->cur++); /* Fast path */
         }
         if (tok->done != E_OK) {
@@ -74,7 +73,6 @@ _PyLexer_backup(struct tok_state *tok, int c)
         if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
             Py_FatalError("tok_backup: wrong character");
         }
-        tok->col_offset--;
     }
 }
 
@@ -88,7 +86,7 @@ verify_identifier(struct tok_state *tok)
         return 1;
     }
     PyObject *s;
-    if (tok->input_error)
+    if (tok_failed(tok))
         return 0;
     s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
     if (s == NULL) {
@@ -165,7 +163,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
     const char *p_end = NULL;
   nextline:
     tok->start = NULL;
-    tok->starting_col_offset = -1;
+    tok->start_loc = (_PyTok_Loc){tok->lineno, -1};
     blankline = 0;
 
 
@@ -181,7 +179,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
                 col++, altcol++;
             }
             else if (c == '\t') {
-                col = (col / tok->tabsize + 1) * tok->tabsize;
+                col = (col / TABSIZE + 1) * TABSIZE;
                 altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
             }
             else if (c == '\014')  {/* Control-L (formfeed) */
@@ -269,7 +267,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
     }
 
     tok->start = tok->cur;
-    tok->starting_col_offset = tok->col_offset;
+    tok->start_loc = (_PyTok_Loc){
+        tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) : -1};
 
     /* Return pending indents/dedents */
     if (tok->pendin != 0) {
@@ -304,7 +303,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
 
     /* Set start of current token */
     tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
-    tok->starting_col_offset = tok->col_offset - 1;
+    tok->start_loc = (_PyTok_Loc){
+        tok->lineno, tok->cur != NULL ? _PyLexer_ByteColumn(tok) - 1 : -1};
 
     /* Skip comment, unless it's a type comment */
     if (c == '#') {
@@ -335,7 +335,7 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
 
         if (tok->type_comments) {
             p = tok->start;
-            current_starting_col_offset = tok->starting_col_offset;
+            current_starting_col_offset = tok->start_loc.byte_col;
             prefix = type_comment_prefix;
             while (*prefix && p < tok->cur) {
                 if (*prefix == ' ') {
@@ -387,7 +387,8 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
                 }
                 _PyLexer_token_setup(tok, token, type, p_start, p_end);
                 token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
-                token->end_loc = (_PyTok_Loc){tok->lineno, tok->col_offset};
+                token->end_loc = (_PyTok_Loc){tok->lineno,
+                                              _PyLexer_ByteColumn(tok)};
                 return type;
             }
         }
@@ -708,7 +709,7 @@ int
 _PyTokenizer_Get(struct tok_state *tok, struct token *token)
 {
     int result = tok_get(tok, token);
-    if (tok->input_error) {
+    if (tok_failed(tok)) {
         result = ERRORTOKEN;
     }
     return result;
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index 9dc70b1e6a42801..68273eb430e6812 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -1,6 +1,7 @@
 #ifndef _PY_LEXER_INTERNAL_H_
 #define _PY_LEXER_INTERNAL_H_
 
+#include "errcode.h"
 #include "lexer.h"
 
 #define is_potential_identifier_start(c) (\
@@ -44,6 +45,13 @@ TOK_NEXT_MODE(struct tok_state *tok)
 #define tok_nextc _PyLexer_nextc
 #define tok_backup _PyLexer_backup
 
+static inline int
+tok_failed(const struct tok_state *tok)
+{
+    return tok->done != E_OK && tok->done != E_EOF &&
+           tok->done != E_INTERACT_STOP;
+}
+
 int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
 void _PyLexer_update_ftstring_expr(struct tok_state *, char);
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index 13ae159476d67f1..ee234767308826f 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -6,9 +6,6 @@
 #include "state.h"
 #include "../tokenizer/reader.h"
 
-/* Never change this */
-#define TABSIZE 8
-
 /* Create and initialize a new tok_state structure */
 struct tok_state *
 _PyTokenizer_tok_new(void)
@@ -28,18 +25,15 @@ _PyTokenizer_tok_new(void)
     tok->start = NULL;
     tok->done = E_OK;
     tok->fp = NULL;
-    tok->tabsize = TABSIZE;
     tok->indent = 0;
     tok->indstack[0] = 0;
     tok->atbol = 1;
     tok->pendin = 0;
     tok->prompt = NULL;
     tok->lineno = 0;
-    tok->starting_col_offset = -1;
-    tok->col_offset = -1;
+    tok->start_loc = (_PyTok_Loc){-1, -1};
     tok->level = 0;
     tok->altindstack[0] = 0;
-    tok->input_error = 0;
     tok->encoding = NULL;
     tok->filename = NULL;
     tok->module = NULL;
@@ -97,13 +91,12 @@ _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const
 {
     token->level = tok->level;
     token->span = _PyLexer_BufferSpan(tok, start, end);
-    int lineno = ISSTRINGLIT(type) ? tok->first_lineno : tok->lineno;
-    token->start_loc = (_PyTok_Loc){lineno, -1};
-    token->end_loc = (_PyTok_Loc){tok->lineno, -1};
-
     if (start != NULL && end != NULL) {
-        token->start_loc.byte_col = tok->starting_col_offset;
-        token->end_loc.byte_col = tok->col_offset;
+        token->start_loc = tok->start_loc;
+        token->end_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
+    }
+    else {
+        token->start_loc = token->end_loc = (_PyTok_Loc){tok->lineno, -1};
     }
     return type;
 }
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 144bffd1a170fc3..520ba2778593735 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -83,17 +83,13 @@ struct tok_state {
     int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
     /* NB If done != E_OK, cur must be == inp!!! */
     FILE *fp;           /* Rest of input; NULL if tokenizing a string */
-    int tabsize;        /* Tab spacing */
     int indent;         /* Current indentation index */
     int indstack[MAXINDENT];            /* Stack of indents */
     int atbol;          /* Nonzero if at begin of new line */
     int pendin;         /* Pending indents (if > 0) or dedents (if < 0) */
     const char *prompt;          /* For interactive prompting */
     int lineno;         /* Current line number */
-    int first_lineno;   /* First line of a single line or multi line string
-                           expression (cf. issue 16806) */
-    int starting_col_offset; /* The column offset at the beginning of a token */
-    int col_offset;     /* Current col offset */
+    _PyTok_Loc start_loc;
     int level;          /* () [] {} Parentheses nesting level */
             /* Used to allow free continuations inside them */
     char parenstack[MAXLEVEL];
@@ -104,12 +100,8 @@ struct tok_state {
     /* Stuff for checking on different tab sizes */
     int altindstack[MAXINDENT];         /* Stack of alternate indents */
     /* Stuff for PEP 0263 */
-    int input_error;
     char *encoding;         /* Source encoding. */
     const char* line_start;     /* pointer to start of current line */
-    const char* multi_line_start; /* pointer to start of first line of
-                                     a single line or multi line string
-                                     expression (cf. issue 16806) */
     char* str;          /* Source string being tokenized (if tokenizing from a string)*/
 
     _PyTok_SourceText source;
@@ -163,6 +155,16 @@ _PyLexer_BufferSpanView(const struct tok_state *tok, _PyTok_Span span,
     return _PyLexer_BufferPointer(tok, span.start);
 }
 
+static inline int
+_PyLexer_ByteColumn(const struct tok_state *tok)
+{
+    assert(tok->line_start != NULL);
+    assert(tok->cur >= tok->line_start);
+    Py_ssize_t column = tok->cur - tok->line_start;
+    assert(column <= INT_MAX);
+    return (int)column;
+}
+
 static inline _PyTok_Span
 _PyLexer_BufferSpan(const struct tok_state *tok, const char *start,
                     const char *end)
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index e2489c34bfe31e3..a75d9f4c5cda8ee 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -7,6 +7,15 @@
 
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
 
+static void
+rewind_to_string_start(struct tok_state *tok, const char *start,
+                       _PyTok_Loc location)
+{
+    tok->cur = (char *)start + 1;
+    tok->line_start = start - location.byte_col;
+    tok->lineno = location.lineno;
+}
+
 int
 _PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start,
                                  const char *end)
@@ -194,13 +203,6 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
     int quote = c;
     int quote_size = 1;             /* 1 or 3 */
 
-    /* Nodes of type STRING, especially multi line strings
-       must be handled differently in order to get both
-       the starting line number and the column offset right.
-       (cf. issue 16806) */
-    tok->first_lineno = tok->lineno;
-    tok->multi_line_start = tok->line_start;
-
     /* Find the quote size and start of string */
     int after_quote = tok_nextc(tok);
     if (after_quote == quote) {
@@ -276,13 +278,6 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
     int end_quote_size = 0;
     int has_escaped_quote = 0;
 
-    /* Nodes of type STRING, especially multi line strings
-       must be handled differently in order to get both
-       the starting line number and the column offset right.
-       (cf. issue 16806) */
-    tok->first_lineno = tok->lineno;
-    tok->multi_line_start = tok->line_start;
-
     /* Find the quote size and start of string */
     c = tok_nextc(tok);
     if (c == quote) {
@@ -308,15 +303,8 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
             break;
         }
         if (c == EOF || (quote_size == 1 && c == '\n')) {
-            assert(tok->multi_line_start != NULL);
-            // shift the tok_state's location into
-            // the start of string, and report the error
-            // from the initial quote character
-            tok->cur = (char *)tok->start;
-            tok->cur++;
-            tok->line_start = tok->multi_line_start;
-            int start = tok->lineno;
-            tok->lineno = tok->first_lineno;
+            int end_lineno = tok->lineno;
+            rewind_to_string_start(tok, tok->start, tok->start_loc);
 
             if (INSIDE_FSTRING(tok)) {
                 /* When we are in an f-string, before raising the
@@ -334,7 +322,7 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
 
             if (quote_size == 3) {
                 _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
-                                 " (detected at line %d)", start);
+                                 " (detected at line %d)", end_lineno);
                 if (c != '\n') {
                     tok->done = E_EOFS;
                 }
@@ -346,11 +334,11 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
                         tok,
                         "unterminated string literal (detected at line %d); "
                         "perhaps you escaped the end quote?",
-                        start
+                        end_lineno
                     );
                 } else {
                     _PyTokenizer_syntaxerror(
-                        tok, "unterminated string literal (detected at line %d)", start
+                        tok, "unterminated string literal (detected at line %d)", end_lineno
                     );
                 }
                 if (c != '\n') {
@@ -390,8 +378,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
     int unicode_escape = 0;
 
     tok->start = tok->cur;
-    tok->first_lineno = tok->lineno;
-    tok->starting_col_offset = tok->col_offset;
+    tok->start_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
 
     // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize
     // before it.
@@ -432,9 +419,6 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
 
 f_string_middle:
 
-    // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle
-    // this.
-    tok->multi_line_start = tok->line_start;
     while (end_quote_size != current_tok->quote_size) {
         int c = tok_nextc(tok);
         if (tok->done == E_ERROR || tok->done == E_DECODE) {
@@ -447,7 +431,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
         );
 
        if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) {
-            if (tok->input_error) {
+            if (tok_failed(tok)) {
                 return MAKE_TOKEN(ERRORTOKEN);
             }
 
@@ -472,7 +456,6 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
                 return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
             }
 
-            assert(tok->multi_line_start != NULL);
             // shift the tok_state's location into
             // the start of string, and report the error
             // from the initial quote character
diff --git a/Parser/pegen.c b/Parser/pegen.c
index d86dd22444e6a7b..20dbbcabec379e9 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -1035,7 +1035,6 @@ _PyPegen_run_parser(Parser *p)
     }
 
     if (p->start_rule == Py_single_input && bad_single_statement(p)) {
-        p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
         return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
     }
 
diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index b13e1c079220a92..0b5614b2faaec5e 100644
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -62,7 +62,7 @@ _Pypegen_tokenizer_error(Parser *p)
             msg = "too many levels of indentation";
             break;
         case E_LINECONT: {
-            col_offset = p->tok->cur - p->tok->buf - 1;
+            col_offset = p->tok->cur - p->tok->line_start - 1;
             msg = "unexpected character after line continuation character";
             break;
         }
diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h
index 5edf5a3dfd2e0d1..51de0cbb156f833 100644
--- a/Parser/tokenizer/helpers.h
+++ b/Parser/tokenizer/helpers.h
@@ -5,10 +5,6 @@
 
 #include "../lexer/state.h"
 
-#define ADVANCE_LINENO() \
-            tok->lineno++; \
-            tok->col_offset = 0;
-
 int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...);
 int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...);
 int _PyTokenizer_indenterror(struct tok_state *tok);
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index 18c55ddb224ce7c..56e50cebf44f98b 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -567,7 +567,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
             tok->done = E_INTR;
         }
         else {
-            tok->input_error = 1;
             if (tok->done == E_OK) {
                 tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
                     ? E_NOMEM : E_ERROR;
@@ -601,7 +600,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
             _PyTok_ChunkClear(&chunk);
             tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
                 ? E_NOMEM : E_ERROR;
-            tok->input_error = 1;
             return 0;
         }
         if (reset_buffer) {
@@ -610,7 +608,6 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
             tok->buf_offset = source_start;
             tok->line_start = tok->buf;
             tok->start = NULL;
-            tok->multi_line_start = NULL;
         }
         else {
             _PyLexer_RestoreBufferPointers(
@@ -633,12 +630,11 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
     }
     tok->implicit_newline = chunk.implicit_newline;
 
-    ADVANCE_LINENO();
+    tok->lineno++;
     if (kind == _PYTOK_READER_FILE &&
             (tok->encoding == NULL || strcmp(tok->encoding, "utf-8") == 0) &&
             !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) {
         _PyTok_ChunkClear(&chunk);
-        tok->input_error = 1;
         return 0;
     }
     _PyTok_ChunkClear(&chunk);
@@ -665,6 +661,7 @@ tokenizer_new_with_reader(_PyTok_ReaderKind kind)
     if (reader_is_streaming(kind)) {
         tok->buf = tok->cur = tok->inp =
             (char *)_PyTok_SourceData(&tok->source);
+        tok->line_start = tok->buf;
     }
     return tok;
 }
@@ -683,6 +680,7 @@ tokenizer_from_string(const char *input, int utf8_only, int exec_input,
         return NULL;
     }
     tok->buf = tok->cur = tok->inp = tok->str;
+    tok->line_start = tok->str;
     return tok;
 }
 
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 71f236b08d93c8f..eb5c0b86a8fbe45 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -287,7 +287,8 @@ tokenizeriter_next(PyObject *op)
         is_trailing_token = 1;
     }
 
-    const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
+    const char *line_start = ISSTRINGLIT(type)
+        ? token_start - token.start_loc.byte_col : it->tok->line_start;
     PyObject* line = NULL;
     int line_changed = 1;
     if (it->tok->tok_extra_tokens && is_trailing_token) {

From 9e645ae4a93090fdc51f1986ca5471322d090e95 Mon Sep 17 00:00:00 2001
From: Pablo Galindo Salgado <pablogsal@gmail.com>
Date: Sun, 6 Sep 2026 22:27:40 +0100
Subject: [PATCH 3/4] gh-153569: keep active formatted-string frames

---
 Lib/test/test_fstring.py      |   1 +
 Lib/test/test_tokenize.py     |  15 ++
 Lib/test/test_tstring.py      |   9 +
 Parser/action_helpers.c       |  34 ++--
 Parser/lexer/lexer.c          | 112 +++++-------
 Parser/lexer/lexer_internal.h |  39 +---
 Parser/lexer/state.c          |  60 +++++-
 Parser/lexer/state.h          |  93 ++++++----
 Parser/lexer/string.c         | 333 +++++++++++++---------------------
 Parser/pegen.h                |   3 -
 Parser/pegen_errors.c         |   6 +-
 Parser/tokenizer/reader.c     |   8 +-
 12 files changed, 360 insertions(+), 353 deletions(-)

diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index ae443b269fe4dde..460adde4956a9f2 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -1657,6 +1657,7 @@ def __repr__(self):
         self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x')
         self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********')
         self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20')
+        self.assertEqual(f"{C():{C():{4=}}}", 'FORMAT-FORMAT-4=4')
 
         self.assertRaises(SyntaxError, eval, "f'{C=]'")
 
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index c4584628b7fc1b4..53215eceeb8aed3 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -2592,6 +2592,21 @@ def test_incomplete_formatted_string_comment_after_carriage_return(self):
                         ("unexpected EOF in multi-line statement", (1, 7)),
                     )
 
+    def test_formatted_string_nesting_limit(self):
+        def nested_string(depth, prefix):
+            source = "'x'"
+            for _ in range(depth):
+                source = f'{prefix}"{{{source}}}"'
+            return source
+
+        for prefix in ("f", "t"):
+            with self.subTest(prefix=prefix):
+                self._get_tokens(nested_string(149, prefix))
+                with self.assertRaisesRegex(
+                        tokenize.TokenError,
+                        "too many nested f-strings or t-strings"):
+                    self._get_tokens(nested_string(150, prefix))
+
     def test_escaped_fstring_brace_has_a_position_gap(self):
         tokens = self._get_tokens('f"a{{"', extra_tokens=True)
         self.assertEqual(
diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py
index c90f18a4ced5296..75d7085c6731d95 100644
--- a/Lib/test/test_tstring.py
+++ b/Lib/test/test_tstring.py
@@ -140,6 +140,15 @@ def test_debug_specifier(self):
         )
         self.assertEqual(fstring(t), "Value: value = 42")
 
+        class C:
+            def __format__(self, spec):
+                return f"FORMAT-{spec}"
+
+        x = y = C()
+        t = t"{x:{y:{value=}}}"
+        self.assertEqual(t.interpolations[0].format_spec,
+                         "FORMAT-value=42")
+
     def test_raw_tstrings(self):
         path = r"C:\Users"
         t = rt"{path}\Documents"
diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c
index 8690dca8331b5ce..8ca6898711f72b0 100644
--- a/Parser/action_helpers.c
+++ b/Parser/action_helpers.c
@@ -1001,6 +1001,13 @@ result_token_with_metadata(Parser *p, void *result, PyObject *metadata)
     return res;
 }
 
+static char
+formatted_string_prefix(const Parser *p)
+{
+    const ftstring_state *state = _PyLexer_CurrentFTString(p->tok);
+    return state == NULL ? 'f' : _PyLexer_StringPrefix(state->kind);
+}
+
 ResultTokenWithMetadata *
 _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
 {
@@ -1008,7 +1015,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
         return RAISE_SYNTAX_ERROR_KNOWN_RANGE(
             conv_token, conv,
             "%c-string: conversion type must come right after the exclamation mark",
-            TOK_GET_STRING_PREFIX(p->tok)
+            formatted_string_prefix(p)
         );
     }
 
@@ -1017,7 +1024,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
             !(first == 's' || first == 'r' || first == 'a')) {
         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(conv,
                                             "%c-string: invalid conversion character %R: expected 's', 'r', or 'a'",
-                                            TOK_GET_STRING_PREFIX(p->tok),
+                                            formatted_string_prefix(p),
                                             conv->v.Name.id);
         return NULL;
     }
@@ -1344,7 +1351,8 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* tok
 }
 
 static asdl_expr_seq *
-_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b, enum string_kind_t string_kind)
+_get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions,
+                   Token *b, ftstring_kind string_kind)
 {
     Py_ssize_t n_items = asdl_seq_LEN(raw_expressions);
     Py_ssize_t total_items = n_items;
@@ -1370,15 +1378,13 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
     for (Py_ssize_t i = 0; i < n_items; i++) {
         expr_ty item = asdl_seq_GET(raw_expressions, i);
 
-        // This should correspond to a JoinedStr node of two elements
-        // created _PyPegen_formatted_value. This situation can only be the result of
-        // a (f|t)-string debug expression where the first element is a constant with the text and the second
-        // a formatted value with the expression.
+        /* Debug expressions arrive as JoinedStr(text, value); flatten them
+           into the surrounding string. */
         if (item->kind == JoinedStr_kind) {
             asdl_expr_seq *values = item->v.JoinedStr.values;
             if (asdl_seq_LEN(values) != 2) {
                 PyErr_Format(PyExc_SystemError,
-                             string_kind == TSTRING
+                             _PyLexer_IsTString(string_kind)
                              ? "unexpected TemplateStr node without debug data in t-string at line %d"
                              : "unexpected JoinedStr node without debug data in f-string at line %d",
                              item->lineno);
@@ -1390,7 +1396,9 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b
             asdl_seq_SET(seq, index++, first);
 
             expr_ty second = asdl_seq_GET(values, 1);
-            assert((string_kind == TSTRING && second->kind == Interpolation_kind) || second->kind == FormattedValue_kind);
+            assert((_PyLexer_IsTString(string_kind) &&
+                    second->kind == Interpolation_kind) ||
+                   second->kind == FormattedValue_kind);
             asdl_seq_SET(seq, index++, second);
 
             continue;
@@ -1460,12 +1468,8 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
         return NULL;
     }
 
-    // Check if we're inside a raw f-string for format spec decoding
-    int is_raw = 0;
-    if (INSIDE_FSTRING(p->tok)) {
-        tokenizer_mode *mode = TOK_GET_MODE(p->tok);
-        is_raw = mode->raw;
-    }
+    const ftstring_state *state = _PyLexer_CurrentFTString(p->tok);
+    int is_raw = state != NULL && _PyLexer_IsRawString(state->kind);
 
     PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);
     if (str == NULL) {
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index fbec2ec92f64103..54dd0123e7ab7bb 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -154,8 +154,11 @@ tok_continuation_line(struct tok_state *tok) {
 
 
 int
-_PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
+_PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token *token)
 {
+    assert(current == NULL ||
+           (current->mode == FTSTRING_MODE_EXPRESSION &&
+            current->replacement_depth > 0));
     int c;
     int blankline, nonascii;
 
@@ -317,13 +320,13 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             c = tok_nextc(tok);
         }
 
-        if (INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
+        if (current != NULL) {
             const char *comment_end = tok->cur;
             if (c == '\n' || c == '\r') {
                 comment_end--;
             }
             if (_PyLexer_record_ftstring_comment(
-                    tok, tok->start, comment_end) < 0) {
+                    tok, current, tok->start, comment_end) < 0) {
                 tok->done = E_NOMEM;
                 return MAKE_TOKEN(ERRORTOKEN);
             }
@@ -547,34 +550,25 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
     }
 
     /* Punctuation character */
-    int is_punctuation = (c == ':' || c == '}' || c == '!' || c == '{');
-    if (is_punctuation && INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
-        /* This code block gets executed before the curly_bracket_depth is incremented
-         * by the `{` case, so for ensuring that we are on the 0th level, we need
-         * to adjust it manually */
-        int cursor = current_tok->curly_bracket_depth - (c != '{');
-        int in_format_spec = current_tok->in_format_spec;
-         int cursor_in_format_with_debug =
-             cursor == 1 && (current_tok->in_debug || in_format_spec);
-         int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
-        if (cursor_valid && c == '!') {
+    int is_punctuation = (c == ':' || c == '}' || c == '!');
+    if (is_punctuation && current != NULL) {
+        int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
+        int at_expression_boundary =
+            bracket_depth == current->replacement_depth;
+        if (at_expression_boundary && c == '!') {
             int c2 = tok_nextc(tok);
             if (c2 == '=') {
-                cursor_valid = 0;
+                at_expression_boundary = 0;
             }
             tok_backup(tok, c2);
         }
-        if (cursor_valid) {
-            _PyLexer_update_ftstring_expr(tok, c);
-        }
-        if (cursor_valid && c != '{' &&
-                _PyLexer_set_ftstring_expr_metadata(tok, token)) {
+        if (at_expression_boundary &&
+                _PyLexer_finish_ftstring_expr(tok, current, token)) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
 
-        if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) {
-            current_tok->kind = TOK_FSTRING_MODE;
-            current_tok->in_format_spec = 1;
+        if (c == ':' && at_expression_boundary) {
+            current->mode = FTSTRING_MODE_FORMAT_SPEC;
             p_start = tok->start;
             p_end = tok->cur;
             return MAKE_TOKEN(_PyToken_OneChar(c));
@@ -613,16 +607,20 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         tok->parenlinenostack[tok->level] = tok->lineno;
         tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
         tok->level++;
-        if (INSIDE_FSTRING(tok)) {
-            current_tok->curly_bracket_depth++;
-        }
         break;
     case ')':
     case ']':
     case '}':
-        if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
-            return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                "%c-string: single '}' is not allowed", TOK_GET_STRING_PREFIX(tok)));
+        if (current != NULL &&
+                _PyLexer_FTStringBracketDepth(tok, current) == 0) {
+            if (c == '}') {
+                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+                    "%c-string: single '}' is not allowed",
+                    _PyLexer_StringPrefix(current->kind)));
+            }
+            return MAKE_TOKEN(_PyTokenizer_syntaxerror(
+                tok, "%c-string: unmatched '%c'",
+                _PyLexer_StringPrefix(current->kind), c));
         }
         if (!tok->tok_extra_tokens && !tok->level) {
             return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
@@ -633,17 +631,15 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
                                             (opening == '[' && c == ']') ||
                                             (opening == '{' && c == '}'))) {
-                /* If the opening bracket belongs to an f-string's expression
-                part (e.g. f"{)}") and the closing bracket is an arbitrary
-                nested expression, then instead of matching a different
-                syntactical construct with it; we'll throw an unmatched
-                parentheses error. */
-                if (INSIDE_FSTRING(tok) && opening == '{') {
-                    assert(current_tok->curly_bracket_depth >= 0);
-                    int previous_bracket = current_tok->curly_bracket_depth - 1;
-                    if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
+                /* Do not match a closer against the brace that opened the
+                 * current replacement field. */
+                if (current != NULL && opening == '{') {
+                    int bracket_depth =
+                        _PyLexer_FTStringBracketDepth(tok, current);
+                    if (bracket_depth == current->replacement_depth - 1) {
                         return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                            "%c-string: unmatched '%c'", TOK_GET_STRING_PREFIX(tok), c));
+                            "%c-string: unmatched '%c'",
+                            _PyLexer_StringPrefix(current->kind), c));
                     }
                 }
                 if (tok->parenlinenostack[tok->level] != tok->lineno) {
@@ -661,17 +657,16 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
             }
         }
 
-        if (INSIDE_FSTRING(tok)) {
-            current_tok->curly_bracket_depth--;
-            if (current_tok->curly_bracket_depth < 0) {
+        if (current != NULL) {
+            int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
+            if (bracket_depth < 0) {
                 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
-                    TOK_GET_STRING_PREFIX(tok), c));
+                    _PyLexer_StringPrefix(current->kind), c));
             }
-            if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) {
-                current_tok->curly_bracket_expr_start_depth--;
-                current_tok->kind = TOK_FSTRING_MODE;
-                current_tok->in_format_spec = 0;
-                current_tok->in_debug = 0;
+            if (c == '}' && bracket_depth == current->replacement_depth - 1) {
+                current->replacement_depth--;
+                current->mode = FTSTRING_MODE_MIDDLE;
+                current->debug_expr = 0;
             }
         }
         break;
@@ -683,8 +678,9 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
     }
 
-    if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) {
-        current_tok->in_debug = 1;
+    if (c == '=' && current != NULL &&
+            _PyLexer_FTStringBracketDepth(tok, current) == current->replacement_depth) {
+        current->debug_expr = 1;
     }
 
     /* Punctuation character */
@@ -694,21 +690,13 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
 }
 
 
-static int
-tok_get(struct tok_state *tok, struct token *token)
-{
-    tokenizer_mode *current_tok = TOK_GET_MODE(tok);
-    if (current_tok->kind == TOK_REGULAR_MODE) {
-        return _PyLexer_get_normal_mode(tok, current_tok, token);
-    } else {
-        return _PyLexer_get_fstring_mode(tok, current_tok, token);
-    }
-}
-
 int
 _PyTokenizer_Get(struct tok_state *tok, struct token *token)
 {
-    int result = tok_get(tok, token);
+    ftstring_state *current = _PyLexer_CurrentFTString(tok);
+    int result = current == NULL || current->mode == FTSTRING_MODE_EXPRESSION
+        ? _PyLexer_get_normal(tok, current, token)
+        : _PyLexer_get_ftstring(tok, current, token);
     if (tok_failed(tok)) {
         result = ERRORTOKEN;
     }
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index 68273eb430e6812..37e05c4fe153f9b 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -17,31 +17,10 @@
                || c == '_'\
                || (c >= 128))
 
-#ifdef Py_DEBUG
-static inline tokenizer_mode *
-TOK_GET_MODE(struct tok_state *tok)
-{
-    assert(tok->tok_mode_stack_index >= 0);
-    assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL);
-    return &tok->tok_mode_stack[tok->tok_mode_stack_index];
-}
-
-static inline tokenizer_mode *
-TOK_NEXT_MODE(struct tok_state *tok)
-{
-    assert(tok->tok_mode_stack_index >= 0);
-    assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL);
-    return &tok->tok_mode_stack[++tok->tok_mode_stack_index];
-}
-#else
-#define TOK_GET_MODE(tok) (&(tok)->tok_mode_stack[(tok)->tok_mode_stack_index])
-#define TOK_NEXT_MODE(tok) (&(tok)->tok_mode_stack[++(tok)->tok_mode_stack_index])
-#endif
-
-#define FTSTRING_MIDDLE(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_MIDDLE : FSTRING_MIDDLE)
-#define FTSTRING_END(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_END : FSTRING_END)
-#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f')
-
+#define FTSTRING_MIDDLE(state) \
+    (_PyLexer_IsTString((state)->kind) ? TSTRING_MIDDLE : FSTRING_MIDDLE)
+#define FTSTRING_END(state) \
+    (_PyLexer_IsTString((state)->kind) ? TSTRING_END : FSTRING_END)
 #define tok_nextc _PyLexer_nextc
 #define tok_backup _PyLexer_backup
 
@@ -54,15 +33,15 @@ tok_failed(const struct tok_state *tok)
 
 int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
-void _PyLexer_update_ftstring_expr(struct tok_state *, char);
 int _PyLexer_record_ftstring_comment(
-    struct tok_state *, const char *, const char *);
-int _PyLexer_set_ftstring_expr_metadata(struct tok_state *, struct token *);
+    struct tok_state *, ftstring_state *, const char *, const char *);
+int _PyLexer_finish_ftstring_expr(
+    struct tok_state *, ftstring_state *, struct token *);
 int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
 int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);
 int _PyLexer_scan_string(struct tok_state *, struct token *, int);
-int _PyLexer_get_normal_mode(struct tok_state *, tokenizer_mode *, struct token *);
-int _PyLexer_get_fstring_mode(struct tok_state *, tokenizer_mode *, struct token *);
+int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *);
+int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *);
 
 #endif
diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c
index ee234767308826f..cdf997363b00038 100644
--- a/Parser/lexer/state.c
+++ b/Parser/lexer/state.c
@@ -4,6 +4,7 @@
 #include "errcode.h"
 
 #include "state.h"
+#include "../tokenizer/helpers.h"
 #include "../tokenizer/reader.h"
 
 /* Create and initialize a new tok_state structure */
@@ -46,14 +47,62 @@ _PyTokenizer_tok_new(void)
     tok->implicit_newline = 0;
     _PyTok_SourceInit(&tok->source);
     tok->reader = NULL;
-    tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .quote='\0', .quote_size = 0, .in_debug=0};
-    tok->tok_mode_stack_index = 0;
+    tok->ftstring_stack = tok->ftstring_stack_inline;
+    tok->ftstring_capacity = FTSTRING_STACK_INLINE_CAPACITY;
 #ifdef Py_DEBUG
     tok->debug = _Py_GetConfig()->parser_debug;
 #endif
     return tok;
 }
 
+ftstring_state *
+_PyLexer_PushFTString(struct tok_state *tok)
+{
+    assert(tok->ftstring_depth >= 0 && tok->ftstring_depth <= tok->ftstring_capacity);
+    int next_depth = tok->ftstring_depth + 1;
+    if (next_depth >= MAXFTSTRINGLEVEL) {
+        _PyTokenizer_syntaxerror(
+            tok, "too many nested f-strings or t-strings");
+        return NULL;
+    }
+    if (tok->ftstring_depth == tok->ftstring_capacity) {
+        int capacity = Py_MIN(Py_MAX(tok->ftstring_capacity * 2, 4),
+                              MAXFTSTRINGLEVEL);
+        size_t size = (size_t)capacity * sizeof(*tok->ftstring_stack);
+        ftstring_state *stack;
+        if (tok->ftstring_stack == tok->ftstring_stack_inline) {
+            stack = PyMem_Malloc(size);
+            if (stack != NULL) {
+                memcpy(stack, tok->ftstring_stack,
+                       (size_t)tok->ftstring_depth * sizeof(*stack));
+            }
+        }
+        else {
+            stack = PyMem_Realloc(tok->ftstring_stack, size);
+        }
+        if (stack == NULL) {
+            PyErr_NoMemory();
+            tok->done = E_NOMEM;
+            return NULL;
+        }
+        tok->ftstring_stack = stack;
+        tok->ftstring_capacity = capacity;
+    }
+    ftstring_state *state = &tok->ftstring_stack[tok->ftstring_depth];
+    tok->ftstring_depth = next_depth;
+    *state = (ftstring_state){0};
+    return state;
+}
+
+void
+_PyLexer_PopFTString(struct tok_state *tok)
+{
+    ftstring_state *state = _PyLexer_CurrentFTString(tok);
+    assert(state != NULL);
+    PyMem_Free(state->comments);
+    tok->ftstring_depth--;
+}
+
 /* Free a tok_state structure */
 void
 _PyTokenizer_Free(struct tok_state *tok)
@@ -65,8 +114,11 @@ _PyTokenizer_Free(struct tok_state *tok)
     Py_XDECREF(tok->module);
     _PyTok_ReaderFree(tok);
     _PyTok_SourceClear(&tok->source);
-    for (int i = 0; i <= tok->tok_mode_stack_index; i++) {
-        PyMem_Free(tok->tok_mode_stack[i].comments);
+    for (int i = 0; i < tok->ftstring_depth; i++) {
+        PyMem_Free(tok->ftstring_stack[i].comments);
+    }
+    if (tok->ftstring_stack != tok->ftstring_stack_inline) {
+        PyMem_Free(tok->ftstring_stack);
     }
     PyMem_Free(tok);
 }
diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h
index 520ba2778593735..6203084f19b54f5 100644
--- a/Parser/lexer/state.h
+++ b/Parser/lexer/state.h
@@ -6,12 +6,8 @@
 
 #define MAXINDENT 100       /* Max indentation level */
 #define MAXLEVEL 200        /* Max parentheses level */
-#define MAXFSTRINGLEVEL 150 /* Max f-string nesting level */
-
-#define INSIDE_FSTRING(tok) (tok->tok_mode_stack_index > 0)
-#define INSIDE_FSTRING_EXPR(tok) (tok->curly_bracket_expr_start_depth >= 0)
-#define INSIDE_FSTRING_EXPR_AT_TOP(tok) \
-    (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1)
+#define MAXFTSTRINGLEVEL 150
+#define FTSTRING_STACK_INLINE_CAPACITY 1
 
 enum interactive_underflow_t {
     /* Normal mode of operation: return a new token when asked in interactive mode */
@@ -29,15 +25,18 @@ struct token {
     PyObject *metadata;
 };
 
-enum tokenizer_mode_kind_t {
-    TOK_REGULAR_MODE,
-    TOK_FSTRING_MODE,
-};
+typedef enum {
+    FTSTRING_MODE_MIDDLE,
+    FTSTRING_MODE_EXPRESSION,
+    FTSTRING_MODE_FORMAT_SPEC,
+} ftstring_mode;
 
-enum string_kind_t {
+typedef enum {
     FSTRING,
+    RAW_FSTRING,
     TSTRING,
-};
+    RAW_TSTRING,
+} ftstring_kind;
 
 #define MAX_EXPR_NESTING 3
 
@@ -47,26 +46,31 @@ typedef struct _tokenizer_comments {
     _PyTok_Span spans[];
 } tokenizer_comments;
 
-typedef struct _tokenizer_mode {
-    enum tokenizer_mode_kind_t kind;
-
-    int curly_bracket_depth;
-    int curly_bracket_expr_start_depth;
-
+typedef struct _ftstring_state {
+    ftstring_mode mode;
+    ftstring_kind kind;
     char quote;
-    int quote_size;
-    int raw;
+    unsigned char quote_size;
+    unsigned char debug_expr;
+    unsigned char replacement_depth;
+    int paren_level;
     _PyTok_Off start;
-    _PyTok_Off multi_line_start;
-    int first_line;
-
+    _PyTok_Loc start_loc;
     _PyTok_Span expr_span;
-    int in_debug;
-    int in_format_spec;
-
-    enum string_kind_t string_kind;
     tokenizer_comments *comments;
-} tokenizer_mode;
+} ftstring_state;
+
+static inline int
+_PyLexer_IsTString(ftstring_kind kind)
+{
+    return kind == TSTRING || kind == RAW_TSTRING;
+}
+
+static inline int
+_PyLexer_IsRawString(ftstring_kind kind)
+{
+    return kind == RAW_FSTRING || kind == RAW_TSTRING;
+}
 
 /* Tokenizer state */
 struct tok_state {
@@ -112,9 +116,10 @@ struct tok_state {
     /* How to proceed when asked for a new token in interactive mode */
     enum interactive_underflow_t interactive_underflow;
     int report_warnings;
-    // TODO: Factor this into its own thing
-    tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL];
-    int tok_mode_stack_index;
+    ftstring_state *ftstring_stack;
+    ftstring_state ftstring_stack_inline[FTSTRING_STACK_INLINE_CAPACITY];
+    int ftstring_depth;
+    int ftstring_capacity;
     int tok_extra_tokens;
     int comment_newline;
     int implicit_newline;
@@ -123,6 +128,30 @@ struct tok_state {
 #endif
 };
 
+static inline ftstring_state *
+_PyLexer_CurrentFTString(struct tok_state *tok)
+{
+    assert(tok->ftstring_stack != NULL);
+    assert(tok->ftstring_depth >= 0 && tok->ftstring_depth <= tok->ftstring_capacity);
+    if (tok->ftstring_depth == 0) {
+        return NULL;
+    }
+    return &tok->ftstring_stack[tok->ftstring_depth - 1];
+}
+
+static inline char
+_PyLexer_StringPrefix(ftstring_kind kind)
+{
+    return _PyLexer_IsTString(kind) ? 't' : 'f';
+}
+
+static inline int
+_PyLexer_FTStringBracketDepth(const struct tok_state *tok,
+                              const ftstring_state *state)
+{
+    return tok->level - state->paren_level;
+}
+
 static inline _PyTok_Off
 _PyLexer_BufferOffset(const struct tok_state *tok, const char *position)
 {
@@ -184,6 +213,8 @@ int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, c
 
 struct tok_state *_PyTokenizer_tok_new(void);
 void _PyTokenizer_Free(struct tok_state *);
+ftstring_state *_PyLexer_PushFTString(struct tok_state *);
+void _PyLexer_PopFTString(struct tok_state *);
 void _PyToken_Free(struct token *);
 void _PyToken_Init(struct token *);
 
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index a75d9f4c5cda8ee..37c816206c2c174 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -17,17 +17,16 @@ rewind_to_string_start(struct tok_state *tok, const char *start,
 }
 
 int
-_PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start,
-                                 const char *end)
+_PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
+                                 const char *start, const char *end)
 {
-    tokenizer_mode *mode = TOK_GET_MODE(tok);
-    if (mode->expr_span.end >= 0) {
+    assert(state == _PyLexer_CurrentFTString(tok) && state->mode == FTSTRING_MODE_EXPRESSION);
+    if (state->expr_span.end >= 0) {
         return 0;
     }
-    assert(mode->expr_span.start >= 0);
-    tokenizer_comments *comments = mode->comments;
+    assert(state->expr_span.start >= 0);
+    tokenizer_comments *comments = state->comments;
     if (comments == NULL || comments->count == comments->capacity) {
-        int create = comments == NULL;
         Py_ssize_t max_capacity = (PY_SSIZE_T_MAX -
             (Py_ssize_t)sizeof(*comments)) /
             (Py_ssize_t)sizeof(*comments->spans);
@@ -44,11 +43,11 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start,
             return -1;
         }
         comments = resized;
-        if (create) {
+        if (state->comments == NULL) {
             comments->count = 0;
         }
         comments->capacity = capacity;
-        mode->comments = comments;
+        state->comments = comments;
     }
     comments->spans[comments->count++] =
         _PyLexer_BufferSpan(tok, start, end);
@@ -56,31 +55,39 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok, const char *start,
 }
 
 int
-_PyLexer_set_ftstring_expr_metadata(struct tok_state *tok, struct token *token)
+_PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
+                              struct token *token)
 {
-    assert(token != NULL);
-    tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
+    assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
+    assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start != NULL);
 
-    if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
+    if (state->expr_span.end >= 0) {
+        return 0;
+    }
+    assert(state->expr_span.start >= 0);
+    state->expr_span.end = _PyLexer_BufferOffset(tok, tok->start);
+    int tstring_interpolation = _PyLexer_IsTString(state->kind) &&
+        state->replacement_depth == 1;
+    if (!(state->debug_expr || tstring_interpolation) || token->metadata) {
         return 0;
     }
     Py_ssize_t expr_len;
     const char *expr = _PyLexer_BufferSpanView(
-        tok, tok_mode->expr_span, &expr_len);
-    tokenizer_comments *comments = tok_mode->comments;
+        tok, state->expr_span, &expr_len);
+    tokenizer_comments *comments = state->comments;
     PyObject *res;
     if (comments != NULL && comments->count > 0) {
         Py_ssize_t stripped_size = expr_len;
-        _PyTok_Off previous_end = tok_mode->expr_span.start;
+        _PyTok_Off previous_end = state->expr_span.start;
         Py_ssize_t comment_count = 0;
         for (Py_ssize_t i = 0; i < comments->count; i++) {
             _PyTok_Span comment = comments->spans[i];
             assert(_PyTok_SpanIsValid(comment));
             assert(comment.start >= previous_end);
-            if (comment.start >= tok_mode->expr_span.end) {
+            if (comment.start >= state->expr_span.end) {
                 break;
             }
-            assert(comment.end <= tok_mode->expr_span.end);
+            assert(comment.end <= state->expr_span.end);
             stripped_size -= comment.end - comment.start;
             previous_end = comment.end;
             comment_count++;
@@ -90,20 +97,20 @@ _PyLexer_set_ftstring_expr_metadata(struct tok_state *tok, struct token *token)
             PyErr_NoMemory();
             return -1;
         }
-        _PyTok_Off copied_to = tok_mode->expr_span.start;
+        _PyTok_Off copied_to = state->expr_span.start;
         Py_ssize_t stripped_len = 0;
         for (Py_ssize_t i = 0; i < comment_count; i++) {
             _PyTok_Span comment = comments->spans[i];
             Py_ssize_t length = comment.start - copied_to;
             memcpy(stripped + stripped_len,
-                   expr + copied_to - tok_mode->expr_span.start,
+                   expr + copied_to - state->expr_span.start,
                    (size_t)length);
             stripped_len += length;
             copied_to = comment.end;
         }
-        Py_ssize_t length = tok_mode->expr_span.end - copied_to;
+        Py_ssize_t length = state->expr_span.end - copied_to;
         memcpy(stripped + stripped_len,
-               expr + copied_to - tok_mode->expr_span.start,
+               expr + copied_to - state->expr_span.start,
                (size_t)length);
         stripped_len += length;
         res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL);
@@ -120,35 +127,6 @@ _PyLexer_set_ftstring_expr_metadata(struct tok_state *tok, struct token *token)
     return 0;
 }
 
-void
-_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
-{
-    tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
-
-    switch (cur) {
-        case '{':
-            tok_mode->expr_span = (_PyTok_Span){
-                _PyLexer_BufferOffset(tok, tok->cur), -1};
-            tokenizer_comments *comments = tok_mode->comments;
-            if (comments != NULL) {
-                comments->count = 0;
-            }
-            break;
-        case '}':
-        case '!':
-            tok_mode->expr_span.end = _PyLexer_BufferOffset(tok, tok->start);
-            break;
-        case ':':
-            if (tok_mode->expr_span.end < 0) {
-                tok_mode->expr_span.end =
-                    _PyLexer_BufferOffset(tok, tok->start);
-            }
-            break;
-        default:
-            Py_UNREACHABLE();
-    }
-}
-
 int
 _PyLexer_check_string_prefixes(struct tok_state *tok,
                                              int saw_b, int saw_r, int saw_u,
@@ -220,51 +198,44 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
         tok_backup(tok, after_quote);
     }
 
-
     p_start = tok->start;
     p_end = tok->cur;
-    if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) {
-        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested f-strings or t-strings"));
+    ftstring_state *state = _PyLexer_PushFTString(tok);
+    if (state == NULL) {
+        return MAKE_TOKEN(ERRORTOKEN);
     }
-    tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok);
-    the_current_tok->kind = TOK_FSTRING_MODE;
-    the_current_tok->quote = quote;
-    the_current_tok->quote_size = quote_size;
-    the_current_tok->start = _PyLexer_BufferOffset(tok, tok->start);
-    the_current_tok->multi_line_start =
-        _PyLexer_BufferOffset(tok, tok->line_start);
-    the_current_tok->first_line = tok->lineno;
-    the_current_tok->expr_span = (_PyTok_Span){-1, -1};
-    the_current_tok->in_format_spec = 0;
-    the_current_tok->in_debug = 0;
-    the_current_tok->comments = NULL;
-
-    enum string_kind_t string_kind = FSTRING;
+    state->mode = FTSTRING_MODE_MIDDLE;
+    state->quote = quote;
+    state->quote_size = quote_size;
+    state->paren_level = tok->level;
+    state->start = _PyLexer_BufferOffset(tok, tok->start);
+    state->start_loc = tok->start_loc;
+    state->expr_span = (_PyTok_Span){-1, -1};
+
+    int raw = 0;
+    int tstring = 0;
     switch (*tok->start) {
         case 'T':
         case 't':
-            the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
-            string_kind = TSTRING;
+            raw = Py_TOLOWER(tok->start[1]) == 'r';
+            tstring = 1;
             break;
         case 'F':
         case 'f':
-            the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
+            raw = Py_TOLOWER(tok->start[1]) == 'r';
             break;
         case 'R':
         case 'r':
-            the_current_tok->raw = 1;
-            if (Py_TOLOWER(*(tok->start + 1)) == 't') {
-                string_kind = TSTRING;
-            }
+            raw = 1;
+            tstring = Py_TOLOWER(tok->start[1]) == 't';
             break;
         default:
             Py_UNREACHABLE();
     }
-
-    the_current_tok->string_kind = string_kind;
-    the_current_tok->curly_bracket_depth = 0;
-    the_current_tok->curly_bracket_expr_start_depth = -1;
-    return string_kind == TSTRING ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
+    state->kind = tstring
+        ? (raw ? RAW_TSTRING : TSTRING)
+        : (raw ? RAW_FSTRING : FSTRING);
+    return tstring ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
 }
 
 int
@@ -306,17 +277,14 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
             int end_lineno = tok->lineno;
             rewind_to_string_start(tok, tok->start, tok->start_loc);
 
-            if (INSIDE_FSTRING(tok)) {
-                /* When we are in an f-string, before raising the
-                 * unterminated string literal error, check whether
-                 * does the initial quote matches with f-strings quotes
-                 * and if it is, then this must be a missing '}' token
-                 * so raise the proper error */
-                tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
-                if (the_current_tok->quote == quote &&
-                    the_current_tok->quote_size == quote_size) {
+            const ftstring_state *state = _PyLexer_CurrentFTString(tok);
+            if (state != NULL) {
+                /* A matching quote belongs to the surrounding formatted
+                 * string, so the expression is missing its closing brace. */
+                if (state->quote == quote && state->quote_size == quote_size) {
                     return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                        "%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok)));
+                        "%c-string: expecting '}'",
+                        _PyLexer_StringPrefix(state->kind)));
                 }
             }
 
@@ -370,108 +338,52 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
 }
 
 int
-_PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
+_PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct token *token)
 {
+    assert(current == _PyLexer_CurrentFTString(tok) && current->mode != FTSTRING_MODE_EXPRESSION);
+    assert((current->quote_size == 1 || current->quote_size == 3) &&
+           current->replacement_depth <= MAX_EXPR_NESTING);
     const char *p_start = NULL;
     const char *p_end = NULL;
     int end_quote_size = 0;
     int unicode_escape = 0;
+    int quote = current->quote;
+    int quote_size = current->quote_size;
+    int in_format_spec = current->mode == FTSTRING_MODE_FORMAT_SPEC;
+    int raw = _PyLexer_IsRawString(current->kind);
 
     tok->start = tok->cur;
     tok->start_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
 
-    // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize
-    // before it.
-    int start_char = tok_nextc(tok);
-    if (start_char == '{') {
-        int peek1 = tok_nextc(tok);
-        tok_backup(tok, peek1);
-        tok_backup(tok, start_char);
-        if (peek1 != '{') {
-            current_tok->curly_bracket_expr_start_depth++;
-            if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
-                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                    "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok)));
-            }
-            TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
-            return _PyLexer_get_normal_mode(tok, current_tok, token);
-        }
-    }
-    else {
-        tok_backup(tok, start_char);
-    }
-
-    // Check if we are at the end of the string
-    for (int i = 0; i < current_tok->quote_size; i++) {
-        int quote = tok_nextc(tok);
-        if (quote != current_tok->quote) {
-            tok_backup(tok, quote);
-            goto f_string_middle;
-        }
-    }
-
-    p_start = tok->start;
-    p_end = tok->cur;
-    PyMem_Free(current_tok->comments);
-    current_tok->comments = NULL;
-    tok->tok_mode_stack_index--;
-    return MAKE_TOKEN(FTSTRING_END(current_tok));
-
-f_string_middle:
-
-    while (end_quote_size != current_tok->quote_size) {
+    while (end_quote_size != quote_size) {
         int c = tok_nextc(tok);
         if (tok->done == E_ERROR || tok->done == E_DECODE) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
-        int in_format_spec = (
-                current_tok->in_format_spec
-                &&
-                INSIDE_FSTRING_EXPR(current_tok)
-        );
 
-       if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) {
+        if (c == EOF || (quote_size == 1 && c == '\n')) {
             if (tok_failed(tok)) {
                 return MAKE_TOKEN(ERRORTOKEN);
             }
 
-            // If we are in a format spec and we found a newline,
-            // it means that the format spec ends here and we should
-            // return to the regular mode.
             if (in_format_spec && c == '\n') {
-                if (current_tok->quote_size == 1) {
-                    return MAKE_TOKEN(
-                        _PyTokenizer_syntaxerror(
-                            tok,
-                            "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings",
-                            TOK_GET_STRING_PREFIX(tok), TOK_GET_STRING_PREFIX(tok)
-                        )
-                    );
-                }
-                tok_backup(tok, c);
-                TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
-                current_tok->in_format_spec = 0;
-                p_start = tok->start;
-                p_end = tok->cur;
-                return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
+                return MAKE_TOKEN(_PyTokenizer_syntaxerror(
+                    tok,
+                    "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings",
+                    _PyLexer_StringPrefix(current->kind),
+                    _PyLexer_StringPrefix(current->kind)));
             }
 
-            // shift the tok_state's location into
-            // the start of string, and report the error
-            // from the initial quote character
-            tok->cur = _PyLexer_BufferPointer(tok, current_tok->start) + 1;
-            tok->line_start = _PyLexer_BufferPointer(
-                tok, current_tok->multi_line_start);
-            int start = tok->lineno;
-
-            tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
-            tok->lineno = the_current_tok->first_line;
+            int end_lineno = tok->lineno;
+            rewind_to_string_start(tok,
+                _PyLexer_BufferPointer(tok, current->start),
+                current->start_loc);
 
-            if (current_tok->quote_size == 3) {
+            if (quote_size == 3) {
                 _PyTokenizer_syntaxerror(tok,
                                     "unterminated triple-quoted %c-string literal"
                                     " (detected at line %d)",
-                                    TOK_GET_STRING_PREFIX(tok), start);
+                                    _PyLexer_StringPrefix(current->kind), end_lineno);
                 if (c != '\n') {
                     tok->done = E_EOFS;
                 }
@@ -480,11 +392,12 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
             else {
                 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                                     "unterminated %c-string literal (detected at"
-                                    " line %d)", TOK_GET_STRING_PREFIX(tok), start));
+                                    " line %d)",
+                                    _PyLexer_StringPrefix(current->kind), end_lineno));
             }
         }
 
-        if (c == current_tok->quote) {
+        if (c == quote) {
             end_quote_size += 1;
             continue;
         } else {
@@ -492,60 +405,73 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
         }
 
         if (c == '{') {
-            _PyLexer_update_ftstring_expr(tok, c);
             int peek = tok_nextc(tok);
             if (peek != '{' || in_format_spec) {
                 tok_backup(tok, peek);
+                current->expr_span = (_PyTok_Span){
+                    _PyLexer_BufferOffset(tok, tok->cur), -1};
+                if (current->comments != NULL) {
+                    current->comments->count = 0;
+                }
                 tok_backup(tok, c);
-                current_tok->curly_bracket_expr_start_depth++;
-                if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
-                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
-                        "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok)));
+                if (current->replacement_depth >= MAX_EXPR_NESTING) {
+                    _PyTokenizer_syntaxerror(
+                        tok, "%c-string: expressions nested too deeply",
+                        _PyLexer_StringPrefix(current->kind));
+                    return MAKE_TOKEN(ERRORTOKEN);
                 }
-                TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
-                current_tok->in_format_spec = 0;
+                current->replacement_depth++;
+                current->mode = FTSTRING_MODE_EXPRESSION;
+                current->debug_expr = 0;
                 p_start = tok->start;
                 p_end = tok->cur;
+                if (p_start == p_end) {
+                    return _PyLexer_get_normal(tok, current, token);
+                }
             } else {
                 p_start = tok->start;
                 p_end = tok->cur - 1;
             }
-            return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
+            goto emit_middle;
         } else if (c == '}') {
             if (unicode_escape) {
                 p_start = tok->start;
                 p_end = tok->cur;
-                return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
+                goto emit_middle;
             }
             int peek = tok_nextc(tok);
 
-            // The tokenizer can only be in the format spec if we have already completed the expression
-            // scanning (indicated by the end of the expression being set) and we are not at the top level
-            // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double
-            // brackets, we can bypass it here.
-            int cursor = current_tok->curly_bracket_depth;
-            if (peek == '}' && !in_format_spec && cursor == 0) {
+            int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
+            if (peek == '}' && !in_format_spec && bracket_depth == 0) {
                 p_start = tok->start;
                 p_end = tok->cur - 1;
-            } else {
+            }
+            else {
                 tok_backup(tok, peek);
+                if (!in_format_spec && bracket_depth == 0) {
+                    if (tok->start == tok->cur - 1) {
+                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(
+                            tok, "%c-string: single '}' is not allowed",
+                            _PyLexer_StringPrefix(current->kind)));
+                    }
+                    tok_backup(tok, c);
+                    p_start = tok->start;
+                    p_end = tok->cur;
+                    goto emit_middle;
+                }
                 tok_backup(tok, c);
-                TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
-                current_tok->in_format_spec = 0;
+                current->mode = FTSTRING_MODE_EXPRESSION;
                 p_start = tok->start;
                 p_end = tok->cur;
             }
-            return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
+            goto emit_middle;
         } else if (c == '\\') {
             int peek = tok_nextc(tok);
             if (peek == '\r') {
                 peek = tok_nextc(tok);
             }
-            // Special case when the backslash is right before a curly
-            // brace. We have to restore and return the control back
-            // to the loop for the next iteration.
             if (peek == '{' || peek == '}') {
-                if (!current_tok->raw) {
+                if (!raw) {
                     if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) {
                         return MAKE_TOKEN(ERRORTOKEN);
                     }
@@ -554,7 +480,7 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
                 continue;
             }
 
-            if (!current_tok->raw) {
+            if (!raw) {
                 if (peek == 'N') {
                     /* Handle named unicode escapes (\N{BULLET}) */
                     peek = tok_nextc(tok);
@@ -564,18 +490,21 @@ _PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, st
                         tok_backup(tok, peek);
                     }
                 }
-            } /* else {
-                skip the escaped character
-            }*/
+            }
         }
     }
 
-    // Backup the f-string quotes to emit a final FSTRING_MIDDLE and
-    // add the quotes to the FSTRING_END in the next tokenizer iteration.
-    for (int i = 0; i < current_tok->quote_size; i++) {
-        tok_backup(tok, current_tok->quote);
-    }
     p_start = tok->start;
     p_end = tok->cur;
-    return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
+    if (p_end - quote_size == p_start) {
+        int end_token = FTSTRING_END(current);
+        _PyLexer_PopFTString(tok);
+        return MAKE_TOKEN(end_token);
+    }
+    for (int i = 0; i < quote_size; i++) {
+        tok_backup(tok, quote);
+    }
+    p_end = tok->cur;
+emit_middle:
+    return MAKE_TOKEN(FTSTRING_MIDDLE(current));
 }
diff --git a/Parser/pegen.h b/Parser/pegen.h
index 3cca698692bf6bf..81137782621daf4 100644
--- a/Parser/pegen.h
+++ b/Parser/pegen.h
@@ -26,9 +26,6 @@
 
 #define CURRENT_POS (-5)
 
-#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index]))
-#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f')
-
 typedef struct _memo {
     int type;
     void *node;
diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c
index 0b5614b2faaec5e..f4c3f988706259d 100644
--- a/Parser/pegen_errors.c
+++ b/Parser/pegen_errors.c
@@ -163,10 +163,8 @@ _PyPegen_tokenize_full_source_to_check_for_errors(Parser *p) {
 
 exit:
     _PyToken_Free(&new_token);
-    // If we're in an f-string, we want the syntax error in the expression part
-    // to propagate, so that tokenizer errors (like expecting '}') that happen afterwards
-    // do not swallow it.
-    if (PyErr_Occurred() && p->tok->tok_mode_stack_index <= 0) {
+    // Preserve expression errors over later formatted-string errors.
+    if (PyErr_Occurred() && _PyLexer_CurrentFTString(p->tok) == NULL) {
         Py_XDECREF(value);
         Py_XDECREF(type);
         Py_XDECREF(traceback);
diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c
index 56e50cebf44f98b..6eb528ca985ed61 100644
--- a/Parser/tokenizer/reader.c
+++ b/Parser/tokenizer/reader.c
@@ -102,6 +102,7 @@ append_implicit_newline(_PyTok_Reader *reader)
 static int
 pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk)
 {
+    assert(reader->decoded_pos >= 0 && reader->decoded_pos <= reader->decoded_len);
     if (reader->decoded_pos == reader->decoded_len) {
         return 0;
     }
@@ -546,10 +547,13 @@ reset_streaming_buffer(struct tok_state *tok)
 int
 _PyTok_ReaderUnderflow(struct tok_state *tok)
 {
+    assert(tok->cur == tok->inp || (tok->buf != NULL &&
+           tok->cur >= tok->buf && tok->cur < tok->inp));
     _PyTok_ReaderKind kind = tok->reader->kind;
     int prepared = kind == _PYTOK_READER_PREPARED;
     int streaming = reader_is_streaming(kind);
-    int reset_buffer = !prepared && tok->start == NULL && !INSIDE_FSTRING(tok);
+    int reset_buffer = !prepared && tok->start == NULL &&
+        _PyLexer_CurrentFTString(tok) == NULL;
 
     _PyTok_Chunk chunk;
     _PyTok_ReadResult result = reader_next(tok, &chunk);
@@ -621,7 +625,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         tok->interactive_src_end = tok->source.bytes + tok->source.len;
     }
     if (prepared) {
-        if (tok->start == NULL && !INSIDE_FSTRING(tok)) {
+        if (tok->start == NULL && _PyLexer_CurrentFTString(tok) == NULL) {
             tok->buf = tok->cur;
             tok->buf_offset = tok->source.base_offset +
                 (chunk.data - tok->source.bytes);

From c0ee2951076ed363dbdeaebba283b4d093b97086 Mon Sep 17 00:00:00 2001
From: Pablo Galindo Salgado <pablogsal@gmail.com>
Date: Sun, 6 Sep 2026 23:10:01 +0100
Subject: [PATCH 4/4] gh-153569: centralize formatted-string transitions

---
 Parser/lexer/lexer.c          | 60 ++++++++++------------
 Parser/lexer/lexer_internal.h |  7 ++-
 Parser/lexer/string.c         | 96 +++++++++++++++++++++++++++++------
 3 files changed, 112 insertions(+), 51 deletions(-)

diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index 54dd0123e7ab7bb..c5157c1c021e012 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -552,26 +552,14 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
     /* Punctuation character */
     int is_punctuation = (c == ':' || c == '}' || c == '!');
     if (is_punctuation && current != NULL) {
-        int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
-        int at_expression_boundary =
-            bracket_depth == current->replacement_depth;
-        if (at_expression_boundary && c == '!') {
-            int c2 = tok_nextc(tok);
-            if (c2 == '=') {
-                at_expression_boundary = 0;
-            }
-            tok_backup(tok, c2);
-        }
-        if (at_expression_boundary &&
-                _PyLexer_finish_ftstring_expr(tok, current, token)) {
+        int type = _PyLexer_ftstring_punctuation(tok, current, token, c);
+        if (type < 0) {
             return MAKE_TOKEN(ERRORTOKEN);
         }
-
-        if (c == ':' && at_expression_boundary) {
-            current->mode = FTSTRING_MODE_FORMAT_SPEC;
+        if (type != 0) {
             p_start = tok->start;
             p_end = tok->cur;
-            return MAKE_TOKEN(_PyToken_OneChar(c));
+            return MAKE_TOKEN(type);
         }
     }
 
@@ -656,18 +644,9 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
                 }
             }
         }
-
-        if (current != NULL) {
-            int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
-            if (bracket_depth < 0) {
-                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
-                    _PyLexer_StringPrefix(current->kind), c));
-            }
-            if (c == '}' && bracket_depth == current->replacement_depth - 1) {
-                current->replacement_depth--;
-                current->mode = FTSTRING_MODE_MIDDLE;
-                current->debug_expr = 0;
-            }
+        if (current != NULL &&
+                _PyLexer_close_ftstring_expr(tok, current, c) < 0) {
+            return MAKE_TOKEN(ERRORTOKEN);
         }
         break;
     default:
@@ -678,9 +657,8 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
         return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
     }
 
-    if (c == '=' && current != NULL &&
-            _PyLexer_FTStringBracketDepth(tok, current) == current->replacement_depth) {
-        current->debug_expr = 1;
+    if (c == '=' && current != NULL) {
+        _PyLexer_mark_ftstring_debug(tok, current);
     }
 
     /* Punctuation character */
@@ -694,9 +672,23 @@ int
 _PyTokenizer_Get(struct tok_state *tok, struct token *token)
 {
     ftstring_state *current = _PyLexer_CurrentFTString(tok);
-    int result = current == NULL || current->mode == FTSTRING_MODE_EXPRESSION
-        ? _PyLexer_get_normal(tok, current, token)
-        : _PyLexer_get_ftstring(tok, current, token);
+    int result;
+    if (current == NULL) {
+        result = _PyLexer_get_normal(tok, NULL, token);
+    }
+    else {
+        switch (current->mode) {
+        case FTSTRING_MODE_EXPRESSION:
+            result = _PyLexer_get_normal(tok, current, token);
+            break;
+        case FTSTRING_MODE_MIDDLE:
+        case FTSTRING_MODE_FORMAT_SPEC:
+            result = _PyLexer_get_ftstring(tok, current, token);
+            break;
+        default:
+            Py_UNREACHABLE();
+        }
+    }
     if (tok_failed(tok)) {
         result = ERRORTOKEN;
     }
diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h
index 37e05c4fe153f9b..95d9754080bedbd 100644
--- a/Parser/lexer/lexer_internal.h
+++ b/Parser/lexer/lexer_internal.h
@@ -35,8 +35,11 @@ int _PyLexer_nextc(struct tok_state *);
 void _PyLexer_backup(struct tok_state *, int);
 int _PyLexer_record_ftstring_comment(
     struct tok_state *, ftstring_state *, const char *, const char *);
-int _PyLexer_finish_ftstring_expr(
-    struct tok_state *, ftstring_state *, struct token *);
+int _PyLexer_ftstring_punctuation(
+    struct tok_state *, ftstring_state *, struct token *, int);
+int _PyLexer_close_ftstring_expr(
+    struct tok_state *, ftstring_state *, int);
+void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *);
 int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
 int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);
diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c
index 37c816206c2c174..ac775e9d61ba096 100644
--- a/Parser/lexer/string.c
+++ b/Parser/lexer/string.c
@@ -54,9 +54,9 @@ _PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
     return 0;
 }
 
-int
-_PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
-                              struct token *token)
+static int
+finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
+                     struct token *token)
 {
     assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
     assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start != NULL);
@@ -127,6 +127,82 @@ _PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
     return 0;
 }
 
+static int
+begin_ftstring_expr(struct tok_state *tok, ftstring_state *state,
+                    _PyTok_Off expr_start)
+{
+    assert(state->mode != FTSTRING_MODE_EXPRESSION);
+    state->expr_span = (_PyTok_Span){expr_start, -1};
+    if (state->comments != NULL) {
+        state->comments->count = 0;
+    }
+    if (state->replacement_depth >= MAX_EXPR_NESTING) {
+        _PyTokenizer_syntaxerror(
+            tok, "%c-string: expressions nested too deeply",
+            _PyLexer_StringPrefix(state->kind));
+        return -1;
+    }
+    state->replacement_depth++;
+    state->mode = FTSTRING_MODE_EXPRESSION;
+    state->debug_expr = 0;
+    return 0;
+}
+
+int
+_PyLexer_ftstring_punctuation(struct tok_state *tok, ftstring_state *state,
+                              struct token *token, int c)
+{
+    assert(state->mode == FTSTRING_MODE_EXPRESSION);
+    assert(c == ':' || c == '}' || c == '!');
+    if (_PyLexer_FTStringBracketDepth(tok, state) != state->replacement_depth) {
+        return 0;
+    }
+    if (c == '!') {
+        int next = tok_nextc(tok);
+        tok_backup(tok, next);
+        if (next == '=') {
+            return 0;
+        }
+    }
+    if (finish_ftstring_expr(tok, state, token) < 0) {
+        return -1;
+    }
+    if (c == ':') {
+        state->mode = FTSTRING_MODE_FORMAT_SPEC;
+        return COLON;
+    }
+    return 0;
+}
+
+int
+_PyLexer_close_ftstring_expr(struct tok_state *tok, ftstring_state *state,
+                             int c)
+{
+    assert(state->mode == FTSTRING_MODE_EXPRESSION);
+    assert(c == ')' || c == ']' || c == '}');
+    int depth = _PyLexer_FTStringBracketDepth(tok, state);
+    if (depth < 0) {
+        _PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
+                                _PyLexer_StringPrefix(state->kind), c);
+        return -1;
+    }
+    if (c == '}' && depth == state->replacement_depth - 1) {
+        state->replacement_depth--;
+        state->mode = FTSTRING_MODE_MIDDLE;
+        state->debug_expr = 0;
+    }
+    return 0;
+}
+
+void
+_PyLexer_mark_ftstring_debug(struct tok_state *tok, ftstring_state *state)
+{
+    assert(state->mode == FTSTRING_MODE_EXPRESSION);
+    if (_PyLexer_FTStringBracketDepth(tok, state) == state->replacement_depth) {
+        state->debug_expr = 1;
+    }
+}
+
 int
 _PyLexer_check_string_prefixes(struct tok_state *tok,
                                              int saw_b, int saw_r, int saw_u,
@@ -408,21 +484,11 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
             int peek = tok_nextc(tok);
             if (peek != '{' || in_format_spec) {
                 tok_backup(tok, peek);
-                current->expr_span = (_PyTok_Span){
-                    _PyLexer_BufferOffset(tok, tok->cur), -1};
-                if (current->comments != NULL) {
-                    current->comments->count = 0;
-                }
+                _PyTok_Off expr_start = _PyLexer_BufferOffset(tok, tok->cur);
                 tok_backup(tok, c);
-                if (current->replacement_depth >= MAX_EXPR_NESTING) {
-                    _PyTokenizer_syntaxerror(
-                        tok, "%c-string: expressions nested too deeply",
-                        _PyLexer_StringPrefix(current->kind));
+                if (begin_ftstring_expr(tok, current, expr_start) < 0) {
                     return MAKE_TOKEN(ERRORTOKEN);
                 }
-                current->replacement_depth++;
-                current->mode = FTSTRING_MODE_EXPRESSION;
-                current->debug_expr = 0;
                 p_start = tok->start;
                 p_end = tok->cur;
                 if (p_start == p_end) {