First pass at update to grammar and action_helpers.c

Squashed commit of the following: commit 0941a3088578c1a8002e58ea6e11527bedf0e81e Author: Dave <davepeck@gmail.com> Date: Thu Jul 10 01:49:24 2025 +0000 Maybe cleaner? commit e5a69c851a0f43a4e84325051a37de20282dd810 Author: Dave <davepeck@gmail.com> Date: Wed Jul 9 23:17:58 2025 +0000 In progress: update grammar
python · lysnikolaou · Jul 21, 2025 · Jun 26, 2025 · Jun 26, 2025 · Jun 30, 2025
commit 860caea455b4a5876b13ed30896547244a697512
@@ -987,7 +987,10 @@ tstring[expr_ty] (memo):
             _PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) }
 
 string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) }
-strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
+strings[expr_ty] (memo):
+    | invalid_string_tstring_concat
+    | a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
+    | a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) }
 
 list[expr_ty]:
     | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) }
@@ -1553,6 +1556,12 @@ invalid_tstring_conversion_character:
     | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") }
     | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") }
 
+invalid_string_tstring_concat:
+    | (fstring|string)+ a=tstring+ {
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(PyPegen_first_item(a, expr_ty), "cannot mix t-strings with strings or f-strings") }
+    | tstring+ a=(fstring|string)+ {
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(PyPegen_first_item(a, expr_ty), "cannot mix t-strings with strings or f-strings") }
+
 invalid_arithmetic:
     | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
 invalid_factor:

@@ -240,30 +240,22 @@ def test_literal_concatenation(self):
         self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
         self.assertEqual(fstring(t), "Hello, Python")
 
-        # Test disallowed mix of t-string and string
+        # Test disallowed mix of t-string and string/f-string (incl. bytes)
         what = 't'
-        expected_msg = 'cannot mix str and Template literals'
+        expected_msg = 'cannot mix t-strings with strings or f-strings'
         for case in (
             "t'{what}-string literal' 'str literal'",
             "t'{what}-string literal' u'unicode literal'",
             "t'{what}-string literal' f'f-string literal'",
             "t'{what}-string literal' r'raw string literal'",
             "t'{what}-string literal' rf'raw f-string literal'",
+            "t'{what}-string literal' b'bytes literal'",
+            "t'{what}-string literal' br'raw bytes literal'",
             "'str literal' t'{what}-string literal'",
             "u'unicode literal' t'{what}-string literal'",
             "f'f-string literal' t'{what}-string literal'",
             "r'raw string literal' t'{what}-string literal'",
             "rf'raw f-string literal' t'{what}-string literal'",
-        ):
-            with self.subTest(case):
-                with self.assertRaisesRegex(SyntaxError, expected_msg):
-                    eval(case)
-
-        # Test disallowed mix of t-string and bytes
-        expected_msg = 'cannot mix bytes and nonbytes literals'
-        for case in (
-            "t'{what}-string literal' b'bytes literal'",
-            "t'{what}-string literal' br'raw bytes literal'",
             "b'bytes literal' t'{what}-string literal'",
             "br'raw bytes literal' t'{what}-string literal'",
         ):

@@ -1834,8 +1834,8 @@ _build_concatenated_joined_str(Parser *p, asdl_expr_seq *strings,
     return _PyAST_JoinedStr(values, lineno, col_offset, end_lineno, end_col_offset, p->arena);
 }
 
-static expr_ty
-_build_concatenated_template_str(Parser *p, asdl_expr_seq *strings,
+expr_ty
+_PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *strings,
                                int lineno, int col_offset, int end_lineno,
                                int end_col_offset, PyArena *arena)
 {
@@ -1853,7 +1853,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
     Py_ssize_t len = asdl_seq_LEN(strings);
     assert(len > 0);
 
-    int t_string_found = 0;
     int f_string_found = 0;
     int unicode_string_found = 0;
     int bytes_found = 0;
@@ -1872,29 +1871,20 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
             case JoinedStr_kind:
                 f_string_found = 1;
                 break;
-            case TemplateStr_kind:
-                t_string_found = 1;
-                break;
             default:
                 f_string_found = 1;
                 break;
         }
     }
 
     // Cannot mix unicode and bytes
-    if ((unicode_string_found || f_string_found || t_string_found) && bytes_found) {
+    if ((unicode_string_found || f_string_found) && bytes_found) {
         RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals");
         return NULL;
     }
 
-    // Cannot mix strings/f-strings and t-strings
-    if ((unicode_string_found || f_string_found) && t_string_found) {
-        RAISE_SYNTAX_ERROR("cannot mix str and Template literals");
-        return NULL;
-    }
-
     // If it's only bytes or only unicode string, do a simple concat
-    if (!f_string_found && !t_string_found) {
+    if (!f_string_found) {
         if (len == 1) {
             return asdl_seq_GET(strings, 0);
         }
@@ -1908,11 +1898,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings,
         }
     }
 
-    if (t_string_found) {
-        return _build_concatenated_template_str(p, strings, lineno,
-            col_offset, end_lineno, end_col_offset, arena);
-    }
-
     return _build_concatenated_joined_str(p, strings, lineno,
         col_offset, end_lineno, end_col_offset, arena);
 }