Skip to content

[3.14] gh-132661: Disallow Template/str concatenation after PEP 750 spec update (#135996) #136901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 21, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion Grammar/python.gram
Original file line number Diff line number Diff line change
Expand Up @@ -985,7 +985,10 @@ tstring[expr_ty] (memo):
_PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) }

string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) }
strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
strings[expr_ty] (memo):
| invalid_string_tstring_concat
| a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
| a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) }

list[expr_ty]:
| '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) }
Expand Down Expand Up @@ -1546,6 +1549,12 @@ invalid_tstring_conversion_character:
| '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") }
| '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") }

invalid_string_tstring_concat:
| a=(fstring|string)+ b[expr_ty]=tstring {
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
| a=tstring+ b[expr_ty]=(fstring|string) {
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }

invalid_arithmetic:
| sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
invalid_factor:
Expand Down
26 changes: 1 addition & 25 deletions Lib/_ast_unparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -626,35 +626,11 @@ def _write_ftstring(self, values, prefix):
)
self._ftstring_helper(fstring_parts)

def _tstring_helper(self, node):
if not node.values:
self._write_ftstring([], "t")
return
last_idx = 0
for i, value in enumerate(node.values):
# This can happen if we have an implicit concat of a t-string
# with an f-string
if isinstance(value, FormattedValue):
if i > last_idx:
# Write t-string until here
self._write_ftstring(node.values[last_idx:i], "t")
self.write(" ")
# Write f-string with the current formatted value
self._write_ftstring([node.values[i]], "f")
if i + 1 < len(node.values):
# Only add a space if there are more values after this
self.write(" ")
last_idx = i + 1

if last_idx < len(node.values):
# Write t-string from last_idx to end
self._write_ftstring(node.values[last_idx:], "t")

def visit_JoinedStr(self, node):
self._write_ftstring(node.values, "f")

def visit_TemplateStr(self, node):
self._tstring_helper(node)
self._write_ftstring(node.values, "t")

def _write_ftstring_inner(self, node, is_format_spec=False):
if isinstance(node, JoinedStr):
Expand Down
7 changes: 0 additions & 7 deletions Lib/test/test_ast/test_ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -999,13 +999,6 @@ def test_tstring(self):
self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)

# Test AST for implicit concat of t-string with f-string
tree = ast.parse('t"Hello {name}" f"{name}"')
self.assertIsInstance(tree.body[0].value, ast.TemplateStr)
self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
self.assertIsInstance(tree.body[0].value.values[2], ast.FormattedValue)


class CopyTests(unittest.TestCase):
"""Test copying and pickling AST nodes."""
Expand Down
73 changes: 25 additions & 48 deletions Lib/test/test_tstring.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,6 @@ def test_raw_tstrings(self):
t = tr"{path}\Documents"
self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])


def test_template_concatenation(self):
# Test template + template
t1 = t"Hello, "
Expand All @@ -161,9 +160,10 @@ def test_template_concatenation(self):

# Test template + string
t1 = t"Hello"
combined = t1 + ", world"
self.assertTStringEqual(combined, ("Hello, world",), ())
self.assertEqual(fstring(combined), "Hello, world")
expected_msg = 'can only concatenate string.templatelib.Template ' \
'\\(not "str"\\) to string.templatelib.Template'
with self.assertRaisesRegex(TypeError, expected_msg):
t1 + ", world"

# Test template + template with interpolation
name = "Python"
Expand All @@ -174,9 +174,10 @@ def test_template_concatenation(self):
self.assertEqual(fstring(combined), "Hello, Python")

# Test string + template
t = "Hello, " + t"{name}"
self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
self.assertEqual(fstring(t), "Hello, Python")
expected_msg = 'can only concatenate str ' \
'\\(not "string.templatelib.Template"\\) to str'
with self.assertRaisesRegex(TypeError, expected_msg):
"Hello, " + t"{name}"

def test_nested_templates(self):
# Test a template inside another template expression
Expand Down Expand Up @@ -241,52 +242,28 @@ def test_literal_concatenation(self):
self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
self.assertEqual(fstring(t), "Hello, Python")

# Test concatenation with string literal
name = "Python"
t = t"Hello, {name}" "and welcome!"
self.assertTStringEqual(
t, ("Hello, ", "and welcome!"), [(name, "name")]
)
self.assertEqual(fstring(t), "Hello, Pythonand welcome!")

# Test concatenation with Unicode literal
name = "Python"
t = t"Hello, {name}" u"and welcome!"
self.assertTStringEqual(
t, ("Hello, ", "and welcome!"), [(name, "name")]
)
self.assertEqual(fstring(t), "Hello, Pythonand welcome!")

# Test concatenation with f-string literal
tab = '\t'
t = t"Tab: {tab}. " f"f-tab: {tab}."
self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")])
self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.")

# Test concatenation with raw string literal
tab = '\t'
t = t"Tab: {tab}. " r"Raw tab: \t."
self.assertTStringEqual(
t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")]
)
self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.")

# Test concatenation with raw f-string literal
tab = '\t'
t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t."
self.assertTStringEqual(
t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")]
)
self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.")

# Test disallowed mix of t-string and string/f-string (incl. bytes)
what = 't'
expected_msg = 'cannot mix bytes and nonbytes literals'
expected_msg = 'cannot mix t-string literals with string or bytes literals'
for case in (
"t'{what}-string literal' 'str literal'",
"t'{what}-string literal' u'unicode literal'",
"t'{what}-string literal' f'f-string literal'",
"t'{what}-string literal' r'raw string literal'",
"t'{what}-string literal' rf'raw f-string literal'",
"t'{what}-string literal' b'bytes literal'",
"t'{what}-string literal' br'raw bytes literal'",
"'str literal' t'{what}-string literal'",
"u'unicode literal' t'{what}-string literal'",
"f'f-string literal' t'{what}-string literal'",
"r'raw string literal' t'{what}-string literal'",
"rf'raw f-string literal' t'{what}-string literal'",
"b'bytes literal' t'{what}-string literal'",
"br'raw bytes literal' t'{what}-string literal'",
):
with self.assertRaisesRegex(SyntaxError, expected_msg):
eval(case)
with self.subTest(case):
with self.assertRaisesRegex(SyntaxError, expected_msg):
eval(case)

def test_triple_quoted(self):
# Test triple-quoted t-strings
Expand Down
4 changes: 0 additions & 4 deletions Lib/test/test_unparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,10 +206,6 @@ def test_tstrings(self):
self.check_ast_roundtrip("t'foo'")
self.check_ast_roundtrip("t'foo {bar}'")
self.check_ast_roundtrip("t'foo {bar!s:.2f}'")
self.check_ast_roundtrip("t'foo {bar}' f'{bar}'")
self.check_ast_roundtrip("f'{bar}' t'foo {bar}'")
self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'")
self.check_ast_roundtrip("t'foo {bar}' u'bar'")

def test_strings(self):
self.check_ast_roundtrip("u'foo'")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Reflect recent :pep:`750` change.

Disallow concatenation of ``string.templatelib.Template`` and :class:`str`.
Also, disallow implicit concatenation of t-string literals with string or
f-string literals.
91 changes: 7 additions & 84 deletions Objects/templateobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ templateiter_next(PyObject *op)
Py_SETREF(item, PyIter_Next(self->interpolationsiter));
self->from_strings = 1;
}
} else {
}
else {
item = PyIter_Next(self->interpolationsiter);
self->from_strings = 1;
}
Expand Down Expand Up @@ -245,54 +246,6 @@ template_iter(PyObject *op)
return (PyObject *)iter;
}

static PyObject *
template_strings_append_str(PyObject *strings, PyObject *str)
{
Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1);
PyObject *concat = PyUnicode_Concat(string, str);
if (concat == NULL) {
return NULL;
}

PyObject *newstrings = PyTuple_New(stringslen);
if (newstrings == NULL) {
Py_DECREF(concat);
return NULL;
}

for (Py_ssize_t i = 0; i < stringslen - 1; i++) {
PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
}
PyTuple_SET_ITEM(newstrings, stringslen - 1, concat);

return newstrings;
}

static PyObject *
template_strings_prepend_str(PyObject *strings, PyObject *str)
{
Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
PyObject *string = PyTuple_GET_ITEM(strings, 0);
PyObject *concat = PyUnicode_Concat(str, string);
if (concat == NULL) {
return NULL;
}

PyObject *newstrings = PyTuple_New(stringslen);
if (newstrings == NULL) {
Py_DECREF(concat);
return NULL;
}

PyTuple_SET_ITEM(newstrings, 0, concat);
for (Py_ssize_t i = 1; i < stringslen; i++) {
PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
}

return newstrings;
}

static PyObject *
template_strings_concat(PyObject *left, PyObject *right)
{
Expand Down Expand Up @@ -344,47 +297,17 @@ template_concat_templates(templateobject *self, templateobject *other)
return newtemplate;
}

static PyObject *
template_concat_template_str(templateobject *self, PyObject *other)
{
PyObject *newstrings = template_strings_append_str(self->strings, other);
if (newstrings == NULL) {
return NULL;
}

PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
Py_DECREF(newstrings);
return newtemplate;
}

static PyObject *
template_concat_str_template(templateobject *self, PyObject *other)
{
PyObject *newstrings = template_strings_prepend_str(self->strings, other);
if (newstrings == NULL) {
return NULL;
}

PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
Py_DECREF(newstrings);
return newtemplate;
}

PyObject *
_PyTemplate_Concat(PyObject *self, PyObject *other)
{
if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) {
return template_concat_templates((templateobject *) self, (templateobject *) other);
}
else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) {
return template_concat_template_str((templateobject *) self, other);
}
else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) {
return template_concat_str_template((templateobject *) other, self);
}
else {
Py_RETURN_NOTIMPLEMENTED;
}

PyErr_Format(PyExc_TypeError,
"can only concatenate string.templatelib.Template (not \"%T\") to string.templatelib.Template",
other);
return NULL;
}

static PyObject *
Expand Down
15 changes: 4 additions & 11 deletions Objects/unicodeobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#include "pycore_pyhash.h" // _Py_HashSecret_t
#include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_template.h" // _PyTemplate_Concat()
#include "pycore_tuple.h" // _PyTuple_FromArray()
#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
#include "pycore_unicodeobject.h" // struct _Py_unicode_state
Expand Down Expand Up @@ -11639,16 +11638,10 @@ PyUnicode_Concat(PyObject *left, PyObject *right)
return NULL;

if (!PyUnicode_Check(right)) {
if (_PyTemplate_CheckExact(right)) {
// str + tstring is implemented in the tstring type
return _PyTemplate_Concat(left, right);
}
else {
PyErr_Format(PyExc_TypeError,
"can only concatenate str (not \"%.200s\") to str",
Py_TYPE(right)->tp_name);
return NULL;
}
PyErr_Format(PyExc_TypeError,
"can only concatenate str (not \"%.200s\") to str",
Py_TYPE(right)->tp_name);
return NULL;
}

/* Shortcuts */
Expand Down
Loading
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy