From 048cd5a7e7f7c841e7f6484cfd039470883e7952 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 11 Mar 2022 21:44:27 +0200 Subject: [PATCH 1/3] bpo-43224: Implement substitution of unpacked TypeVarTuple in C --- Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_runtime_init.h | 1 + Lib/test/test_typing.py | 2 +- Lib/typing.py | 8 +- Objects/genericaliasobject.c | 112 ++++++++++++++++++++--- 5 files changed, 107 insertions(+), 17 deletions(-) diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 35bffa7aff9493..3232ab368973d2 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -200,6 +200,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__truediv__) STRUCT_FOR_ID(__trunc__) STRUCT_FOR_ID(__typing_subst__) + STRUCT_FOR_ID(__typing_unpacked__) STRUCT_FOR_ID(__warningregistry__) STRUCT_FOR_ID(__weakref__) STRUCT_FOR_ID(__xor__) diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 20d543a8cbc565..cf9f256fac788d 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -823,6 +823,7 @@ extern "C" { INIT_ID(__truediv__), \ INIT_ID(__trunc__), \ INIT_ID(__typing_subst__), \ + INIT_ID(__typing_unpacked__), \ INIT_ID(__warningregistry__), \ INIT_ID(__weakref__), \ INIT_ID(__xor__), \ diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a6936653bc566f..84b4b60549265b 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -467,7 +467,7 @@ def test_var_substitution(self): T2 = TypeVar('T2') class G(Generic[Unpack[Ts]]): pass - for A in G, Tuple: + for A in G, Tuple, tuple: B = A[Unpack[Ts]] if A != Tuple: self.assertEqual(B[()], A[()]) diff --git a/Lib/typing.py b/Lib/typing.py index 842554f193ca79..40032ad041f311 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1622,11 +1622,15 @@ def __repr__(self): return '*' + repr(self.__args__[0]) def __getitem__(self, args): - if (len(self.__parameters__) == 1 and - isinstance(self.__parameters__[0], TypeVarTuple)): + if self.__typing_unpacked__(): return args return super().__getitem__(args) + def __typing_unpacked__(self): + # If x is Unpack[tuple[...]], __parameters__ will be empty. + return bool(self.__parameters__ and + isinstance(self.__parameters__[0], TypeVarTuple)) + class Generic: """Abstract base class for generic types. diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 45caf2e2ee7db0..64e55c703d3cdd 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -175,6 +175,23 @@ tuple_add(PyObject *self, Py_ssize_t len, PyObject *item) return 0; } +static Py_ssize_t +tuple_extend(PyObject **dst, Py_ssize_t dstindex, + PyObject **src, Py_ssize_t count) +{ + assert(count >= 0); + if (_PyTuple_Resize(dst, PyTuple_GET_SIZE(*dst) + count - 1) != 0) { + return -1; + } + assert(dstindex + count <= PyTuple_GET_SIZE(*dst)); + for (Py_ssize_t i = 0; i < count; ++i) { + PyObject *item = src[i]; + Py_INCREF(item); + PyTuple_SET_ITEM(*dst, dstindex + i, item); + } + return dstindex + count; +} + PyObject * _Py_make_parameters(PyObject *args) { @@ -236,7 +253,8 @@ _Py_make_parameters(PyObject *args) If obj doesn't have a __parameters__ attribute or that's not a non-empty tuple, return a new reference to obj. */ static PyObject * -subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems) +subs_tvars(PyObject *obj, PyObject *params, + PyObject **argitems, Py_ssize_t nargs, Py_ssize_t varparam) { PyObject *subparams; if (_PyObject_LookupAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) { @@ -250,14 +268,27 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems) Py_DECREF(subparams); return NULL; } - for (Py_ssize_t i = 0; i < nsubargs; ++i) { + for (Py_ssize_t i = 0, j = 0; i < nsubargs; ++i) { PyObject *arg = PyTuple_GET_ITEM(subparams, i); Py_ssize_t iparam = tuple_index(params, nparams, arg); - if (iparam >= 0) { - arg = argitems[iparam]; + if (iparam == varparam) { + j = tuple_extend(&subargs, j, + argitems + iparam, nargs - nparams + 1); + if (j < 0) { + return NULL; + } + } + else { + if (iparam >= 0) { + if (iparam > varparam) { + iparam += nargs - nsubargs; + } + arg = argitems[iparam]; + } + Py_INCREF(arg); + PyTuple_SET_ITEM(subargs, j, arg); + j++; } - Py_INCREF(arg); - PyTuple_SET_ITEM(subargs, i, arg); } obj = PyObject_GetItem(obj, subargs); @@ -271,6 +302,23 @@ subs_tvars(PyObject *obj, PyObject *params, PyObject **argitems) return obj; } +static int +_is_unpacked_typevartuple(PyObject *arg) +{ + PyObject *meth; + int res = _PyObject_LookupAttr(arg, &_Py_ID(__typing_unpacked__), &meth); + if (res > 0) { + PyObject *tmp = PyObject_CallNoArgs(meth); + Py_DECREF(meth); + if (tmp == NULL) { + return -1; + } + res = PyObject_IsTrue(tmp); + Py_DECREF(tmp); + } + return res; +} + PyObject * _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObject *item) { @@ -283,11 +331,27 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje int is_tuple = PyTuple_Check(item); Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1; PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item; - if (nitems != nparams) { - return PyErr_Format(PyExc_TypeError, - "Too %s arguments for %R", - nitems > nparams ? "many" : "few", - self); + Py_ssize_t varparam = 0; + for (; varparam < nparams; varparam++) { + PyObject *param = PyTuple_GET_ITEM(parameters, varparam); + if (Py_TYPE(param)->tp_iter) { // TypeVarTuple + break; + } + } + if (varparam < nparams) { + if (nitems < nparams - 1) { + return PyErr_Format(PyExc_TypeError, + "!Too few arguments for %R", + self); + } + } + else { + if (nitems != nparams) { + return PyErr_Format(PyExc_TypeError, + "Too %s arguments for %R", + nitems > nparams ? "many" : "few", + self); + } } /* Replace all type variables (specified by parameters) with corresponding values specified by argitems. @@ -300,8 +364,13 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje if (newargs == NULL) { return NULL; } - for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { + for (Py_ssize_t iarg = 0, jarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(args, iarg); + int unpack = _is_unpacked_typevartuple(arg); + if (unpack < 0) { + Py_DECREF(newargs); + return NULL; + } PyObject *subst; if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) { Py_DECREF(newargs); @@ -310,17 +379,32 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje if (subst) { Py_ssize_t iparam = tuple_index(parameters, nparams, arg); assert(iparam >= 0); + assert(iparam != varparam); + if (iparam > varparam) { + iparam += nitems - nparams; + } arg = PyObject_CallOneArg(subst, argitems[iparam]); Py_DECREF(subst); } else { - arg = subs_tvars(arg, parameters, argitems); + arg = subs_tvars(arg, parameters, argitems, nitems, varparam); } if (arg == NULL) { Py_DECREF(newargs); return NULL; } - PyTuple_SET_ITEM(newargs, iarg, arg); + if (unpack) { + jarg = tuple_extend(&newargs, jarg, + &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg)); + Py_DECREF(arg); + if (jarg < 0) { + return NULL; + } + } + else { + PyTuple_SET_ITEM(newargs, jarg, arg); + jarg++; + } } return newargs; From f4dd9b3c51f96af455bf4444622e53b5243cea94 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 12 Mar 2022 09:19:24 +0200 Subject: [PATCH 2/3] Raise explicit error if substitute a bare TypeVarTuple --- Lib/test/test_typing.py | 15 +++++++++++++++ Lib/typing.py | 2 +- Objects/genericaliasobject.c | 10 ++++++++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 84b4b60549265b..cf5687f6291349 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -502,6 +502,21 @@ class G(Generic[Unpack[Ts]]): pass self.assertEqual(E[float, str, int, bytes], Tuple[List[float], A[str, int], List[bytes]]) + def test_bad_var_substitution(self): + Ts = TypeVarTuple('Ts') + T = TypeVar('T') + T2 = TypeVar('T2') + class G(Generic[Unpack[Ts]]): pass + + for A in G, Tuple, tuple: + B = A[Ts] + with self.assertRaises(TypeError): + B[int, str] + + C = A[T, T2] + with self.assertRaises(TypeError): + C[Unpack[Ts]] + def test_repr_is_correct(self): Ts = TypeVarTuple('Ts') self.assertEqual(repr(Ts), 'Ts') diff --git a/Lib/typing.py b/Lib/typing.py index 40032ad041f311..008417905a9271 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -950,7 +950,7 @@ def __repr__(self): return self._name def __typing_subst__(self, arg): - raise AssertionError + raise TypeError("Substitution of bare TypeVarTuple is not supported") class ParamSpecArgs(_Final, _Immutable, _root=True): diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 64e55c703d3cdd..2eb40c033a486a 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -341,7 +341,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje if (varparam < nparams) { if (nitems < nparams - 1) { return PyErr_Format(PyExc_TypeError, - "!Too few arguments for %R", + "Too few arguments for %R", self); } } @@ -379,7 +379,13 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje if (subst) { Py_ssize_t iparam = tuple_index(parameters, nparams, arg); assert(iparam >= 0); - assert(iparam != varparam); + if (iparam == varparam) { + Py_DECREF(subst); + Py_DECREF(newargs); + PyErr_SetString(PyExc_TypeError, + "Substitution of bare TypeVarTuple is not supported"); + return NULL; + } if (iparam > varparam) { iparam += nitems - nparams; } From 0b9a4d3effc89a10fe30d3f7b1ab63409a8b56a3 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 29 Apr 2022 15:09:17 -0600 Subject: [PATCH 3/3] Update unpack test cases (Code by Matthew Rahtz) Co-authored-by: Matthew Rahtz --- Lib/test/test_typing.py | 46 ++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 9eb8e76a962176..30d362284fdb34 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -754,89 +754,89 @@ class C(Generic[*Ts]): pass tests = [ # Alias # Args # Expected result ('C[*Ts]', '[()]', 'C[()]'), - ('tuple[*Ts]', '[()]', 'TypeError'), # Should be tuple[()] + ('tuple[*Ts]', '[()]', 'tuple[()]'), ('Tuple[*Ts]', '[()]', 'Tuple[()]'), ('C[*Ts]', '[int]', 'C[int]'), - ('tuple[*Ts]', '[int]', 'tuple[(int,),]'), # Should be tuple[int] + ('tuple[*Ts]', '[int]', 'tuple[int]'), ('Tuple[*Ts]', '[int]', 'Tuple[int]'), ('C[*Ts]', '[int, str]', 'C[int, str]'), - ('tuple[*Ts]', '[int, str]', 'TypeError'), # Should be tuple[int, str] + ('tuple[*Ts]', '[int, str]', 'tuple[int, str]'), ('Tuple[*Ts]', '[int, str]', 'Tuple[int, str]'), ('C[*Ts]', '[*tuple_type[int]]', 'C[*tuple_type[int]]'), # Should be C[int] - ('tuple[*Ts]', '[*tuple_type[int]]', 'tuple[(*tuple_type[int],),]'), # Should be tuple[int] + ('tuple[*Ts]', '[*tuple_type[int]]', 'tuple[*tuple_type[int]]'), # Should be tuple[int] ('Tuple[*Ts]', '[*tuple_type[int]]', 'Tuple[*tuple_type[int]]'), # Should be Tuple[int] ('C[*Ts]', '[*tuple_type[*Ts]]', 'C[*tuple_type[*Ts]]'), # Should be C[*Ts] - ('tuple[*Ts]', '[*tuple_type[*Ts]]', 'tuple[(*tuple_type[*Ts],),]'), # Should be tuple[*Ts] + ('tuple[*Ts]', '[*tuple_type[*Ts]]', 'tuple[*tuple_type[*Ts]]'), # Should be tuple[*Ts] ('Tuple[*Ts]', '[*tuple_type[*Ts]]', 'Tuple[*tuple_type[*Ts]]'), # Should be Tuple[*Ts] ('C[*Ts]', '[*tuple_type[int, str]]', 'C[*tuple_type[int, str]]'), # Should be C[int, str] - ('tuple[*Ts]', '[*tuple_type[int, str]]', 'tuple[(*tuple_type[int, str],),]'), # Should be tuple[int, str] + ('tuple[*Ts]', '[*tuple_type[int, str]]', 'tuple[*tuple_type[int, str]]'), # Should be tuple[int, str] ('Tuple[*Ts]', '[*tuple_type[int, str]]', 'Tuple[*tuple_type[int, str]]'), # Should be Tuple[int, str] ('C[*Ts]', '[tuple_type[int, ...]]', 'C[tuple_type[int, ...]]'), - ('tuple[*Ts]', '[tuple_type[int, ...]]', 'tuple[(tuple_type[int, ...],),]'), # Should be tuple[tuple_type[int, ...]] + ('tuple[*Ts]', '[tuple_type[int, ...]]', 'tuple[tuple_type[int, ...]]'), ('Tuple[*Ts]', '[tuple_type[int, ...]]', 'Tuple[tuple_type[int, ...]]'), ('C[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'C[tuple_type[int, ...], tuple_type[str, ...]]'), - ('tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'TypeError'), # Should be tuple[tuple_type[int, ...], tuple_type[str, ...]] + ('tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'tuple[tuple_type[int, ...], tuple_type[str, ...]]'), ('Tuple[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'Tuple[tuple_type[int, ...], tuple_type[str, ...]]'), ('C[*Ts]', '[*tuple_type[int, ...]]', 'C[*tuple_type[int, ...]]'), - ('tuple[*Ts]', '[*tuple_type[int, ...]]', 'tuple[(*tuple_type[int, ...],),]'), # Should be tuple[*tuple_type[int, ...]] + ('tuple[*Ts]', '[*tuple_type[int, ...]]', 'tuple[*tuple_type[int, ...]]'), ('Tuple[*Ts]', '[*tuple_type[int, ...]]', 'Tuple[*tuple_type[int, ...]]'), # Technically, multiple unpackings are forbidden by PEP 646, but we # choose to be less restrictive at runtime, to allow folks room # to experiment. So all three of these should be valid. ('C[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'C[*tuple_type[int, ...], *tuple_type[str, ...]]'), - # Should be tuple[*tuple_type[int, ...], *tuple_type[str, ...]], to match the other two. - ('tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'), + ('tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'), ('Tuple[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'Tuple[*tuple_type[int, ...], *tuple_type[str, ...]]'), ('C[*Ts]', '[*Ts]', 'C[*Ts]'), - ('tuple[*Ts]', '[*Ts]', 'tuple[(*Ts,),]'), # Should be tuple[*Ts] + ('tuple[*Ts]', '[*Ts]', 'tuple[*Ts]'), ('Tuple[*Ts]', '[*Ts]', 'Tuple[*Ts]'), ('C[*Ts]', '[T, *Ts]', 'C[T, *Ts]'), - ('tuple[*Ts]', '[T, *Ts]', 'TypeError'), # Should be tuple[T, *Ts] + ('tuple[*Ts]', '[T, *Ts]', 'tuple[T, *Ts]'), ('Tuple[*Ts]', '[T, *Ts]', 'Tuple[T, *Ts]'), ('C[*Ts]', '[*Ts, T]', 'C[*Ts, T]'), - ('tuple[*Ts]', '[*Ts, T]', 'TypeError'), # Should be tuple[*Ts, T] + ('tuple[*Ts]', '[*Ts, T]', 'tuple[*Ts, T]'), ('Tuple[*Ts]', '[*Ts, T]', 'Tuple[*Ts, T]'), ('C[T, *Ts]', '[int]', 'C[int]'), - ('tuple[T, *Ts]', '[int]', 'TypeError'), # Should be tuple[int] + ('tuple[T, *Ts]', '[int]', 'tuple[int]'), ('Tuple[T, *Ts]', '[int]', 'Tuple[int]'), ('C[T, *Ts]', '[int, str]', 'C[int, str]'), - ('tuple[T, *Ts]', '[int, str]', 'tuple[int, (str,)]'), # Should be tuple[int, str] + ('tuple[T, *Ts]', '[int, str]', 'tuple[int, str]'), ('Tuple[T, *Ts]', '[int, str]', 'Tuple[int, str]'), ('C[T, *Ts]', '[int, str, bool]', 'C[int, str, bool]'), - ('tuple[T, *Ts]', '[int, str, bool]', 'TypeError'), # Should be tuple[int, str, bool] + ('tuple[T, *Ts]', '[int, str, bool]', 'tuple[int, str, bool]'), ('Tuple[T, *Ts]', '[int, str, bool]', 'Tuple[int, str, bool]'), ('C[T, *Ts]', '[*tuple[int, ...]]', 'C[*tuple[int, ...]]'), # Should be C[int, *tuple[int, ...]] ('C[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Ditto - ('tuple[T, *Ts]', '[*tuple_type[int, ...]]', 'TypeError'), # Should be tuple[int, *tuple[int, ...]] - ('Tuple[T, *Ts]', '[*tuple[int, ...]]', 'Tuple[*tuple[int, ...]]'), # Ditto - ('Tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Ditto + ('tuple[T, *Ts]', '[*tuple[int, ...]]', 'tuple[*tuple[int, ...]]'), # Should be tuple[int, *tuple[int, ...]] + ('tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be tuple[int, *Tuple[int, ...]] + ('Tuple[T, *Ts]', '[*tuple[int, ...]]', 'Tuple[*tuple[int, ...]]'), # Should be Tuple[int, *tuple[int, ...]] + ('Tuple[T, *Ts]', '[*Tuple[int, ...]]', 'TypeError'), # Should be Tuple[int, *Tuple[int, ...]] ('C[*Ts, T]', '[int]', 'C[int]'), - ('tuple[*Ts, T]', '[int]', 'TypeError'), # Should be tuple[int] + ('tuple[*Ts, T]', '[int]', 'tuple[int]'), ('Tuple[*Ts, T]', '[int]', 'Tuple[int]'), ('C[*Ts, T]', '[int, str]', 'C[int, str]'), - ('tuple[*Ts, T]', '[int, str]', 'tuple[(int,), str]'), # Should be tuple[int, str] + ('tuple[*Ts, T]', '[int, str]', 'tuple[int, str]'), ('Tuple[*Ts, T]', '[int, str]', 'Tuple[int, str]'), ('C[*Ts, T]', '[int, str, bool]', 'C[int, str, bool]'), - ('tuple[*Ts, T]', '[int, str, bool]', 'TypeError'), # Should be tuple[int, str, bool] + ('tuple[*Ts, T]', '[int, str, bool]', 'tuple[int, str, bool]'), ('Tuple[*Ts, T]', '[int, str, bool]', 'Tuple[int, str, bool]'), ('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'), pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy