From 197e61c6bbca7b35414f341f0596391ecf17a31f Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 9 Jun 2023 21:27:40 +0200 Subject: [PATCH 01/27] API: Switch to "weak" promotion state by default --- numpy/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/__init__.py b/numpy/__init__.py index 065a1fe14a91..f51db94f0954 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -517,9 +517,9 @@ def hugepage_setup(): # it is tidier organized. _core.multiarray._multiarray_umath._reload_guard() - # TODO: Switch to defaulting to "weak". + # TODO: Remove the environment variable entirely now that it is "weak" _core._set_promotion_state( - os.environ.get("NPY_PROMOTION_STATE", "legacy")) + os.environ.get("NPY_PROMOTION_STATE", "weak")) # Tell PyInstaller where to find hook-numpy.py def _pyinstaller_hooks_dir(): From af5fe2ee3ac2128766fb7a6a61964143796c3994 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 9 Jun 2023 13:09:22 +0200 Subject: [PATCH 02/27] MAINT,TST: Fix issues due to out-of-range integers This contains two fixes: 1. a somewhat hackish solution to keep allowing very large integers for unary ufuncs. This is partially wrong, but keeps isnan and other functions working (reducing fallout in the tests quite a bit) 2. Generally adapt changes that now fail for the same reason, this is mainly due to comparisons like `-1 < uint8` now failing in generally. That is fixable in principle, but it isn't super easy. Neither is perfect, both are the pragmatic solutions at this time until they proof to be too problematic for some reason. (Getting rid of going to object for large integers is tougher and needs deeper work. It certainly is possible but too difficult at this stage: we need to remove some old branches first. We could, and maybe should, get rid of the `uint` step though, it is even more of a trap I think!) --- numpy/_core/src/umath/ufunc_object.c | 13 +++++++++ numpy/_core/tests/test_multiarray.py | 28 +++++++++++++++---- numpy/_core/tests/test_numeric.py | 2 +- numpy/_core/tests/test_scalar_ctors.py | 2 +- numpy/_core/tests/test_scalarmath.py | 16 ++++++----- numpy/_core/tests/test_umath.py | 37 +++++++++++++++++--------- 6 files changed, 71 insertions(+), 27 deletions(-) diff --git a/numpy/_core/src/umath/ufunc_object.c b/numpy/_core/src/umath/ufunc_object.c index eacd8bc9969e..856f16785485 100644 --- a/numpy/_core/src/umath/ufunc_object.c +++ b/numpy/_core/src/umath/ufunc_object.c @@ -935,6 +935,19 @@ convert_ufunc_arguments(PyUFuncObject *ufunc, out_op_DTypes[i] = NPY_DTYPE(PyArray_DESCR(out_op[i])); Py_INCREF(out_op_DTypes[i]); + if (nin == 1) { + /* + * TODO: If nin == 1 we don't promote! This has exactly the effect + * that right now integers can still go to object/uint64 and + * their behavior is thus unchanged for unary ufuncs (like + * isnan). This is not ideal, but pragmatic... + * We should eventually have special loops for isnan and once + * we do, we may just deprecate all remaining ones (e.g. + * `negative(2**100)` not working as it is an object.) + */ + break; + } + if (!NPY_DT_is_legacy(out_op_DTypes[i])) { *allow_legacy_promotion = NPY_FALSE; // TODO: A subclass of int, float, complex could reach here and diff --git a/numpy/_core/tests/test_multiarray.py b/numpy/_core/tests/test_multiarray.py index df4134f8d29d..24993f010c29 100644 --- a/numpy/_core/tests/test_multiarray.py +++ b/numpy/_core/tests/test_multiarray.py @@ -4125,7 +4125,8 @@ def test_extension_incref_elide_stack(self): def test_temporary_with_cast(self): # check that we don't elide into a temporary which would need casting d = np.ones(200000, dtype=np.int64) - assert_equal(((d + d) + 2**222).dtype, np.dtype('O')) + r = ((d + d) + np.array(2**222, dtype='O')) + assert_equal(r.dtype, np.dtype('O')) r = ((d + d) / 2) assert_equal(r.dtype, np.dtype('f8')) @@ -5012,7 +5013,19 @@ def test_basic(self): self._clip_type( 'uint', 1024, 0, 0, inplace=inplace) self._clip_type( - 'uint', 1024, -120, 100, inplace=inplace, expected_min=0) + 'uint', 1024, 10, 100, inplace=inplace) + + @pytest.mark.parametrize("inplace", [False, True]) + def test_int_range_error(self, inplace): + # E.g. clipping uint with negative integers fails to promote + # (changed with NEP 50 and may be adaptable) + # Similar to last check in `test_basic` + x = (np.random.random(1000) * 255).astype("uint8") + with pytest.raises(OverflowError): + x.clip(-1, 10, out=x if inplace else None) + + with pytest.raises(OverflowError): + x.clip(0, 256, out=x if inplace else None) def test_record_array(self): rec = np.array([(-5, 2.0, 3.0), (5.0, 4.0, 3.0)], @@ -8694,9 +8707,14 @@ def test_array_scalar_relational_operation(self): # Unsigned integers for dt1 in 'BHILQP': - assert_(-1 < np.array(1, dtype=dt1), "type %s failed" % (dt1,)) - assert_(not -1 > np.array(1, dtype=dt1), "type %s failed" % (dt1,)) - assert_(-1 != np.array(1, dtype=dt1), "type %s failed" % (dt1,)) + # NEP 50 broke comparison of unsigned with -1 for the time being + # (this may be fixed in the future of course). + with pytest.raises(OverflowError): + -1 < np.array(1, dtype=dt1) + with pytest.raises(OverflowError): + -1 > np.array(1, dtype=dt1) + with pytest.raises(OverflowError): + -1 != np.array(1, dtype=dt1) # Unsigned vs signed for dt2 in 'bhilqp': diff --git a/numpy/_core/tests/test_numeric.py b/numpy/_core/tests/test_numeric.py index e7a656671674..2a0798b2d211 100644 --- a/numpy/_core/tests/test_numeric.py +++ b/numpy/_core/tests/test_numeric.py @@ -2600,7 +2600,7 @@ def test_clip_value_min_max_flip(self, amin, amax): @pytest.mark.parametrize("arr, amin, amax, exp", [ # for a bug in npy_ObjectClip, based on a # case produced by hypothesis - (np.zeros(10, dtype=np.int64), + (np.zeros(10, dtype=object), 0, -2**64+1, np.full(10, -2**64+1, dtype=object)), diff --git a/numpy/_core/tests/test_scalar_ctors.py b/numpy/_core/tests/test_scalar_ctors.py index bb0e619ccada..a4d61c127624 100644 --- a/numpy/_core/tests/test_scalar_ctors.py +++ b/numpy/_core/tests/test_scalar_ctors.py @@ -78,7 +78,7 @@ def test_intp(self): assert_equal(1024, np.intp(1024)) def test_uint64_from_negative(self): - with pytest.warns(DeprecationWarning): + with pytest.raises(OverflowError): assert_equal(np.uint64(-2), np.uint64(18446744073709551614)) diff --git a/numpy/_core/tests/test_scalarmath.py b/numpy/_core/tests/test_scalarmath.py index 12a58711026d..5d339f193318 100644 --- a/numpy/_core/tests/test_scalarmath.py +++ b/numpy/_core/tests/test_scalarmath.py @@ -485,10 +485,8 @@ def test_int_from_long(self): def test_iinfo_long_values(self): for code in 'bBhH': - with pytest.warns(DeprecationWarning): - res = np.array(np.iinfo(code).max + 1, dtype=code) - tgt = np.iinfo(code).min - assert_(res == tgt) + with pytest.raises(OverflowError): + np.array(np.iinfo(code).max + 1, dtype=code) for code in np.typecodes['AllInteger']: res = np.array(np.iinfo(code).max, dtype=code) @@ -560,9 +558,13 @@ def test_numpy_scalar_relational_operators(self): #Unsigned integers for dt1 in 'BHILQP': - assert_(-1 < np.array(1, dtype=dt1)[()], "type %s failed" % (dt1,)) - assert_(not -1 > np.array(1, dtype=dt1)[()], "type %s failed" % (dt1,)) - assert_(-1 != np.array(1, dtype=dt1)[()], "type %s failed" % (dt1,)) + # NEP 50 means the -1 is currently rejected (we may change this) + with pytest.raises(OverflowError): + -1 < np.array(1, dtype=dt1)[()] + with pytest.raises(OverflowError): + -1 > np.array(1, dtype=dt1)[()] + with pytest.raises(OverflowError): + -1 != np.array(1, dtype=dt1)[()] #unsigned vs signed for dt2 in 'bhilqp': diff --git a/numpy/_core/tests/test_umath.py b/numpy/_core/tests/test_umath.py index 1f33083f59cb..b27ee23c6708 100644 --- a/numpy/_core/tests/test_umath.py +++ b/numpy/_core/tests/test_umath.py @@ -418,15 +418,16 @@ def test_unsigned_signed_direct_comparison( np_comp = np_comp_func arr = np.array([np.iinfo(dtype).max], dtype=dtype) - expected = py_comp(int(arr[0]), -1) - assert py_comp(arr, -1) == expected - assert np_comp(arr, -1) == expected + # NEP 50 broke this (for now), this could be salvaged eventually and + # the test should check equivalence to Python (no overflow). + # If not, the test can be simplified a bit more eventually. + with pytest.raises(OverflowError, match="Python integer -1"): + np_comp(arr, -1) + scalar = arr[0] - assert isinstance(scalar, np.integer) - # The Python operator here is mainly interesting: - assert py_comp(scalar, -1) == expected - assert np_comp(scalar, -1) == expected + with pytest.raises(OverflowError): + np_comp(scalar, -1) class TestAdd: @@ -4056,18 +4057,28 @@ def test_float(self): assert_raises(TypeError, np.gcd, 0.3, 0.4) assert_raises(TypeError, np.lcm, 0.3, 0.4) - def test_builtin_long(self): - # sanity check that array coercion is alright for builtin longs - assert_equal(np.array(2**200).item(), 2**200) + def test_huge_integers(self): + # As of now, you can still convert to an array first and then + # call a ufunc with huge integers: + assert_equal(np.array(2**200), 2**200) + # but promotion rules mean that we try to use a normal integer + # in the following case: + with pytest.raises(OverflowError): + np.equal(2**200, 2**200) + + with pytest.raises(OverflowError): + np.gcd(2**100, 3**100) - # expressed as prime factors + # Asking for `object` explicitly is fine, though: + assert np.gcd(2**100, 3**100, dtype=object) == 1 + + # As of now, the below work, because it is using arrays (which + # will be object arrays) a = np.array(2**100 * 3**5) b = np.array([2**100 * 5**7, 2**50 * 3**10]) assert_equal(np.gcd(a, b), [2**100, 2**50 * 3**5]) assert_equal(np.lcm(a, b), [2**100 * 3**5 * 5**7, 2**100 * 3**10]) - assert_equal(np.gcd(2**100, 3**100), 1) - class TestRoundingFunctions: From 14f6aa1ab292f65efcb6e260b542e34bbd2e6139 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 9 Jun 2023 14:19:07 +0200 Subject: [PATCH 03/27] MAINT: Robustify histogram unsigned-subtract and fix test for NEP 50 --- numpy/_core/tests/test_dtype.py | 11 +++++------ numpy/lib/_histograms_impl.py | 8 +++++--- numpy/lib/tests/test_histograms.py | 11 ++++------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/numpy/_core/tests/test_dtype.py b/numpy/_core/tests/test_dtype.py index 03f1dd8f93f9..d94ba34dcdbc 100644 --- a/numpy/_core/tests/test_dtype.py +++ b/numpy/_core/tests/test_dtype.py @@ -1465,13 +1465,12 @@ def test_complex_pyscalar_promote_rational(self): @pytest.mark.parametrize("val", [2, 2**32, 2**63, 2**64, 2*100]) def test_python_integer_promotion(self, val): - # If we only path scalars (mainly python ones!), the result must take - # into account that the integer may be considered int32, int64, uint64, - # or object depending on the input value. So test those paths! - expected_dtype = np.result_type(np.array(val).dtype, np.array(0).dtype) + # If we only pass scalars (mainly python ones!), NEP 50 means + # that we get either the default integer + expected_dtype = np.dtype(int) # the default integer assert np.result_type(val, 0) == expected_dtype - # For completeness sake, also check with a NumPy scalar as second arg: - assert np.result_type(val, np.int8(0)) == expected_dtype + # With NEP 50, the NumPy scalar wins though: + assert np.result_type(val, np.int8(0)) == np.int8 @pytest.mark.parametrize(["other", "expected"], [(1, rational), (1., np.float64)]) diff --git a/numpy/lib/_histograms_impl.py b/numpy/lib/_histograms_impl.py index ced0c5601144..e647c2ebe96d 100644 --- a/numpy/lib/_histograms_impl.py +++ b/numpy/lib/_histograms_impl.py @@ -348,13 +348,15 @@ def _unsigned_subtract(a, b): } dt = np.result_type(a, b) try: - dt = signed_to_unsigned[dt.type] + unsigned_dt = signed_to_unsigned[dt.type] except KeyError: return np.subtract(a, b, dtype=dt) else: # we know the inputs are integers, and we are deliberately casting - # signed to unsigned - return np.subtract(a, b, casting='unsafe', dtype=dt) + # signed to unsigned. The input may be negative python integers so + # ensure we pass in arrays with the initial dtype (related to NEP 50). + return np.subtract(np.asarray(a, dtype=dt), np.asarray(b, dtype=dt), + casting='unsafe', dtype=unsigned_dt) def _get_bin_edges(a, bins, range, weights): diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py index f715156efb3d..330229c528a0 100644 --- a/numpy/lib/tests/test_histograms.py +++ b/numpy/lib/tests/test_histograms.py @@ -349,10 +349,8 @@ def do_precision_lower_bound(self, float_small, float_large): # previously crashed count, x_loc = np.histogram(arr, bins=1, range=range) - assert_equal(count, [1]) - - # gh-10322 means that the type comes from arr - this may change - assert_equal(x_loc.dtype, float_small) + assert_equal(count, [0]) + assert_equal(x_loc.dtype, float_large) def do_precision_upper_bound(self, float_small, float_large): eps = np.finfo(float_large).eps @@ -366,10 +364,9 @@ def do_precision_upper_bound(self, float_small, float_large): # previously crashed count, x_loc = np.histogram(arr, bins=1, range=range) - assert_equal(count, [1]) + assert_equal(count, [0]) - # gh-10322 means that the type comes from arr - this may change - assert_equal(x_loc.dtype, float_small) + assert_equal(x_loc.dtype, float_large) def do_precision(self, float_small, float_large): self.do_precision_lower_bound(float_small, float_large) From 0cc124d9e53ea53fd11be2b6adda8eeef4e3bb00 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 9 Jun 2023 14:58:28 +0200 Subject: [PATCH 04/27] TST: Adjust tests for direct NEP 50 changes --- numpy/_core/tests/test_nditer.py | 16 +++++++++------- numpy/_core/tests/test_numeric.py | 27 +++++++++++---------------- numpy/_core/tests/test_ufunc.py | 8 ++++---- numpy/lib/tests/test_index_tricks.py | 4 +++- 4 files changed, 27 insertions(+), 28 deletions(-) diff --git a/numpy/_core/tests/test_nditer.py b/numpy/_core/tests/test_nditer.py index 7653f232acd0..8fc90e0ea39e 100644 --- a/numpy/_core/tests/test_nditer.py +++ b/numpy/_core/tests/test_nditer.py @@ -1131,6 +1131,7 @@ def test_iter_object_arrays_conversions(): def test_iter_common_dtype(): # Check that the iterator finds a common data type correctly + # (some checks are somewhat duplicate after adopting NEP 50) i = nditer([array([3], dtype='f4'), array([0], dtype='f8')], ['common_dtype'], @@ -1148,14 +1149,14 @@ def test_iter_common_dtype(): ['common_dtype'], [['readonly', 'copy']]*2, casting='same_kind') - assert_equal(i.dtypes[0], np.dtype('f4')) - assert_equal(i.dtypes[1], np.dtype('f4')) + assert_equal(i.dtypes[0], np.dtype('f8')) + assert_equal(i.dtypes[1], np.dtype('f8')) i = nditer([array([3], dtype='u4'), array(0, dtype='i4')], ['common_dtype'], [['readonly', 'copy']]*2, casting='safe') - assert_equal(i.dtypes[0], np.dtype('u4')) - assert_equal(i.dtypes[1], np.dtype('u4')) + assert_equal(i.dtypes[0], np.dtype('i8')) + assert_equal(i.dtypes[1], np.dtype('i8')) i = nditer([array([3], dtype='u4'), array(-12, dtype='i4')], ['common_dtype'], [['readonly', 'copy']]*2, @@ -1554,7 +1555,8 @@ def test_iter_allocate_output_opaxes(): assert_equal(i.operands[0].dtype, np.dtype('u4')) def test_iter_allocate_output_types_promotion(): - # Check type promotion of automatic outputs + # Check type promotion of automatic outputs (this was more interesting + # before NEP 50...) i = nditer([array([3], dtype='f4'), array([0], dtype='f8'), None], [], [['readonly']]*2+[['writeonly', 'allocate']]) @@ -1564,10 +1566,10 @@ def test_iter_allocate_output_types_promotion(): assert_equal(i.dtypes[2], np.dtype('f8')) i = nditer([array([3], dtype='f4'), array(0, dtype='f8'), None], [], [['readonly']]*2+[['writeonly', 'allocate']]) - assert_equal(i.dtypes[2], np.dtype('f4')) + assert_equal(i.dtypes[2], np.dtype('f8')) i = nditer([array([3], dtype='u4'), array(0, dtype='i4'), None], [], [['readonly']]*2+[['writeonly', 'allocate']]) - assert_equal(i.dtypes[2], np.dtype('u4')) + assert_equal(i.dtypes[2], np.dtype('i8')) i = nditer([array([3], dtype='u4'), array(-12, dtype='i4'), None], [], [['readonly']]*2+[['writeonly', 'allocate']]) assert_equal(i.dtypes[2], np.dtype('i8')) diff --git a/numpy/_core/tests/test_numeric.py b/numpy/_core/tests/test_numeric.py index 2a0798b2d211..a27ff3f5cc5d 100644 --- a/numpy/_core/tests/test_numeric.py +++ b/numpy/_core/tests/test_numeric.py @@ -1039,20 +1039,25 @@ def check_promotion_cases(self, promote_func): assert_equal(promote_func(np.array([b]), u8), np.dtype(np.uint8)) assert_equal(promote_func(np.array([b]), i32), np.dtype(np.int32)) assert_equal(promote_func(np.array([b]), u32), np.dtype(np.uint32)) - assert_equal(promote_func(np.array([i8]), i64), np.dtype(np.int8)) - assert_equal(promote_func(u64, np.array([i32])), np.dtype(np.int32)) - assert_equal(promote_func(i64, np.array([u32])), np.dtype(np.uint32)) + assert_equal(promote_func(np.array([i8]), i64), np.dtype(np.int64)) + # unsigned and signed unfortunately tend to promote to float64: + assert_equal(promote_func(u64, np.array([i32])), np.dtype(np.float64)) + assert_equal(promote_func(i64, np.array([u32])), np.dtype(np.int64)) + assert_equal(promote_func(np.array([u16]), i32), np.dtype(np.int32)) assert_equal(promote_func(np.int32(-1), np.array([u64])), np.dtype(np.float64)) - assert_equal(promote_func(f64, np.array([f32])), np.dtype(np.float32)) - assert_equal(promote_func(fld, np.array([f32])), np.dtype(np.float32)) + assert_equal(promote_func(f64, np.array([f32])), np.dtype(np.float64)) + assert_equal(promote_func(fld, np.array([f32])), + np.dtype(np.longdouble)) assert_equal(promote_func(np.array([f64]), fld), np.dtype(np.float64)) assert_equal(promote_func(fld, np.array([c64])), - np.dtype(np.complex64)) + np.dtype(np.clongdouble)) assert_equal(promote_func(c64, np.array([f64])), np.dtype(np.complex128)) assert_equal(promote_func(np.complex64(3j), np.array([f64])), np.dtype(np.complex128)) + assert_equal(promote_func(np.array([f32]), c128), + np.dtype(np.complex128)) # coercion between scalars and 1-D arrays, where # the scalar has greater kind than the array @@ -1062,16 +1067,6 @@ def check_promotion_cases(self, promote_func): assert_equal(promote_func(np.array([i8]), f64), np.dtype(np.float64)) assert_equal(promote_func(np.array([u16]), f64), np.dtype(np.float64)) - # uint and int are treated as the same "kind" for - # the purposes of array-scalar promotion. - assert_equal(promote_func(np.array([u16]), i32), np.dtype(np.uint16)) - - # float and complex are treated as the same "kind" for - # the purposes of array-scalar promotion, so that you can do - # (0j + float32array) to get a complex64 array instead of - # a complex128 array. - assert_equal(promote_func(np.array([f32]), c128), - np.dtype(np.complex64)) def test_coercion(self): def res_type(a, b): diff --git a/numpy/_core/tests/test_ufunc.py b/numpy/_core/tests/test_ufunc.py index 09354d90fb07..c6f2b838bd9d 100644 --- a/numpy/_core/tests/test_ufunc.py +++ b/numpy/_core/tests/test_ufunc.py @@ -1887,10 +1887,10 @@ def test_ufunc_custom_out(self): result = _rational_tests.test_add(a, b.astype(np.uint16), out=c) assert_equal(result, target) - # But, it can be fooled, e.g. (use scalars, which forces legacy - # type resolution to kick in, which then fails): - with assert_raises(TypeError): - _rational_tests.test_add(a, np.uint16(2)) + # This path used to go into legacy promotion, but doesn't now: + result = _rational_tests.test_add(a, np.uint16(2)) + target = np.array([2, 3, 4], dtype=_rational_tests.rational) + assert_equal(result, target) def test_operand_flags(self): a = np.arange(16, dtype='l').reshape(4, 4) diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py index a963801fea8e..fe1cfce2eaf8 100644 --- a/numpy/lib/tests/test_index_tricks.py +++ b/numpy/lib/tests/test_index_tricks.py @@ -245,8 +245,10 @@ def test_accepts_npfloating(self): # regression test for #16466 grid64 = mgrid[0.1:0.33:0.1, ] grid32 = mgrid[np.float32(0.1):np.float32(0.33):np.float32(0.1), ] - assert_(grid32.dtype == np.float64) assert_array_almost_equal(grid64, grid32) + # At some point this was float64, but NEP 50 changed it: + assert grid32.dtype == np.float32 + assert grid64.dtype == np.float64 # different code path for single slice grid64 = mgrid[0.1:0.33:0.1] From e19fd644a3554baa4005ee33e0f74adf550a6d20 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 9 Jun 2023 15:28:21 +0200 Subject: [PATCH 05/27] TST: Straight forward NEP 50 fixup or longdouble+int scalar tests --- numpy/_core/tests/test_scalarmath.py | 33 ++++++++++++++-------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/numpy/_core/tests/test_scalarmath.py b/numpy/_core/tests/test_scalarmath.py index 5d339f193318..b7aeb03edf02 100644 --- a/numpy/_core/tests/test_scalarmath.py +++ b/numpy/_core/tests/test_scalarmath.py @@ -911,31 +911,32 @@ def test_operator_scalars(op, type1, type2): @pytest.mark.parametrize("op", reasonable_operators_for_scalars) -@pytest.mark.parametrize("val", [None, 2**64]) -def test_longdouble_inf_loop(op, val): - # Note: The 2**64 value will pass once NEP 50 is adopted. +@pytest.mark.parametrize("sctype", [np.longdouble, np.clongdouble]) +def test_longdouble_inf_loop(sctype, op): try: - op(np.longdouble(3), val) + op(sctype(3), None) except TypeError: pass try: - op(val, np.longdouble(3)) + op(None, sctype(3)) except TypeError: pass @pytest.mark.parametrize("op", reasonable_operators_for_scalars) -@pytest.mark.parametrize("val", [None, 2**64]) -def test_clongdouble_inf_loop(op, val): - # Note: The 2**64 value will pass once NEP 50 is adopted. - try: - op(np.clongdouble(3), val) - except TypeError: - pass - try: - op(val, np.longdouble(3)) - except TypeError: - pass +@pytest.mark.parametrize("sctype", [np.longdouble, np.clongdouble]) +@np.errstate(all="ignore") +def test_longdouble_inf_loop(sctype, op): + # NEP 50 means that the result is clearly a (c)longdouble here: + if sctype == np.clongdouble and op in [operator.mod, operator.floordiv]: + # The above operators are not support for complex though... + with pytest.raises(TypeError): + op(sctype(3), 2**64) + with pytest.raises(TypeError): + op(sctype(3), 2**64) + else: + assert op(sctype(3), 2**64) == op(sctype(3), sctype(2**64)) + assert op(2**64, sctype(3)) == op(sctype(2**64), sctype(3)) @pytest.mark.parametrize("dtype", np.typecodes["AllInteger"]) From 1f07c939bee9cb0ad3ef300c7d61cd48290d8ad0 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 26 Jun 2023 17:02:49 +0200 Subject: [PATCH 06/27] STY: Remove unnecessary indentation level This used to make a bit of sense in C99, but now just distracts --- numpy/_core/src/multiarray/multiarraymodule.c | 249 +++++++++--------- 1 file changed, 124 insertions(+), 125 deletions(-) diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index d50064b1250c..6c06882e2bfd 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -3278,153 +3278,152 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) if (ax == NULL || ay == NULL) { goto fail; } - else { - npy_uint32 flags = NPY_ITER_EXTERNAL_LOOP | NPY_ITER_BUFFERED | - NPY_ITER_REFS_OK | NPY_ITER_ZEROSIZE_OK; - PyArrayObject * op_in[4] = { - NULL, arr, ax, ay - }; - npy_uint32 op_flags[4] = { - NPY_ITER_WRITEONLY | NPY_ITER_ALLOCATE | NPY_ITER_NO_SUBTYPE, - NPY_ITER_READONLY, - NPY_ITER_READONLY | NPY_ITER_ALIGNED, - NPY_ITER_READONLY | NPY_ITER_ALIGNED - }; - PyArray_Descr * common_dt = PyArray_ResultType(2, &op_in[0] + 2, - 0, NULL); - PyArray_Descr * op_dt[4] = {common_dt, PyArray_DescrFromType(NPY_BOOL), - common_dt, common_dt}; - NpyIter * iter; - NPY_BEGIN_THREADS_DEF; - - if (common_dt == NULL || op_dt[1] == NULL) { - Py_XDECREF(op_dt[1]); - Py_XDECREF(common_dt); - goto fail; - } - iter = NpyIter_MultiNew(4, op_in, flags, - NPY_KEEPORDER, NPY_UNSAFE_CASTING, - op_flags, op_dt); - Py_DECREF(op_dt[1]); - Py_DECREF(common_dt); - if (iter == NULL) { - goto fail; - } - /* Get the result from the iterator object array */ - ret = (PyObject*)NpyIter_GetOperandArray(iter)[0]; + npy_uint32 flags = NPY_ITER_EXTERNAL_LOOP | NPY_ITER_BUFFERED | + NPY_ITER_REFS_OK | NPY_ITER_ZEROSIZE_OK; + PyArrayObject * op_in[4] = { + NULL, arr, ax, ay + }; + npy_uint32 op_flags[4] = { + NPY_ITER_WRITEONLY | NPY_ITER_ALLOCATE | NPY_ITER_NO_SUBTYPE, + NPY_ITER_READONLY, + NPY_ITER_READONLY | NPY_ITER_ALIGNED, + NPY_ITER_READONLY | NPY_ITER_ALIGNED + }; + PyArray_Descr * common_dt = PyArray_ResultType(2, &op_in[0] + 2, + 0, NULL); + PyArray_Descr * op_dt[4] = {common_dt, PyArray_DescrFromType(NPY_BOOL), + common_dt, common_dt}; + NpyIter * iter; + NPY_BEGIN_THREADS_DEF; - npy_intp itemsize = common_dt->elsize; + if (common_dt == NULL || op_dt[1] == NULL) { + Py_XDECREF(op_dt[1]); + Py_XDECREF(common_dt); + goto fail; + } + iter = NpyIter_MultiNew(4, op_in, flags, + NPY_KEEPORDER, NPY_UNSAFE_CASTING, + op_flags, op_dt); + Py_DECREF(op_dt[1]); + Py_DECREF(common_dt); + if (iter == NULL) { + goto fail; + } - int has_ref = PyDataType_REFCHK(common_dt); + /* Get the result from the iterator object array */ + ret = (PyObject*)NpyIter_GetOperandArray(iter)[0]; - NPY_ARRAYMETHOD_FLAGS transfer_flags = 0; + npy_intp itemsize = common_dt->elsize; - npy_intp transfer_strides[2] = {itemsize, itemsize}; - npy_intp one = 1; + int has_ref = PyDataType_REFCHK(common_dt); - if (has_ref || ((itemsize != 16) && (itemsize != 8) && (itemsize != 4) && - (itemsize != 2) && (itemsize != 1))) { - // The iterator has NPY_ITER_ALIGNED flag so no need to check alignment - // of the input arrays. - // - // There's also no need to set up a cast for y, since the iterator - // ensures both casts are identical. - if (PyArray_GetDTypeTransferFunction( - 1, itemsize, itemsize, common_dt, common_dt, 0, - &cast_info, &transfer_flags) != NPY_SUCCEED) { - goto fail; - } - } + NPY_ARRAYMETHOD_FLAGS transfer_flags = 0; - transfer_flags = PyArrayMethod_COMBINED_FLAGS( - transfer_flags, NpyIter_GetTransferFlags(iter)); + npy_intp transfer_strides[2] = {itemsize, itemsize}; + npy_intp one = 1; - if (!(transfer_flags & NPY_METH_REQUIRES_PYAPI)) { - NPY_BEGIN_THREADS_THRESHOLDED(NpyIter_GetIterSize(iter)); + if (has_ref || ((itemsize != 16) && (itemsize != 8) && (itemsize != 4) && + (itemsize != 2) && (itemsize != 1))) { + // The iterator has NPY_ITER_ALIGNED flag so no need to check alignment + // of the input arrays. + // + // There's also no need to set up a cast for y, since the iterator + // ensures both casts are identical. + if (PyArray_GetDTypeTransferFunction( + 1, itemsize, itemsize, common_dt, common_dt, 0, + &cast_info, &transfer_flags) != NPY_SUCCEED) { + goto fail; } + } - if (NpyIter_GetIterSize(iter) != 0) { - NpyIter_IterNextFunc *iternext = NpyIter_GetIterNext(iter, NULL); - npy_intp * innersizeptr = NpyIter_GetInnerLoopSizePtr(iter); - char **dataptrarray = NpyIter_GetDataPtrArray(iter); - npy_intp *strides = NpyIter_GetInnerStrideArray(iter); + transfer_flags = PyArrayMethod_COMBINED_FLAGS( + transfer_flags, NpyIter_GetTransferFlags(iter)); - do { - npy_intp n = (*innersizeptr); - char * dst = dataptrarray[0]; - char * csrc = dataptrarray[1]; - char * xsrc = dataptrarray[2]; - char * ysrc = dataptrarray[3]; + if (!(transfer_flags & NPY_METH_REQUIRES_PYAPI)) { + NPY_BEGIN_THREADS_THRESHOLDED(NpyIter_GetIterSize(iter)); + } - // the iterator might mutate these pointers, - // so need to update them every iteration - npy_intp cstride = strides[1]; - npy_intp xstride = strides[2]; - npy_intp ystride = strides[3]; + if (NpyIter_GetIterSize(iter) != 0) { + NpyIter_IterNextFunc *iternext = NpyIter_GetIterNext(iter, NULL); + npy_intp * innersizeptr = NpyIter_GetInnerLoopSizePtr(iter); + char **dataptrarray = NpyIter_GetDataPtrArray(iter); + npy_intp *strides = NpyIter_GetInnerStrideArray(iter); - /* constant sizes so compiler replaces memcpy */ - if (!has_ref && itemsize == 16) { - INNER_WHERE_LOOP(16); - } - else if (!has_ref && itemsize == 8) { - INNER_WHERE_LOOP(8); - } - else if (!has_ref && itemsize == 4) { - INNER_WHERE_LOOP(4); - } - else if (!has_ref && itemsize == 2) { - INNER_WHERE_LOOP(2); - } - else if (!has_ref && itemsize == 1) { - INNER_WHERE_LOOP(1); - } - else { - npy_intp i; - for (i = 0; i < n; i++) { - if (*csrc) { - char *args[2] = {xsrc, dst}; - - if (cast_info.func( - &cast_info.context, args, &one, - transfer_strides, cast_info.auxdata) < 0) { - goto fail; - } + do { + npy_intp n = (*innersizeptr); + char * dst = dataptrarray[0]; + char * csrc = dataptrarray[1]; + char * xsrc = dataptrarray[2]; + char * ysrc = dataptrarray[3]; + + // the iterator might mutate these pointers, + // so need to update them every iteration + npy_intp cstride = strides[1]; + npy_intp xstride = strides[2]; + npy_intp ystride = strides[3]; + + /* constant sizes so compiler replaces memcpy */ + if (!has_ref && itemsize == 16) { + INNER_WHERE_LOOP(16); + } + else if (!has_ref && itemsize == 8) { + INNER_WHERE_LOOP(8); + } + else if (!has_ref && itemsize == 4) { + INNER_WHERE_LOOP(4); + } + else if (!has_ref && itemsize == 2) { + INNER_WHERE_LOOP(2); + } + else if (!has_ref && itemsize == 1) { + INNER_WHERE_LOOP(1); + } + else { + npy_intp i; + for (i = 0; i < n; i++) { + if (*csrc) { + char *args[2] = {xsrc, dst}; + + if (cast_info.func( + &cast_info.context, args, &one, + transfer_strides, cast_info.auxdata) < 0) { + goto fail; } - else { - char *args[2] = {ysrc, dst}; - - if (cast_info.func( - &cast_info.context, args, &one, - transfer_strides, cast_info.auxdata) < 0) { - goto fail; - } + } + else { + char *args[2] = {ysrc, dst}; + + if (cast_info.func( + &cast_info.context, args, &one, + transfer_strides, cast_info.auxdata) < 0) { + goto fail; } - dst += itemsize; - xsrc += xstride; - ysrc += ystride; - csrc += cstride; } + dst += itemsize; + xsrc += xstride; + ysrc += ystride; + csrc += cstride; } - } while (iternext(iter)); - } - - NPY_END_THREADS; + } + } while (iternext(iter)); + } - Py_INCREF(ret); - Py_DECREF(arr); - Py_DECREF(ax); - Py_DECREF(ay); - NPY_cast_info_xfree(&cast_info); + NPY_END_THREADS; - if (NpyIter_Deallocate(iter) != NPY_SUCCEED) { - Py_DECREF(ret); - return NULL; - } + Py_INCREF(ret); + Py_DECREF(arr); + Py_DECREF(ax); + Py_DECREF(ay); + NPY_cast_info_xfree(&cast_info); - return ret; + if (NpyIter_Deallocate(iter) != NPY_SUCCEED) { + Py_DECREF(ret); + return NULL; } + return ret; + fail: Py_DECREF(arr); Py_XDECREF(ax); From e445e8097f3c1248ba12625cb4d881606ca6a498 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 26 Jun 2023 17:04:25 +0200 Subject: [PATCH 07/27] API: Use weak scalar logic in where (and fix up error paths slightly) --- numpy/_core/src/multiarray/multiarraymodule.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index 6c06882e2bfd..005543a6a26a 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -3252,7 +3252,7 @@ array_set_datetimeparse_function(PyObject *NPY_UNUSED(self), NPY_NO_EXPORT PyObject * PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) { - PyArrayObject *arr, *ax, *ay; + PyArrayObject *arr, *ax, *ay = NULL; PyObject *ret = NULL; arr = (PyArrayObject *)PyArray_FROM_O(condition); @@ -3274,10 +3274,15 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) NPY_cast_info cast_info = {.func = NULL}; ax = (PyArrayObject*)PyArray_FROM_O(x); + if (ax == NULL) { + goto fail; + } ay = (PyArrayObject*)PyArray_FROM_O(y); - if (ax == NULL || ay == NULL) { + if (ay == NULL) { goto fail; } + npy_mark_tmp_array_if_pyscalar(x, ax, NULL); + npy_mark_tmp_array_if_pyscalar(y, ay, NULL); npy_uint32 flags = NPY_ITER_EXTERNAL_LOOP | NPY_ITER_BUFFERED | NPY_ITER_REFS_OK | NPY_ITER_ZEROSIZE_OK; From 346a146d4bf622cd36673d4472c7aabf8b253525 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 26 Jun 2023 17:09:30 +0200 Subject: [PATCH 08/27] TST: Adept where test to NEP 50 --- numpy/_core/tests/test_multiarray.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/numpy/_core/tests/test_multiarray.py b/numpy/_core/tests/test_multiarray.py index 24993f010c29..e07a33c23ea0 100644 --- a/numpy/_core/tests/test_multiarray.py +++ b/numpy/_core/tests/test_multiarray.py @@ -8864,9 +8864,11 @@ def test_exotic(self): assert_equal(np.where(True, d, e).dtype, np.float32) e = float('-Infinity') assert_equal(np.where(True, d, e).dtype, np.float32) - # also check upcast + # With NEP 50 adopted, the float will overflow here: e = float(1e150) - assert_equal(np.where(True, d, e).dtype, np.float64) + with pytest.warns(RuntimeWarning, match="overflow"): + res = np.where(True, d, e) + assert res.dtype == np.float32 def test_ndim(self): c = [True, False] From 166e806e4828e58f547a76c3170e1ee85e21caed Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 12 Jul 2023 12:26:18 +0200 Subject: [PATCH 09/27] MAINT: Require `select` for NEP 50 support This isn't quite ideally, it would be nice to grow specific code for it (and actually, we currently allow some subclasses in other paths, but I am planning to undo that). --- numpy/lib/_function_base_impl.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/numpy/lib/_function_base_impl.py b/numpy/lib/_function_base_impl.py index 22923ad553bf..123719df5a6b 100644 --- a/numpy/lib/_function_base_impl.py +++ b/numpy/lib/_function_base_impl.py @@ -812,22 +812,19 @@ def select(condlist, choicelist, default=0): if len(condlist) == 0: raise ValueError("select with an empty condition list is not possible") - choicelist = [np.asarray(choice) for choice in choicelist] + # TODO: This preserves the Python int, float, complex manually to get the + # right `result_type` with NEP 50. Most likely we will grow a better + # way to spell this (and this can be replaced). + choicelist = [ + choice if type(choice) in (int, float, complex) else np.asarray(choice) + for choice in choicelist] + choicelist.append(default if type(default) in (int, float, complex) + else np.asarray(default)) try: - intermediate_dtype = np.result_type(*choicelist) + dtype = np.result_type(*choicelist) except TypeError as e: - msg = f'Choicelist elements do not have a common dtype: {e}' - raise TypeError(msg) from None - default_array = np.asarray(default) - choicelist.append(default_array) - - # need to get the result type before broadcasting for correct scalar - # behaviour - try: - dtype = np.result_type(intermediate_dtype, default_array) - except TypeError as e: - msg = f'Choicelists and default value do not have a common dtype: {e}' + msg = f'Choicelist and default value do not have a common dtype: {e}' raise TypeError(msg) from None # Convert conditions to arrays and broadcast conditions and choices From 6cf68f36888b475ae2b25d480493fb84dcec6f35 Mon Sep 17 00:00:00 2001 From: Marten van Kerkwijk Date: Wed, 12 Jul 2023 14:10:44 -0400 Subject: [PATCH 10/27] Fix percentile and quantile --- numpy/lib/_function_base_impl.py | 15 ++++++++++++--- numpy/lib/tests/test_function_base.py | 20 +++++++++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/numpy/lib/_function_base_impl.py b/numpy/lib/_function_base_impl.py index 123719df5a6b..a2b0146df99c 100644 --- a/numpy/lib/_function_base_impl.py +++ b/numpy/lib/_function_base_impl.py @@ -4197,7 +4197,9 @@ def percentile(a, if a.dtype.kind == "c": raise TypeError("a must be an array of real numbers") - q = np.true_divide(q, 100) + # Use dtype of array if possible (e.g., if q is a python int or float) + # by making the divisor have the dtype of the data array. + q = np.true_divide(q, a.dtype.type(100) if a.dtype.kind == "f" else 100) q = asanyarray(q) # undo any decay that the ufunc performed (see gh-13105) if not _quantile_is_valid(q): raise ValueError("Percentiles must be in the range [0, 100]") @@ -4458,7 +4460,12 @@ def quantile(a, if a.dtype.kind == "c": raise TypeError("a must be an array of real numbers") - q = np.asanyarray(q) + # Use dtype of array if possible (e.g., if q is a python int or float). + if isinstance(q, (int, float)) and a.dtype.kind == "f": + q = np.asanyarray(q, dtype=np.result_type(a, q)) + else: + q = np.asanyarray(q) + if not _quantile_is_valid(q): raise ValueError("Quantiles must be in the range [0, 1]") return _quantile_unchecked( @@ -4556,7 +4563,9 @@ def _get_gamma(virtual_indexes, previous_indexes, method): """ gamma = np.asanyarray(virtual_indexes - previous_indexes) gamma = method["fix_gamma"](gamma, virtual_indexes) - return np.asanyarray(gamma) + # Ensure both that we have an array, and that we keep the dtype + # (which may have been matched to the input array). + return np.asanyarray(gamma, dtype=virtual_indexes.dtype) def _lerp(a, b, t, out=None): diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 87ec64cbfe0b..72d7e34f4a26 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -3063,6 +3063,9 @@ def test_linear_nan_1D(self, dtype): (np.longdouble, np.longdouble), (np.dtype("O"), np.float64)] + @pytest.mark.parametrize(["function", "quantile"], + [(np.quantile, 0.4), + (np.percentile, 40.0)]) @pytest.mark.parametrize(["input_dtype", "expected_dtype"], H_F_TYPE_CODES) @pytest.mark.parametrize(["method", "expected"], [("inverted_cdf", 20), @@ -3076,6 +3079,8 @@ def test_linear_nan_1D(self, dtype): ("normal_unbiased", 27.125), ]) def test_linear_interpolation(self, + function, + quantile, method, expected, input_dtype, @@ -3085,10 +3090,19 @@ def test_linear_interpolation(self, expected_dtype = np.promote_types(expected_dtype, np.float64) arr = np.asarray([15.0, 20.0, 35.0, 40.0, 50.0], dtype=input_dtype) - actual = np.percentile(arr, 40.0, method=method) + if input_dtype is np.longdouble: + if function is np.quantile: + # 0.4 is not exactly representable and it matters + # for "averaged_inverted_cdf", so we need to cheat. + quantile = input_dtype("0.4") + # We want to use nulp, but that does not work for longdouble + test_function = np.testing.assert_almost_equal + else: + test_function = np.testing.assert_array_almost_equal_nulp + + actual = function(arr, quantile, method=method) - np.testing.assert_almost_equal( - actual, expected_dtype.type(expected), 14) + test_function(actual, expected_dtype.type(expected)) if method in ["inverted_cdf", "closest_observation"]: if input_dtype == "O": From 623e0f052dd2c8ae224e770f2502922931f743b4 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 28 Sep 2023 12:24:15 +0200 Subject: [PATCH 11/27] TST: xfail test that checks `can_cast()` beahvior with pyints --- numpy/_core/tests/test_numeric.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/numpy/_core/tests/test_numeric.py b/numpy/_core/tests/test_numeric.py index a27ff3f5cc5d..1f3721cc265a 100644 --- a/numpy/_core/tests/test_numeric.py +++ b/numpy/_core/tests/test_numeric.py @@ -1421,6 +1421,8 @@ def test_can_cast_structured_to_simple(self): assert_(not np.can_cast([('f0', ('i4,i4'), (2,))], 'i4', casting='unsafe')) + @pytest.mark.xfail(np._get_promotion_state() != "legacy", + reason="NEP 50: no int/float/complex (yet)") def test_can_cast_values(self): # gh-5917 for dt in sctypes['int'] + sctypes['uint']: From 0f18c6f7f7a0646286087fabe2f37e2a6e6eacde Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 28 Sep 2023 12:24:31 +0200 Subject: [PATCH 12/27] MAINT: Also adopt `q` handling from the main quantile/percnetile to na version --- numpy/lib/_nanfunctions_impl.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/numpy/lib/_nanfunctions_impl.py b/numpy/lib/_nanfunctions_impl.py index 66b55eb1c34c..e1a601d5214f 100644 --- a/numpy/lib/_nanfunctions_impl.py +++ b/numpy/lib/_nanfunctions_impl.py @@ -1375,9 +1375,8 @@ def nanpercentile( if a.dtype.kind == "c": raise TypeError("a must be an array of real numbers") - q = np.true_divide(q, 100.0) - # undo any decay that the ufunc performed (see gh-13105) - q = np.asanyarray(q) + q = np.true_divide(q, a.dtype.type(100) if a.dtype.kind == "f" else 100) + q = np.asanyarray(q) # undo any decay that the ufunc performed (see gh-13105) if not fnb._quantile_is_valid(q): raise ValueError("Percentiles must be in the range [0, 100]") return _nanquantile_unchecked( @@ -1538,7 +1537,12 @@ def nanquantile( if a.dtype.kind == "c": raise TypeError("a must be an array of real numbers") - q = np.asanyarray(q) + # Use dtype of array if possible (e.g., if q is a python int or float). + if isinstance(q, (int, float)) and a.dtype.kind == "f": + q = np.asanyarray(q, dtype=np.result_type(a, q)) + else: + q = np.asanyarray(q) + if not fnb._quantile_is_valid(q): raise ValueError("Quantiles must be in the range [0, 1]") return _nanquantile_unchecked( From 94b7f905cfe94424892cd2af6fe25e7db3ce33e1 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 28 Sep 2023 13:04:42 +0200 Subject: [PATCH 13/27] MAINT: Address review comments by Marten --- numpy/_core/src/multiarray/multiarraymodule.c | 2 +- numpy/_core/tests/test_scalar_ctors.py | 2 +- numpy/_core/tests/test_scalarmath.py | 15 +++++++++++++-- numpy/_core/tests/test_ufunc.py | 5 ++++- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index 005543a6a26a..2d5735baa4b2 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -3295,7 +3295,7 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) NPY_ITER_READONLY | NPY_ITER_ALIGNED, NPY_ITER_READONLY | NPY_ITER_ALIGNED }; - PyArray_Descr * common_dt = PyArray_ResultType(2, &op_in[0] + 2, + PyArray_Descr * common_dt = PyArray_ResultType(2, &op_in[2], 0, NULL); PyArray_Descr * op_dt[4] = {common_dt, PyArray_DescrFromType(NPY_BOOL), common_dt, common_dt}; diff --git a/numpy/_core/tests/test_scalar_ctors.py b/numpy/_core/tests/test_scalar_ctors.py index a4d61c127624..13cf3b6430ea 100644 --- a/numpy/_core/tests/test_scalar_ctors.py +++ b/numpy/_core/tests/test_scalar_ctors.py @@ -79,7 +79,7 @@ def test_intp(self): def test_uint64_from_negative(self): with pytest.raises(OverflowError): - assert_equal(np.uint64(-2), np.uint64(18446744073709551614)) + np.uint64(-2) int_types = [np.byte, np.short, np.intc, np.long, np.longlong] diff --git a/numpy/_core/tests/test_scalarmath.py b/numpy/_core/tests/test_scalarmath.py index b7aeb03edf02..73a8b0d430b3 100644 --- a/numpy/_core/tests/test_scalarmath.py +++ b/numpy/_core/tests/test_scalarmath.py @@ -912,7 +912,17 @@ def test_operator_scalars(op, type1, type2): @pytest.mark.parametrize("op", reasonable_operators_for_scalars) @pytest.mark.parametrize("sctype", [np.longdouble, np.clongdouble]) -def test_longdouble_inf_loop(sctype, op): +def test_longdouble_operators_with_obj(sctype, op): + # This is/used to be tricky, because NumPy generally falls back to + # using the ufunc via `np.asarray()`, this effectively might do: + # longdouble + None + # -> asarray(longdouble) + np.array(None, dtype=object) + # -> asarray(longdouble).astype(object) + np.array(None, dtype=object) + # And after getting the scalars in the inner loop: + # -> longdouble + None + # + # That would recurse infinitely. Other scalars return the python object + # on cast, so this type of things works OK. try: op(sctype(3), None) except TypeError: @@ -926,7 +936,8 @@ def test_longdouble_inf_loop(sctype, op): @pytest.mark.parametrize("op", reasonable_operators_for_scalars) @pytest.mark.parametrize("sctype", [np.longdouble, np.clongdouble]) @np.errstate(all="ignore") -def test_longdouble_inf_loop(sctype, op): +def test_longdouble_operators_with_large_int(sctype, op): + # (See `test_longdouble_operators_with_obj` for why longdouble is special) # NEP 50 means that the result is clearly a (c)longdouble here: if sctype == np.clongdouble and op in [operator.mod, operator.floordiv]: # The above operators are not support for complex though... diff --git a/numpy/_core/tests/test_ufunc.py b/numpy/_core/tests/test_ufunc.py index c6f2b838bd9d..415893f5adcd 100644 --- a/numpy/_core/tests/test_ufunc.py +++ b/numpy/_core/tests/test_ufunc.py @@ -547,6 +547,9 @@ def test_partial_signature_mismatch_with_cache(self): with pytest.raises(TypeError): np.add(np.float16(1), np.uint64(2), sig=("e", "d", None)) + @pytest.mark.xfail(np._get_promotion_state() != "legacy", + reason="NEP 50 impl breaks casting checks when `dtype=` is used " + "together with python scalars.") def test_use_output_signature_for_all_arguments(self): # Test that providing only `dtype=` or `signature=(None, None, dtype)` # is sufficient if falling back to a homogeneous signature works. @@ -1887,7 +1890,7 @@ def test_ufunc_custom_out(self): result = _rational_tests.test_add(a, b.astype(np.uint16), out=c) assert_equal(result, target) - # This path used to go into legacy promotion, but doesn't now: + # This scalar path used to go into legacy promotion, but doesn't now: result = _rational_tests.test_add(a, np.uint16(2)) target = np.array([2, 3, 4], dtype=_rational_tests.rational) assert_equal(result, target) From 841a280b920312679bc2bf34b181b6d9e39fa976 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 28 Sep 2023 13:20:47 +0200 Subject: [PATCH 14/27] TST: Fix one more tests Was hidden locally due to longdouble having the same size as double --- numpy/_core/tests/test_numeric.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/_core/tests/test_numeric.py b/numpy/_core/tests/test_numeric.py index 1f3721cc265a..7494dcedb36f 100644 --- a/numpy/_core/tests/test_numeric.py +++ b/numpy/_core/tests/test_numeric.py @@ -1049,7 +1049,8 @@ def check_promotion_cases(self, promote_func): assert_equal(promote_func(f64, np.array([f32])), np.dtype(np.float64)) assert_equal(promote_func(fld, np.array([f32])), np.dtype(np.longdouble)) - assert_equal(promote_func(np.array([f64]), fld), np.dtype(np.float64)) + assert_equal(promote_func(np.array([f64]), fld), + np.dtype(np.longdouble)) assert_equal(promote_func(fld, np.array([c64])), np.dtype(np.clongdouble)) assert_equal(promote_func(c64, np.array([f64])), From 09418fc148f14b4aa9c95046841a6336a30e72de Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 28 Sep 2023 13:37:51 +0200 Subject: [PATCH 15/27] TYP: Fix typing passing test which cannot pass anymore Note that we don't seem to type signed/unsigned mixing for the cases where they promote to floats correctly, so there cannot be a new test case. (This is unrelated to NEP 50, as it already applied to array or scalar only cases.) --- numpy/typing/tests/data/fail/bitwise_ops.pyi | 1 + numpy/typing/tests/data/pass/bitwise_ops.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/numpy/typing/tests/data/fail/bitwise_ops.pyi b/numpy/typing/tests/data/fail/bitwise_ops.pyi index ee9090007924..924a3c8dc698 100644 --- a/numpy/typing/tests/data/fail/bitwise_ops.pyi +++ b/numpy/typing/tests/data/fail/bitwise_ops.pyi @@ -14,6 +14,7 @@ i | f8 # E: Unsupported operand types i8 ^ f8 # E: No overload variant u8 & f8 # E: No overload variant ~f8 # E: Unsupported operand type +# TODO: Certain mixes like i4 << u8 go to float and thus should fail # mypys' error message for `NoReturn` is unfortunately pretty bad # TODO: Re-enable this once we add support for numerical precision for `number`s diff --git a/numpy/typing/tests/data/pass/bitwise_ops.py b/numpy/typing/tests/data/pass/bitwise_ops.py index 67449e2c21d8..8bc934f58a64 100644 --- a/numpy/typing/tests/data/pass/bitwise_ops.py +++ b/numpy/typing/tests/data/pass/bitwise_ops.py @@ -63,11 +63,11 @@ u8 ^ u8 u8 & u8 -u8 << AR -u8 >> AR -u8 | AR -u8 ^ AR -u8 & AR +i << AR +i >> AR +i | AR +i ^ AR +i & AR u4 << u4 u4 >> u4 From df8f9d6d9c0b2aa7248ecc204ea5cf26e540d05d Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 28 Sep 2023 13:46:13 +0200 Subject: [PATCH 16/27] DOC: Fix docs that cause an integer overflow --- doc/source/user/basics.creation.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/doc/source/user/basics.creation.rst b/doc/source/user/basics.creation.rst index 3ee501889666..069b91bef482 100644 --- a/doc/source/user/basics.creation.rst +++ b/doc/source/user/basics.creation.rst @@ -43,13 +43,18 @@ When you use :func:`numpy.array` to define a new array, you should consider the :doc:`dtype ` of the elements in the array, which can be specified explicitly. This feature gives you more control over the underlying data structures and how the elements -are handled in C/C++ functions. If you are not careful with ``dtype`` -assignments, you can get unwanted overflow, as such +are handled in C/C++ functions. +When values do not fit and you are using a ``dtype`` NumPy may raise an +error:: -:: + >>> np.array([127, 128, 129], dtype=np.int8) + Traceback (most recent call last): + ... + OverflowError: Python integer 128 out of bounds for int8 + +However, NumPy integers are force-cast and may silently overflow:: - >>> a = np.array([127, 128, 129], dtype=np.int8) - >>> a + >>> np.array([127, np.int64(128), np.int64(129)], dtype=np.int8) array([ 127, -128, -127], dtype=int8) An 8-bit signed integer represents integers from -128 to 127. From 8a1497f26850c12f60c9b880b23684cd338c9d91 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 28 Sep 2023 14:29:23 +0200 Subject: [PATCH 17/27] MAINT: Avoid overflow in comparison with int64 on shorter platforms Maybe not the best way, but its functional and easy to grok and I doubt it matters speed wise. --- numpy/random/_generator.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx index b9f4e467ad48..89842ed5a2aa 100644 --- a/numpy/random/_generator.pyx +++ b/numpy/random/_generator.pyx @@ -4229,7 +4229,7 @@ cdef class Generator: elif colors.size > 0 and not np.issubdtype(colors.dtype, np.integer): invalid_colors = True - elif np.any((colors < 0) | (colors > INT64_MAX)): + elif np.any((colors < 0) | (colors > np.int64(INT64_MAX))): invalid_colors = True except ValueError: invalid_colors = True From 0aa93fa49bc3e08e22e811dd62d3708a6acd2246 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 28 Sep 2023 15:03:29 +0200 Subject: [PATCH 18/27] DOC: Fix docstring pointing out quirks that got less bad with NEP 50 --- numpy/_core/code_generators/ufunc_docstrings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/_core/code_generators/ufunc_docstrings.py b/numpy/_core/code_generators/ufunc_docstrings.py index b2f92ffad539..70066e6e7323 100644 --- a/numpy/_core/code_generators/ufunc_docstrings.py +++ b/numpy/_core/code_generators/ufunc_docstrings.py @@ -1893,7 +1893,7 @@ def add_newdoc(place, name, doc): result and can lead to unexpected results in some cases (see :ref:`Casting Rules `): - >>> a = np.left_shift(np.uint8(255), 1) # Expect 254 + >>> a = np.left_shift(np.uint8(255), np.int64(1)) # Expect 254 >>> print(a, type(a)) # Unexpected result due to upcasting 510 >>> b = np.left_shift(np.uint8(255), np.uint8(1)) From 375de5d4b342bab699006e24ca08ead5be7f03b3 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 12 Oct 2023 13:18:30 +0200 Subject: [PATCH 19/27] TST: Undo test changes related to comparising to out-of-bound ints --- numpy/_core/tests/test_multiarray.py | 11 +++------- numpy/_core/tests/test_scalarmath.py | 10 +++------- numpy/_core/tests/test_umath.py | 30 +++++++++++++++------------- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/numpy/_core/tests/test_multiarray.py b/numpy/_core/tests/test_multiarray.py index e07a33c23ea0..918d6b24eac3 100644 --- a/numpy/_core/tests/test_multiarray.py +++ b/numpy/_core/tests/test_multiarray.py @@ -8707,14 +8707,9 @@ def test_array_scalar_relational_operation(self): # Unsigned integers for dt1 in 'BHILQP': - # NEP 50 broke comparison of unsigned with -1 for the time being - # (this may be fixed in the future of course). - with pytest.raises(OverflowError): - -1 < np.array(1, dtype=dt1) - with pytest.raises(OverflowError): - -1 > np.array(1, dtype=dt1) - with pytest.raises(OverflowError): - -1 != np.array(1, dtype=dt1) + assert_(-1 < np.array(1, dtype=dt1), "type %s failed" % (dt1,)) + assert_(not -1 > np.array(1, dtype=dt1), "type %s failed" % (dt1,)) + assert_(-1 != np.array(1, dtype=dt1), "type %s failed" % (dt1,)) # Unsigned vs signed for dt2 in 'bhilqp': diff --git a/numpy/_core/tests/test_scalarmath.py b/numpy/_core/tests/test_scalarmath.py index 73a8b0d430b3..1d41fd2123ff 100644 --- a/numpy/_core/tests/test_scalarmath.py +++ b/numpy/_core/tests/test_scalarmath.py @@ -558,13 +558,9 @@ def test_numpy_scalar_relational_operators(self): #Unsigned integers for dt1 in 'BHILQP': - # NEP 50 means the -1 is currently rejected (we may change this) - with pytest.raises(OverflowError): - -1 < np.array(1, dtype=dt1)[()] - with pytest.raises(OverflowError): - -1 > np.array(1, dtype=dt1)[()] - with pytest.raises(OverflowError): - -1 != np.array(1, dtype=dt1)[()] + assert_(-1 < np.array(1, dtype=dt1)[()], "type %s failed" % (dt1,)) + assert_(not -1 > np.array(1, dtype=dt1)[()], "type %s failed" % (dt1,)) + assert_(-1 != np.array(1, dtype=dt1)[()], "type %s failed" % (dt1,)) #unsigned vs signed for dt2 in 'bhilqp': diff --git a/numpy/_core/tests/test_umath.py b/numpy/_core/tests/test_umath.py index b27ee23c6708..e03a1fae5289 100644 --- a/numpy/_core/tests/test_umath.py +++ b/numpy/_core/tests/test_umath.py @@ -418,17 +418,17 @@ def test_unsigned_signed_direct_comparison( np_comp = np_comp_func arr = np.array([np.iinfo(dtype).max], dtype=dtype) + expected = py_comp(int(arr[0]), -1) - # NEP 50 broke this (for now), this could be salvaged eventually and - # the test should check equivalence to Python (no overflow). - # If not, the test can be simplified a bit more eventually. - with pytest.raises(OverflowError, match="Python integer -1"): - np_comp(arr, -1) + assert py_comp(arr, -1) == expected + assert np_comp(arr, -1) == expected - scalar = arr[0] - with pytest.raises(OverflowError): - np_comp(scalar, -1) + scalar = arr[0] + assert isinstance(scalar, np.integer) + # The Python operator here is mainly interesting: + assert py_comp(scalar, -1) == expected + assert np_comp(scalar, -1) == expected class TestAdd: def test_reduce_alignment(self): @@ -4058,14 +4058,16 @@ def test_float(self): assert_raises(TypeError, np.lcm, 0.3, 0.4) def test_huge_integers(self): - # As of now, you can still convert to an array first and then - # call a ufunc with huge integers: + # Converting to an array first is a bit different as it means we + # have an explicit object dtype: assert_equal(np.array(2**200), 2**200) - # but promotion rules mean that we try to use a normal integer - # in the following case: - with pytest.raises(OverflowError): - np.equal(2**200, 2**200) + # Special promotion rules should ensure that this also works for + # two Python integers (even if slow). + # (We do this for comparisons, as the result is always bool and + # we also special case array comparisons with Python integers) + np.equal(2**200, 2**200) + # But, we cannot do this when it would affect the result dtype: with pytest.raises(OverflowError): np.gcd(2**100, 3**100) From 0e6d569c3ad2ea232726e5b592340a225598f190 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 12 Oct 2023 13:35:29 +0200 Subject: [PATCH 20/27] MAINT: Undo now unnecessary random number cast to int64 --- numpy/random/_generator.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx index 89842ed5a2aa..b9f4e467ad48 100644 --- a/numpy/random/_generator.pyx +++ b/numpy/random/_generator.pyx @@ -4229,7 +4229,7 @@ cdef class Generator: elif colors.size > 0 and not np.issubdtype(colors.dtype, np.integer): invalid_colors = True - elif np.any((colors < 0) | (colors > np.int64(INT64_MAX))): + elif np.any((colors < 0) | (colors > INT64_MAX)): invalid_colors = True except ValueError: invalid_colors = True From a907624d7cdd7b9b04e3a4253c1e45bed101fe4c Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 12 Oct 2023 14:58:28 +0200 Subject: [PATCH 21/27] API: Disable `can_cast` with Python scalars (for now) --- numpy/_core/multiarray.py | 58 +++---------------- numpy/_core/src/multiarray/multiarraymodule.c | 11 +++- 2 files changed, 16 insertions(+), 53 deletions(-) diff --git a/numpy/_core/multiarray.py b/numpy/_core/multiarray.py index 770d8879531b..19b4e18ca8dd 100644 --- a/numpy/_core/multiarray.py +++ b/numpy/_core/multiarray.py @@ -511,14 +511,12 @@ def can_cast(from_, to, casting=None): can_cast(from_, to, casting='safe') Returns True if cast between data types can occur according to the - casting rule. If from is a scalar or array scalar, also returns - True if the scalar value can be cast without overflow or truncation - to an integer. + casting rule. Parameters ---------- - from_ : dtype, dtype specifier, scalar, or array - Data type, scalar, or array to cast from. + from_ : dtype, dtype specifier, NumPy scalar, or array + Data type, NumPy scalar, or array to cast from. to : dtype or dtype specifier Data type to cast to. casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional @@ -548,6 +546,10 @@ def can_cast(from_, to, casting=None): that the string dtype length is long enough to store the maximum integer/float value converted. + .. versionchanged:: 2.0 + This function does not support Python scalars anymore and does not + apply any value-based logic for 0-D arrays and NumPy scalars. + See also -------- dtype, result_type @@ -570,52 +572,6 @@ def can_cast(from_, to, casting=None): >>> np.can_cast('i4', 'S4') False - Casting scalars - - >>> np.can_cast(100, 'i1') - True - >>> np.can_cast(150, 'i1') - False - >>> np.can_cast(150, 'u1') - True - - >>> np.can_cast(3.5e100, np.float32) - False - >>> np.can_cast(1000.0, np.float32) - True - - Array scalar checks the value, array does not - - >>> np.can_cast(np.array(1000.0), np.float32) - True - >>> np.can_cast(np.array([1000.0]), np.float32) - False - - Using the casting rules - - >>> np.can_cast('i8', 'i8', 'no') - True - >>> np.can_cast('i8', 'no') - False - - >>> np.can_cast('i8', 'equiv') - True - >>> np.can_cast('i8', 'equiv') - False - - >>> np.can_cast('i8', 'safe') - True - >>> np.can_cast('i4', 'safe') - False - - >>> np.can_cast('i4', 'same_kind') - True - >>> np.can_cast('u4', 'same_kind') - False - - >>> np.can_cast('u4', 'unsafe') - True - """ return (from_,) diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index 2d5735baa4b2..ae2f12a5f1df 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -3502,8 +3502,15 @@ array_can_cast_safely(PyObject *NPY_UNUSED(self), if (PyArray_Check(from_obj)) { ret = PyArray_CanCastArrayTo((PyArrayObject *)from_obj, d2, casting); } - else if (PyArray_IsScalar(from_obj, Generic) || - PyArray_IsPythonNumber(from_obj)) { + else if (PyArray_IsPythonNumber(from_obj)) { + PyErr_SetString(PyExc_TypeError, + "can_cast() does not support Python ints, floats, and " + "complex because the result used to depend on the value.\n" + "This change was part of adopting NEP 50, we may " + "explicitly allow them again in the future."); + goto finish; + } + else if (PyArray_IsScalar(from_obj, Generic)) { PyArrayObject *arr; arr = (PyArrayObject *)PyArray_FROM_O(from_obj); if (arr == NULL) { From dcb8de260237284dd351237a421940702c9c03f6 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 12 Oct 2023 16:02:11 +0200 Subject: [PATCH 22/27] TST: Adjust test to avoid overflow (and bad results on BSD) Readding invalid warning ignoring, even though it should not matter (it does seem to for macos arm64 tests, maybe some M2/clang version issue...). --- numpy/_core/tests/test_scalarmath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/_core/tests/test_scalarmath.py b/numpy/_core/tests/test_scalarmath.py index 1d41fd2123ff..651a7214995c 100644 --- a/numpy/_core/tests/test_scalarmath.py +++ b/numpy/_core/tests/test_scalarmath.py @@ -942,7 +942,7 @@ def test_longdouble_operators_with_large_int(sctype, op): with pytest.raises(TypeError): op(sctype(3), 2**64) else: - assert op(sctype(3), 2**64) == op(sctype(3), sctype(2**64)) + assert op(sctype(3), -2**64) == op(sctype(3), sctype(-2**64)) assert op(2**64, sctype(3)) == op(sctype(2**64), sctype(3)) From 6707f39277d48b2053b590b750d46682ea8f0b9d Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 12 Oct 2023 17:38:05 +0200 Subject: [PATCH 23/27] BENCH: Explicitly skip reversed (arg)partition case because they fail This doesn't seem like an ideal solution, but is "minimal"? --- benchmarks/benchmarks/bench_function_base.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/benchmarks/benchmarks/bench_function_base.py b/benchmarks/benchmarks/bench_function_base.py index 8591bf1cd773..d4b08a3a0e65 100644 --- a/benchmarks/benchmarks/bench_function_base.py +++ b/benchmarks/benchmarks/bench_function_base.py @@ -2,6 +2,13 @@ import numpy as np +try: + # SkipNotImplemented is available since 6.0 + from asv_runner.benchmarks.mark import SkipNotImplemented +except ImportError: + SkipNotImplemented = NotImplementedError + + class Linspace(Benchmark): def setup(self): self.d = np.array([1, 2, 3]) @@ -169,6 +176,13 @@ def reversed(size, dtype): """ Returns an array that's in descending order. """ + dtype = np.dtype(dtype) + try: + with np.errstate(over="raise"): + res = dtype.type(size-1) + except (OverflowError, FloatingPointError): + raise SkipNotImplemented("Cannot construct arange for this size.") + return np.arange(size-1, -1, -1, dtype=dtype) @staticmethod From ba47e60841d7b3e0d08090a99650166ff66d6f3e Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 20 Oct 2023 11:01:23 +0200 Subject: [PATCH 24/27] MAINT: Address review comments --- numpy/_core/src/multiarray/multiarraymodule.c | 14 +++++++------- numpy/_core/tests/test_dtype.py | 2 +- numpy/_core/tests/test_numeric.py | 2 +- numpy/_core/tests/test_umath.py | 2 +- numpy/lib/_function_base_impl.py | 2 +- numpy/lib/_nanfunctions_impl.py | 2 +- numpy/typing/tests/data/pass/bitwise_ops.py | 12 ++++++------ 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index ae2f12a5f1df..1630418e9145 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -3254,6 +3254,7 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) { PyArrayObject *arr, *ax, *ay = NULL; PyObject *ret = NULL; + PyArray_Descr *common_dt = NULL; arr = (PyArrayObject *)PyArray_FROM_O(condition); if (arr == NULL) { @@ -3295,8 +3296,7 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) NPY_ITER_READONLY | NPY_ITER_ALIGNED, NPY_ITER_READONLY | NPY_ITER_ALIGNED }; - PyArray_Descr * common_dt = PyArray_ResultType(2, &op_in[2], - 0, NULL); + common_dt = PyArray_ResultType(2, &op_in[2], 0, NULL); PyArray_Descr * op_dt[4] = {common_dt, PyArray_DescrFromType(NPY_BOOL), common_dt, common_dt}; NpyIter * iter; @@ -3304,14 +3304,12 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) if (common_dt == NULL || op_dt[1] == NULL) { Py_XDECREF(op_dt[1]); - Py_XDECREF(common_dt); goto fail; } - iter = NpyIter_MultiNew(4, op_in, flags, - NPY_KEEPORDER, NPY_UNSAFE_CASTING, - op_flags, op_dt); + iter = NpyIter_MultiNew( + 4, op_in, flags, NPY_KEEPORDER, NPY_UNSAFE_CASTING, + op_flags, op_dt); Py_DECREF(op_dt[1]); - Py_DECREF(common_dt); if (iter == NULL) { goto fail; } @@ -3420,6 +3418,7 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) Py_DECREF(arr); Py_DECREF(ax); Py_DECREF(ay); + Py_DECREF(common_dt); NPY_cast_info_xfree(&cast_info); if (NpyIter_Deallocate(iter) != NPY_SUCCEED) { @@ -3433,6 +3432,7 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) Py_DECREF(arr); Py_XDECREF(ax); Py_XDECREF(ay); + Py_XDECREF(common_dt); NPY_cast_info_xfree(&cast_info); return NULL; } diff --git a/numpy/_core/tests/test_dtype.py b/numpy/_core/tests/test_dtype.py index d94ba34dcdbc..b39f914b3b99 100644 --- a/numpy/_core/tests/test_dtype.py +++ b/numpy/_core/tests/test_dtype.py @@ -1466,7 +1466,7 @@ def test_complex_pyscalar_promote_rational(self): @pytest.mark.parametrize("val", [2, 2**32, 2**63, 2**64, 2*100]) def test_python_integer_promotion(self, val): # If we only pass scalars (mainly python ones!), NEP 50 means - # that we get either the default integer + # that we get the default integer expected_dtype = np.dtype(int) # the default integer assert np.result_type(val, 0) == expected_dtype # With NEP 50, the NumPy scalar wins though: diff --git a/numpy/_core/tests/test_numeric.py b/numpy/_core/tests/test_numeric.py index 7494dcedb36f..8668b96bf708 100644 --- a/numpy/_core/tests/test_numeric.py +++ b/numpy/_core/tests/test_numeric.py @@ -1423,7 +1423,7 @@ def test_can_cast_structured_to_simple(self): casting='unsafe')) @pytest.mark.xfail(np._get_promotion_state() != "legacy", - reason="NEP 50: no int/float/complex (yet)") + reason="NEP 50: no python int/float/complex support (yet)") def test_can_cast_values(self): # gh-5917 for dt in sctypes['int'] + sctypes['uint']: diff --git a/numpy/_core/tests/test_umath.py b/numpy/_core/tests/test_umath.py index e03a1fae5289..39a4f7534e50 100644 --- a/numpy/_core/tests/test_umath.py +++ b/numpy/_core/tests/test_umath.py @@ -423,13 +423,13 @@ def test_unsigned_signed_direct_comparison( assert py_comp(arr, -1) == expected assert np_comp(arr, -1) == expected - scalar = arr[0] assert isinstance(scalar, np.integer) # The Python operator here is mainly interesting: assert py_comp(scalar, -1) == expected assert np_comp(scalar, -1) == expected + class TestAdd: def test_reduce_alignment(self): # gh-9876 diff --git a/numpy/lib/_function_base_impl.py b/numpy/lib/_function_base_impl.py index a2b0146df99c..e6f4fdf021a5 100644 --- a/numpy/lib/_function_base_impl.py +++ b/numpy/lib/_function_base_impl.py @@ -4462,7 +4462,7 @@ def quantile(a, # Use dtype of array if possible (e.g., if q is a python int or float). if isinstance(q, (int, float)) and a.dtype.kind == "f": - q = np.asanyarray(q, dtype=np.result_type(a, q)) + q = np.asanyarray(q, dtype=a.dtype) else: q = np.asanyarray(q) diff --git a/numpy/lib/_nanfunctions_impl.py b/numpy/lib/_nanfunctions_impl.py index e1a601d5214f..b2e5b3ea38f4 100644 --- a/numpy/lib/_nanfunctions_impl.py +++ b/numpy/lib/_nanfunctions_impl.py @@ -1539,7 +1539,7 @@ def nanquantile( # Use dtype of array if possible (e.g., if q is a python int or float). if isinstance(q, (int, float)) and a.dtype.kind == "f": - q = np.asanyarray(q, dtype=np.result_type(a, q)) + q = np.asanyarray(q, dtype=a.dtype) else: q = np.asanyarray(q) diff --git a/numpy/typing/tests/data/pass/bitwise_ops.py b/numpy/typing/tests/data/pass/bitwise_ops.py index 8bc934f58a64..e7bbc31a5578 100644 --- a/numpy/typing/tests/data/pass/bitwise_ops.py +++ b/numpy/typing/tests/data/pass/bitwise_ops.py @@ -21,6 +21,12 @@ i8 ^ i8 i8 & i8 +i << AR +i >> AR +i | AR +i ^ AR +i & AR + i8 << AR i8 >> AR i8 | AR @@ -63,12 +69,6 @@ u8 ^ u8 u8 & u8 -i << AR -i >> AR -i | AR -i ^ AR -i & AR - u4 << u4 u4 >> u4 u4 | u4 From bf0ff21419f2d0f43c646e72b758e73a9cdf1b4b Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 23 Oct 2023 13:08:33 +0200 Subject: [PATCH 25/27] MAINT: Clean up error path (DescrFromType cannot fail for simple builtins) But move the error check to be higher up and thus more directly relevant. If `PyArray_DescrFromType` could have failed, the code would have been incorrect anyway. --- numpy/_core/src/multiarray/multiarraymodule.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index 1630418e9145..b989c7c09762 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -3297,15 +3297,14 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) NPY_ITER_READONLY | NPY_ITER_ALIGNED }; common_dt = PyArray_ResultType(2, &op_in[2], 0, NULL); + if (common_dt == NULL) { + goto fail; + } PyArray_Descr * op_dt[4] = {common_dt, PyArray_DescrFromType(NPY_BOOL), common_dt, common_dt}; NpyIter * iter; NPY_BEGIN_THREADS_DEF; - if (common_dt == NULL || op_dt[1] == NULL) { - Py_XDECREF(op_dt[1]); - goto fail; - } iter = NpyIter_MultiNew( 4, op_in, flags, NPY_KEEPORDER, NPY_UNSAFE_CASTING, op_flags, op_dt); From 35e105f92e6778d273b7ece2d863599effa347a1 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 24 Oct 2023 14:45:19 +0200 Subject: [PATCH 26/27] MAINT: Address Nathans review comments --- doc/source/user/basics.creation.rst | 7 +------ numpy/_core/src/multiarray/multiarraymodule.c | 3 ++- numpy/_core/src/umath/ufunc_object.c | 2 ++ 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/doc/source/user/basics.creation.rst b/doc/source/user/basics.creation.rst index 069b91bef482..b45857f4f0e8 100644 --- a/doc/source/user/basics.creation.rst +++ b/doc/source/user/basics.creation.rst @@ -44,7 +44,7 @@ consider the :doc:`dtype ` of the elements in the array, which can be specified explicitly. This feature gives you more control over the underlying data structures and how the elements are handled in C/C++ functions. -When values do not fit and you are using a ``dtype`` NumPy may raise an +When values do not fit and you are using a ``dtype``, NumPy may raise an error:: >>> np.array([127, 128, 129], dtype=np.int8) @@ -52,11 +52,6 @@ error:: ... OverflowError: Python integer 128 out of bounds for int8 -However, NumPy integers are force-cast and may silently overflow:: - - >>> np.array([127, np.int64(128), np.int64(129)], dtype=np.int8) - array([ 127, -128, -127], dtype=int8) - An 8-bit signed integer represents integers from -128 to 127. Assigning the ``int8`` array to integers outside of this range results in overflow. This feature can often be misunderstood. If you diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index b989c7c09762..acc9f2cc0997 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -3252,7 +3252,7 @@ array_set_datetimeparse_function(PyObject *NPY_UNUSED(self), NPY_NO_EXPORT PyObject * PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) { - PyArrayObject *arr, *ax, *ay = NULL; + PyArrayObject *arr = NULL, *ax = NULL, *ay = NULL; PyObject *ret = NULL; PyArray_Descr *common_dt = NULL; @@ -3300,6 +3300,7 @@ PyArray_Where(PyObject *condition, PyObject *x, PyObject *y) if (common_dt == NULL) { goto fail; } + /* `PyArray_DescrFromType` cannot fail for simple builtin types: */ PyArray_Descr * op_dt[4] = {common_dt, PyArray_DescrFromType(NPY_BOOL), common_dt, common_dt}; NpyIter * iter; diff --git a/numpy/_core/src/umath/ufunc_object.c b/numpy/_core/src/umath/ufunc_object.c index 856f16785485..58edac7f7143 100644 --- a/numpy/_core/src/umath/ufunc_object.c +++ b/numpy/_core/src/umath/ufunc_object.c @@ -944,6 +944,8 @@ convert_ufunc_arguments(PyUFuncObject *ufunc, * We should eventually have special loops for isnan and once * we do, we may just deprecate all remaining ones (e.g. * `negative(2**100)` not working as it is an object.) + * + * This is issue is part of the NEP 50 adoption. */ break; } From b5c8398ec77fbba93226e677974650514704183e Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Tue, 24 Oct 2023 12:05:50 -0600 Subject: [PATCH 27/27] MAINT: appease linter --- numpy/lib/_nanfunctions_impl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/lib/_nanfunctions_impl.py b/numpy/lib/_nanfunctions_impl.py index b2e5b3ea38f4..e6bcdc6102cb 100644 --- a/numpy/lib/_nanfunctions_impl.py +++ b/numpy/lib/_nanfunctions_impl.py @@ -1376,7 +1376,8 @@ def nanpercentile( raise TypeError("a must be an array of real numbers") q = np.true_divide(q, a.dtype.type(100) if a.dtype.kind == "f" else 100) - q = np.asanyarray(q) # undo any decay that the ufunc performed (see gh-13105) + # undo any decay that the ufunc performed (see gh-13105) + q = np.asanyarray(q) if not fnb._quantile_is_valid(q): raise ValueError("Percentiles must be in the range [0, 100]") return _nanquantile_unchecked( pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy