diff --git a/.github/workflows/imports.yml b/.github/workflows/imports.yml index a9863a213..2b0b0ed9f 100644 --- a/.github/workflows/imports.yml +++ b/.github/workflows/imports.yml @@ -7,13 +7,24 @@ on: - main jobs: - test_imports: + rngs: runs-on: ubuntu-latest - # strategy: - # matrix: - # python-version: ["3.8", "3.9", "3.10"] + outputs: + os: ${{ steps.os.outputs.selected }} + pyver: ${{ steps.pyver.outputs.selected }} steps: - - uses: actions/checkout@v3 + - name: RNG for os + uses: ddradar/choose-random-action@v2.0.2 + id: os + with: + contents: | + ubuntu-latest + macos-latest + windows-latest + weights: | + 1 + 1 + 1 - name: RNG for Python version uses: ddradar/choose-random-action@v2.0.2 id: pyver @@ -22,14 +33,26 @@ jobs: 3.8 3.9 3.10 + 3.11 weights: | 1 1 1 + 1 + test_imports: + needs: rngs + runs-on: ${{ needs.rngs.outputs.os }} + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # python-version: ["3.8", "3.9", "3.10", "3.11"] + # os: ["ubuntu-latest", "macos-latest", "windows-latest"] + steps: + - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: ${{ steps.pyver.outputs.selected }} + python-version: ${{ needs.rngs.outputs.pyver }} # python-version: ${{ matrix.python-version }} - run: python -m pip install --upgrade pip - - run: pip install -e . + - run: pip install -e .[default] - run: ./scripts/test_imports.sh diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 04ffd3eb5..6aa692155 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -102,17 +102,19 @@ jobs: id: pyver with: # We should support major Python versions for at least 36-42 months - # We could probably support pypy if numba were optional + # We may be able to support pypy if anybody asks for it # 3.8.16 0_73_pypy # 3.9.16 0_73_pypy contents: | 3.8 3.9 3.10 + 3.11 weights: | 1 1 1 + 1 - name: RNG for source of python-suitesparse-graphblas uses: ddradar/choose-random-action@v2.0.2 id: sourcetype @@ -138,8 +140,8 @@ jobs: miniforge-version: latest use-mamba: true python-version: ${{ steps.pyver.outputs.selected }} - channels: conda-forge,nodefaults - channel-priority: strict + channels: conda-forge,${{ contains(steps.pyver.outputs.selected, 'pypy') && 'defaults' || 'nodefaults' }} + channel-priority: ${{ contains(steps.pyver.outputs.selected, 'pypy') && 'flexible' || 'strict' }} activate-environment: graphblas auto-activate-base: false - name: Setup conda @@ -150,8 +152,8 @@ jobs: with: auto-update-conda: true python-version: ${{ steps.pyver.outputs.selected }} - channels: conda-forge,nodefaults - channel-priority: strict + channels: conda-forge,${{ contains(steps.pyver.outputs.selected, 'pypy') && 'defaults' || 'nodefaults' }} + channel-priority: ${{ contains(steps.pyver.outputs.selected, 'pypy') && 'flexible' || 'strict' }} activate-environment: graphblas auto-activate-base: false - name: Update env @@ -161,29 +163,29 @@ jobs: # # First let's randomly get versions of dependencies to install. # Consider removing old versions when they become problematic or very old (>=2 years). - nxver=$(python -c 'import random ; print(random.choice(["=2.7", "=2.8", "=3.0", ""]))') + nxver=$(python -c 'import random ; print(random.choice(["=2.7", "=2.8", "=3.0", "=3.1", ""]))') yamlver=$(python -c 'import random ; print(random.choice(["=5.4", "=6.0", ""]))') - sparsever=$(python -c 'import random ; print(random.choice(["=0.12", "=0.13", "=0.14", ""]))') - fmmver=$(python -c 'import random ; print(random.choice(["=1.4", ""]))') + sparsever=$(python -c 'import random ; print(random.choice(["=0.13", "=0.14", ""]))') + fmmver=$(python -c 'import random ; print(random.choice(["=1.4", "=1.5", ""]))') if [[ ${{ startsWith(steps.pyver.outputs.selected, '3.8') }} == true ]]; then - npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", ""]))') + npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", "=1.24", ""]))') spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", ""]))') - pdver=$(python -c 'import random ; print(random.choice(["=1.2", "=1.3", "=1.4", "=1.5", ""]))') + pdver=$(python -c 'import random ; print(random.choice(["=1.2", "=1.3", "=1.4", "=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=2.0", "=2.1", ""]))') elif [[ ${{ startsWith(steps.pyver.outputs.selected, '3.9') }} == true ]]; then - npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", ""]))') + npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", "=1.24", ""]))') spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", ""]))') - pdver=$(python -c 'import random ; print(random.choice(["=1.2", "=1.3", "=1.4", "=1.5", ""]))') + pdver=$(python -c 'import random ; print(random.choice(["=1.2", "=1.3", "=1.4", "=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=2.0", "=2.1", ""]))') elif [[ ${{ startsWith(steps.pyver.outputs.selected, '3.10') }} == true ]]; then - npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", ""]))') + npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", "=1.24", ""]))') spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", ""]))') - pdver=$(python -c 'import random ; print(random.choice(["=1.3", "=1.4", "=1.5", ""]))') + pdver=$(python -c 'import random ; print(random.choice(["=1.3", "=1.4", "=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=2.0", "=2.1", ""]))') else # Python 3.11 - npver=$(python -c 'import random ; print(random.choice(["=1.23", ""]))') + npver=$(python -c 'import random ; print(random.choice(["=1.23", "=1.24", ""]))') spver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", ""]))') - pdver=$(python -c 'import random ; print(random.choice(["=1.5", ""]))') + pdver=$(python -c 'import random ; print(random.choice(["=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.10", "=2.0", "=2.1", ""]))') fi if [[ ${{ steps.sourcetype.outputs.selected }} == "source" || ${{ steps.sourcetype.outputs.selected }} == "upstream" ]]; then @@ -208,16 +210,54 @@ jobs: else psgver="" fi - if [[ $npver == "=1.21" ]] ; then - numbaver=$(python -c 'import random ; print(random.choice(["=0.55", "=0.56", ""]))') + # TODO: drop 0.57.0rc1 and use 0.57 once numba 0.57 is properly released + if [[ ${npver} == "=1.24" || ${{ startsWith(steps.pyver.outputs.selected, '3.11') }} == true ]] ; then + numbaver=$(python -c 'import random ; print(random.choice(["=0.57.0rc1", ""]))') + elif [[ ${npver} == "=1.21" ]] ; then + numbaver=$(python -c 'import random ; print(random.choice(["=0.55", "=0.56", "=0.57.0rc1", ""]))') + else + numbaver=$(python -c 'import random ; print(random.choice(["=0.56", "=0.57.0rc1", ""]))') + fi + if [[ ${{ matrix.os == 'windows-latest' }} == true && ( ${npver} == "=1.24" || ${numbaver} == "=0.57.0rc1" ) ]] ; then + # TODO: numba 0.57.0rc1 currently crashes sometimes on windows, so skip it for now + npver="" + numbaver="" + fi + fmm=fast_matrix_market${fmmver} + awkward=awkward${akver} + if [[ ${{ contains(steps.pyver.outputs.selected, 'pypy') || + startsWith(steps.pyver.outputs.selected, '3.12') }} == true || + ( ${{ matrix.slowtask != 'notebooks'}} == true && ( + ( ${{ matrix.os == 'windows-latest' }} == true && $(python -c 'import random ; print(random.random() < .2)') == True ) || + ( ${{ matrix.os == 'windows-latest' }} == false && $(python -c 'import random ; print(random.random() < .4)') == True ))) ]] + then + # Some packages aren't available for pypy or Python 3.12; randomly otherwise (if not running notebooks) + echo "skipping numba" + numba="" + numbaver=NA + sparse="" + sparsever=NA + if [[ ${{ contains(steps.pyver.outputs.selected, 'pypy') }} ]]; then + awkward="" + akver=NA + fmm="" + fmmver=NA + # Be more flexible until we determine what versions are supported by pypy + npver="" + spver="" + pdver="" + yamlver="" + fi else - numbaver=$(python -c 'import random ; print(random.choice(["=0.56", ""]))') + numba=numba${numbaver} + sparse=sparse${sparsever} fi echo "versions: np${npver} sp${spver} pd${pdver} ak${akver} nx${nxver} numba${numbaver} yaml${yamlver} sparse${sparsever} psgver${psgver}" - $(command -v mamba || command -v conda) install packaging pytest coverage coveralls=3.3.1 pytest-randomly cffi donfig tomli \ - pyyaml${yamlver} sparse${sparsever} pandas${pdver} scipy${spver} numpy${npver} awkward${akver} \ - networkx${nxver} numba${numbaver} fast_matrix_market${fmmver} ${psg} \ + # TODO: remove `-c numba` when numba 0.57 is properly released + $(command -v mamba || command -v conda) install -c numba packaging pytest coverage coveralls=3.3.1 pytest-randomly cffi donfig tomli \ + pyyaml${yamlver} ${sparse} pandas${pdver} scipy${spver} numpy${npver} ${awkward} \ + networkx${nxver} ${numba} ${fmm} ${psg} \ ${{ matrix.slowtask == 'pytest_bizarro' && 'black' || '' }} \ ${{ matrix.slowtask == 'notebooks' && 'matplotlib nbconvert jupyter "ipython>=7"' || '' }} \ ${{ steps.sourcetype.outputs.selected == 'upstream' && 'cython' || '' }} \ @@ -269,9 +309,9 @@ jobs: if [[ $G && $bizarro ]] ; then if [[ $ubuntu ]] ; then echo " $suitesparse" ; elif [[ $windows ]] ; then echo " $vanilla" ; fi ; fi)$( \ if [[ $H && $normal ]] ; then if [[ $macos ]] ; then echo " $vanilla" ; elif [[ $windows ]] ; then echo " $suitesparse" ; fi ; fi)$( \ if [[ $H && $bizarro ]] ; then if [[ $macos ]] ; then echo " $suitesparse" ; elif [[ $windows ]] ; then echo " $vanilla" ; fi ; fi) - echo $args + echo ${args} pytest -v --pyargs suitesparse_graphblas - coverage run -m pytest --color=yes --randomly -v $args \ + coverage run -m pytest --color=yes --randomly -v ${args} \ ${{ matrix.slowtask == 'pytest_normal' && '--runslow' || '' }} - name: Unit tests (bizarro scalars) run: | @@ -305,8 +345,8 @@ jobs: if [[ $G && $bizarro ]] ; then if [[ $ubuntu ]] ; then echo " $vanilla" ; elif [[ $windows ]] ; then echo " $suitesparse" ; fi ; fi)$( \ if [[ $H && $normal ]] ; then if [[ $macos ]] ; then echo " $suitesparse" ; elif [[ $windows ]] ; then echo " $vanilla" ; fi ; fi)$( \ if [[ $H && $bizarro ]] ; then if [[ $macos ]] ; then echo " $vanilla" ; elif [[ $windows ]] ; then echo " $suitesparse" ; fi ; fi) - echo $args - coverage run -a -m pytest --color=yes --randomly -v $args \ + echo ${args} + coverage run -a -m pytest --color=yes --randomly -v ${args} \ ${{ matrix.slowtask == 'pytest_bizarro' && '--runslow' || '' }} git checkout . # Undo changes to scalar default - name: Miscellaneous tests @@ -329,6 +369,13 @@ jobs: # TODO: understand why these are order-dependent and try to fix coverage run -a -m pytest --color=yes -x --no-mapnumpy --runslow -k test_binaryop_attributes_numpy graphblas/tests/test_op.py # coverage run -a -m pytest --color=yes -x --no-mapnumpy -k test_npmonoid graphblas/tests/test_numpyops.py --runslow + - name: More tests for coverage + if: matrix.slowtask == 'notebooks' && matrix.os == 'windows-latest' + run: | + # We use 'notebooks' slow task b/c it should have numba installed + coverage run -a -m pytest --color=yes --runslow --no-mapnumpy -p no:randomly -v -k 'test_commutes or test_bool_doesnt_get_too_large or test_npbinary or test_npmonoid or test_npsemiring' + coverage run -a -m pytest --color=yes --runslow --mapnumpy -p no:randomly -k 'test_bool_doesnt_get_too_large or test_npunary or test_binaryop_monoid_numpy' + coverage run -a -m pytest --color=yes -x --no-mapnumpy --runslow -k test_binaryop_attributes_numpy graphblas/tests/test_op.py - name: Auto-generated code check if: matrix.slowtask == 'pytest_bizarro' run: | @@ -364,7 +411,11 @@ jobs: uses: codecov/codecov-action@v3 - name: Notebooks Execution check if: matrix.slowtask == 'notebooks' - run: jupyter nbconvert --to notebook --execute notebooks/*ipynb + run: | + # Run notebooks only if numba is installed + if python -c 'import numba' 2> /dev/null ; then + jupyter nbconvert --to notebook --execute notebooks/*ipynb + fi finish: needs: build_and_test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b00aee30..426153fee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,7 +58,7 @@ repos: - id: black - id: black-jupyter - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.260 + rev: v0.0.261 hooks: - id: ruff args: [--fix-only, --show-fixes] @@ -86,7 +86,7 @@ repos: additional_dependencies: [tomli] files: ^(graphblas|docs)/ - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.260 + rev: v0.0.261 hooks: - id: ruff - repo: https://github.com/sphinx-contrib/sphinx-lint diff --git a/README.md b/README.md index 23fc3650d..f07fdea12 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ $ conda install -c conda-forge python-graphblas ``` or pip: ``` -$ pip install python-graphblas +$ pip install python-graphblas[default] ``` This will also install the [SuiteSparse:GraphBLAS](https://github.com/DrTimothyAldenDavis/GraphBLAS) compiled C library. diff --git a/docs/_static/img/GraphBLAS-API-example.png b/docs/_static/img/GraphBLAS-API-example.png index c6dd48182..1edc91988 100644 Binary files a/docs/_static/img/GraphBLAS-API-example.png and b/docs/_static/img/GraphBLAS-API-example.png differ diff --git a/docs/_static/img/GraphBLAS-mapping.png b/docs/_static/img/GraphBLAS-mapping.png index 7ef73c88d..c5d1a1d4e 100644 Binary files a/docs/_static/img/GraphBLAS-mapping.png and b/docs/_static/img/GraphBLAS-mapping.png differ diff --git a/docs/_static/img/Matrix-A-strictly-upper.png b/docs/_static/img/Matrix-A-strictly-upper.png index 9b127aa84..0fedf2617 100644 Binary files a/docs/_static/img/Matrix-A-strictly-upper.png and b/docs/_static/img/Matrix-A-strictly-upper.png differ diff --git a/docs/_static/img/Matrix-A-upper.png b/docs/_static/img/Matrix-A-upper.png index 1b930a9a3..e3703710a 100644 Binary files a/docs/_static/img/Matrix-A-upper.png and b/docs/_static/img/Matrix-A-upper.png differ diff --git a/docs/_static/img/Recorder-output.png b/docs/_static/img/Recorder-output.png index 355cc1376..525221c55 100644 Binary files a/docs/_static/img/Recorder-output.png and b/docs/_static/img/Recorder-output.png differ diff --git a/docs/_static/img/adj-graph.png b/docs/_static/img/adj-graph.png index da9f36447..13a05fcc2 100644 Binary files a/docs/_static/img/adj-graph.png and b/docs/_static/img/adj-graph.png differ diff --git a/docs/_static/img/draw-example.png b/docs/_static/img/draw-example.png index 3c5e6c008..90c5917d9 100644 Binary files a/docs/_static/img/draw-example.png and b/docs/_static/img/draw-example.png differ diff --git a/docs/_static/img/logo-name-dark.svg b/docs/_static/img/logo-name-dark.svg index 039eb7e25..cdf5227c7 100644 --- a/docs/_static/img/logo-name-dark.svg +++ b/docs/_static/img/logo-name-dark.svg @@ -1,6 +1,4 @@ - - - - - - - - - - - - __EXPR__", topline) + # BRANCH NOT COVERED keys = [] values = [] diff --git a/graphblas/core/matrix.py b/graphblas/core/matrix.py index 1935fcee7..0183893fd 100644 --- a/graphblas/core/matrix.py +++ b/graphblas/core/matrix.py @@ -7,7 +7,7 @@ from .. import backend, binary, monoid, select, semiring from ..dtypes import _INDEX, FP64, INT64, lookup_dtype, unify from ..exceptions import DimensionMismatch, InvalidValue, NoValue, check_status -from . import automethods, ffi, lib, utils +from . import _supports_udfs, automethods, ffi, lib, utils from .base import BaseExpression, BaseType, _check_mask, call from .descriptor import lookup as descriptor_lookup from .expr import _ALL_INDICES, AmbiguousAssignOrExtract, IndexerResolver, Updater @@ -33,7 +33,7 @@ values_to_numpy_buffer, wrapdoc, ) -from .vector import Vector, VectorExpression, VectorIndexExpr, _select_mask +from .vector import Vector, VectorExpression, VectorIndexExpr, _isclose_recipe, _select_mask if backend == "suitesparse": from .ss.matrix import ss @@ -368,6 +368,8 @@ def isclose(self, other, *, rel_tol=1e-7, abs_tol=0.0, check_dtype=False, **opts return False if self._nvals != other._nvals: return False + if not _supports_udfs: + return _isclose_recipe(self, other, rel_tol, abs_tol, **opts) matches = self.ewise_mult(other, binary.isclose(rel_tol, abs_tol)).new( bool, name="M_isclose", **opts @@ -611,14 +613,15 @@ def build(self, rows, columns, values, *, dup_op=None, clear=False, nrows=None, if not dup_op_given: if not self.dtype._is_udt: dup_op = binary.plus - else: + elif backend != "suitesparse": dup_op = binary.any - # SS:SuiteSparse-specific: we could use NULL for dup_op - dup_op = get_typed_op(dup_op, self.dtype, kind="binary") - if dup_op.opclass == "Monoid": - dup_op = dup_op.binaryop - else: - self._expect_op(dup_op, "BinaryOp", within="build", argname="dup_op") + # SS:SuiteSparse-specific: we use NULL for dup_op + if dup_op is not None: + dup_op = get_typed_op(dup_op, self.dtype, kind="binary") + if dup_op.opclass == "Monoid": + dup_op = dup_op.binaryop + else: + self._expect_op(dup_op, "BinaryOp", within="build", argname="dup_op") rows = _CArray(rows) columns = _CArray(columns) @@ -1584,7 +1587,7 @@ def from_dicts( # If we know the dtype, then using `np.fromiter` is much faster dtype = lookup_dtype(dtype) if dtype.np_type.subdtype is not None and np.__version__[:5] in {"1.21.", "1.22."}: - values, dtype = values_to_numpy_buffer(list(iter_values), dtype) + values, dtype = values_to_numpy_buffer(list(iter_values), dtype) # FLAKY COVERAGE else: values = np.fromiter(iter_values, dtype.np_type) return getattr(cls, methodname)( @@ -2466,6 +2469,7 @@ def select(self, op, thunk=None): self._expect_op(op, ("SelectOp", "IndexUnaryOp"), within=method_name, argname="op") if thunk._is_cscalar: if thunk.dtype._is_udt: + # NOT COVERED dtype_name = "UDT" thunk = _Pointer(thunk) else: diff --git a/graphblas/core/operator/agg.py b/graphblas/core/operator/agg.py index 036149b1f..09d644c32 100644 --- a/graphblas/core/operator/agg.py +++ b/graphblas/core/operator/agg.py @@ -5,6 +5,7 @@ from ... import agg, backend, binary, monoid, semiring, unary from ...dtypes import INT64, lookup_dtype +from .. import _supports_udfs from ..utils import output_type @@ -38,6 +39,7 @@ def __init__( semiring=None, switch=False, semiring2=None, + applybegin=None, finalize=None, composite=None, custom=None, @@ -52,6 +54,7 @@ def __init__( self._semiring = semiring self._semiring2 = semiring2 self._switch = switch + self._applybegin = applybegin self._finalize = finalize self._composite = composite self._custom = custom @@ -152,8 +155,11 @@ def __repr__(self): def _new(self, updater, expr, *, in_composite=False): agg = self.parent + opts = updater.opts if agg._monoid is not None: x = expr.args[0] + if agg._applybegin is not None: # pragma: no cover (unused) + x = agg._applybegin(x).new(**opts) method = getattr(x, expr.method_name) if expr.output_type.__name__ == "Scalar": expr = method(agg._monoid[self.type], allow_empty=not expr._is_cscalar) @@ -167,7 +173,6 @@ def _new(self, updater, expr, *, in_composite=False): return parent._as_vector() return - opts = updater.opts if agg._composite is not None: # Masks are applied throughout the aggregation, including composite aggregations. # Aggregations done while `in_composite is True` should return the updater parent @@ -203,6 +208,8 @@ def _new(self, updater, expr, *, in_composite=False): if expr.cfunc_name == "GrB_Matrix_reduce_Aggregator": # Matrix -> Vector A = expr.args[0] + if agg._applybegin is not None: + A = agg._applybegin(A).new(**opts) orig_updater = updater if agg._finalize is not None: step1 = expr.construct_output(semiring.return_type) @@ -223,6 +230,8 @@ def _new(self, updater, expr, *, in_composite=False): elif expr.cfunc_name.startswith("GrB_Vector_reduce"): # Vector -> Scalar v = expr.args[0] + if agg._applybegin is not None: + v = agg._applybegin(v).new(**opts) step1 = expr._new_vector(semiring.return_type, size=1) init = expr._new_matrix(agg._initdtype, nrows=v._size, ncols=1) init(**opts)[...] = agg._initval # O(1) dense column vector in SuiteSparse 5 @@ -242,6 +251,8 @@ def _new(self, updater, expr, *, in_composite=False): elif expr.cfunc_name.startswith("GrB_Matrix_reduce"): # Matrix -> Scalar A = expr.args[0] + if agg._applybegin is not None: + A = agg._applybegin(A).new(**opts) # We need to compute in two steps: Matrix -> Vector -> Scalar. # This has not been benchmarked or optimized. # We may be able to intelligently choose the faster path. @@ -339,11 +350,21 @@ def __reduce__(self): # logaddexp2 = Aggregator('logaddexp2', monoid=semiring.numpy.logaddexp2) # hypot as monoid doesn't work if single negative element! # hypot = Aggregator('hypot', monoid=semiring.numpy.hypot) +# hypot = Aggregator('hypot', applybegin=unary.abs, monoid=semiring.numpy.hypot) agg.L0norm = agg.count_nonzero -agg.L1norm = Aggregator("L1norm", semiring="plus_absfirst", semiring2=semiring.plus_first) agg.L2norm = agg.hypot -agg.Linfnorm = Aggregator("Linfnorm", semiring="max_absfirst", semiring2=semiring.max_first) +if _supports_udfs: + agg.L1norm = Aggregator("L1norm", semiring="plus_absfirst", semiring2=semiring.plus_first) + agg.Linfnorm = Aggregator("Linfnorm", semiring="max_absfirst", semiring2=semiring.max_first) +else: + # Are these always better? + agg.L1norm = Aggregator( + "L1norm", applybegin=unary.abs, semiring=semiring.plus_first, semiring2=semiring.plus_first + ) + agg.Linfnorm = Aggregator( + "Linfnorm", applybegin=unary.abs, semiring=semiring.max_first, semiring2=semiring.max_first + ) # Composite diff --git a/graphblas/core/operator/base.py b/graphblas/core/operator/base.py index 38a76cbcf..a40438f14 100644 --- a/graphblas/core/operator/base.py +++ b/graphblas/core/operator/base.py @@ -1,16 +1,19 @@ -from functools import lru_cache, reduce -from operator import getitem, mul +from functools import lru_cache +from operator import getitem from types import BuiltinFunctionType, ModuleType -import numba -import numpy as np - from ... import _STANDARD_OPERATOR_NAMES, backend, op from ...dtypes import BOOL, INT8, UINT64, _supports_complex, lookup_dtype -from .. import lib +from .. import _has_numba, _supports_udfs, lib from ..expr import InfixExprBase from ..utils import output_type +if _has_numba: + import numba + from numba import NumbaError +else: + NumbaError = TypeError + UNKNOWN_OPCLASS = "UnknownOpClass" # These now live as e.g. `gb.unary.ss.positioni` @@ -158,96 +161,69 @@ def _call_op(op, left, right=None, thunk=None, **kwargs): ) -_udt_mask_cache = {} - - -def _udt_mask(dtype): - """Create mask to determine which bytes of UDTs to use for equality check.""" - if dtype in _udt_mask_cache: - return _udt_mask_cache[dtype] - if dtype.subdtype is not None: - mask = _udt_mask(dtype.subdtype[0]) - N = reduce(mul, dtype.subdtype[1]) - rv = np.concatenate([mask] * N) - elif dtype.names is not None: - prev_offset = mask = None - masks = [] - for name in dtype.names: - dtype2, offset = dtype.fields[name] - if mask is not None: - masks.append(np.pad(mask, (0, offset - prev_offset - mask.size))) - mask = _udt_mask(dtype2) - prev_offset = offset - masks.append(np.pad(mask, (0, dtype.itemsize - prev_offset - mask.size))) - rv = np.concatenate(masks) - else: - rv = np.ones(dtype.itemsize, dtype=bool) - # assert rv.size == dtype.itemsize - _udt_mask_cache[dtype] = rv - return rv - - -def _get_udt_wrapper(numba_func, return_type, dtype, dtype2=None, *, include_indexes=False): - ztype = INT8 if return_type == BOOL else return_type - xtype = INT8 if dtype == BOOL else dtype - nt = numba.types - wrapper_args = [nt.CPointer(ztype.numba_type), nt.CPointer(xtype.numba_type)] - if include_indexes: - wrapper_args.extend([UINT64.numba_type, UINT64.numba_type]) - if dtype2 is not None: - ytype = INT8 if dtype2 == BOOL else dtype2 - wrapper_args.append(nt.CPointer(ytype.numba_type)) - wrapper_sig = nt.void(*wrapper_args) - - zarray = xarray = yarray = BL = BR = yarg = yname = rcidx = "" - if return_type._is_udt: - if return_type.np_type.subdtype is None: - zarray = " z = numba.carray(z_ptr, 1)\n" - zname = "z[0]" +if _has_numba: + + def _get_udt_wrapper(numba_func, return_type, dtype, dtype2=None, *, include_indexes=False): + ztype = INT8 if return_type == BOOL else return_type + xtype = INT8 if dtype == BOOL else dtype + nt = numba.types + wrapper_args = [nt.CPointer(ztype.numba_type), nt.CPointer(xtype.numba_type)] + if include_indexes: + wrapper_args.extend([UINT64.numba_type, UINT64.numba_type]) + if dtype2 is not None: + ytype = INT8 if dtype2 == BOOL else dtype2 + wrapper_args.append(nt.CPointer(ytype.numba_type)) + wrapper_sig = nt.void(*wrapper_args) + + zarray = xarray = yarray = BL = BR = yarg = yname = rcidx = "" + if return_type._is_udt: + if return_type.np_type.subdtype is None: + zarray = " z = numba.carray(z_ptr, 1)\n" + zname = "z[0]" + else: + zname = "z_ptr[0]" + BR = "[0]" else: zname = "z_ptr[0]" - BR = "[0]" - else: - zname = "z_ptr[0]" - if return_type == BOOL: - BL = "bool(" - BR = ")" - - if dtype._is_udt: - if dtype.np_type.subdtype is None: - xarray = " x = numba.carray(x_ptr, 1)\n" - xname = "x[0]" - else: - xname = "x_ptr" - elif dtype == BOOL: - xname = "bool(x_ptr[0])" - else: - xname = "x_ptr[0]" - - if dtype2 is not None: - yarg = ", y_ptr" - if dtype2._is_udt: - if dtype2.np_type.subdtype is None: - yarray = " y = numba.carray(y_ptr, 1)\n" - yname = ", y[0]" + if return_type == BOOL: + BL = "bool(" + BR = ")" + + if dtype._is_udt: + if dtype.np_type.subdtype is None: + xarray = " x = numba.carray(x_ptr, 1)\n" + xname = "x[0]" else: - yname = ", y_ptr" - elif dtype2 == BOOL: - yname = ", bool(y_ptr[0])" + xname = "x_ptr" + elif dtype == BOOL: + xname = "bool(x_ptr[0])" else: - yname = ", y_ptr[0]" + xname = "x_ptr[0]" + + if dtype2 is not None: + yarg = ", y_ptr" + if dtype2._is_udt: + if dtype2.np_type.subdtype is None: + yarray = " y = numba.carray(y_ptr, 1)\n" + yname = ", y[0]" + else: + yname = ", y_ptr" + elif dtype2 == BOOL: + yname = ", bool(y_ptr[0])" + else: + yname = ", y_ptr[0]" - if include_indexes: - rcidx = ", row, col" + if include_indexes: + rcidx = ", row, col" - d = {"numba": numba, "numba_func": numba_func} - text = ( - f"def wrapper(z_ptr, x_ptr{rcidx}{yarg}):\n" - f"{zarray}{xarray}{yarray}" - f" {zname} = {BL}numba_func({xname}{rcidx}{yname}){BR}\n" - ) - exec(text, d) # pylint: disable=exec-used - return d["wrapper"], wrapper_sig + d = {"numba": numba, "numba_func": numba_func} + text = ( + f"def wrapper(z_ptr, x_ptr{rcidx}{yarg}):\n" + f"{zarray}{xarray}{yarray}" + f" {zname} = {BL}numba_func({xname}{rcidx}{yname}){BR}\n" + ) + exec(text, d) # pylint: disable=exec-used + return d["wrapper"], wrapper_sig class TypedOpBase: @@ -360,6 +336,8 @@ def __getitem__(self, type_): raise KeyError(f"{self.name} does not work with {type_}") else: return self._typed_ops[type_] + if not _supports_udfs: + raise KeyError(f"{self.name} does not work with {type_}") # This is a UDT or is able to operate on UDTs such as `first` any `any` dtype = lookup_dtype(type_) return self._compile_udt(dtype, dtype) @@ -376,7 +354,7 @@ def __delitem__(self, type_): def __contains__(self, type_): try: self[type_] - except (TypeError, KeyError, numba.NumbaError): + except (TypeError, KeyError, NumbaError): return False return True @@ -487,7 +465,7 @@ def _initialize(cls, include_in_ops=True): if type_ is None: type_ = BOOL else: - if type_ is None: # pragma: no cover + if type_ is None: # pragma: no cover (safety) raise TypeError(f"Unable to determine return type for {varname}") if return_prefix is None: return_type = type_ @@ -513,6 +491,13 @@ def _deserialize(cls, name, *args): return rv # Should we verify this is what the user expects? return cls.register_new(name, *args) + @classmethod + def _check_supports_udf(cls, method_name): + if not _supports_udfs: + raise RuntimeError( + f"{cls.__name__}.{method_name}(...) unavailable; install numba for UDF support" + ) + _builtin_to_op = {} # Populated in .utils diff --git a/graphblas/core/operator/binary.py b/graphblas/core/operator/binary.py index eeb72ea3b..8d41a097e 100644 --- a/graphblas/core/operator/binary.py +++ b/graphblas/core/operator/binary.py @@ -1,9 +1,9 @@ import inspect import re -from functools import lru_cache +from functools import lru_cache, reduce +from operator import mul from types import FunctionType -import numba import numpy as np from ... import _STANDARD_OPERATOR_NAMES, backend, binary, monoid, op @@ -24,7 +24,7 @@ lookup_dtype, ) from ...exceptions import UdfParseError, check_status_carg -from .. import ffi, lib +from .. import _has_numba, _supports_udfs, ffi, lib from ..expr import InfixExprBase from .base import ( _SS_OPERATORS, @@ -33,16 +33,46 @@ TypedOpBase, _call_op, _deserialize_parameterized, - _get_udt_wrapper, _hasop, - _udt_mask, ) +if _has_numba: + import numba + + from .base import _get_udt_wrapper if _supports_complex: from ...dtypes import FC32, FC64 ffi_new = ffi.new +if _has_numba: + _udt_mask_cache = {} + + def _udt_mask(dtype): + """Create mask to determine which bytes of UDTs to use for equality check.""" + if dtype in _udt_mask_cache: + return _udt_mask_cache[dtype] + if dtype.subdtype is not None: + mask = _udt_mask(dtype.subdtype[0]) + N = reduce(mul, dtype.subdtype[1]) + rv = np.concatenate([mask] * N) + elif dtype.names is not None: + prev_offset = mask = None + masks = [] + for name in dtype.names: + dtype2, offset = dtype.fields[name] + if mask is not None: + masks.append(np.pad(mask, (0, offset - prev_offset - mask.size))) + mask = _udt_mask(dtype2) + prev_offset = offset + masks.append(np.pad(mask, (0, dtype.itemsize - prev_offset - mask.size))) + rv = np.concatenate(masks) + else: + rv = np.ones(dtype.itemsize, dtype=bool) + # assert rv.size == dtype.itemsize + _udt_mask_cache[dtype] = rv + return rv + class TypedBuiltinBinaryOp(TypedOpBase): __slots__ = () @@ -601,6 +631,7 @@ def register_anonymous(cls, func, name=None, *, parameterized=False, is_udt=Fals Because it is not registered in the namespace, the name is optional. """ + cls._check_supports_udf("register_anonymous") if parameterized: return ParameterizedBinaryOp(name, func, anonymous=True, is_udt=is_udt) return cls._build(name, func, anonymous=True, is_udt=is_udt) @@ -621,6 +652,7 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal >>> dir(gb.binary) [..., 'max_zero', ...] """ + cls._check_supports_udf("register_new") module, funcname = cls._remove_nesting(name) if lazy: module._delayed[funcname] = ( @@ -681,21 +713,22 @@ def _initialize(cls): orig_op.gb_name, ) new_op._add(cur_op) - # Add floordiv - # cdiv truncates towards 0, while floordiv truncates towards -inf - BinaryOp.register_new("floordiv", _floordiv, lazy=True) # cast to integer - BinaryOp.register_new("rfloordiv", _rfloordiv, lazy=True) # cast to integer + if _supports_udfs: + # Add floordiv + # cdiv truncates towards 0, while floordiv truncates towards -inf + BinaryOp.register_new("floordiv", _floordiv, lazy=True) # cast to integer + BinaryOp.register_new("rfloordiv", _rfloordiv, lazy=True) # cast to integer - # For aggregators - BinaryOp.register_new("absfirst", _absfirst, lazy=True) - BinaryOp.register_new("abssecond", _abssecond, lazy=True) - BinaryOp.register_new("rpow", _rpow, lazy=True) + # For aggregators + BinaryOp.register_new("absfirst", _absfirst, lazy=True) + BinaryOp.register_new("abssecond", _abssecond, lazy=True) + BinaryOp.register_new("rpow", _rpow, lazy=True) - # For algorithms - binary._delayed["binom"] = (_register_binom, {}) # Lazy with custom creation - op._delayed["binom"] = binary + # For algorithms + binary._delayed["binom"] = (_register_binom, {}) # Lazy with custom creation + op._delayed["binom"] = binary - BinaryOp.register_new("isclose", _isclose, parameterized=True) + BinaryOp.register_new("isclose", _isclose, parameterized=True) # Update type information with sane coercion position_dtypes = [ @@ -777,14 +810,23 @@ def _initialize(cls): if right_name not in binary._delayed: if right_name in _SS_OPERATORS: right = binary._deprecated[right_name] - else: + elif _supports_udfs: right = getattr(binary, right_name) + else: + right = getattr(binary, right_name, None) + if right is None: + continue if backend == "suitesparse" and left_name in _SS_OPERATORS: right._commutes_to = f"ss.{left_name}" else: right._commutes_to = left_name for name in cls._commutative: - cur_op = getattr(binary, name) + if _supports_udfs: + cur_op = getattr(binary, name) + else: + cur_op = getattr(binary, name, None) + if cur_op is None: + continue cur_op._commutes_to = name for left_name, right_name in cls._commutes_to_in_semiring.items(): if left_name in _SS_OPERATORS: @@ -805,7 +847,10 @@ def _initialize(cls): (binary.any, _first), ]: binop.orig_func = func - binop._numba_func = numba.njit(func) + if _has_numba: + binop._numba_func = numba.njit(func) + else: + binop._numba_func = None binop._udt_types = {} binop._udt_ops = {} binary.any._numba_func = binary.first._numba_func diff --git a/graphblas/core/operator/indexunary.py b/graphblas/core/operator/indexunary.py index 5fdafb62a..ad5d841d0 100644 --- a/graphblas/core/operator/indexunary.py +++ b/graphblas/core/operator/indexunary.py @@ -2,21 +2,16 @@ import re from types import FunctionType -import numba - from ... import _STANDARD_OPERATOR_NAMES, indexunary, select from ...dtypes import BOOL, FP64, INT8, INT64, UINT64, _sample_values, lookup_dtype from ...exceptions import UdfParseError, check_status_carg -from .. import ffi, lib -from .base import ( - OpBase, - ParameterizedUdf, - TypedOpBase, - _call_op, - _deserialize_parameterized, - _get_udt_wrapper, -) +from .. import _has_numba, ffi, lib +from .base import OpBase, ParameterizedUdf, TypedOpBase, _call_op, _deserialize_parameterized + +if _has_numba: + import numba + from .base import _get_udt_wrapper ffi_new = ffi.new @@ -65,6 +60,7 @@ def _call(self, *args, **kwargs): return IndexUnaryOp.register_anonymous(indexunary, self.name, is_udt=self._is_udt) def __reduce__(self): + # NOT COVERED name = f"indexunary.{self.name}" if not self._anonymous and name in _STANDARD_OPERATOR_NAMES: return name @@ -72,6 +68,7 @@ def __reduce__(self): @staticmethod def _deserialize(name, func, anonymous): + # NOT COVERED if anonymous: return IndexUnaryOp.register_anonymous(func, name, parameterized=True) if (rv := IndexUnaryOp._find(name)) is not None: @@ -249,6 +246,7 @@ def register_anonymous(cls, func, name=None, *, parameterized=False, is_udt=Fals Because it is not registered in the namespace, the name is optional. """ + cls._check_supports_udf("register_anonymous") if parameterized: return ParameterizedIndexUnaryOp(name, func, anonymous=True, is_udt=is_udt) return cls._build(name, func, anonymous=True, is_udt=is_udt) @@ -265,6 +263,7 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal >>> dir(gb.indexunary) [..., 'row_mod', ...] """ + cls._check_supports_udf("register_new") module, funcname = cls._remove_nesting(name) if lazy: module._delayed[funcname] = ( @@ -281,9 +280,12 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal if all(x == BOOL for x in indexunary_op.types.values()): from .select import SelectOp - setattr(select, funcname, SelectOp._from_indexunary(indexunary_op)) + select_module, funcname = SelectOp._remove_nesting(name, strict=False) + setattr(select_module, funcname, SelectOp._from_indexunary(indexunary_op)) + if not cls._initialized: # pragma: no cover (safety) + _STANDARD_OPERATOR_NAMES.add(f"{SelectOp._modname}.{name}") - if not cls._initialized: + if not cls._initialized: # pragma: no cover (safety) _STANDARD_OPERATOR_NAMES.add(f"{cls._modname}.{name}") if not lazy: return indexunary_op @@ -323,6 +325,7 @@ def _initialize(cls): "valueeq", "valuene", "valuegt", "valuege", "valuelt", "valuele"]: iop = getattr(indexunary, name) setattr(select, name, SelectOp._from_indexunary(iop)) + _STANDARD_OPERATOR_NAMES.add(f"{SelectOp._modname}.{name}") # fmt: on cls._initialized = True @@ -348,10 +351,12 @@ def __init__( def __reduce__(self): if self._anonymous: if hasattr(self.orig_func, "_parameterized_info"): + # NOT COVERED return (_deserialize_parameterized, self.orig_func._parameterized_info) return (self.register_anonymous, (self.orig_func, self.name)) if (name := f"indexunary.{self.name}") in _STANDARD_OPERATOR_NAMES: return name + # NOT COVERED return (self._deserialize, (self.name, self.orig_func)) __call__ = TypedBuiltinIndexUnaryOp.__call__ diff --git a/graphblas/core/operator/select.py b/graphblas/core/operator/select.py index 844565f3a..27567eb2f 100644 --- a/graphblas/core/operator/select.py +++ b/graphblas/core/operator/select.py @@ -51,6 +51,7 @@ def _call(self, *args, **kwargs): return SelectOp.register_anonymous(sel, self.name, is_udt=self._is_udt) def __reduce__(self): + # NOT COVERED name = f"select.{self.name}" if not self._anonymous and name in _STANDARD_OPERATOR_NAMES: return name @@ -58,6 +59,7 @@ def __reduce__(self): @staticmethod def _deserialize(name, func, anonymous): + # NOT COVERED if anonymous: return SelectOp.register_anonymous(func, name, parameterized=True) if (rv := SelectOp._find(name)) is not None: @@ -124,6 +126,7 @@ def register_anonymous(cls, func, name=None, *, parameterized=False, is_udt=Fals Because it is not registered in the namespace, the name is optional. """ + cls._check_supports_udf("register_anonymous") if parameterized: return ParameterizedSelectOp(name, func, anonymous=True, is_udt=is_udt) iop = IndexUnaryOp._build(name, func, anonymous=True, is_udt=is_udt) @@ -140,13 +143,36 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal >>> dir(gb.select) [..., 'upper_left_triangle', ...] """ + cls._check_supports_udf("register_new") iop = IndexUnaryOp.register_new( name, func, parameterized=parameterized, is_udt=is_udt, lazy=lazy ) + module, funcname = cls._remove_nesting(name, strict=False) + if lazy: + module._delayed[funcname] = ( + cls._get_delayed, + {"name": name}, + ) + elif parameterized: + op = ParameterizedSelectOp(funcname, func, is_udt=is_udt) + setattr(module, funcname, op) + return op + elif not all(x == BOOL for x in iop.types.values()): + # Undo registration of indexunaryop + imodule, funcname = IndexUnaryOp._remove_nesting(name, strict=False) + delattr(imodule, funcname) + raise ValueError("SelectOp must have BOOL return type") + else: + return getattr(module, funcname) + + @classmethod + def _get_delayed(cls, name): + imodule, funcname = IndexUnaryOp._remove_nesting(name, strict=False) + iop = getattr(imodule, name) if not all(x == BOOL for x in iop.types.values()): raise ValueError("SelectOp must have BOOL return type") - if lazy: - return getattr(select, iop.name) + module, funcname = cls._remove_nesting(name, strict=False) + return getattr(module, funcname) @classmethod def _initialize(cls): @@ -172,16 +198,19 @@ def __init__( self.is_positional = is_positional self._is_udt = is_udt if is_udt: + # NOT COVERED self._udt_types = {} # {dtype: DataType} self._udt_ops = {} # {dtype: TypedUserIndexUnaryOp} def __reduce__(self): if self._anonymous: if hasattr(self.orig_func, "_parameterized_info"): + # NOT COVERED return (_deserialize_parameterized, self.orig_func._parameterized_info) return (self.register_anonymous, (self.orig_func, self.name)) if (name := f"select.{self.name}") in _STANDARD_OPERATOR_NAMES: return name + # NOT COVERED return (self._deserialize, (self.name, self.orig_func)) __call__ = TypedBuiltinSelectOp.__call__ diff --git a/graphblas/core/operator/semiring.py b/graphblas/core/operator/semiring.py index 06450e007..ac716b9dd 100644 --- a/graphblas/core/operator/semiring.py +++ b/graphblas/core/operator/semiring.py @@ -17,7 +17,7 @@ _supports_complex, ) from ...exceptions import check_status_carg -from .. import ffi, lib +from .. import _supports_udfs, ffi, lib from .base import _SS_OPERATORS, OpBase, ParameterizedUdf, TypedOpBase, _call_op, _hasop from .binary import BinaryOp, ParameterizedBinaryOp from .monoid import Monoid, ParameterizedMonoid @@ -358,15 +358,17 @@ def _initialize(cls): for orig_name, orig in div_semirings.items(): cls.register_new(f"{orig_name[:-3]}truediv", orig.monoid, binary.truediv, lazy=True) cls.register_new(f"{orig_name[:-3]}rtruediv", orig.monoid, "rtruediv", lazy=True) - cls.register_new(f"{orig_name[:-3]}floordiv", orig.monoid, "floordiv", lazy=True) - cls.register_new(f"{orig_name[:-3]}rfloordiv", orig.monoid, "rfloordiv", lazy=True) + if _supports_udfs: + cls.register_new(f"{orig_name[:-3]}floordiv", orig.monoid, "floordiv", lazy=True) + cls.register_new(f"{orig_name[:-3]}rfloordiv", orig.monoid, "rfloordiv", lazy=True) # For aggregators cls.register_new("plus_pow", monoid.plus, binary.pow) - cls.register_new("plus_rpow", monoid.plus, "rpow", lazy=True) - cls.register_new("plus_absfirst", monoid.plus, "absfirst", lazy=True) - cls.register_new("max_absfirst", monoid.max, "absfirst", lazy=True) - cls.register_new("plus_abssecond", monoid.plus, "abssecond", lazy=True) - cls.register_new("max_abssecond", monoid.max, "abssecond", lazy=True) + if _supports_udfs: + cls.register_new("plus_rpow", monoid.plus, "rpow", lazy=True) + cls.register_new("plus_absfirst", monoid.plus, "absfirst", lazy=True) + cls.register_new("max_absfirst", monoid.max, "absfirst", lazy=True) + cls.register_new("plus_abssecond", monoid.plus, "abssecond", lazy=True) + cls.register_new("max_abssecond", monoid.max, "abssecond", lazy=True) # Update type information with sane coercion for lname in ["any", "eq", "land", "lor", "lxnor", "lxor"]: diff --git a/graphblas/core/operator/unary.py b/graphblas/core/operator/unary.py index 6b1319057..1432a9387 100644 --- a/graphblas/core/operator/unary.py +++ b/graphblas/core/operator/unary.py @@ -2,8 +2,6 @@ import re from types import FunctionType -import numba - from ... import _STANDARD_OPERATOR_NAMES, op, unary from ...dtypes import ( BOOL, @@ -22,7 +20,7 @@ lookup_dtype, ) from ...exceptions import UdfParseError, check_status_carg -from .. import ffi, lib +from .. import _has_numba, ffi, lib from ..utils import output_type from .base import ( _SS_OPERATORS, @@ -30,12 +28,15 @@ ParameterizedUdf, TypedOpBase, _deserialize_parameterized, - _get_udt_wrapper, _hasop, ) if _supports_complex: from ...dtypes import FC32, FC64 +if _has_numba: + import numba + + from .base import _get_udt_wrapper ffi_new = ffi.new @@ -276,6 +277,7 @@ def register_anonymous(cls, func, name=None, *, parameterized=False, is_udt=Fals Because it is not registered in the namespace, the name is optional. """ + cls._check_supports_udf("register_anonymous") if parameterized: return ParameterizedUnaryOp(name, func, anonymous=True, is_udt=is_udt) return cls._build(name, func, anonymous=True, is_udt=is_udt) @@ -289,6 +291,7 @@ def register_new(cls, name, func, *, parameterized=False, is_udt=False, lazy=Fal >>> dir(gb.unary) [..., 'plus_one', ...] """ + cls._check_supports_udf("register_new") module, funcname = cls._remove_nesting(name) if lazy: module._delayed[funcname] = ( @@ -372,7 +375,10 @@ def _initialize(cls): (unary.one, _one), ]: unop.orig_func = func - unop._numba_func = numba.njit(func) + if _has_numba: + unop._numba_func = numba.njit(func) + else: + unop._numba_func = None unop._udt_types = {} unop._udt_ops = {} cls._initialized = True diff --git a/graphblas/core/recorder.py b/graphblas/core/recorder.py index ce79c85ff..2268c31eb 100644 --- a/graphblas/core/recorder.py +++ b/graphblas/core/recorder.py @@ -137,10 +137,10 @@ def _repr_base_(self): tail = "\n\n\n" return "\n".join(head), tail - def _repr_html_(self): # pragma: no cover + def _repr_html_(self): try: from IPython.display import Code - except ImportError as exc: + except ImportError as exc: # pragma: no cover (import) raise NotImplementedError from exc lines = self._get_repr_lines() code = Code("\n".join(lines), language="C") diff --git a/graphblas/core/scalar.py b/graphblas/core/scalar.py index 93b5ebb4b..a7a251a1d 100644 --- a/graphblas/core/scalar.py +++ b/graphblas/core/scalar.py @@ -3,15 +3,19 @@ import numpy as np from .. import backend, binary, config, monoid -from ..binary import isclose from ..dtypes import _INDEX, FP64, lookup_dtype, unify from ..exceptions import EmptyObject, check_status -from . import automethods, ffi, lib, utils +from . import _has_numba, _supports_udfs, automethods, ffi, lib, utils from .base import BaseExpression, BaseType, call from .expr import AmbiguousAssignOrExtract from .operator import get_typed_op from .utils import _Pointer, output_type, wrapdoc +if _supports_udfs: + from ..binary import isclose +else: + from .operator.binary import _isclose as isclose + ffi_new = ffi.new @@ -261,6 +265,17 @@ def isclose(self, other, *, rel_tol=1e-7, abs_tol=0.0, check_dtype=False): return False # We can't yet call a UDF on a scalar as part of the spec, so let's do it ourselves isclose_func = isclose(rel_tol, abs_tol) + if not _has_numba: + # Check if types are compatible + get_typed_op( + binary.eq, + self.dtype, + other.dtype, + is_left_scalar=True, + is_right_scalar=True, + kind="binary", + ) + return isclose_func(self.value, other.value) isclose_func = get_typed_op( isclose_func, self.dtype, diff --git a/graphblas/core/ss/matrix.py b/graphblas/core/ss/matrix.py index b1869f198..cac0296c7 100644 --- a/graphblas/core/ss/matrix.py +++ b/graphblas/core/ss/matrix.py @@ -1,9 +1,7 @@ import itertools import warnings -import numba import numpy as np -from numba import njit from suitesparse_graphblas.utils import claim_buffer, claim_buffer_2d, unclaim_buffer import graphblas as gb @@ -11,7 +9,7 @@ from ... import binary, monoid from ...dtypes import _INDEX, BOOL, INT64, UINT64, _string_to_dtype, lookup_dtype from ...exceptions import _error_code_lookup, check_status, check_status_carg -from .. import NULL, ffi, lib +from .. import NULL, _has_numba, ffi, lib from ..base import call from ..operator import get_typed_op from ..scalar import Scalar, _as_scalar, _scalar_index @@ -30,6 +28,16 @@ from .config import BaseConfig from .descriptor import get_descriptor +if _has_numba: + from numba import njit, prange +else: + + def njit(func=None, **kwargs): + if func is not None: + return func + return njit + + prange = range ffi_new = ffi.new @@ -888,7 +896,7 @@ def _export(self, format=None, *, sort=False, give_ownership=False, raw=False, m col_indices = claim_buffer(ffi, Aj[0], Aj_size[0] // index_dtype.itemsize, index_dtype) values = claim_buffer(ffi, Ax[0], Ax_size[0] // dtype.itemsize, dtype) if not raw: - if indptr.size > nrows + 1: + if indptr.size > nrows + 1: # pragma: no cover (suitesparse) indptr = indptr[: nrows + 1] if col_indices.size > nvals: col_indices = col_indices[:nvals] @@ -929,7 +937,7 @@ def _export(self, format=None, *, sort=False, give_ownership=False, raw=False, m row_indices = claim_buffer(ffi, Ai[0], Ai_size[0] // index_dtype.itemsize, index_dtype) values = claim_buffer(ffi, Ax[0], Ax_size[0] // dtype.itemsize, dtype) if not raw: - if indptr.size > ncols + 1: + if indptr.size > ncols + 1: # pragma: no cover (suitesparse) indptr = indptr[: ncols + 1] if row_indices.size > nvals: row_indices = row_indices[:nvals] @@ -1786,6 +1794,7 @@ def import_hypercsc( ---------- nrows : int ncols : int + cols : array-like indptr : array-like values : array-like row_indices : array-like @@ -4371,28 +4380,28 @@ def deserialize(cls, data, dtype=None, *, name=None, **opts): return rv -@numba.njit(parallel=True) +@njit(parallel=True) def argsort_values(indptr, indices, values): # pragma: no cover (numba) rv = np.empty(indptr[-1], dtype=np.uint64) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): rv[indptr[i] : indptr[i + 1]] = indices[ np.int64(indptr[i]) + np.argsort(values[indptr[i] : indptr[i + 1]]) ] return rv -@numba.njit(parallel=True) +@njit(parallel=True) def sort_values(indptr, values): # pragma: no cover (numba) rv = np.empty(indptr[-1], dtype=values.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): rv[indptr[i] : indptr[i + 1]] = np.sort(values[indptr[i] : indptr[i + 1]]) return rv -@numba.njit(parallel=True) +@njit(parallel=True) def compact_values(old_indptr, new_indptr, values): # pragma: no cover (numba) rv = np.empty(new_indptr[-1], dtype=values.dtype) - for i in numba.prange(new_indptr.size - 1): + for i in prange(new_indptr.size - 1): start = np.int64(new_indptr[i]) offset = np.int64(old_indptr[i]) - start for j in range(start, new_indptr[i + 1]): @@ -4400,17 +4409,17 @@ def compact_values(old_indptr, new_indptr, values): # pragma: no cover (numba) return rv -@numba.njit(parallel=True) +@njit(parallel=True) def reverse_values(indptr, values): # pragma: no cover (numba) rv = np.empty(indptr[-1], dtype=values.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): offset = np.int64(indptr[i]) + np.int64(indptr[i + 1]) - 1 for j in range(indptr[i], indptr[i + 1]): rv[j] = values[offset - j] return rv -@numba.njit(parallel=True) +@njit(parallel=True) def compact_indices(indptr, k): # pragma: no cover (numba) """Given indptr from hypercsr, create a new col_indices array that is compact. @@ -4420,7 +4429,7 @@ def compact_indices(indptr, k): # pragma: no cover (numba) indptr = create_indptr(indptr, k) col_indices = np.empty(indptr[-1], dtype=np.uint64) N = np.int64(0) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): start = np.int64(indptr[i]) deg = np.int64(indptr[i + 1]) - start N = max(N, deg) @@ -4433,7 +4442,7 @@ def compact_indices(indptr, k): # pragma: no cover (numba) def choose_random1(indptr): # pragma: no cover (numba) choices = np.empty(indptr.size - 1, dtype=indptr.dtype) new_indptr = np.arange(indptr.size, dtype=indptr.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): idx = np.int64(indptr[i]) deg = np.int64(indptr[i + 1]) - idx if deg == 1: @@ -4470,7 +4479,7 @@ def choose_random(indptr, k): # pragma: no cover (numba) # be nice to have them sorted if convenient to do so. new_indptr = create_indptr(indptr, k) choices = np.empty(new_indptr[-1], dtype=indptr.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): idx = np.int64(indptr[i]) deg = np.int64(indptr[i + 1]) - idx if k < deg: @@ -4551,7 +4560,7 @@ def choose_first(indptr, k): # pragma: no cover (numba) new_indptr = create_indptr(indptr, k) choices = np.empty(new_indptr[-1], dtype=indptr.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): idx = np.int64(indptr[i]) deg = np.int64(indptr[i + 1]) - idx if k < deg: @@ -4575,7 +4584,7 @@ def choose_last(indptr, k): # pragma: no cover (numba) new_indptr = create_indptr(indptr, k) choices = np.empty(new_indptr[-1], dtype=indptr.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): idx = np.int64(indptr[i]) deg = np.int64(indptr[i + 1]) - idx if k < deg: @@ -4608,19 +4617,20 @@ def indices_to_indptr(indices, size): # pragma: no cover (numba) """Calculate the indptr for e.g. CSR from sorted COO rows.""" indptr = np.zeros(size, dtype=indices.dtype) index = np.uint64(0) + one = np.uint64(1) for i in range(indices.size): row = indices[i] if row != index: - indptr[index + 1] = i + indptr[index + one] = i index = row - indptr[index + 1] = indices.size + indptr[index + one] = indices.size return indptr @njit(parallel=True) def indptr_to_indices(indptr): # pragma: no cover (numba) indices = np.empty(indptr[-1], dtype=indptr.dtype) - for i in numba.prange(indptr.size - 1): + for i in prange(indptr.size - 1): for j in range(indptr[i], indptr[i + 1]): indices[j] = i return indices diff --git a/graphblas/core/ss/vector.py b/graphblas/core/ss/vector.py index 343335773..2b1e8bf05 100644 --- a/graphblas/core/ss/vector.py +++ b/graphblas/core/ss/vector.py @@ -1,7 +1,6 @@ import itertools import numpy as np -from numba import njit from suitesparse_graphblas.utils import claim_buffer, unclaim_buffer import graphblas as gb @@ -23,7 +22,7 @@ ) from .config import BaseConfig from .descriptor import get_descriptor -from .matrix import _concat_mn +from .matrix import _concat_mn, njit from .prefix_scan import prefix_scan ffi_new = ffi.new @@ -588,7 +587,7 @@ def _export(self, format=None, *, sort=False, give_ownership=False, raw=False, m if is_iso: if values.size > 1: # pragma: no cover (suitesparse) values = values[:1] - elif values.size > size: # pragma: no branch (suitesparse) + elif values.size > size: # pragma: no cover (suitesparse) values = values[:size] rv = { "bitmap": bitmap, diff --git a/graphblas/core/utils.py b/graphblas/core/utils.py index 0beeb4a2a..77c64a7ac 100644 --- a/graphblas/core/utils.py +++ b/graphblas/core/utils.py @@ -131,6 +131,7 @@ def get_shape(nrows, ncols, dtype=None, **arrays): # We could be smarter and determine the shape of the dtype sub-arrays if arr.ndim >= 3: break + # BRANCH NOT COVERED elif arr.ndim == 2: break else: diff --git a/graphblas/core/vector.py b/graphblas/core/vector.py index 8231691c6..57851420d 100644 --- a/graphblas/core/vector.py +++ b/graphblas/core/vector.py @@ -3,10 +3,10 @@ import numpy as np -from .. import backend, binary, monoid, select, semiring +from .. import backend, binary, monoid, select, semiring, unary from ..dtypes import _INDEX, FP64, INT64, lookup_dtype, unify from ..exceptions import DimensionMismatch, NoValue, check_status -from . import automethods, ffi, lib, utils +from . import _supports_udfs, automethods, ffi, lib, utils from .base import BaseExpression, BaseType, _check_mask, call from .descriptor import lookup as descriptor_lookup from .expr import _ALL_INDICES, AmbiguousAssignOrExtract, IndexerResolver, Updater @@ -93,6 +93,45 @@ def _select_mask(updater, obj, mask): updater << obj.dup(mask=mask) +def _isclose_recipe(self, other, rel_tol, abs_tol, **opts): + # x == y or abs(x - y) <= max(rel_tol * max(abs(x), abs(y)), abs_tol) + isequal = self.ewise_mult(other, binary.eq).new(bool, name="isclose", **opts) + if isequal._nvals != self._nvals: + return False + if type(isequal) is Vector: + val = isequal.reduce(monoid.land, allow_empty=False).new(**opts).value + else: + val = isequal.reduce_scalar(monoid.land, allow_empty=False).new(**opts).value + if val: + return True + # So we can use structural mask below + isequal(**opts) << select.value(isequal == True) # noqa: E712 + + # abs(x) + x = self.apply(unary.abs).new(FP64, mask=~isequal.S, **opts) + # abs(y) + y = other.apply(unary.abs).new(FP64, mask=~isequal.S, **opts) + # max(abs(x), abs(y)) + x(**opts) << x.ewise_mult(y, binary.max) + max_x_y = x + # rel_tol * max(abs(x), abs(y)) + max_x_y(**opts) << max_x_y.apply(binary.times, rel_tol) + # max(rel_tol * max(abs(x), abs(y)), abs_tol) + max_x_y(**opts) << max_x_y.apply(binary.max, abs_tol) + + # x - y + y(~isequal.S, replace=True, **opts) << self.ewise_mult(other, binary.minus) + abs_x_y = y + # abs(x - y) + abs_x_y(**opts) << abs_x_y.apply(unary.abs) + + # abs(x - y) <= max(rel_tol * max(abs(x), abs(y)), abs_tol) + isequal(**opts) << abs_x_y.ewise_mult(max_x_y, binary.le) + if isequal.ndim == 1: + return isequal.reduce(monoid.land, allow_empty=False).new(**opts).value + return isequal.reduce_scalar(monoid.land, allow_empty=False).new(**opts).value + + class Vector(BaseType): """Create a new GraphBLAS Sparse Vector. @@ -354,6 +393,8 @@ def isclose(self, other, *, rel_tol=1e-7, abs_tol=0.0, check_dtype=False, **opts return False if self._nvals != other._nvals: return False + if not _supports_udfs: + return _isclose_recipe(self, other, rel_tol, abs_tol, **opts) matches = self.ewise_mult(other, binary.isclose(rel_tol, abs_tol)).new( bool, name="M_isclose", **opts @@ -520,14 +561,15 @@ def build(self, indices, values, *, dup_op=None, clear=False, size=None): if not dup_op_given: if not self.dtype._is_udt: dup_op = binary.plus - else: + elif backend != "suitesparse": dup_op = binary.any - # SS:SuiteSparse-specific: we could use NULL for dup_op - dup_op = get_typed_op(dup_op, self.dtype, kind="binary") - if dup_op.opclass == "Monoid": - dup_op = dup_op.binaryop - else: - self._expect_op(dup_op, "BinaryOp", within="build", argname="dup_op") + # SS:SuiteSparse-specific: we use NULL for dup_op + if dup_op is not None: + dup_op = get_typed_op(dup_op, self.dtype, kind="binary") + if dup_op.opclass == "Monoid": + dup_op = dup_op.binaryop + else: + self._expect_op(dup_op, "BinaryOp", within="build", argname="dup_op") indices = _CArray(indices) values = _CArray(values, self.dtype) @@ -1500,6 +1542,7 @@ def select(self, op, thunk=None): if thunk.dtype._is_udt: dtype_name = "UDT" thunk = _Pointer(thunk) + # NOT COVERED else: dtype_name = thunk.dtype.name cfunc_name = f"GrB_Vector_select_{dtype_name}" @@ -1817,13 +1860,14 @@ def _prep_for_assign(self, resolved_indexes, value, mask, is_submask, replace, o shape = values.shape try: vals = Vector.from_dense(values, dtype=dtype) - except Exception: # pragma: no cover (safety) + except Exception: vals = None else: if dtype.np_type.subdtype is not None: shape = vals.shape if vals is None or shape != (size,): if dtype.np_type.subdtype is not None: + # NOT COVERED extra = ( " (this is assigning to a vector with sub-array dtype " f"({dtype}), so array shape should include dtype shape)" @@ -1943,7 +1987,7 @@ def from_dict(cls, d, dtype=None, *, size=None, name=None): # If we know the dtype, then using `np.fromiter` is much faster dtype = lookup_dtype(dtype) if dtype.np_type.subdtype is not None and np.__version__[:5] in {"1.21.", "1.22."}: - values, dtype = values_to_numpy_buffer(list(d.values()), dtype) + values, dtype = values_to_numpy_buffer(list(d.values()), dtype) # FLAKY COVERAGE else: values = np.fromiter(d.values(), dtype.np_type) if size is None and indices.size == 0: diff --git a/graphblas/dtypes.py b/graphblas/dtypes.py index 22d98b8f1..920610b95 100644 --- a/graphblas/dtypes.py +++ b/graphblas/dtypes.py @@ -1,15 +1,18 @@ import warnings as _warnings -import numba as _numba import numpy as _np from numpy import find_common_type as _find_common_type from numpy import promote_types as _promote_types from . import backend from .core import NULL as _NULL +from .core import _has_numba from .core import ffi as _ffi from .core import lib as _lib +if _has_numba: + import numba as _numba + # Default assumption unless FC32/FC64 are found in lib _supports_complex = hasattr(_lib, "GrB_FC64") or hasattr(_lib, "GxB_FC64") @@ -140,44 +143,126 @@ def register_anonymous(dtype, name=None): # For now, let's use "opaque" unsigned bytes for the c type. if name is None: name = _default_name(dtype) - numba_type = _numba.typeof(dtype).dtype + numba_type = _numba.typeof(dtype).dtype if _has_numba else None rv = DataType(name, gb_obj, None, f"uint8_t[{dtype.itemsize}]", numba_type, dtype) _registry[gb_obj] = rv _registry[dtype] = rv - _registry[numba_type] = rv - _registry[numba_type.name] = rv + if _has_numba: + _registry[numba_type] = rv + _registry[numba_type.name] = rv return rv -BOOL = DataType("BOOL", _lib.GrB_BOOL, "GrB_BOOL", "_Bool", _numba.types.bool_, _np.bool_) -INT8 = DataType("INT8", _lib.GrB_INT8, "GrB_INT8", "int8_t", _numba.types.int8, _np.int8) -UINT8 = DataType("UINT8", _lib.GrB_UINT8, "GrB_UINT8", "uint8_t", _numba.types.uint8, _np.uint8) -INT16 = DataType("INT16", _lib.GrB_INT16, "GrB_INT16", "int16_t", _numba.types.int16, _np.int16) +BOOL = DataType( + "BOOL", + _lib.GrB_BOOL, + "GrB_BOOL", + "_Bool", + _numba.types.bool_ if _has_numba else None, + _np.bool_, +) +INT8 = DataType( + "INT8", _lib.GrB_INT8, "GrB_INT8", "int8_t", _numba.types.int8 if _has_numba else None, _np.int8 +) +UINT8 = DataType( + "UINT8", + _lib.GrB_UINT8, + "GrB_UINT8", + "uint8_t", + _numba.types.uint8 if _has_numba else None, + _np.uint8, +) +INT16 = DataType( + "INT16", + _lib.GrB_INT16, + "GrB_INT16", + "int16_t", + _numba.types.int16 if _has_numba else None, + _np.int16, +) UINT16 = DataType( - "UINT16", _lib.GrB_UINT16, "GrB_UINT16", "uint16_t", _numba.types.uint16, _np.uint16 + "UINT16", + _lib.GrB_UINT16, + "GrB_UINT16", + "uint16_t", + _numba.types.uint16 if _has_numba else None, + _np.uint16, +) +INT32 = DataType( + "INT32", + _lib.GrB_INT32, + "GrB_INT32", + "int32_t", + _numba.types.int32 if _has_numba else None, + _np.int32, ) -INT32 = DataType("INT32", _lib.GrB_INT32, "GrB_INT32", "int32_t", _numba.types.int32, _np.int32) UINT32 = DataType( - "UINT32", _lib.GrB_UINT32, "GrB_UINT32", "uint32_t", _numba.types.uint32, _np.uint32 + "UINT32", + _lib.GrB_UINT32, + "GrB_UINT32", + "uint32_t", + _numba.types.uint32 if _has_numba else None, + _np.uint32, +) +INT64 = DataType( + "INT64", + _lib.GrB_INT64, + "GrB_INT64", + "int64_t", + _numba.types.int64 if _has_numba else None, + _np.int64, ) -INT64 = DataType("INT64", _lib.GrB_INT64, "GrB_INT64", "int64_t", _numba.types.int64, _np.int64) # _Index (like UINT64) is for internal use only and shouldn't be exposed to the user _INDEX = DataType( - "UINT64", _lib.GrB_UINT64, "GrB_Index", "GrB_Index", _numba.types.uint64, _np.uint64 + "UINT64", + _lib.GrB_UINT64, + "GrB_Index", + "GrB_Index", + _numba.types.uint64 if _has_numba else None, + _np.uint64, ) UINT64 = DataType( - "UINT64", _lib.GrB_UINT64, "GrB_UINT64", "uint64_t", _numba.types.uint64, _np.uint64 + "UINT64", + _lib.GrB_UINT64, + "GrB_UINT64", + "uint64_t", + _numba.types.uint64 if _has_numba else None, + _np.uint64, +) +FP32 = DataType( + "FP32", + _lib.GrB_FP32, + "GrB_FP32", + "float", + _numba.types.float32 if _has_numba else None, + _np.float32, +) +FP64 = DataType( + "FP64", + _lib.GrB_FP64, + "GrB_FP64", + "double", + _numba.types.float64 if _has_numba else None, + _np.float64, ) -FP32 = DataType("FP32", _lib.GrB_FP32, "GrB_FP32", "float", _numba.types.float32, _np.float32) -FP64 = DataType("FP64", _lib.GrB_FP64, "GrB_FP64", "double", _numba.types.float64, _np.float64) if _supports_complex and hasattr(_lib, "GxB_FC32"): FC32 = DataType( - "FC32", _lib.GxB_FC32, "GxB_FC32", "float _Complex", _numba.types.complex64, _np.complex64 + "FC32", + _lib.GxB_FC32, + "GxB_FC32", + "float _Complex", + _numba.types.complex64 if _has_numba else None, + _np.complex64, ) if _supports_complex and hasattr(_lib, "GrB_FC32"): # pragma: no cover (unused) FC32 = DataType( - "FC32", _lib.GrB_FC32, "GrB_FC32", "float _Complex", _numba.types.complex64, _np.complex64 + "FC32", + _lib.GrB_FC32, + "GrB_FC32", + "float _Complex", + _numba.types.complex64 if _has_numba else None, + _np.complex64, ) if _supports_complex and hasattr(_lib, "GxB_FC64"): FC64 = DataType( @@ -185,7 +270,7 @@ def register_anonymous(dtype, name=None): _lib.GxB_FC64, "GxB_FC64", "double _Complex", - _numba.types.complex128, + _numba.types.complex128 if _has_numba else None, _np.complex128, ) if _supports_complex and hasattr(_lib, "GrB_FC64"): # pragma: no cover (unused) @@ -194,7 +279,7 @@ def register_anonymous(dtype, name=None): _lib.GrB_FC64, "GrB_FC64", "double _Complex", - _numba.types.complex128, + _numba.types.complex128 if _has_numba else None, _np.complex128, ) @@ -246,8 +331,9 @@ def register_anonymous(dtype, name=None): _registry[dtype.gb_name.lower()] = dtype _registry[dtype.c_type] = dtype _registry[dtype.c_type.upper()] = dtype - _registry[dtype.numba_type] = dtype - _registry[dtype.numba_type.name] = dtype + if _has_numba: + _registry[dtype.numba_type] = dtype + _registry[dtype.numba_type.name] = dtype val = _sample_values[dtype] _registry[val.dtype] = dtype _registry[val.dtype.name] = dtype diff --git a/graphblas/io.py b/graphblas/io.py index bc57c2084..23b9b30b7 100644 --- a/graphblas/io.py +++ b/graphblas/io.py @@ -11,7 +11,7 @@ from .exceptions import GraphblasException as _GraphblasException -def draw(m): # pragma: no cover +def draw(m): # pragma: no cover (deprecated) """Draw a square adjacency Matrix as a graph. Requires `networkx `_ and @@ -455,7 +455,7 @@ def to_awkward(A, format=None): indices, values = A.to_coo() form = RecordForm( contents=[ - NumpyForm(A.dtype.numba_type.name, form_key="node1"), + NumpyForm(A.dtype.np_type.name, form_key="node1"), NumpyForm("int64", form_key="node0"), ], fields=["values", "indices"], @@ -489,7 +489,7 @@ def to_awkward(A, format=None): RecordForm( contents=[ NumpyForm("int64", form_key="node3"), - NumpyForm(A.dtype.numba_type.name, form_key="node4"), + NumpyForm(A.dtype.np_type.name, form_key="node4"), ], fields=["indices", "values"], ), @@ -502,11 +502,11 @@ def to_awkward(A, format=None): @ak.behaviors.mixins.mixin_class(ak.behavior) class _AwkwardDoublyCompressedMatrix: @property - def values(self): + def values(self): # pragma: no branch (???) return self.data.values @property - def indices(self): + def indices(self): # pragma: no branch (???) return self.data.indices form = RecordForm( diff --git a/graphblas/monoid/numpy.py b/graphblas/monoid/numpy.py index 1d687443f..f46d57143 100644 --- a/graphblas/monoid/numpy.py +++ b/graphblas/monoid/numpy.py @@ -5,15 +5,18 @@ https://numba.readthedocs.io/en/stable/reference/numpysupported.html#math-operations """ -import numba as _numba import numpy as _np from .. import _STANDARD_OPERATOR_NAMES from .. import binary as _binary from .. import config as _config from .. import monoid as _monoid +from ..core import _has_numba, _supports_udfs from ..dtypes import _supports_complex +if _has_numba: + import numba as _numba + _delayed = {} _complex_dtypes = {"FC32", "FC64"} _float_dtypes = {"FP32", "FP64"} @@ -86,7 +89,8 @@ # To increase import speed, only call njit when `_config.get("mapnumpy")` is False if ( _config.get("mapnumpy") - or type(_numba.njit(lambda x, y: _np.fmax(x, y))(1, 2)) # pragma: no branch (numba) + or _has_numba + and type(_numba.njit(lambda x, y: _np.fmax(x, y))(1, 2)) # pragma: no branch (numba) is not float ): # Incorrect behavior was introduced in numba 0.56.2 and numpy 1.23 @@ -155,7 +159,12 @@ def __dir__(): - return globals().keys() | _delayed.keys() | _monoid_identities.keys() + if not _supports_udfs and not _config.get("mapnumpy"): + return globals().keys() # FLAKY COVERAGE + attrs = _delayed.keys() | _monoid_identities.keys() + if not _supports_udfs: + attrs &= _numpy_to_graphblas.keys() + return attrs | globals().keys() def __getattr__(name): diff --git a/graphblas/op/__init__.py b/graphblas/op/__init__.py index af05cbef4..1eb2b51d7 100644 --- a/graphblas/op/__init__.py +++ b/graphblas/op/__init__.py @@ -39,10 +39,18 @@ def __getattr__(key): ss = import_module(".ss", __name__) globals()["ss"] = ss return ss + if not _supports_udfs: + from .. import binary, semiring + + if key in binary._udfs or key in semiring._udfs: + raise AttributeError( + f"module {__name__!r} unable to compile UDF for {key!r}; " + "install numba for UDF support" + ) raise AttributeError(f"module {__name__!r} has no attribute {key!r}") -from ..core import operator # noqa: E402 isort:skip +from ..core import operator, _supports_udfs # noqa: E402 isort:skip from . import numpy # noqa: E402 isort:skip del operator diff --git a/graphblas/op/numpy.py b/graphblas/op/numpy.py index 497a6037c..cadba17eb 100644 --- a/graphblas/op/numpy.py +++ b/graphblas/op/numpy.py @@ -1,4 +1,5 @@ from ..binary import numpy as _np_binary +from ..core import _supports_udfs from ..semiring import numpy as _np_semiring from ..unary import numpy as _np_unary @@ -10,7 +11,10 @@ def __dir__(): - return globals().keys() | _delayed.keys() | _op_to_mod.keys() + attrs = _delayed.keys() | _op_to_mod.keys() + if not _supports_udfs: + attrs &= _np_unary.__dir__() | _np_binary.__dir__() | _np_semiring.__dir__() + return attrs | globals().keys() def __getattr__(name): diff --git a/graphblas/semiring/__init__.py b/graphblas/semiring/__init__.py index 904ae192f..538136406 100644 --- a/graphblas/semiring/__init__.py +++ b/graphblas/semiring/__init__.py @@ -1,7 +1,29 @@ # All items are dynamically added by classes in operator.py # This module acts as a container of Semiring instances +from ..core import _supports_udfs + _delayed = {} _deprecated = {} +_udfs = { + # Used by aggregators + "max_absfirst", + "max_abssecond", + "plus_absfirst", + "plus_abssecond", + "plus_rpow", + # floordiv + "any_floordiv", + "max_floordiv", + "min_floordiv", + "plus_floordiv", + "times_floordiv", + # rfloordiv + "any_rfloordiv", + "max_rfloordiv", + "min_rfloordiv", + "plus_rfloordiv", + "times_rfloordiv", +} def __dir__(): @@ -47,6 +69,11 @@ def __getattr__(key): ss = import_module(".ss", __name__) globals()["ss"] = ss return ss + if not _supports_udfs and key in _udfs: + raise AttributeError( + f"module {__name__!r} unable to compile UDF for {key!r}; " + "install numba for UDF support" + ) raise AttributeError(f"module {__name__!r} has no attribute {key!r}") diff --git a/graphblas/semiring/numpy.py b/graphblas/semiring/numpy.py index e47ac0336..3a59090cc 100644 --- a/graphblas/semiring/numpy.py +++ b/graphblas/semiring/numpy.py @@ -12,6 +12,7 @@ from .. import config as _config from .. import monoid as _monoid from ..binary.numpy import _binary_names +from ..core import _supports_udfs from ..monoid.numpy import _fmin_is_float, _monoid_identities _delayed = {} @@ -132,7 +133,17 @@ def __dir__(): - return globals().keys() | _delayed.keys() | _semiring_names + if not _supports_udfs and not _config.get("mapnumpy"): + return globals().keys() # FLAKY COVERAGE + attrs = _delayed.keys() | _semiring_names + if not _supports_udfs: + attrs &= { + f"{monoid_name}_{binary_name}" + for monoid_name, binary_name in _itertools.product( + dir(_monoid.numpy), dir(_binary.numpy) + ) + } + return attrs | globals().keys() def __getattr__(name): diff --git a/graphblas/tests/conftest.py b/graphblas/tests/conftest.py index 24aba085f..a4df5d336 100644 --- a/graphblas/tests/conftest.py +++ b/graphblas/tests/conftest.py @@ -1,16 +1,20 @@ import atexit import functools import itertools +import platform from pathlib import Path import numpy as np import pytest import graphblas as gb +from graphblas.core import _supports_udfs as supports_udfs orig_binaryops = set() orig_semirings = set() +pypy = platform.python_implementation() == "PyPy" + def pytest_configure(config): rng = np.random.default_rng() @@ -48,7 +52,7 @@ def pytest_configure(config): rec.start() def save_records(): - with Path("record.txt").open("w") as f: # pragma: no cover + with Path("record.txt").open("w") as f: # pragma: no cover (???) f.write("\n".join(rec.data)) # I'm sure there's a `pytest` way to do this... @@ -116,3 +120,8 @@ def inner(*args, **kwargs): def compute(x): return x + + +def shouldhave(module, opname): + """Whether an "operator" module should have the given operator.""" + return supports_udfs or hasattr(module, opname) diff --git a/graphblas/tests/test_core.py b/graphblas/tests/test_core.py index 71d0bd8a3..ae2051145 100644 --- a/graphblas/tests/test_core.py +++ b/graphblas/tests/test_core.py @@ -80,7 +80,7 @@ def test_packages(): pkgs.append("graphblas") pkgs.sort() pyproject = path.parent / "pyproject.toml" - if not pyproject.exists(): + if not pyproject.exists(): # pragma: no cover (safety) pytest.skip("Did not find pyproject.toml") with pyproject.open("rb") as f: pkgs2 = sorted(tomli.load(f)["tool"]["setuptools"]["packages"]) diff --git a/graphblas/tests/test_dtype.py b/graphblas/tests/test_dtype.py index 64e6d69ab..66c19cce5 100644 --- a/graphblas/tests/test_dtype.py +++ b/graphblas/tests/test_dtype.py @@ -252,7 +252,4 @@ def test_has_complex(): import suitesparse_graphblas as ssgb from packaging.version import parse - if parse(ssgb.__version__) < parse("7.4.3.1"): - assert not dtypes._supports_complex - else: - assert dtypes._supports_complex + assert dtypes._supports_complex == (parse(ssgb.__version__) >= parse("7.4.3.1")) diff --git a/graphblas/tests/test_formatting.py b/graphblas/tests/test_formatting.py index 3094aea91..faadc983b 100644 --- a/graphblas/tests/test_formatting.py +++ b/graphblas/tests/test_formatting.py @@ -40,9 +40,8 @@ def _printer(text, name, repr_name, indent): # line = f"f'{{CSS_STYLE}}'" in_style = False is_style = True - else: # pragma: no cover (???) - # This definitely gets covered, but why is it not picked up? - continue + else: + continue # FLAKY COVERAGE if repr_name == "repr_html" and line.startswith("
" + f"{CSS_STYLE}" + '
vA
\n' + '\n' + " \n" + ' \n' + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "
gb.Vector
nvals
size
dtype
format
12INT64bitmap (iso)
\n" + "
\n" + "
\n" + "\n" + '\n' + " \n" + ' \n' + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "
01
2
\n" + "
" + ) diff --git a/graphblas/tests/test_infix.py b/graphblas/tests/test_infix.py index 14af6108c..72e1c8a42 100644 --- a/graphblas/tests/test_infix.py +++ b/graphblas/tests/test_infix.py @@ -360,3 +360,10 @@ def test_infix_expr_value_types(): assert expr._expr is not None assert expr._value is None assert type(expr.new()) is Matrix + assert type(expr._get_value()) is Matrix + assert expr._expr is not None + assert expr._value is not None + assert expr._expr._value is not None + expr._value = None + assert expr._value is None + assert expr._expr._value is None diff --git a/graphblas/tests/test_io.py b/graphblas/tests/test_io.py index ada092025..24df55e9d 100644 --- a/graphblas/tests/test_io.py +++ b/graphblas/tests/test_io.py @@ -267,9 +267,13 @@ def test_mmread_mmwrite(engine): # fast_matrix_market v1.4.5 raises ValueError instead of OverflowError M = gb.io.mmread(mm_in, engine) else: - if example == "_empty_lines_example" and engine in {"fmm", "auto"} and fmm is not None: - # TODO MAINT: is this a bug in fast_matrix_market, or does scipy.io.mmread - # read an invalid file? `fast_matrix_market` v1.4.5 does not handle this. + if ( + example == "_empty_lines_example" + and engine in {"fmm", "auto"} + and fmm is not None + and fmm.__version__ in {"1.4.5"} + ): + # `fast_matrix_market` __version__ v1.4.5 does not handle this, but v1.5.0 does continue M = gb.io.mmread(mm_in, engine) if not M.isequal(expected): # pragma: no cover (debug) diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py index 1d42035a3..26017f364 100644 --- a/graphblas/tests/test_matrix.py +++ b/graphblas/tests/test_matrix.py @@ -11,6 +11,7 @@ import graphblas as gb from graphblas import agg, backend, binary, dtypes, indexunary, monoid, select, semiring, unary +from graphblas.core import _supports_udfs as supports_udfs from graphblas.core import lib from graphblas.exceptions import ( DimensionMismatch, @@ -23,7 +24,7 @@ OutputNotEmpty, ) -from .conftest import autocompute, compute +from .conftest import autocompute, compute, pypy, shouldhave from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) @@ -1230,6 +1231,8 @@ def test_apply_indexunary(A): assert w4.isequal(A3) with pytest.raises(TypeError, match="left"): A.apply(select.valueeq, left=s3) + assert pickle.loads(pickle.dumps(indexunary.tril)) is indexunary.tril + assert pickle.loads(pickle.dumps(indexunary.tril[int])) is indexunary.tril[int] def test_select(A): @@ -1259,6 +1262,16 @@ def test_select(A): with pytest.raises(TypeError, match="thunk"): A.select(select.valueeq, object()) + A3rows = Matrix.from_coo([0, 0, 1, 1, 2], [1, 3, 4, 6, 5], [2, 3, 8, 4, 1], nrows=7, ncols=7) + w8 = select.rowle(A, 2).new() + w9 = A.select("row<=", 2).new() + w10 = select.row(A < 3).new() + assert w8.isequal(A3rows) + assert w9.isequal(A3rows) + assert w10.isequal(A3rows) + assert pickle.loads(pickle.dumps(select.tril)) is select.tril + assert pickle.loads(pickle.dumps(select.tril[bool])) is select.tril[bool] + @autocompute def test_select_bools_and_masks(A): @@ -1283,16 +1296,27 @@ def test_select_bools_and_masks(A): A.select(A[0, :].new().S) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_indexunary_udf(A): def threex_minusthunk(x, row, col, thunk): # pragma: no cover (numba) return 3 * x - thunk - indexunary.register_new("threex_minusthunk", threex_minusthunk) + assert indexunary.register_new("threex_minusthunk", threex_minusthunk) is not None assert hasattr(indexunary, "threex_minusthunk") assert not hasattr(select, "threex_minusthunk") with pytest.raises(ValueError, match="SelectOp must have BOOL return type"): select.register_anonymous(threex_minusthunk) + with pytest.raises(ValueError, match="SelectOp must have BOOL return type"): + select.register_new("bad_select", threex_minusthunk) + assert not hasattr(indexunary, "bad_select") + assert not hasattr(select, "bad_select") + assert select.register_new("bad_select", threex_minusthunk, lazy=True) is None + with pytest.raises(ValueError, match="SelectOp must have BOOL return type"): + select.bad_select + assert not hasattr(select, "bad_select") + assert hasattr(indexunary, "bad_select") # Keep it + expected = Matrix.from_coo( [3, 0, 3, 5, 6, 0, 6, 1, 6, 2, 4, 1], [0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6], @@ -1308,6 +1332,8 @@ def iii(x, row, col, thunk): # pragma: no cover (numba) select.register_new("iii", iii) assert hasattr(indexunary, "iii") assert hasattr(select, "iii") + assert indexunary.iii[int].orig_func is select.iii[int].orig_func is select.iii.orig_func + assert indexunary.iii[int]._numba_func is select.iii[int]._numba_func is select.iii._numba_func iii_apply = indexunary.register_anonymous(iii) expected = Matrix.from_coo( [3, 0, 3, 5, 6, 0, 6, 1, 6, 2, 4, 1], @@ -1353,15 +1379,17 @@ def test_reduce_agg(A): expected = unary.sqrt[float](squared).new() w5 = A.reduce_rowwise(agg.hypot).new() assert w5.isclose(expected) - w6 = A.reduce_rowwise(monoid.numpy.hypot[float]).new() - assert w6.isclose(expected) + if shouldhave(monoid.numpy, "hypot"): + w6 = A.reduce_rowwise(monoid.numpy.hypot[float]).new() + assert w6.isclose(expected) w7 = Vector(w5.dtype, size=w5.size) w7 << A.reduce_rowwise(agg.hypot) assert w7.isclose(expected) w8 = A.reduce_rowwise(agg.logaddexp).new() - expected = A.reduce_rowwise(monoid.numpy.logaddexp[float]).new() - assert w8.isclose(w8) + if shouldhave(monoid.numpy, "logaddexp"): + expected = A.reduce_rowwise(monoid.numpy.logaddexp[float]).new() + assert w8.isclose(w8) result = Vector.from_coo([0, 1, 2, 3, 4, 5, 6], [3, 2, 9, 10, 11, 8, 4]) w9 = A.reduce_columnwise(agg.sum).new() @@ -1598,6 +1626,7 @@ def test_reduce_agg_empty(): assert compute(s.value) is None +@pytest.mark.skipif("not supports_udfs") def test_reduce_row_udf(A): result = Vector.from_coo([0, 1, 2, 3, 4, 5, 6], [5, 12, 1, 6, 7, 1, 15]) @@ -2007,6 +2036,12 @@ def test_ss_import_export(A, do_iso, methods): B4 = Matrix.ss.import_any(**d) assert B4.isequal(A) assert B4.ss.is_iso is do_iso + if do_iso: + d["values"] = 1 + d["is_iso"] = False + B4b = Matrix.ss.import_any(**d) + assert B4b.isequal(A) + assert B4b.ss.is_iso is True else: A4.ss.pack_any(**d) assert A4.isequal(A) @@ -2262,6 +2297,11 @@ def test_ss_import_on_view(): A = Matrix.from_coo([0, 0, 1, 1], [0, 1, 0, 1], [1, 2, 3, 4]) B = Matrix.ss.import_any(nrows=2, ncols=2, values=np.array([1, 2, 3, 4, 99, 99, 99])[:4]) assert A.isequal(B) + values = np.arange(16).reshape(4, 4)[::2, ::2] + bitmap = np.ones((4, 4), dtype=bool)[::2, ::2] + C = Matrix.ss.import_any(values=values, bitmap=bitmap) + D = Matrix.ss.import_any(values=values.copy(), bitmap=bitmap.copy()) + assert C.isequal(D) @pytest.mark.skipif("not suitesparse") @@ -2902,18 +2942,19 @@ def test_expr_is_like_matrix(A): "resize", "update", } - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Matrix. You may need to " "add an entry to `matrix` or `matrix_vector` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " "methods, then you may need to run `python -m graphblas.core.infixmethods`." ) - assert attrs - infix_attrs == expected + assert attrs - infix_attrs - ignore == expected # TransposedMatrix is used differently than other expressions, # so maybe it shouldn't support everything. if suitesparse: expected.add("ss") - assert attrs - transposed_attrs == (expected | {"_as_vector", "S", "V"}) - { + assert attrs - transposed_attrs - ignore == (expected | {"_as_vector", "S", "V"}) - { "_prep_for_extract", "_extract_element", } @@ -2965,7 +3006,8 @@ def test_index_expr_is_like_matrix(A): "from_scalar", "resize", } - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Matrix. You may need to " "add an entry to `matrix` or `matrix_vector` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " @@ -3110,10 +3152,12 @@ def test_infix_sugar(A): assert binary.times(2, A).isequal(2 * A) assert binary.truediv(A, 2).isequal(A / 2) assert binary.truediv(5, A).isequal(5 / A) - assert binary.floordiv(A, 2).isequal(A // 2) - assert binary.floordiv(5, A).isequal(5 // A) - assert binary.numpy.mod(A, 2).isequal(A % 2) - assert binary.numpy.mod(5, A).isequal(5 % A) + if shouldhave(binary, "floordiv"): + assert binary.floordiv(A, 2).isequal(A // 2) + assert binary.floordiv(5, A).isequal(5 // A) + if shouldhave(binary.numpy, "mod"): + assert binary.numpy.mod(A, 2).isequal(A % 2) + assert binary.numpy.mod(5, A).isequal(5 % A) assert binary.pow(A, 2).isequal(A**2) assert binary.pow(2, A).isequal(2**A) assert binary.pow(A, 2).isequal(pow(A, 2)) @@ -3140,26 +3184,27 @@ def test_infix_sugar(A): assert binary.ge(A, 4).isequal(A >= 4) assert binary.eq(A, 4).isequal(A == 4) assert binary.ne(A, 4).isequal(A != 4) - x, y = divmod(A, 3) - assert binary.floordiv(A, 3).isequal(x) - assert binary.numpy.mod(A, 3).isequal(y) - assert binary.fmod(A, 3).isequal(y) - assert A.isequal(binary.plus((3 * x) & y)) - x, y = divmod(-A, 3) - assert binary.floordiv(-A, 3).isequal(x) - assert binary.numpy.mod(-A, 3).isequal(y) - # assert binary.fmod(-A, 3).isequal(y) # The reason we use numpy.mod - assert (-A).isequal(binary.plus((3 * x) & y)) - x, y = divmod(3, A) - assert binary.floordiv(3, A).isequal(x) - assert binary.numpy.mod(3, A).isequal(y) - assert binary.fmod(3, A).isequal(y) - assert binary.plus(binary.times(A & x) & y).isequal(3 * unary.one(A)) - x, y = divmod(-3, A) - assert binary.floordiv(-3, A).isequal(x) - assert binary.numpy.mod(-3, A).isequal(y) - # assert binary.fmod(-3, A).isequal(y) # The reason we use numpy.mod - assert binary.plus(binary.times(A & x) & y).isequal(-3 * unary.one(A)) + if shouldhave(binary, "floordiv") and shouldhave(binary.numpy, "mod"): + x, y = divmod(A, 3) + assert binary.floordiv(A, 3).isequal(x) + assert binary.numpy.mod(A, 3).isequal(y) + assert binary.fmod(A, 3).isequal(y) + assert A.isequal(binary.plus((3 * x) & y)) + x, y = divmod(-A, 3) + assert binary.floordiv(-A, 3).isequal(x) + assert binary.numpy.mod(-A, 3).isequal(y) + # assert binary.fmod(-A, 3).isequal(y) # The reason we use numpy.mod + assert (-A).isequal(binary.plus((3 * x) & y)) + x, y = divmod(3, A) + assert binary.floordiv(3, A).isequal(x) + assert binary.numpy.mod(3, A).isequal(y) + assert binary.fmod(3, A).isequal(y) + assert binary.plus(binary.times(A & x) & y).isequal(3 * unary.one(A)) + x, y = divmod(-3, A) + assert binary.floordiv(-3, A).isequal(x) + assert binary.numpy.mod(-3, A).isequal(y) + # assert binary.fmod(-3, A).isequal(y) # The reason we use numpy.mod + assert binary.plus(binary.times(A & x) & y).isequal(-3 * unary.one(A)) assert binary.eq(A & A).isequal(A == A) assert binary.ne(A.T & A.T).isequal(A.T != A.T) @@ -3182,14 +3227,16 @@ def test_infix_sugar(A): B /= 2 assert type(B) is Matrix assert binary.truediv(A, 2).isequal(B) - B = A.dup() - B //= 2 - assert type(B) is Matrix - assert binary.floordiv(A, 2).isequal(B) - B = A.dup() - B %= 2 - assert type(B) is Matrix - assert binary.numpy.mod(A, 2).isequal(B) + if shouldhave(binary, "floordiv"): + B = A.dup() + B //= 2 + assert type(B) is Matrix + assert binary.floordiv(A, 2).isequal(B) + if shouldhave(binary.numpy, "mod"): + B = A.dup() + B %= 2 + assert type(B) is Matrix + assert binary.numpy.mod(A, 2).isequal(B) B = A.dup() B **= 2 assert type(B) is Matrix @@ -3520,7 +3567,7 @@ def test_ndim(A): def test_sizeof(A): - if suitesparse: + if suitesparse and not pypy: assert sys.getsizeof(A) > A.nvals * 16 else: with pytest.raises(TypeError): @@ -3607,6 +3654,7 @@ def test_ss_iteration(A): assert next(A.ss.iteritems()) is not None +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_udt(): record_dtype = np.dtype([("x", np.bool_), ("y", np.float64)], align=True) @@ -3917,7 +3965,7 @@ def test_ss_config(A): def test_to_csr_from_csc(A): - assert Matrix.from_csr(*A.to_csr(dtype=int)).isequal(A, check_dtype=True) + assert Matrix.from_csr(*A.to_csr(sort=False, dtype=int)).isequal(A, check_dtype=True) assert Matrix.from_csr(*A.T.to_csc()).isequal(A, check_dtype=True) assert Matrix.from_csc(*A.to_csc()).isequal(A) assert Matrix.from_csc(*A.T.to_csr()).isequal(A) @@ -4126,7 +4174,11 @@ def test_from_scalar(): A = Matrix.from_scalar(1, dtype="INT64[2]", nrows=3, ncols=4) B = Matrix("INT64[2]", nrows=3, ncols=4) B << [1, 1] - assert A.isequal(B, check_dtype=True) + if supports_udfs: + assert A.isequal(B, check_dtype=True) + else: + with pytest.raises(KeyError, match="eq does not work with"): + assert A.isequal(B, check_dtype=True) def test_to_dense_from_dense(): @@ -4252,7 +4304,7 @@ def test_ss_descriptors(A): (A @ A).new(nthreads=4, Nthreads=5) with pytest.raises(ValueError, match="escriptor"): A[0, 0].new(bad_opt=True) - A[0, 0].new(nthreads=4) # ignored, but okay + A[0, 0].new(nthreads=4, sort=None) # ignored, but okay with pytest.raises(ValueError, match="escriptor"): A.__setitem__((0, 0), 1, bad_opt=True) A.__setitem__((0, 0), 1, nthreads=4) # ignored, but okay @@ -4286,6 +4338,7 @@ def test_wait_chains(A): assert result == 47 +@pytest.mark.skipif("not supports_udfs") def test_subarray_dtypes(): a = np.arange(3 * 4, dtype=np.int64).reshape(3, 4) A = Matrix.from_coo([1, 3, 5], [0, 1, 3], a) diff --git a/graphblas/tests/test_numpyops.py b/graphblas/tests/test_numpyops.py index 5b7e797f3..25c52d7fd 100644 --- a/graphblas/tests/test_numpyops.py +++ b/graphblas/tests/test_numpyops.py @@ -11,22 +11,25 @@ import graphblas.monoid.numpy as npmonoid import graphblas.semiring.numpy as npsemiring import graphblas.unary.numpy as npunary -from graphblas import Vector, backend +from graphblas import Vector, backend, config +from graphblas.core import _supports_udfs as supports_udfs from graphblas.dtypes import _supports_complex -from .conftest import compute +from .conftest import compute, shouldhave is_win = sys.platform.startswith("win") suitesparse = backend == "suitesparse" def test_numpyops_dir(): - assert "exp2" in dir(npunary) - assert "logical_and" in dir(npbinary) - assert "logaddexp" in dir(npmonoid) - assert "add_add" in dir(npsemiring) + udf_or_mapped = supports_udfs or config["mapnumpy"] + assert ("exp2" in dir(npunary)) == udf_or_mapped + assert ("logical_and" in dir(npbinary)) == udf_or_mapped + assert ("logaddexp" in dir(npmonoid)) == supports_udfs + assert ("add_add" in dir(npsemiring)) == udf_or_mapped +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_bool_doesnt_get_too_large(): a = Vector.from_coo([0, 1, 2, 3], [True, False, True, False]) @@ -70,9 +73,12 @@ def test_npunary(): # due to limitation of MSVC with complex blocklist["FC64"].update({"arcsin", "arcsinh"}) blocklist["FC32"] = {"arcsin", "arcsinh"} - isclose = gb.binary.isclose(1e-6, 0) + if shouldhave(gb.binary, "isclose"): + isclose = gb.binary.isclose(1e-6, 0) + else: + isclose = None for gb_input, np_input in data: - for unary_name in sorted(npunary._unary_names): + for unary_name in sorted(npunary._unary_names & npunary.__dir__()): op = getattr(npunary, unary_name) if gb_input.dtype not in op.types or unary_name in blocklist.get( gb_input.dtype.name, () @@ -99,6 +105,8 @@ def test_npunary(): list(range(np_input.size)), list(np_result), dtype=gb_result.dtype ) assert gb_result.nvals == np_result.size + if compare_op is None: + continue # FLAKY COVERAGE match = gb_result.ewise_mult(np_result, compare_op).new() if gb_result.dtype.name.startswith("F"): match(accum=gb.binary.lor) << gb_result.apply(npunary.isnan) @@ -149,9 +157,24 @@ def test_npbinary(): "FP64": {"floor_divide"}, # numba/numpy difference for 1.0 / 0.0 "BOOL": {"gcd", "lcm", "subtract"}, # not supported by numpy } - isclose = gb.binary.isclose(1e-7, 0) + if shouldhave(gb.binary, "isclose"): + isclose = gb.binary.isclose(1e-7, 0) + else: + isclose = None + if shouldhave(npbinary, "equal"): + equal = npbinary.equal + else: + equal = gb.binary.eq + if shouldhave(npbinary, "isnan"): + isnan = npunary.isnan + else: + isnan = gb.unary.isnan + if shouldhave(npbinary, "isinf"): + isinf = npunary.isinf + else: + isinf = gb.unary.isinf for (gb_left, gb_right), (np_left, np_right) in data: - for binary_name in sorted(npbinary._binary_names): + for binary_name in sorted(npbinary._binary_names & npbinary.__dir__()): op = getattr(npbinary, binary_name) if gb_left.dtype not in op.types or binary_name in blocklist.get( gb_left.dtype.name, () @@ -171,7 +194,7 @@ def test_npbinary(): if binary_name in {"arctan2"}: compare_op = isclose else: - compare_op = npbinary.equal + compare_op = equal except Exception: # pragma: no cover (debug) print(f"Error computing numpy result for {binary_name}") print(f"dtypes: ({gb_left.dtype}, {gb_right.dtype}) -> {gb_result.dtype}") @@ -179,12 +202,14 @@ def test_npbinary(): np_result = Vector.from_coo(np.arange(np_left.size), np_result, dtype=gb_result.dtype) assert gb_result.nvals == np_result.size + if compare_op is None: + continue # FLAKY COVERAGE match = gb_result.ewise_mult(np_result, compare_op).new() if gb_result.dtype.name.startswith("F"): - match(accum=gb.binary.lor) << gb_result.apply(npunary.isnan) + match(accum=gb.binary.lor) << gb_result.apply(isnan) if gb_result.dtype.name.startswith("FC"): # Divide by 0j sometimes result in different behavior, such as `nan` or `(inf+0j)` - match(accum=gb.binary.lor) << gb_result.apply(npunary.isinf) + match(accum=gb.binary.lor) << gb_result.apply(isinf) compare = match.reduce(gb.monoid.land).new() if not compare: # pragma: no cover (debug) print(compare_op) @@ -223,7 +248,7 @@ def test_npmonoid(): ], ] # Complex monoids not working yet (they segfault upon creation in gb.core.operators) - if _supports_complex: # pragma: no branch + if _supports_complex: data.append( [ [ @@ -241,13 +266,13 @@ def test_npmonoid(): "BOOL": {"add"}, } for (gb_left, gb_right), (np_left, np_right) in data: - for binary_name in sorted(npmonoid._monoid_identities): + for binary_name in sorted(npmonoid._monoid_identities.keys() & npmonoid.__dir__()): op = getattr(npmonoid, binary_name) assert len(op.types) > 0, op.name if gb_left.dtype not in op.types or binary_name in blocklist.get( gb_left.dtype.name, () - ): # pragma: no cover (flaky) - continue + ): + continue # FLAKY COVERAGE with np.errstate(divide="ignore", over="ignore", under="ignore", invalid="ignore"): gb_result = gb_left.ewise_mult(gb_right, op).new() np_result = getattr(np, binary_name)(np_left, np_right) @@ -279,7 +304,8 @@ def test_npmonoid(): @pytest.mark.slow def test_npsemiring(): for monoid_name, binary_name in itertools.product( - sorted(npmonoid._monoid_identities), sorted(npbinary._binary_names) + sorted(npmonoid._monoid_identities.keys() & npmonoid.__dir__()), + sorted(npbinary._binary_names & npbinary.__dir__()), ): monoid = getattr(npmonoid, monoid_name) binary = getattr(npbinary, binary_name) diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index 3a80dbe52..c9a176afd 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -4,7 +4,20 @@ import pytest import graphblas as gb -from graphblas import agg, backend, binary, dtypes, indexunary, monoid, op, select, semiring, unary +from graphblas import ( + agg, + backend, + binary, + config, + dtypes, + indexunary, + monoid, + op, + select, + semiring, + unary, +) +from graphblas.core import _supports_udfs as supports_udfs from graphblas.core import lib, operator from graphblas.core.operator import BinaryOp, IndexUnaryOp, Monoid, Semiring, UnaryOp, get_semiring from graphblas.dtypes import ( @@ -22,6 +35,8 @@ ) from graphblas.exceptions import DomainMismatch, UdfParseError +from .conftest import shouldhave + if dtypes._supports_complex: from graphblas.dtypes import FC32, FC64 @@ -142,6 +157,36 @@ def test_get_typed_op(): operator.get_typed_op(binary.plus, dtypes.INT64, "bad dtype") +@pytest.mark.skipif("supports_udfs") +def test_udf_mentions_numba(): + with pytest.raises(AttributeError, match="install numba"): + binary.rfloordiv + assert "rfloordiv" not in dir(binary) + with pytest.raises(AttributeError, match="install numba"): + semiring.any_rfloordiv + assert "any_rfloordiv" not in dir(semiring) + with pytest.raises(AttributeError, match="install numba"): + op.absfirst + assert "absfirst" not in dir(op) + with pytest.raises(AttributeError, match="install numba"): + op.plus_rpow + assert "plus_rpow" not in dir(op) + with pytest.raises(AttributeError, match="install numba"): + binary.numpy.gcd + assert "gcd" not in dir(binary.numpy) + assert "gcd" not in dir(op.numpy) + + +@pytest.mark.skipif("supports_udfs") +def test_unaryop_udf_no_support(): + def plus_one(x): # pragma: no cover (numba) + return x + 1 + + with pytest.raises(RuntimeError, match="UnaryOp.register_new.* unavailable"): + unary.register_new("plus_one", plus_one) + + +@pytest.mark.skipif("not supports_udfs") def test_unaryop_udf(): def plus_one(x): return x + 1 # pragma: no cover (numba) @@ -150,6 +195,7 @@ def plus_one(x): assert hasattr(unary, "plus_one") assert unary.plus_one.orig_func is plus_one assert unary.plus_one[int].orig_func is plus_one + assert unary.plus_one[int]._numba_func(1) == 2 comp_set = { INT8, INT16, @@ -182,6 +228,7 @@ def plus_one(x): UnaryOp.register_new("bad", lambda x: v) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_unaryop_parameterized(): def plus_x(x=0): @@ -207,6 +254,7 @@ def inner(val): assert r10.isequal(v11, check_dtype=True) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_binaryop_parameterized(): def plus_plus_x(x=0): @@ -268,6 +316,7 @@ def my_add(x, y): assert op.name == "my_add" +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_monoid_parameterized(): def plus_plus_x(x=0): @@ -363,6 +412,7 @@ def bad_identity(x=0): assert monoid.is_idempotent +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_semiring_parameterized(): def plus_plus_x(x=0): @@ -490,6 +540,7 @@ def inner(y): assert B.isequal(A.kronecker(A, binary.plus).new()) +@pytest.mark.skipif("not supports_udfs") def test_unaryop_udf_bool_result(): # numba has trouble compiling this, but we have a work-around def is_positive(x): @@ -516,12 +567,14 @@ def is_positive(x): assert w.isequal(result) +@pytest.mark.skipif("not supports_udfs") def test_binaryop_udf(): def times_minus_sum(x, y): return x * y - (x + y) # pragma: no cover (numba) BinaryOp.register_new("bin_test_func", times_minus_sum) assert hasattr(binary, "bin_test_func") + assert binary.bin_test_func[int].orig_func is times_minus_sum comp_set = { BOOL, # goes to INT64 INT8, @@ -545,6 +598,7 @@ def times_minus_sum(x, y): assert w.isequal(result) +@pytest.mark.skipif("not supports_udfs") def test_monoid_udf(): def plus_plus_one(x, y): return x + y + 1 # pragma: no cover (numba) @@ -579,6 +633,7 @@ def plus_plus_one(x, y): Monoid.register_anonymous(binary.plus_plus_one, {"BOOL": -1}) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_semiring_udf(): def plus_plus_two(x, y): @@ -608,10 +663,12 @@ def test_binary_updates(): vec4 = Vector.from_coo([0], [-3], dtype=dtypes.INT64) result2 = vec4.ewise_mult(vec2, binary.cdiv).new() assert result2.isequal(Vector.from_coo([0], [-1], dtype=dtypes.INT64), check_dtype=True) - result3 = vec4.ewise_mult(vec2, binary.floordiv).new() - assert result3.isequal(Vector.from_coo([0], [-2], dtype=dtypes.INT64), check_dtype=True) + if shouldhave(binary, "floordiv"): + result3 = vec4.ewise_mult(vec2, binary.floordiv).new() + assert result3.isequal(Vector.from_coo([0], [-2], dtype=dtypes.INT64), check_dtype=True) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_nested_names(): def plus_three(x): @@ -671,12 +728,17 @@ def test_op_namespace(): assert op.plus is binary.plus assert op.plus_times is semiring.plus_times - assert op.numpy.fabs is unary.numpy.fabs - assert op.numpy.subtract is binary.numpy.subtract - assert op.numpy.add is binary.numpy.add - assert op.numpy.add_add is semiring.numpy.add_add + if shouldhave(unary.numpy, "fabs"): + assert op.numpy.fabs is unary.numpy.fabs + if shouldhave(binary.numpy, "subtract"): + assert op.numpy.subtract is binary.numpy.subtract + if shouldhave(binary.numpy, "add"): + assert op.numpy.add is binary.numpy.add + if shouldhave(semiring.numpy, "add_add"): + assert op.numpy.add_add is semiring.numpy.add_add assert len(dir(op)) > 300 - assert len(dir(op.numpy)) > 500 + if supports_udfs: + assert len(dir(op.numpy)) > 500 with pytest.raises( AttributeError, match="module 'graphblas.op.numpy' has no attribute 'bad_attr'" @@ -740,10 +802,18 @@ def test_op_namespace(): @pytest.mark.slow def test_binaryop_attributes_numpy(): # Some coverage from this test depends on order of tests - assert binary.numpy.add[int].monoid is monoid.numpy.add[int] - assert binary.numpy.subtract[int].monoid is None - assert binary.numpy.add.monoid is monoid.numpy.add - assert binary.numpy.subtract.monoid is None + if shouldhave(monoid.numpy, "add"): + assert binary.numpy.add[int].monoid is monoid.numpy.add[int] + assert binary.numpy.add.monoid is monoid.numpy.add + if shouldhave(binary.numpy, "subtract"): + assert binary.numpy.subtract[int].monoid is None + assert binary.numpy.subtract.monoid is None + + +@pytest.mark.skipif("not supports_udfs") +@pytest.mark.slow +def test_binaryop_monoid_numpy(): + assert gb.binary.numpy.minimum[int].monoid is gb.monoid.numpy.minimum[int] @pytest.mark.slow @@ -756,18 +826,21 @@ def test_binaryop_attributes(): def plus(x, y): return x + y # pragma: no cover (numba) - op = BinaryOp.register_anonymous(plus, name="plus") - assert op.monoid is None - assert op[int].monoid is None + if supports_udfs: + op = BinaryOp.register_anonymous(plus, name="plus") + assert op.monoid is None + assert op[int].monoid is None + assert op[int].parent is op assert binary.plus[int].parent is binary.plus - assert binary.numpy.add[int].parent is binary.numpy.add - assert op[int].parent is op + if shouldhave(binary.numpy, "add"): + assert binary.numpy.add[int].parent is binary.numpy.add # bad type assert binary.plus[bool].monoid is None - assert binary.numpy.equal[int].monoid is None - assert binary.numpy.equal[bool].monoid is monoid.numpy.equal[bool] # sanity + if shouldhave(binary.numpy, "equal"): + assert binary.numpy.equal[int].monoid is None + assert binary.numpy.equal[bool].monoid is monoid.numpy.equal[bool] # sanity for attr, val in vars(binary).items(): if not isinstance(val, BinaryOp): @@ -790,22 +863,25 @@ def test_monoid_attributes(): assert monoid.plus.binaryop is binary.plus assert monoid.plus.identities == {typ: 0 for typ in monoid.plus.types} - assert monoid.numpy.add[int].binaryop is binary.numpy.add[int] - assert monoid.numpy.add[int].identity == 0 - assert monoid.numpy.add.binaryop is binary.numpy.add - assert monoid.numpy.add.identities == {typ: 0 for typ in monoid.numpy.add.types} + if shouldhave(monoid.numpy, "add"): + assert monoid.numpy.add[int].binaryop is binary.numpy.add[int] + assert monoid.numpy.add[int].identity == 0 + assert monoid.numpy.add.binaryop is binary.numpy.add + assert monoid.numpy.add.identities == {typ: 0 for typ in monoid.numpy.add.types} def plus(x, y): # pragma: no cover (numba) return x + y - binop = BinaryOp.register_anonymous(plus, name="plus") - op = Monoid.register_anonymous(binop, 0, name="plus") - assert op.binaryop is binop - assert op[int].binaryop is binop[int] + if supports_udfs: + binop = BinaryOp.register_anonymous(plus, name="plus") + op = Monoid.register_anonymous(binop, 0, name="plus") + assert op.binaryop is binop + assert op[int].binaryop is binop[int] + assert op[int].parent is op assert monoid.plus[int].parent is monoid.plus - assert monoid.numpy.add[int].parent is monoid.numpy.add - assert op[int].parent is op + if shouldhave(monoid.numpy, "add"): + assert monoid.numpy.add[int].parent is monoid.numpy.add for attr, val in vars(monoid).items(): if not isinstance(val, Monoid): @@ -826,25 +902,27 @@ def test_semiring_attributes(): assert semiring.min_plus.monoid is monoid.min assert semiring.min_plus.binaryop is binary.plus - assert semiring.numpy.add_subtract[int].monoid is monoid.numpy.add[int] - assert semiring.numpy.add_subtract[int].binaryop is binary.numpy.subtract[int] - assert semiring.numpy.add_subtract.monoid is monoid.numpy.add - assert semiring.numpy.add_subtract.binaryop is binary.numpy.subtract + if shouldhave(semiring.numpy, "add_subtract"): + assert semiring.numpy.add_subtract[int].monoid is monoid.numpy.add[int] + assert semiring.numpy.add_subtract[int].binaryop is binary.numpy.subtract[int] + assert semiring.numpy.add_subtract.monoid is monoid.numpy.add + assert semiring.numpy.add_subtract.binaryop is binary.numpy.subtract + assert semiring.numpy.add_subtract[int].parent is semiring.numpy.add_subtract def plus(x, y): return x + y # pragma: no cover (numba) - binop = BinaryOp.register_anonymous(plus, name="plus") - mymonoid = Monoid.register_anonymous(binop, 0, name="plus") - op = Semiring.register_anonymous(mymonoid, binop, name="plus_plus") - assert op.binaryop is binop - assert op.binaryop[int] is binop[int] - assert op.monoid is mymonoid - assert op.monoid[int] is mymonoid[int] + if supports_udfs: + binop = BinaryOp.register_anonymous(plus, name="plus") + mymonoid = Monoid.register_anonymous(binop, 0, name="plus") + op = Semiring.register_anonymous(mymonoid, binop, name="plus_plus") + assert op.binaryop is binop + assert op.binaryop[int] is binop[int] + assert op.monoid is mymonoid + assert op.monoid[int] is mymonoid[int] + assert op[int].parent is op assert semiring.min_plus[int].parent is semiring.min_plus - assert semiring.numpy.add_subtract[int].parent is semiring.numpy.add_subtract - assert op[int].parent is op for attr, val in vars(semiring).items(): if not isinstance(val, Semiring): @@ -881,9 +959,10 @@ def test_div_semirings(): assert result[0, 0].new() == -2 assert result.dtype == dtypes.FP64 - result = A1.T.mxm(A2, semiring.plus_floordiv).new() - assert result[0, 0].new() == -3 - assert result.dtype == dtypes.INT64 + if shouldhave(semiring, "plus_floordiv"): + result = A1.T.mxm(A2, semiring.plus_floordiv).new() + assert result[0, 0].new() == -3 + assert result.dtype == dtypes.INT64 @pytest.mark.slow @@ -902,25 +981,27 @@ def test_get_semiring(): def myplus(x, y): return x + y # pragma: no cover (numba) - binop = BinaryOp.register_anonymous(myplus, name="myplus") - st = get_semiring(monoid.plus, binop) - assert st.monoid is monoid.plus - assert st.binaryop is binop + if supports_udfs: + binop = BinaryOp.register_anonymous(myplus, name="myplus") + st = get_semiring(monoid.plus, binop) + assert st.monoid is monoid.plus + assert st.binaryop is binop - binop = BinaryOp.register_new("myplus", myplus) - assert binop is binary.myplus - st = get_semiring(monoid.plus, binop) - assert st.monoid is monoid.plus - assert st.binaryop is binop + binop = BinaryOp.register_new("myplus", myplus) + assert binop is binary.myplus + st = get_semiring(monoid.plus, binop) + assert st.monoid is monoid.plus + assert st.binaryop is binop with pytest.raises(TypeError, match="Monoid"): get_semiring(None, binary.times) with pytest.raises(TypeError, match="Binary"): get_semiring(monoid.plus, None) - sr = get_semiring(monoid.plus, binary.numpy.copysign) - assert sr.monoid is monoid.plus - assert sr.binaryop is binary.numpy.copysign + if shouldhave(binary.numpy, "copysign"): + sr = get_semiring(monoid.plus, binary.numpy.copysign) + assert sr.monoid is monoid.plus + assert sr.binaryop is binary.numpy.copysign def test_create_semiring(): @@ -958,17 +1039,22 @@ def test_commutes(): assert semiring.plus_times.is_commutative if suitesparse: assert semiring.ss.min_secondi.commutes_to is semiring.ss.min_firstj - assert semiring.plus_pow.commutes_to is semiring.plus_rpow + if shouldhave(semiring, "plus_pow") and shouldhave(semiring, "plus_rpow"): + assert semiring.plus_pow.commutes_to is semiring.plus_rpow assert not semiring.plus_pow.is_commutative - assert binary.isclose.commutes_to is binary.isclose - assert binary.isclose.is_commutative - assert binary.isclose(0.1).commutes_to is binary.isclose(0.1) - assert binary.floordiv.commutes_to is binary.rfloordiv - assert not binary.floordiv.is_commutative - assert binary.numpy.add.commutes_to is binary.numpy.add - assert binary.numpy.add.is_commutative - assert binary.numpy.less.commutes_to is binary.numpy.greater - assert not binary.numpy.less.is_commutative + if shouldhave(binary, "isclose"): + assert binary.isclose.commutes_to is binary.isclose + assert binary.isclose.is_commutative + assert binary.isclose(0.1).commutes_to is binary.isclose(0.1) + if shouldhave(binary, "floordiv") and shouldhave(binary, "rfloordiv"): + assert binary.floordiv.commutes_to is binary.rfloordiv + assert not binary.floordiv.is_commutative + if shouldhave(binary.numpy, "add"): + assert binary.numpy.add.commutes_to is binary.numpy.add + assert binary.numpy.add.is_commutative + if shouldhave(binary.numpy, "less") and shouldhave(binary.numpy, "greater"): + assert binary.numpy.less.commutes_to is binary.numpy.greater + assert not binary.numpy.less.is_commutative # Typed assert binary.plus[int].commutes_to is binary.plus[int] @@ -985,15 +1071,20 @@ def test_commutes(): assert semiring.plus_times[int].is_commutative if suitesparse: assert semiring.ss.min_secondi[int].commutes_to is semiring.ss.min_firstj[int] - assert semiring.plus_pow[int].commutes_to is semiring.plus_rpow[int] + if shouldhave(semiring, "plus_rpow"): + assert semiring.plus_pow[int].commutes_to is semiring.plus_rpow[int] assert not semiring.plus_pow[int].is_commutative - assert binary.isclose(0.1)[int].commutes_to is binary.isclose(0.1)[int] - assert binary.floordiv[int].commutes_to is binary.rfloordiv[int] - assert not binary.floordiv[int].is_commutative - assert binary.numpy.add[int].commutes_to is binary.numpy.add[int] - assert binary.numpy.add[int].is_commutative - assert binary.numpy.less[int].commutes_to is binary.numpy.greater[int] - assert not binary.numpy.less[int].is_commutative + if shouldhave(binary, "isclose"): + assert binary.isclose(0.1)[int].commutes_to is binary.isclose(0.1)[int] + if shouldhave(binary, "floordiv") and shouldhave(binary, "rfloordiv"): + assert binary.floordiv[int].commutes_to is binary.rfloordiv[int] + assert not binary.floordiv[int].is_commutative + if shouldhave(binary.numpy, "add"): + assert binary.numpy.add[int].commutes_to is binary.numpy.add[int] + assert binary.numpy.add[int].is_commutative + if shouldhave(binary.numpy, "less") and shouldhave(binary.numpy, "greater"): + assert binary.numpy.less[int].commutes_to is binary.numpy.greater[int] + assert not binary.numpy.less[int].is_commutative # Stress test (this can create extra semirings) names = dir(semiring) @@ -1014,9 +1105,12 @@ def test_from_string(): assert unary.from_string("abs[float]") is unary.abs[float] assert binary.from_string("+") is binary.plus assert binary.from_string("-[int]") is binary.minus[int] - assert binary.from_string("true_divide") is binary.numpy.true_divide - assert binary.from_string("//") is binary.floordiv - assert binary.from_string("%") is binary.numpy.mod + if config["mapnumpy"] or shouldhave(binary.numpy, "true_divide"): + assert binary.from_string("true_divide") is binary.numpy.true_divide + if shouldhave(binary, "floordiv"): + assert binary.from_string("//") is binary.floordiv + if shouldhave(binary.numpy, "mod"): + assert binary.from_string("%") is binary.numpy.mod assert monoid.from_string("*[FP64]") is monoid.times["FP64"] assert semiring.from_string("min.plus") is semiring.min_plus assert semiring.from_string("min.+") is semiring.min_plus @@ -1053,6 +1147,7 @@ def test_from_string(): agg.from_string("bad_agg") +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_lazy_op(): UnaryOp.register_new("lazy", lambda x: x, lazy=True) # pragma: no branch (numba) @@ -1115,6 +1210,7 @@ def test_positional(): assert semiring.ss.any_secondj[int].is_positional +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_udt(): record_dtype = np.dtype([("x", np.bool_), ("y", np.float64)], align=True) @@ -1280,6 +1376,7 @@ def test_binaryop_commute_exists(): raise AssertionError("Missing binaryops: " + ", ".join(sorted(missing))) +@pytest.mark.skipif("not supports_udfs") def test_binom(): v = Vector.from_coo([0, 1, 2], [3, 4, 5]) result = v.apply(binary.binom, 2).new() @@ -1341,9 +1438,11 @@ def test_is_idempotent(): assert monoid.max[int].is_idempotent assert monoid.lor.is_idempotent assert monoid.band.is_idempotent - assert monoid.numpy.gcd.is_idempotent + if shouldhave(monoid.numpy, "gcd"): + assert monoid.numpy.gcd.is_idempotent assert not monoid.plus.is_idempotent assert not monoid.times[float].is_idempotent - assert not monoid.numpy.equal.is_idempotent + if config["mapnumpy"] or shouldhave(monoid.numpy, "equal"): + assert not monoid.numpy.equal.is_idempotent with pytest.raises(AttributeError): binary.min.is_idempotent diff --git a/graphblas/tests/test_operator_types.py b/graphblas/tests/test_operator_types.py index 522b42ad2..027f02fcc 100644 --- a/graphblas/tests/test_operator_types.py +++ b/graphblas/tests/test_operator_types.py @@ -2,6 +2,7 @@ from collections import defaultdict from graphblas import backend, binary, dtypes, monoid, semiring, unary +from graphblas.core import _supports_udfs as supports_udfs from graphblas.core import operator from graphblas.dtypes import ( BOOL, @@ -83,6 +84,11 @@ BINARY[(ALL, POS)] = { "firsti", "firsti1", "firstj", "firstj1", "secondi", "secondi1", "secondj", "secondj1", } +if not supports_udfs: + udfs = {"absfirst", "abssecond", "binom", "floordiv", "rfloordiv", "rpow"} + for funcnames in BINARY.values(): + funcnames -= udfs + BINARY = {key: val for key, val in BINARY.items() if val} MONOID = { (UINT, UINT): {"band", "bor", "bxnor", "bxor"}, diff --git a/graphblas/tests/test_pickle.py b/graphblas/tests/test_pickle.py index de2d9cfda..724f43d76 100644 --- a/graphblas/tests/test_pickle.py +++ b/graphblas/tests/test_pickle.py @@ -5,6 +5,7 @@ import pytest import graphblas as gb +from graphblas.core import _supports_udfs as supports_udfs # noqa: F401 suitesparse = gb.backend == "suitesparse" @@ -36,6 +37,7 @@ def extra(): return "" +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_deserialize(extra): path = Path(__file__).parent / f"pickle1{extra}.pkl" @@ -62,6 +64,7 @@ def test_deserialize(extra): assert d3["semiring_pickle"] is gb.semiring.semiring_pickle +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_serialize(): v = gb.Vector.from_coo([1], 2) @@ -232,6 +235,7 @@ def identity_par(z): return -z +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_serialize_parameterized(): # unary_pickle = gb.core.operator.UnaryOp.register_new( @@ -285,6 +289,7 @@ def test_serialize_parameterized(): pickle.loads(pkl) # TODO: check results +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_deserialize_parameterized(extra): path = Path(__file__).parent / f"pickle2{extra}.pkl" @@ -295,6 +300,7 @@ def test_deserialize_parameterized(extra): pickle.load(f) # TODO: check results +@pytest.mark.skipif("not supports_udfs") def test_udt(extra): record_dtype = np.dtype([("x", np.bool_), ("y", np.int64)], align=True) udt = gb.dtypes.register_new("PickleUDT", record_dtype) diff --git a/graphblas/tests/test_scalar.py b/graphblas/tests/test_scalar.py index 6ee70311c..7b7c77177 100644 --- a/graphblas/tests/test_scalar.py +++ b/graphblas/tests/test_scalar.py @@ -12,7 +12,7 @@ from graphblas import backend, binary, dtypes, monoid, replace, select, unary from graphblas.exceptions import EmptyObject -from .conftest import autocompute, compute +from .conftest import autocompute, compute, pypy from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) @@ -209,12 +209,12 @@ def test_unsupported_ops(s): s[0] with pytest.raises(TypeError, match="does not support"): s[0] = 0 - with pytest.raises(TypeError, match="doesn't support"): + with pytest.raises(TypeError, match="doesn't support|does not support"): del s[0] def test_is_empty(s): - with pytest.raises(AttributeError, match="can't set attribute"): + with pytest.raises(AttributeError, match="can't set attribute|object has no setter"): s.is_empty = True @@ -226,7 +226,7 @@ def test_update(s): s << Scalar.from_value(3) assert s == 3 if s._is_cscalar: - with pytest.raises(TypeError, match="an integer is required"): + with pytest.raises(TypeError, match="an integer is required|expected integer"): s << Scalar.from_value(4.4) else: s << Scalar.from_value(4.4) @@ -358,14 +358,15 @@ def test_expr_is_like_scalar(s): } if s.is_cscalar: expected.add("_empty") - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Scalar. You may need to " "add an entry to `scalar` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " "methods, then you may need to run `python -m graphblas.core.infixmethods`." ) - assert attrs - infix_attrs == expected - assert attrs - scalar_infix_attrs == expected + assert attrs - infix_attrs - ignore == expected + assert attrs - scalar_infix_attrs - ignore == expected # Make sure signatures actually match. `expr.dup` has `**opts` skip = {"__init__", "__repr__", "_repr_html_", "dup"} for expr in [v.inner(v), v @ v, t & t]: @@ -399,7 +400,8 @@ def test_index_expr_is_like_scalar(s): } if s.is_cscalar: expected.add("_empty") - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Scalar. You may need to " "add an entry to `scalar` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " @@ -505,10 +507,10 @@ def test_scalar_expr(s): def test_sizeof(s): - if suitesparse or s._is_cscalar: + if (suitesparse or s._is_cscalar) and not pypy: assert 1 < sys.getsizeof(s) < 1000 else: - with pytest.raises(TypeError): + with pytest.raises(TypeError): # flakey coverage (why?!) sys.getsizeof(s) diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index 8505313e4..ab019b734 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -11,6 +11,7 @@ import graphblas as gb from graphblas import agg, backend, binary, dtypes, indexunary, monoid, select, semiring, unary +from graphblas.core import _supports_udfs as supports_udfs from graphblas.exceptions import ( DimensionMismatch, DomainMismatch, @@ -19,9 +20,10 @@ InvalidObject, InvalidValue, OutputNotEmpty, + UdfParseError, ) -from .conftest import autocompute, compute +from .conftest import autocompute, compute, pypy from graphblas import Matrix, Scalar, Vector # isort:skip (for dask-graphblas) @@ -798,12 +800,13 @@ def test_select_bools_and_masks(v): assert w8.isequal(w9) +@pytest.mark.skipif("not supports_udfs") @pytest.mark.slow def test_indexunary_udf(v): def twox_minusthunk(x, row, col, thunk): # pragma: no cover (numba) return 2 * x - thunk - indexunary.register_new("twox_minusthunk", twox_minusthunk) + indexunary.register_new("twox_minusthunk", twox_minusthunk, lazy=True) assert hasattr(indexunary, "twox_minusthunk") assert not hasattr(select, "twox_minusthunk") with pytest.raises(ValueError, match="SelectOp must have BOOL return type"): @@ -813,24 +816,49 @@ def twox_minusthunk(x, row, col, thunk): # pragma: no cover (numba) expected = Vector.from_coo([1, 3, 4, 6], [-2, -2, 0, -4], size=7) result = indexunary.twox_minusthunk(v, 4).new() assert result.isequal(expected) + assert pickle.loads(pickle.dumps(indexunary.triu)) is indexunary.triu + assert indexunary.twox_minusthunk[int]._numba_func(1, 2, 3, 4) == twox_minusthunk(1, 2, 3, 4) + assert indexunary.twox_minusthunk[int].orig_func is twox_minusthunk delattr(indexunary, "twox_minusthunk") def ii(x, idx, _, thunk): # pragma: no cover (numba) return idx // 2 >= thunk - select.register_new("ii", ii) - assert hasattr(indexunary, "ii") + def iin(n): + def inner(x, idx, _, thunk): # pragma: no cover (numba) + return idx // n >= thunk + + return inner + + select.register_new("ii", ii, lazy=True) + select.register_new("iin", iin, parameterized=True) + assert "ii" in dir(select) + assert "ii" in dir(indexunary) assert hasattr(select, "ii") + assert hasattr(indexunary, "ii") ii_apply = indexunary.register_anonymous(ii) expected = Vector.from_coo([1, 3, 4, 6], [False, False, True, True], size=7) result = ii_apply(v, 2).new() assert result.isequal(expected) + result = v.apply(indexunary.iin(2), 2).new() + assert result.isequal(expected) + result = v.apply(indexunary.register_anonymous(iin, parameterized=True)(2), 2).new() + assert result.isequal(expected) + ii_select = select.register_anonymous(ii) expected = Vector.from_coo([4, 6], [2, 0], size=7) result = ii_select(v, 2).new() assert result.isequal(expected) + result = v.select(select.iin(2), 2).new() + assert result.isequal(expected) + result = v.select(select.register_anonymous(iin, parameterized=True)(2), 2).new() + assert result.isequal(expected) delattr(indexunary, "ii") delattr(select, "ii") + delattr(indexunary, "iin") + delattr(select, "iin") + with pytest.raises(UdfParseError, match="Unable to parse function using Numba"): + indexunary.register_new("bad", lambda x, row, col, thunk: result) def test_reduce(v): @@ -1624,13 +1652,14 @@ def test_expr_is_like_vector(v): "resize", "update", } - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Vector. You may need to " "add an entry to `vector` or `matrix_vector` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " "methods, then you may need to run `python -m graphblas.core.infixmethods`." ) - assert attrs - infix_attrs == expected + assert attrs - infix_attrs - ignore == expected # Make sure signatures actually match skip = {"__init__", "__repr__", "_repr_html_"} for expr in [binary.times(w & w), w & w]: @@ -1672,7 +1701,8 @@ def test_index_expr_is_like_vector(v): "from_values", "resize", } - assert attrs - expr_attrs == expected, ( + ignore = {"__sizeof__"} + assert attrs - expr_attrs - ignore == expected, ( "If you see this message, you probably added a method to Vector. You may need to " "add an entry to `vector` or `matrix_vector` set in `graphblas.core.automethods` " "and then run `python -m graphblas.core.automethods`. If you're messing with infix " @@ -1963,7 +1993,7 @@ def test_ndim(A, v): def test_sizeof(v): - if suitesparse: + if suitesparse and not pypy: assert sys.getsizeof(v) > v.nvals * 16 else: with pytest.raises(TypeError): @@ -2006,6 +2036,7 @@ def test_delete_via_scalar(v): assert v.nvals == 0 +@pytest.mark.skipif("not supports_udfs") def test_udt(): record_dtype = np.dtype([("x", np.bool_), ("y", np.float64)], align=True) udt = dtypes.register_anonymous(record_dtype, "VectorUDT") @@ -2380,6 +2411,7 @@ def test_to_coo_subset(v): assert vals.dtype == np.int64 +@pytest.mark.skipif("not supports_udfs") def test_lambda_udfs(v): result = v.apply(lambda x: x + 1).new() # pragma: no branch (numba) expected = binary.plus(v, 1).new() @@ -2506,7 +2538,8 @@ def test_from_scalar(): v = Vector.from_scalar(1, dtype="INT64[2]", size=3) w = Vector("INT64[2]", size=3) w << [1, 1] - assert v.isequal(w, check_dtype=True) + if supports_udfs: + assert v.isequal(w, check_dtype=True) def test_to_dense_from_dense(): @@ -2559,9 +2592,10 @@ def test_ss_sort(v): v.ss.sort(binary.plus) # Like compactify - _, p = v.ss.sort(lambda x, y: False, values=False) # pragma: no branch (numba) - expected_p = Vector.from_coo([0, 1, 2, 3], [1, 3, 4, 6], size=7) - assert p.isequal(expected_p) + if supports_udfs: + _, p = v.ss.sort(lambda x, y: False, values=False) # pragma: no branch (numba) + expected_p = Vector.from_coo([0, 1, 2, 3], [1, 3, 4, 6], size=7) + assert p.isequal(expected_p) # reversed _, p = v.ss.sort(binary.pair[bool], values=False) expected_p = Vector.from_coo([0, 1, 2, 3], [6, 4, 3, 1], size=7) @@ -2569,6 +2603,7 @@ def test_ss_sort(v): w, p = v.ss.sort(monoid.lxor) # Weird, but user-defined monoids may not commute, so okay +@pytest.mark.skipif("not supports_udfs") def test_subarray_dtypes(): a = np.arange(3 * 4, dtype=np.int64).reshape(3, 4) v = Vector.from_coo([1, 3, 5], a) diff --git a/graphblas/unary/numpy.py b/graphblas/unary/numpy.py index 836da2024..9b742d8bc 100644 --- a/graphblas/unary/numpy.py +++ b/graphblas/unary/numpy.py @@ -10,6 +10,7 @@ from .. import _STANDARD_OPERATOR_NAMES from .. import config as _config from .. import unary as _unary +from ..core import _supports_udfs from ..dtypes import _supports_complex _delayed = {} @@ -119,7 +120,12 @@ def __dir__(): - return globals().keys() | _delayed.keys() | _unary_names + if not _supports_udfs and not _config.get("mapnumpy"): + return globals().keys() # FLAKY COVERAGE + attrs = _delayed.keys() | _unary_names + if not _supports_udfs: + attrs &= _numpy_to_graphblas.keys() + return attrs | globals().keys() def __getattr__(name): @@ -132,6 +138,11 @@ def __getattr__(name): raise AttributeError(f"module {__name__!r} has no attribute {name!r}") if _config.get("mapnumpy") and name in _numpy_to_graphblas: globals()[name] = getattr(_unary, _numpy_to_graphblas[name]) + elif not _supports_udfs: + raise AttributeError( + f"module {__name__!r} unable to compile UDF for {name!r}; " + "install numba for UDF support" + ) else: numpy_func = getattr(_np, name) diff --git a/pyproject.toml b/pyproject.toml index 47cf1e67f..1eaa942e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3 :: Only", "Intended Audience :: Developers", "Intended Audience :: Other Audience", @@ -57,11 +58,13 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - "suitesparse-graphblas >=7.4.0.0, <7.5", "numpy >=1.21", - "numba >=0.55", "donfig >=0.6", "pyyaml >=5.4", + # These won't be installed by default after 2024.3.0 + # Use e.g. "python-graphblas[suitesparse]" or "python-graphblas[default]" instead + "suitesparse-graphblas >=7.4.0.0, <7.5", + "numba >=0.55; python_version<'3.11'", # make optional where numba is not supported ] [project.urls] @@ -71,37 +74,56 @@ repository = "https://github.com/python-graphblas/python-graphblas" changelog = "https://github.com/python-graphblas/python-graphblas/releases" [project.optional-dependencies] -repr = [ - "pandas >=1.2", +suitesparse = [ + "suitesparse-graphblas >=7.4.0.0, <7.5", ] -io = [ +networkx = [ "networkx >=2.8", +] +numba = [ + "numba >=0.55", +] +pandas = [ + "pandas >=1.2", +] +scipy = [ "scipy >=1.8", +] +suitesparse-udf = [ # udf requires numba + "python-graphblas[suitesparse,numba]", +] +repr = [ + "python-graphblas[pandas]", +] +io = [ + "python-graphblas[networkx,scipy]", + "python-graphblas[numba]; python_version<'3.11'", "awkward >=1.9", - "sparse >=0.12", + "sparse >=0.13; python_version<'3.11'", # make optional, b/c sparse needs numba "fast-matrix-market >=1.4.5", ] viz = [ + "python-graphblas[networkx,scipy]", "matplotlib >=3.5", ] +datashade = [ # datashade requires numba + "python-graphblas[numba,pandas,scipy]", + "datashader >=0.12", + "hvplot >=0.7", +] test = [ - "pytest", - "packaging", - "pandas >=1.2", - "scipy >=1.8", - "tomli", + "python-graphblas[suitesparse,pandas,scipy]", + "packaging >=21", + "pytest >=6.2", + "tomli >=1", +] +default = [ + "python-graphblas[suitesparse,pandas,scipy]", + "python-graphblas[numba]; python_version<'3.11'", # make optional where numba is not supported ] complete = [ - "pandas >=1.2", - "networkx >=2.8", - "scipy >=1.8", - "awkward >=1.9", - "sparse >=0.12", - "fast-matrix-market >=1.4.5", - "matplotlib >=3.5", - "pytest", - "packaging", - "tomli", + "python-graphblas[default,io,viz,test]", + "python-graphblas[datashade]; python_version<'3.11'", # make optional, b/c datashade needs numba ] [tool.setuptools] @@ -154,8 +176,6 @@ filterwarnings = [ # See: https://docs.python.org/3/library/warnings.html#describing-warning-filters # and: https://docs.pytest.org/en/7.2.x/how-to/capture-warnings.html#controlling-warnings "error", - # MAINT: we can drop support for sparse <0.13 at any time - "ignore:`np.bool` is a deprecated alias:DeprecationWarning:sparse._umath", # sparse <0.13 # sparse 0.14.0 (2022-02-24) began raising this warning; it has been reported and fixed upstream. "ignore:coords should be an ndarray. This will raise a ValueError:DeprecationWarning:sparse._coo.core", @@ -166,6 +186,13 @@ filterwarnings = [ # And this deprecation warning was added in setuptools v67.5.0 (8 Mar 2023). See: # https://setuptools.pypa.io/en/latest/history.html#v67-5-0 "ignore:pkg_resources is deprecated as an API:DeprecationWarning:pkg_resources", + + # sre_parse deprecated in 3.11; this is triggered by awkward 0.10 + "ignore:module 'sre_parse' is deprecated:DeprecationWarning:", + "ignore:module 'sre_constants' is deprecated:DeprecationWarning:", + + # pypy gives this warning + "ignore:can't resolve package from __spec__ or __package__:ImportWarning:", ] [tool.coverage.run] diff --git a/scripts/check_versions.sh b/scripts/check_versions.sh index 54b02d1f9..026f3a656 100755 --- a/scripts/check_versions.sh +++ b/scripts/check_versions.sh @@ -4,12 +4,12 @@ # This may be helpful when updating dependency versions in CI. # Tip: add `--json` for more information. conda search 'numpy[channel=conda-forge]>=1.24.2' -conda search 'pandas[channel=conda-forge]>=1.5.3' +conda search 'pandas[channel=conda-forge]>=2.0.0' conda search 'scipy[channel=conda-forge]>=1.10.1' -conda search 'networkx[channel=conda-forge]>=3.0' -conda search 'awkward[channel=conda-forge]>=2.1.1' +conda search 'networkx[channel=conda-forge]>=3.1' +conda search 'awkward[channel=conda-forge]>=2.1.2' conda search 'sparse[channel=conda-forge]>=0.14.0' -conda search 'fast_matrix_market[channel=conda-forge]>=1.4.5' +conda search 'fast_matrix_market[channel=conda-forge]>=1.5.1' conda search 'numba[channel=conda-forge]>=0.56.4' conda search 'pyyaml[channel=conda-forge]>=6.0' conda search 'flake8-bugbear[channel=conda-forge]>=23.3.23' pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy