From 3767e782aebcbfc98cd433c1dc90529b043620a1 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Thu, 2 Mar 2023 16:47:23 +0100 Subject: [PATCH 1/3] Testing and timing compared to NumPy --- .github/workflows/profile.yml | 53 +++++++++++++++++++ .gitignore | 2 + python-module/CMakeLists.txt | 63 +++++++++++++++++++++++ python-module/module/main.cpp | 58 +++++++++++++++++++++ python-module/module/xt/__init__.py | 1 + python-module/profiling/test_a.py | 79 +++++++++++++++++++++++++++++ python-module/setup.py | 24 +++++++++ python-module/tests/__init__.py | 0 python-module/tests/test_a.py | 61 ++++++++++++++++++++++ 9 files changed, 341 insertions(+) create mode 100644 .github/workflows/profile.yml create mode 100644 python-module/CMakeLists.txt create mode 100644 python-module/module/main.cpp create mode 100644 python-module/module/xt/__init__.py create mode 100644 python-module/profiling/test_a.py create mode 100644 python-module/setup.py create mode 100644 python-module/tests/__init__.py create mode 100644 python-module/tests/test_a.py diff --git a/.github/workflows/profile.yml b/.github/workflows/profile.yml new file mode 100644 index 0000000..f838a48 --- /dev/null +++ b/.github/workflows/profile.yml @@ -0,0 +1,53 @@ +name: profile + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + +jobs: + + standard: + + strategy: + fail-fast: false + matrix: + runs-on: [ubuntu-latest, macos-latest, windows-latest] + + defaults: + run: + shell: bash -l {0} + + name: ${{ matrix.runs-on }} • x64 ${{ matrix.args }} + runs-on: ${{ matrix.runs-on }} + + steps: + + - name: Basic GitHub action setup + uses: actions/checkout@v3 + + - name: Set conda environment + uses: mamba-org/provision-with-micromamba@main + with: + environment-file: environment-dev.yml + environment-name: myenv + cache-env: true + extra-specs: | + prettytable + setuptools_scm + scikit-build + xsimd + + - name: Set dummy version + run: echo "SETUPTOOLS_SCM_PRETEND_VERSION=0.0" >> $GITHUB_ENV + + - name: Build and install Python module + working-directory: python-module + run: | + SKBUILD_CONFIGURE_OPTIONS="-DUSE_XSIMD=1" python -m pip install . -v + + - name: Run Python profiling + working-directory: python-module + run: python -m unittest discover profiling diff --git a/.gitignore b/.gitignore index 9413a44..72c0e07 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +_skbuild + # Prerequisites *.d diff --git a/python-module/CMakeLists.txt b/python-module/CMakeLists.txt new file mode 100644 index 0000000..44391a6 --- /dev/null +++ b/python-module/CMakeLists.txt @@ -0,0 +1,63 @@ +cmake_minimum_required(VERSION 3.18..3.21) + +project(xt) + +# The C++ functions are build to a library with name "_${PROJECT_NAME}" +# The Python library simply loads all functions +set(PYPROJECT_NAME "_${PROJECT_NAME}") + +option(USE_WARNINGS "${PROJECT_NAME}: Build with runtime warnings" ON) +option(USE_XSIMD "${PROJECT_NAME}: Build with hardware optimization" OFF) + +if (DEFINED ENV{SETUPTOOLS_SCM_PRETEND_VERSION}) + set(PROJECT_VERSION $ENV{SETUPTOOLS_SCM_PRETEND_VERSION}) + message(STATUS "Building ${PROJECT_NAME} ${PROJECT_VERSION} (read from SETUPTOOLS_SCM_PRETEND_VERSION)") +else() + execute_process( + COMMAND python -c "from setuptools_scm import get_version; print(get_version(root='..'))" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE PROJECT_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + + message(STATUS "Building ${PROJECT_NAME} ${PROJECT_VERSION}") +endif() + +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +find_package(xtensor REQUIRED) +find_package(pybind11 REQUIRED CONFIG) + +if (SKBUILD) + find_package(NumPy REQUIRED) +else() + find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) +endif() + +pybind11_add_module(${PYPROJECT_NAME} module/main.cpp) + +target_compile_definitions(${PYPROJECT_NAME} PUBLIC VERSION_INFO=${PROJECT_VERSION}) +target_link_libraries(${PYPROJECT_NAME} PUBLIC xtensor) +target_include_directories(${PYPROJECT_NAME} PUBLIC "../include") + +if (SKBUILD) + target_include_directories(${PYPROJECT_NAME} PUBLIC ${NumPy_INCLUDE_DIRS}) +else() + target_link_libraries(${PYPROJECT_NAME} PUBLIC pybind11::module Python::NumPy) +endif() + +if (USE_SIMD) + find_package(xsimd REQUIRED) + target_link_libraries(${PYPROJECT_NAME} PUBLIC xtensor::optimize xtensor::use_xsimd) + message(STATUS "Compiling ${PROJECT_NAME}-Python with hardware optimization") +endif() + +if (SKBUILD) + if(APPLE) + set_target_properties(${PYPROJECT_NAME} PROPERTIES INSTALL_RPATH "@loader_path/${CMAKE_INSTALL_LIBDIR}") + else() + set_target_properties(${PYPROJECT_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/${CMAKE_INSTALL_LIBDIR}") + endif() + install(TARGETS ${PYPROJECT_NAME} DESTINATION .) +endif() diff --git a/python-module/module/main.cpp b/python-module/module/main.cpp new file mode 100644 index 0000000..98a6e99 --- /dev/null +++ b/python-module/module/main.cpp @@ -0,0 +1,58 @@ +/** + * @file + * @copyright Copyright 2020. Tom de Geus. All rights reserved. + * @license This project is released under the GNU Public License (MIT). + */ + +#include +#include + +#define FORCE_IMPORT_ARRAY +#include +#include +#include + +namespace py = pybind11; + +/** + * Overrides the `__name__` of a module. + * Classes defined by pybind11 use the `__name__` of the module as of the time they are defined, + * which affects the `__repr__` of the class type objects. + */ +class ScopedModuleNameOverride { +public: + explicit ScopedModuleNameOverride(py::module m, std::string name) : module_(std::move(m)) + { + original_name_ = module_.attr("__name__"); + module_.attr("__name__") = name; + } + ~ScopedModuleNameOverride() + { + module_.attr("__name__") = original_name_; + } + +private: + py::module module_; + py::object original_name_; +}; + +PYBIND11_MODULE(_xt, m) +{ + // Ensure members to display as `xt.X` (not `xt._xt.X`) + ScopedModuleNameOverride name_override(m, "xt"); + + xt::import_numpy(); + + m.doc() = "Python bindings of xtensor"; + + m.def("mean", [](const xt::pyarray& a) -> xt::pyarray { return xt::mean(a); }); + + m.def("flip", [](const xt::pyarray& a, ptrdiff_t axis) -> xt::pyarray { return xt::flip(a, axis); }); + + m.def("cos", [](const xt::pyarray& a) -> xt::pyarray { return xt::cos(a); }); + + m.def("isin", [](const xt::pyarray& a, const xt::pyarray& b) -> xt::pyarray { return xt::isin(a, b); }); + m.def("in1d", [](const xt::pyarray& a, const xt::pyarray& b) -> xt::pyarray { return xt::in1d(a, b); }); + + +} diff --git a/python-module/module/xt/__init__.py b/python-module/module/xt/__init__.py new file mode 100644 index 0000000..eadaf1c --- /dev/null +++ b/python-module/module/xt/__init__.py @@ -0,0 +1 @@ +from ._xt import * # noqa: F401, F403 diff --git a/python-module/profiling/test_a.py b/python-module/profiling/test_a.py new file mode 100644 index 0000000..ac8b976 --- /dev/null +++ b/python-module/profiling/test_a.py @@ -0,0 +1,79 @@ +import unittest +import timeit +import warnings +from prettytable import PrettyTable + +import xt +import numpy as np + + +class test_a(unittest.TestCase): + """ + ?? + """ + + @classmethod + def setUpClass(self): + + self.a = np.random.random([103, 102, 101]) + self.axis = int(np.random.randint(0, high=3)) + self.data = [] + + @classmethod + def tearDownClass(self): + + self.data = sorted(self.data, key=lambda x: x[1]) + + table = PrettyTable(["function", "xtensor / numpy"]) + + for row in self.data: + table.add_row(row) + + print("") + print(table) + print("") + + # code = table.get_html_string() + # with open('profile.html', 'w') as file: + # file.write(code) + + def test_mean(self): + + n = timeit.timeit(lambda: np.mean(self.a), number=10) + x = timeit.timeit(lambda: xt.mean(self.a), number=10) + self.data.append(("mean", x / n)) + + def test_flip(self): + + n = timeit.timeit(lambda: np.flip(self.a, self.axis), number=10) + x = timeit.timeit(lambda: xt.flip(self.a, self.axis), number=10) + self.data.append(("flip", x / n)) + + def test_cos(self): + + n = timeit.timeit(lambda: np.cos(self.a), number=10) + x = timeit.timeit(lambda: xt.cos(self.a), number=10) + self.data.append(("cos", x / n)) + + def test_isin(self): + + a = (np.random.random([103, 102]) * 1000).astype(int) + b = (np.random.random([103, 102]) * 1000).astype(int) + + n = timeit.timeit(lambda: np.isin(a, b), number=10) + x = timeit.timeit(lambda: xt.isin(a, b), number=10) + self.data.append(("isin", x / n)) + + def test_in1d(self): + + a = (np.random.random([1003]) * 1000).astype(int) + b = (np.random.random([1003]) * 1000).astype(int) + + n = timeit.timeit(lambda: np.in1d(a, b), number=10) + x = timeit.timeit(lambda: xt.in1d(a, b), number=10) + self.data.append(("in1d", x / n)) + + +if __name__ == "__main__": + + unittest.main() diff --git a/python-module/setup.py b/python-module/setup.py new file mode 100644 index 0000000..a8e9f26 --- /dev/null +++ b/python-module/setup.py @@ -0,0 +1,24 @@ +from pathlib import Path + +from setuptools_scm import get_version +from skbuild import setup + +project_name = "xt" + +this_directory = Path(__file__).parent +long_description = (this_directory / ".." / "README.md").read_text() +license = (this_directory / ".." / "LICENSE").read_text() + +setup( + name=project_name, + description="xt - Python bindings of xtensor", + long_description=long_description, + version=get_version(root="..", relative_to=__file__), + license=license, + author="Tom de Geus", + url=f"ttps://github.com/xtensor-stack/{project_name}", + packages=[f"{project_name}"], + package_dir={"": "module"}, + cmake_install_dir=f"module/{project_name}", + cmake_minimum_required_version="3.13", +) diff --git a/python-module/tests/__init__.py b/python-module/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python-module/tests/test_a.py b/python-module/tests/test_a.py new file mode 100644 index 0000000..93c04a2 --- /dev/null +++ b/python-module/tests/test_a.py @@ -0,0 +1,61 @@ +import unittest +import timeit +import warnings + +import xt +import numpy as np + + +class test_a(unittest.TestCase): + """ + ?? + """ + + def test_mean(self): + + a = np.random.random([103, 102, 101]) + n = np.mean(a) + x = xt.mean(a) + + self.assertTrue(np.allclose(n, x)) + + n = timeit.timeit(lambda: np.mean(a), number=10) + x = timeit.timeit(lambda: xt.mean(a), number=10) + + if x / n > 1.1: + warnings.warn(f"efficiency xt.mean {x / n:.2e}") + + def test_flip(self): + + axis = int(np.random.randint(0, high=3)) + a = np.random.random([103, 102, 101]) + n = np.flip(a, axis) + x = xt.flip(a, axis) + + self.assertTrue(np.allclose(n, x)) + + n = timeit.timeit(lambda: np.flip(a, axis), number=10) + x = timeit.timeit(lambda: xt.flip(a, axis), number=10) + + if x / n > 1.1: + warnings.warn(f"efficiency xt.flip {x / n:.2e}") + + def test_cos(self): + + a = np.random.random([103, 102, 101]) + n = np.cos(a) + x = xt.cos(a) + + self.assertTrue(np.allclose(n, x)) + + n = timeit.timeit(lambda: np.cos(a), number=10) + x = timeit.timeit(lambda: xt.cos(a), number=10) + + if x / n > 1.1: + warnings.warn(f"efficiency xt.cos {x / n:.2e}") + + + +if __name__ == "__main__": + + unittest.main() From d2584a423d0a83d1a5a1817875bd3118feeda04f Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Thu, 2 Mar 2023 17:56:40 +0100 Subject: [PATCH 2/3] Reproducing https://github.com/xtensor-stack/xtensor/pull/2569 --- .github/workflows/check.yml | 53 +++++++++++++++++++++++++++++++++++ python-module/module/main.cpp | 2 ++ python-module/tests/test_a.py | 36 +++++++++++------------- 3 files changed, 72 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/check.yml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..0d2f11a --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,53 @@ +name: check + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + +jobs: + + standard: + + strategy: + fail-fast: false + matrix: + runs-on: [ubuntu-latest, macos-latest, windows-latest] + + defaults: + run: + shell: bash -l {0} + + name: ${{ matrix.runs-on }} • x64 ${{ matrix.args }} + runs-on: ${{ matrix.runs-on }} + + steps: + + - name: Basic GitHub action setup + uses: actions/checkout@v3 + + - name: Set conda environment + uses: mamba-org/provision-with-micromamba@main + with: + environment-file: environment-dev.yml + environment-name: myenv + cache-env: true + extra-specs: | + prettytable + setuptools_scm + scikit-build + xsimd + + - name: Set dummy version + run: echo "SETUPTOOLS_SCM_PRETEND_VERSION=0.0" >> $GITHUB_ENV + + - name: Build and install Python module + working-directory: python-module + run: | + SKBUILD_CONFIGURE_OPTIONS="-DUSE_XSIMD=1" python -m pip install . -v + + - name: Run Python tests + working-directory: python-module + run: python -m unittest discover tests diff --git a/python-module/module/main.cpp b/python-module/module/main.cpp index 98a6e99..4a6e89a 100644 --- a/python-module/module/main.cpp +++ b/python-module/module/main.cpp @@ -46,6 +46,8 @@ PYBIND11_MODULE(_xt, m) m.doc() = "Python bindings of xtensor"; m.def("mean", [](const xt::pyarray& a) -> xt::pyarray { return xt::mean(a); }); + m.def("average", [](const xt::pyarray& a, const xt::pyarray& w) -> xt::pyarray { return xt::average(a, w); }); + m.def("average", [](const xt::pyarray& a, const xt::pyarray& w, const std::vector& axes) -> xt::pyarray { return xt::average(a, w, axes); }); m.def("flip", [](const xt::pyarray& a, ptrdiff_t axis) -> xt::pyarray { return xt::flip(a, axis); }); diff --git a/python-module/tests/test_a.py b/python-module/tests/test_a.py index 93c04a2..98d18eb 100644 --- a/python-module/tests/test_a.py +++ b/python-module/tests/test_a.py @@ -1,6 +1,4 @@ import unittest -import timeit -import warnings import xt import numpy as np @@ -19,11 +17,24 @@ def test_mean(self): self.assertTrue(np.allclose(n, x)) - n = timeit.timeit(lambda: np.mean(a), number=10) - x = timeit.timeit(lambda: xt.mean(a), number=10) + def test_average(self): - if x / n > 1.1: - warnings.warn(f"efficiency xt.mean {x / n:.2e}") + a = np.random.random([103, 102, 101]) + w = np.random.random([103, 102, 101]) + n = np.average(a, weights=w) + x = xt.average(a, w) + + self.assertTrue(np.allclose(n, x)) + + def test_average_axes(self): + + a = np.random.random([103, 102, 101]) + w = np.random.random([103, 102, 101]) + axis = int(np.random.randint(0, high=3)) + n = np.average(a, weights=w, axis=(axis,)) + x = xt.average(a, w, [axis]) + + self.assertTrue(np.allclose(n, x)) def test_flip(self): @@ -34,12 +45,6 @@ def test_flip(self): self.assertTrue(np.allclose(n, x)) - n = timeit.timeit(lambda: np.flip(a, axis), number=10) - x = timeit.timeit(lambda: xt.flip(a, axis), number=10) - - if x / n > 1.1: - warnings.warn(f"efficiency xt.flip {x / n:.2e}") - def test_cos(self): a = np.random.random([103, 102, 101]) @@ -48,13 +53,6 @@ def test_cos(self): self.assertTrue(np.allclose(n, x)) - n = timeit.timeit(lambda: np.cos(a), number=10) - x = timeit.timeit(lambda: xt.cos(a), number=10) - - if x / n > 1.1: - warnings.warn(f"efficiency xt.cos {x / n:.2e}") - - if __name__ == "__main__": From 8fa769759d3614c4ab3f7733d1883a4efe4391c5 Mon Sep 17 00:00:00 2001 From: Tom de Geus Date: Fri, 3 Mar 2023 10:12:47 +0100 Subject: [PATCH 3/3] debugging --- environment-dev.yml | 4 ++-- python-module/profiling/test_a.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 6dff19f..2386b82 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -5,9 +5,9 @@ dependencies: # Build dependencies - cmake # Host dependencies - - xtensor=0.24.0 + - xtensor >=0.24.0 - numpy - - pybind11=2.4.3 + - pybind11 >=2.4.3 # Test dependencies - pytest diff --git a/python-module/profiling/test_a.py b/python-module/profiling/test_a.py index ac8b976..a35957d 100644 --- a/python-module/profiling/test_a.py +++ b/python-module/profiling/test_a.py @@ -57,8 +57,8 @@ def test_cos(self): def test_isin(self): - a = (np.random.random([103, 102]) * 1000).astype(int) - b = (np.random.random([103, 102]) * 1000).astype(int) + a = (np.random.random([103, 102]) * 1000).astype(np.intc) + b = (np.random.random([103, 102]) * 1000).astype(np.intc) n = timeit.timeit(lambda: np.isin(a, b), number=10) x = timeit.timeit(lambda: xt.isin(a, b), number=10) @@ -66,8 +66,8 @@ def test_isin(self): def test_in1d(self): - a = (np.random.random([1003]) * 1000).astype(int) - b = (np.random.random([1003]) * 1000).astype(int) + a = (np.random.random([1003]) * 1000).astype(np.intc) + b = (np.random.random([1003]) * 1000).astype(np.intc) n = timeit.timeit(lambda: np.in1d(a, b), number=10) x = timeit.timeit(lambda: xt.in1d(a, b), number=10) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy