diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb5c47e82..e9b0b1412 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: python3-pip \ python3 \ lld - sudo pip3 install -r requirements.txt + sudo pip3 install -r requirements.linux.txt - name: Cache dependencies uses: buildjet/cache@v3 if: steps.pgml_extension_changed.outputs.PGML_EXTENSION_CHANGED_FILES != '0' diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml index 27970f03b..953c5d969 100644 --- a/.github/workflows/ubuntu-packages-and-docker-image.yml +++ b/.github/workflows/ubuntu-packages-and-docker-image.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: packageVersion: - default: "2.8.0" + default: "2.8.1" jobs: # # PostgresML extension. diff --git a/.github/workflows/ubuntu-postgresml-python-package.yaml b/.github/workflows/ubuntu-postgresml-python-package.yaml index 1af8ef614..0e4be9b21 100644 --- a/.github/workflows/ubuntu-postgresml-python-package.yaml +++ b/.github/workflows/ubuntu-postgresml-python-package.yaml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: packageVersion: - default: "2.7.13" + default: "2.8.1" jobs: postgresml-python: diff --git a/packages/postgresml-python/DEBIAN/postinst b/packages/postgresml-python/DEBIAN/postinst index ae680b4ea..d62a53350 100755 --- a/packages/postgresml-python/DEBIAN/postinst +++ b/packages/postgresml-python/DEBIAN/postinst @@ -7,6 +7,5 @@ set -e # Setup virtualenv virtualenv /var/lib/postgresml-python/pgml-venv source "/var/lib/postgresml-python/pgml-venv/bin/activate" -python -m pip install -r "/etc/postgresml-python/requirements.txt" -python -m pip install -r "/etc/postgresml-python/requirements-xformers.txt" --no-dependencies +python -m pip install -r "/etc/postgresml-python/requirements.linux.txt" deactivate diff --git a/packages/postgresml-python/build.sh b/packages/postgresml-python/build.sh index f13fb241e..f559547f5 100644 --- a/packages/postgresml-python/build.sh +++ b/packages/postgresml-python/build.sh @@ -28,19 +28,12 @@ rm "$deb_dir/release.sh" (cat ${SCRIPT_DIR}/DEBIAN/prerm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/prerm" (cat ${SCRIPT_DIR}/DEBIAN/postrm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postrm" -cp ${SCRIPT_DIR}/../../pgml-extension/requirements.txt "$deb_dir/etc/postgresml-python/requirements.txt" +cp ${SCRIPT_DIR}/../../pgml-extension/requirements.linux.txt "$deb_dir/etc/postgresml-python/requirements.linux.txt" virtualenv --python="python$PYTHON_VERSION" "$deb_dir/var/lib/postgresml-python/pgml-venv" source "$deb_dir/var/lib/postgresml-python/pgml-venv/bin/activate" -python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.txt" - -# No source included, can't build on non x86 platforms -set +e -python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements-autogptq.txt" -set -e - -python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements-xformers.txt" --no-dependencies +python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.linux.txt" deactivate diff --git a/pgml-cms/docs/resources/developer-docs/installation.md b/pgml-cms/docs/resources/developer-docs/installation.md index 990cec5a8..40bd4ab57 100644 --- a/pgml-cms/docs/resources/developer-docs/installation.md +++ b/pgml-cms/docs/resources/developer-docs/installation.md @@ -63,8 +63,7 @@ To install the necessary Python packages into a virtual environment, use the `vi ```bash virtualenv pgml-venv && \ source pgml-venv/bin/activate && \ -pip install -r requirements.txt && \ -pip install -r requirements-xformers.txt --no-dependencies +pip install -r requirements.linux.txt ``` {% endtab %} @@ -72,7 +71,7 @@ pip install -r requirements-xformers.txt --no-dependencies Installing Python packages globally can cause issues with your system. If you wish to proceed nonetheless, you can do so: ```bash -pip3 install -r requirements.txt +pip3 install -r requirements.linux.txt ``` {% endtab %} {% endtabs %} diff --git a/pgml-extension/Cargo.lock b/pgml-extension/Cargo.lock index ec0c7e847..1697813d8 100644 --- a/pgml-extension/Cargo.lock +++ b/pgml-extension/Cargo.lock @@ -503,7 +503,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", + "memoffset", "scopeguard", ] @@ -1070,9 +1070,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "instant" @@ -1335,15 +1335,6 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.9.0" @@ -1833,7 +1824,7 @@ dependencies = [ "clang-sys", "eyre", "libc", - "memoffset 0.9.0", + "memoffset", "once_cell", "pgrx-macros", "pgrx-pg-config", @@ -2018,14 +2009,14 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543" +checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" dependencies = [ "cfg-if", "indoc", "libc", - "memoffset 0.6.5", + "memoffset", "parking_lot", "pyo3-build-config", "pyo3-ffi", @@ -2035,9 +2026,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8" +checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" dependencies = [ "once_cell", "target-lexicon", @@ -2045,9 +2036,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc" +checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" dependencies = [ "libc", "pyo3-build-config", @@ -2055,25 +2046,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28" +checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote 1.0.33", - "syn 1.0.109", + "syn 2.0.40", ] [[package]] name = "pyo3-macros-backend" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f" +checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" dependencies = [ + "heck", "proc-macro2", "quote 1.0.33", - "syn 1.0.109", + "syn 2.0.40", ] [[package]] @@ -3028,9 +3020,9 @@ checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "ureq" diff --git a/pgml-extension/Cargo.toml b/pgml-extension/Cargo.toml index 1acce9f58..aaf78ff9c 100644 --- a/pgml-extension/Cargo.toml +++ b/pgml-extension/Cargo.toml @@ -26,7 +26,6 @@ blas = { version = "0.22" } blas-src = { version = "0.9", features = ["openblas"] } indexmap = { version = "1.0", features = ["serde"] } itertools = "0.11" -pyo3 = { version = "0.17", features = ["auto-initialize"], optional = true } heapless = "0.7" lightgbm = { git = "https://github.com/postgresml/lightgbm-rs", branch = "main" } linfa = { path = "deps/linfa" } @@ -42,6 +41,7 @@ ndarray-stats = "0.5.1" parking_lot = "0.12" pgrx = "=0.11.2" pgrx-pg-sys = "=0.11.2" +pyo3 = { version = "0.20.0", features = ["auto-initialize"], optional = true } rand = "0.8" rmp-serde = { version = "1.1" } signal-hook = "0.3" diff --git a/pgml-extension/build.rs b/pgml-extension/build.rs index 355c19293..ca4ab1faf 100644 --- a/pgml-extension/build.rs +++ b/pgml-extension/build.rs @@ -9,7 +9,7 @@ fn main() { // of symbols into the previous opened .so file, but the others will use a // relative offset in pgml.so, and will cause a null-pointer crash. // - // hid all symbol to avoid symbol conflicts. + // hide all symbol to avoid symbol conflicts. // // append mode (link-args) only works with clang ld (lld) println!( diff --git a/pgml-extension/requirements.base.txt b/pgml-extension/requirements.base.txt deleted file mode 100644 index aeffd168c..000000000 --- a/pgml-extension/requirements.base.txt +++ /dev/null @@ -1,41 +0,0 @@ -# The immediate dependencies of PostgresML are maintained here. - -# Locked because newer versions have bugs -transformers-stream-generator==0.0.4 -optimum==1.13.2 -peft==0.6.2 -pyarrow==11.0.0 -torch==2.0.1 # 2.1.1 breaks sentence-transformers==2.2.2 - -# ML -catboost -lightgbm -torchaudio -torchvision -xgboost - -# Transformers -accelerate -auto-gptq; sys_platform == 'linux' -bitsandbytes -ctransformers -huggingface-hub -deepspeed -einops -tokenizers -transformers -xformers; sys_platform == 'linux' - -# Embeddings -InstructorEmbedding -sentence-transformers - -# Ratings -rouge -sacrebleu -sacremoses - -# Utils -datasets -orjson -langchain diff --git a/pgml-extension/requirements.linux.txt b/pgml-extension/requirements.linux.txt new file mode 100644 index 000000000..067036d25 --- /dev/null +++ b/pgml-extension/requirements.linux.txt @@ -0,0 +1,123 @@ +accelerate==0.25.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +anyio==4.2.0 +async-timeout==4.0.3 +attrs==23.1.0 +auto-gptq==0.6.0 +bitsandbytes==0.41.3.post2 +catboost==1.2.2 +certifi==2023.11.17 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +coloredlogs==15.0.1 +contourpy==1.2.0 +ctransformers==0.2.27 +cycler==0.12.1 +dataclasses-json==0.6.3 +datasets==2.15.0 +deepspeed==0.12.5 +dill==0.3.7 +einops==0.7.0 +exceptiongroup==1.2.0 +filelock==3.13.1 +fonttools==4.47.0 +frozenlist==1.4.1 +fsspec==2023.10.0 +gekko==1.0.6 +graphviz==0.20.1 +greenlet==3.0.2 +hjson==3.1.0 +huggingface-hub==0.19.4 +humanfriendly==10.0 +idna==3.6 +InstructorEmbedding==1.0.1 +Jinja2==3.1.2 +joblib==1.3.2 +jsonpatch==1.33 +jsonpointer==2.4 +kiwisolver==1.4.5 +langchain==0.0.351 +langchain-community==0.0.4 +langchain-core==0.1.1 +langsmith==0.0.72 +lightgbm==4.1.0 +lxml==4.9.3 +MarkupSafe==2.1.3 +marshmallow==3.20.1 +matplotlib==3.8.2 +mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.15 +mypy-extensions==1.0.0 +networkx==3.2.1 +ninja==1.11.1.1 +nltk==3.8.1 +numpy==1.26.2 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.3.101 +nvidia-nvtx-cu12==12.1.105 +optimum==1.16.1 +orjson==3.9.10 +packaging==23.2 +pandas==2.1.4 +peft==0.7.1 +Pillow==10.1.0 +plotly==5.18.0 +portalocker==2.8.2 +protobuf==4.25.1 +psutil==5.9.7 +py-cpuinfo==9.0.0 +pyarrow==11.0.0 +pyarrow-hotfix==0.6 +pydantic==2.5.2 +pydantic_core==2.14.5 +pynvml==11.5.0 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.10.3 +requests==2.31.0 +rouge==1.0.1 +sacrebleu==2.4.0 +sacremoses==0.1.1 +safetensors==0.4.1 +scikit-learn==1.3.2 +scipy==1.11.4 +sentence-transformers==2.2.2 +sentencepiece==0.1.99 +six==1.16.0 +sniffio==1.3.0 +SQLAlchemy==2.0.23 +sympy==1.12 +tabulate==0.9.0 +tenacity==8.2.3 +threadpoolctl==3.2.0 +tokenizers==0.15.0 +torch==2.1.2 +torchaudio==2.1.2 +torchvision==0.16.2 +tqdm==4.66.1 +transformers==4.36.2 +transformers-stream-generator==0.0.4 +triton==2.1.0 +typing-inspect==0.9.0 +typing_extensions==4.9.0 +tzdata==2023.3 +urllib3==2.1.0 +xformers==0.0.23.post1 +xgboost==2.0.2 +xxhash==3.4.1 +yarl==1.9.4 diff --git a/pgml-extension/requirements.macos.txt b/pgml-extension/requirements.macos.txt new file mode 100644 index 000000000..b41533af7 --- /dev/null +++ b/pgml-extension/requirements.macos.txt @@ -0,0 +1,104 @@ +accelerate==0.25.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +anyio==4.2.0 +attrs==23.1.0 +bitsandbytes==0.41.3.post2 +catboost==1.2.2 +certifi==2023.11.17 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +coloredlogs==15.0.1 +contourpy==1.2.0 +ctransformers==0.2.27 +cycler==0.12.1 +dataclasses-json==0.6.3 +datasets==2.15.0 +deepspeed==0.12.5 +dill==0.3.7 +einops==0.7.0 +filelock==3.13.1 +fonttools==4.47.0 +frozenlist==1.4.1 +fsspec==2023.10.0 +graphviz==0.20.1 +hjson==3.1.0 +huggingface-hub==0.19.4 +humanfriendly==10.0 +idna==3.6 +InstructorEmbedding==1.0.1 +Jinja2==3.1.2 +joblib==1.3.2 +jsonpatch==1.33 +jsonpointer==2.4 +kiwisolver==1.4.5 +langchain==0.0.351 +langchain-community==0.0.4 +langchain-core==0.1.1 +langsmith==0.0.72 +lightgbm==4.1.0 +lxml==4.9.3 +MarkupSafe==2.1.3 +marshmallow==3.20.1 +matplotlib==3.8.2 +mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.15 +mypy-extensions==1.0.0 +networkx==3.2.1 +ninja==1.11.1.1 +nltk==3.8.1 +numpy==1.26.2 +optimum==1.16.1 +orjson==3.9.10 +packaging==23.2 +pandas==2.1.4 +peft==0.7.1 +Pillow==10.1.0 +plotly==5.18.0 +portalocker==2.8.2 +protobuf==4.25.1 +psutil==5.9.7 +py-cpuinfo==9.0.0 +pyarrow==11.0.0 +pyarrow-hotfix==0.6 +pydantic==2.5.2 +pydantic_core==2.14.5 +pynvml==11.5.0 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.10.3 +requests==2.31.0 +rouge==1.0.1 +sacrebleu==2.4.0 +sacremoses==0.1.1 +safetensors==0.4.1 +scikit-learn==1.3.2 +scipy==1.11.4 +sentence-transformers==2.2.2 +sentencepiece==0.1.99 +six==1.16.0 +sniffio==1.3.0 +SQLAlchemy==2.0.23 +sympy==1.12 +tabulate==0.9.0 +tenacity==8.2.3 +threadpoolctl==3.2.0 +tokenizers==0.15.0 +torch==2.1.2 +torchaudio==2.1.2 +torchvision==0.16.2 +tqdm==4.66.1 +transformers==4.36.2 +transformers-stream-generator==0.0.4 +typing-inspect==0.9.0 +typing_extensions==4.9.0 +tzdata==2023.3 +urllib3==2.1.0 +xgboost==2.0.2 +xxhash==3.4.1 +yarl==1.9.4 diff --git a/pgml-extension/requirements.txt b/pgml-extension/requirements.txt index ddc7229b6..8f37b28b3 100644 --- a/pgml-extension/requirements.txt +++ b/pgml-extension/requirements.txt @@ -1,104 +1,52 @@ -accelerate==0.25.0 -aiohttp==3.9.1 -aiosignal==1.3.1 -annotated-types==0.6.0 -anyio==4.1.0 -attrs==23.1.0 -auto-gptq==0.5.1; sys_platform == 'linux' -bitsandbytes==0.41.3.post2 -catboost==1.2.2 -certifi==2023.11.17 -charset-normalizer==3.3.2 -click==8.1.7 -colorama==0.4.6 -contourpy==1.2.0 -ctransformers==0.2.27 -cycler==0.12.1 -dataclasses-json==0.6.3 -datasets==2.15.0 -deepspeed==0.12.4 -dill==0.3.7 -einops==0.7.0 -filelock==3.13.1 -fonttools==4.46.0 -frozenlist==1.4.0 -fsspec==2023.10.0 -gekko==1.0.6 -graphviz==0.20.1 -hjson==3.1.0 -huggingface-hub==0.19.4 -idna==3.6 -InstructorEmbedding==1.0.1 -Jinja2==3.1.2 -joblib==1.3.2 -jsonpatch==1.33 -jsonpointer==2.4 -kiwisolver==1.4.5 -langchain==0.0.349 -langchain-community==0.0.1 -langchain-core==0.0.13 -langsmith==0.0.69 -lightgbm==4.1.0 -lxml==4.9.3 -MarkupSafe==2.1.3 -marshmallow==3.20.1 -matplotlib==3.8.2 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.15 -mypy-extensions==1.0.0 -networkx==3.2.1 -ninja==1.11.1.1 -nltk==3.8.1 -numpy==1.26.2 -optimum==1.13.2 -orjson==3.9.10 -packaging==23.2 -pandas==2.1.4 -peft==0.6.2 -Pillow==10.1.0 -plotly==5.18.0 -portalocker==2.8.2 -psutil==5.9.6 -py-cpuinfo==9.0.0 -pyarrow==14.0.1 -pyarrow-hotfix==0.6 -pydantic==2.5.2 -pydantic_core==2.14.5 -pynvml==11.5.0 -pyparsing==3.1.1 -python-dateutil==2.8.2 -pytz==2023.3.post1 -PyYAML==6.0.1 -regex==2023.10.3 -requests==2.31.0 -rouge==1.0.1 -sacrebleu==2.3.3 -sacremoses==0.1.1 -safetensors==0.4.1 -scikit-learn==1.3.2 -scipy==1.11.4 -sentence-transformers==2.2.2 -sentencepiece==0.1.99 -six==1.16.0 -sniffio==1.3.0 -SQLAlchemy==2.0.23 -sympy==1.12 -tabulate==0.9.0 -tenacity==8.2.3 -threadpoolctl==3.2.0 -tokenizers==0.15.0 -torch==2.0.1 -torchaudio==2.0.2 -torchvision==0.15.2 -tqdm==4.66.1 -transformers==4.36.0 -transformers-stream-generator==0.0.4 -typing-inspect==0.9.0 -typing_extensions==4.9.0 -tzdata==2023.3 -urllib3==2.1.0 -xformers==0.0.22; sys_platform == 'linux' -xgboost==2.0.2 -xxhash==3.4.1 -yarl==1.9.4 +# If you update this file, `pip freeze` the full locked requirements as well to prevent +# future dependency incompatibility on all supported platforms. We'd use Poetry or some +# other sane lockfile resolution mechanism other than pip, except we have to maintain +# this isn't really a Python project, so Poetry conventions don't work, and we need +# different dependencies on platforms that have different hardware acceleration. + +# Only the immediate dependencies of PostgresML are maintained here. +# See requirements.linux.txt or requirements.macos.txt for complete and locked versions. + +# Python 3.12 can't currently resolve all dependencies: +# catboost fails to build +# virtualenv -p python3.11 pgml-venv + +# These packages are specifically locked to avoid known bugs +pyarrow==11.0.0 # newer versions cause Postgres segfaults on OOM + +# ML +catboost +lightgbm +torch +torchaudio +torchvision +xgboost + +# Transformers +accelerate +auto-gptq; sys_platform == 'linux' # only runs on nvidia hardware +bitsandbytes +ctransformers +huggingface-hub +deepspeed +einops +optimum +peft +tokenizers +transformers +transformers-stream-generator +xformers; sys_platform == 'linux' # only runs on nvidia hardware + +# Embeddings +InstructorEmbedding +sentence-transformers + +# Ratings +rouge +sacrebleu +sacremoses + +# Utils +datasets +orjson +langchain
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies: