Content-Length: 25287 | pFad | http://github.com/postgresml/postgresml/pull/1246.patch

thub.com From 39ccb685cfb9f880ec53491cddc0e50ebb21ebf8 Mon Sep 17 00:00:00 2001 From: Montana Low Date: Mon, 18 Dec 2023 16:49:43 -0800 Subject: [PATCH 1/4] love me some deps --- .github/workflows/ci.yml | 2 +- packages/postgresml-python/DEBIAN/postinst | 3 +- packages/postgresml-python/build.sh | 11 +- .../resources/developer-docs/installation.md | 5 +- pgml-extension/Cargo.lock | 48 +++--- pgml-extension/Cargo.toml | 2 +- pgml-extension/build.rs | 2 +- pgml-extension/requirements.base.txt | 41 ----- pgml-extension/requirements.linux.txt | 123 +++++++++++++++ pgml-extension/requirements.macos.txt | 123 +++++++++++++++ pgml-extension/requirements.txt | 146 +++++------------- 11 files changed, 316 insertions(+), 190 deletions(-) delete mode 100644 pgml-extension/requirements.base.txt create mode 100644 pgml-extension/requirements.linux.txt create mode 100644 pgml-extension/requirements.macos.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb5c47e82..e9b0b1412 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: python3-pip \ python3 \ lld - sudo pip3 install -r requirements.txt + sudo pip3 install -r requirements.linux.txt - name: Cache dependencies uses: buildjet/cache@v3 if: steps.pgml_extension_changed.outputs.PGML_EXTENSION_CHANGED_FILES != '0' diff --git a/packages/postgresml-python/DEBIAN/postinst b/packages/postgresml-python/DEBIAN/postinst index ae680b4ea..d62a53350 100755 --- a/packages/postgresml-python/DEBIAN/postinst +++ b/packages/postgresml-python/DEBIAN/postinst @@ -7,6 +7,5 @@ set -e # Setup virtualenv virtualenv /var/lib/postgresml-python/pgml-venv source "/var/lib/postgresml-python/pgml-venv/bin/activate" -python -m pip install -r "/etc/postgresml-python/requirements.txt" -python -m pip install -r "/etc/postgresml-python/requirements-xformers.txt" --no-dependencies +python -m pip install -r "/etc/postgresml-python/requirements.linux.txt" deactivate diff --git a/packages/postgresml-python/build.sh b/packages/postgresml-python/build.sh index f13fb241e..f559547f5 100644 --- a/packages/postgresml-python/build.sh +++ b/packages/postgresml-python/build.sh @@ -28,19 +28,12 @@ rm "$deb_dir/release.sh" (cat ${SCRIPT_DIR}/DEBIAN/prerm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/prerm" (cat ${SCRIPT_DIR}/DEBIAN/postrm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postrm" -cp ${SCRIPT_DIR}/../../pgml-extension/requirements.txt "$deb_dir/etc/postgresml-python/requirements.txt" +cp ${SCRIPT_DIR}/../../pgml-extension/requirements.linux.txt "$deb_dir/etc/postgresml-python/requirements.linux.txt" virtualenv --python="python$PYTHON_VERSION" "$deb_dir/var/lib/postgresml-python/pgml-venv" source "$deb_dir/var/lib/postgresml-python/pgml-venv/bin/activate" -python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.txt" - -# No source included, can't build on non x86 platforms -set +e -python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements-autogptq.txt" -set -e - -python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements-xformers.txt" --no-dependencies +python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.linux.txt" deactivate diff --git a/pgml-cms/docs/resources/developer-docs/installation.md b/pgml-cms/docs/resources/developer-docs/installation.md index 990cec5a8..40bd4ab57 100644 --- a/pgml-cms/docs/resources/developer-docs/installation.md +++ b/pgml-cms/docs/resources/developer-docs/installation.md @@ -63,8 +63,7 @@ To install the necessary Python packages into a virtual environment, use the `vi ```bash virtualenv pgml-venv && \ source pgml-venv/bin/activate && \ -pip install -r requirements.txt && \ -pip install -r requirements-xformers.txt --no-dependencies +pip install -r requirements.linux.txt ``` {% endtab %} @@ -72,7 +71,7 @@ pip install -r requirements-xformers.txt --no-dependencies Installing Python packages globally can cause issues with your system. If you wish to proceed nonetheless, you can do so: ```bash -pip3 install -r requirements.txt +pip3 install -r requirements.linux.txt ``` {% endtab %} {% endtabs %} diff --git a/pgml-extension/Cargo.lock b/pgml-extension/Cargo.lock index ec0c7e847..1697813d8 100644 --- a/pgml-extension/Cargo.lock +++ b/pgml-extension/Cargo.lock @@ -503,7 +503,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", + "memoffset", "scopeguard", ] @@ -1070,9 +1070,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "instant" @@ -1335,15 +1335,6 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.9.0" @@ -1833,7 +1824,7 @@ dependencies = [ "clang-sys", "eyre", "libc", - "memoffset 0.9.0", + "memoffset", "once_cell", "pgrx-macros", "pgrx-pg-config", @@ -2018,14 +2009,14 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543" +checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" dependencies = [ "cfg-if", "indoc", "libc", - "memoffset 0.6.5", + "memoffset", "parking_lot", "pyo3-build-config", "pyo3-ffi", @@ -2035,9 +2026,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8" +checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" dependencies = [ "once_cell", "target-lexicon", @@ -2045,9 +2036,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc" +checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" dependencies = [ "libc", "pyo3-build-config", @@ -2055,25 +2046,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28" +checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote 1.0.33", - "syn 1.0.109", + "syn 2.0.40", ] [[package]] name = "pyo3-macros-backend" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f" +checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" dependencies = [ + "heck", "proc-macro2", "quote 1.0.33", - "syn 1.0.109", + "syn 2.0.40", ] [[package]] @@ -3028,9 +3020,9 @@ checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "ureq" diff --git a/pgml-extension/Cargo.toml b/pgml-extension/Cargo.toml index 1acce9f58..aaf78ff9c 100644 --- a/pgml-extension/Cargo.toml +++ b/pgml-extension/Cargo.toml @@ -26,7 +26,6 @@ blas = { version = "0.22" } blas-src = { version = "0.9", features = ["openblas"] } indexmap = { version = "1.0", features = ["serde"] } itertools = "0.11" -pyo3 = { version = "0.17", features = ["auto-initialize"], optional = true } heapless = "0.7" lightgbm = { git = "https://github.com/postgresml/lightgbm-rs", branch = "main" } linfa = { path = "deps/linfa" } @@ -42,6 +41,7 @@ ndarray-stats = "0.5.1" parking_lot = "0.12" pgrx = "=0.11.2" pgrx-pg-sys = "=0.11.2" +pyo3 = { version = "0.20.0", features = ["auto-initialize"], optional = true } rand = "0.8" rmp-serde = { version = "1.1" } signal-hook = "0.3" diff --git a/pgml-extension/build.rs b/pgml-extension/build.rs index 355c19293..ca4ab1faf 100644 --- a/pgml-extension/build.rs +++ b/pgml-extension/build.rs @@ -9,7 +9,7 @@ fn main() { // of symbols into the previous opened .so file, but the others will use a // relative offset in pgml.so, and will cause a null-pointer crash. // - // hid all symbol to avoid symbol conflicts. + // hide all symbol to avoid symbol conflicts. // // append mode (link-args) only works with clang ld (lld) println!( diff --git a/pgml-extension/requirements.base.txt b/pgml-extension/requirements.base.txt deleted file mode 100644 index aeffd168c..000000000 --- a/pgml-extension/requirements.base.txt +++ /dev/null @@ -1,41 +0,0 @@ -# The immediate dependencies of PostgresML are maintained here. - -# Locked because newer versions have bugs -transformers-stream-generator==0.0.4 -optimum==1.13.2 -peft==0.6.2 -pyarrow==11.0.0 -torch==2.0.1 # 2.1.1 breaks sentence-transformers==2.2.2 - -# ML -catboost -lightgbm -torchaudio -torchvision -xgboost - -# Transformers -accelerate -auto-gptq; sys_platform == 'linux' -bitsandbytes -ctransformers -huggingface-hub -deepspeed -einops -tokenizers -transformers -xformers; sys_platform == 'linux' - -# Embeddings -InstructorEmbedding -sentence-transformers - -# Ratings -rouge -sacrebleu -sacremoses - -# Utils -datasets -orjson -langchain diff --git a/pgml-extension/requirements.linux.txt b/pgml-extension/requirements.linux.txt new file mode 100644 index 000000000..067036d25 --- /dev/null +++ b/pgml-extension/requirements.linux.txt @@ -0,0 +1,123 @@ +accelerate==0.25.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +anyio==4.2.0 +async-timeout==4.0.3 +attrs==23.1.0 +auto-gptq==0.6.0 +bitsandbytes==0.41.3.post2 +catboost==1.2.2 +certifi==2023.11.17 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +coloredlogs==15.0.1 +contourpy==1.2.0 +ctransformers==0.2.27 +cycler==0.12.1 +dataclasses-json==0.6.3 +datasets==2.15.0 +deepspeed==0.12.5 +dill==0.3.7 +einops==0.7.0 +exceptiongroup==1.2.0 +filelock==3.13.1 +fonttools==4.47.0 +frozenlist==1.4.1 +fsspec==2023.10.0 +gekko==1.0.6 +graphviz==0.20.1 +greenlet==3.0.2 +hjson==3.1.0 +huggingface-hub==0.19.4 +humanfriendly==10.0 +idna==3.6 +InstructorEmbedding==1.0.1 +Jinja2==3.1.2 +joblib==1.3.2 +jsonpatch==1.33 +jsonpointer==2.4 +kiwisolver==1.4.5 +langchain==0.0.351 +langchain-community==0.0.4 +langchain-core==0.1.1 +langsmith==0.0.72 +lightgbm==4.1.0 +lxml==4.9.3 +MarkupSafe==2.1.3 +marshmallow==3.20.1 +matplotlib==3.8.2 +mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.15 +mypy-extensions==1.0.0 +networkx==3.2.1 +ninja==1.11.1.1 +nltk==3.8.1 +numpy==1.26.2 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.3.101 +nvidia-nvtx-cu12==12.1.105 +optimum==1.16.1 +orjson==3.9.10 +packaging==23.2 +pandas==2.1.4 +peft==0.7.1 +Pillow==10.1.0 +plotly==5.18.0 +portalocker==2.8.2 +protobuf==4.25.1 +psutil==5.9.7 +py-cpuinfo==9.0.0 +pyarrow==11.0.0 +pyarrow-hotfix==0.6 +pydantic==2.5.2 +pydantic_core==2.14.5 +pynvml==11.5.0 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.10.3 +requests==2.31.0 +rouge==1.0.1 +sacrebleu==2.4.0 +sacremoses==0.1.1 +safetensors==0.4.1 +scikit-learn==1.3.2 +scipy==1.11.4 +sentence-transformers==2.2.2 +sentencepiece==0.1.99 +six==1.16.0 +sniffio==1.3.0 +SQLAlchemy==2.0.23 +sympy==1.12 +tabulate==0.9.0 +tenacity==8.2.3 +threadpoolctl==3.2.0 +tokenizers==0.15.0 +torch==2.1.2 +torchaudio==2.1.2 +torchvision==0.16.2 +tqdm==4.66.1 +transformers==4.36.2 +transformers-stream-generator==0.0.4 +triton==2.1.0 +typing-inspect==0.9.0 +typing_extensions==4.9.0 +tzdata==2023.3 +urllib3==2.1.0 +xformers==0.0.23.post1 +xgboost==2.0.2 +xxhash==3.4.1 +yarl==1.9.4 diff --git a/pgml-extension/requirements.macos.txt b/pgml-extension/requirements.macos.txt new file mode 100644 index 000000000..b08bde5e9 --- /dev/null +++ b/pgml-extension/requirements.macos.txt @@ -0,0 +1,123 @@ +accelerate==0.25.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +anyio==4.1.0 +async-timeout==4.0.3 +attrs==23.1.0 +auto-gptq==0.5.1 +bitsandbytes==0.41.3.post2 +catboost==1.2.2 +certifi==2023.11.17 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +coloredlogs==15.0.1 +contourpy==1.2.0 +ctransformers==0.2.27 +cycler==0.12.1 +dataclasses-json==0.6.3 +datasets==2.15.0 +deepspeed==0.12.4 +dill==0.3.7 +einops==0.7.0 +exceptiongroup==1.2.0 +filelock==3.13.1 +fonttools==4.46.0 +frozenlist==1.4.0 +fsspec==2023.10.0 +gekko==1.0.6 +graphviz==0.20.1 +greenlet==3.0.2 +hjson==3.1.0 +huggingface-hub==0.19.4 +humanfriendly==10.0 +idna==3.6 +InstructorEmbedding==1.0.1 +Jinja2==3.1.2 +joblib==1.3.2 +jsonpatch==1.33 +jsonpointer==2.4 +kiwisolver==1.4.5 +langchain==0.0.350 +langchain-community==0.0.3 +langchain-core==0.1.1 +langsmith==0.0.70 +lightgbm==4.1.0 +lxml==4.9.3 +MarkupSafe==2.1.3 +marshmallow==3.20.1 +matplotlib==3.8.2 +mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.15 +mypy-extensions==1.0.0 +networkx==3.2.1 +ninja==1.11.1.1 +nltk==3.8.1 +numpy==1.26.2 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.3.101 +nvidia-nvtx-cu12==12.1.105 +optimum==1.13.2 +orjson==3.9.10 +packaging==23.2 +pandas==2.1.4 +peft==0.6.2 +Pillow==10.1.0 +plotly==5.18.0 +portalocker==2.8.2 +protobuf==4.25.1 +psutil==5.9.6 +py-cpuinfo==9.0.0 +pyarrow==11.0.0 +pyarrow-hotfix==0.6 +pydantic==2.5.2 +pydantic_core==2.14.5 +pynvml==11.5.0 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.10.3 +requests==2.31.0 +rouge==1.0.1 +sacrebleu==2.4.0 +sacremoses==0.1.1 +safetensors==0.4.1 +scikit-learn==1.3.2 +scipy==1.11.4 +sentence-transformers==2.2.2 +sentencepiece==0.1.99 +six==1.16.0 +sniffio==1.3.0 +SQLAlchemy==2.0.23 +sympy==1.12 +tabulate==0.9.0 +tenacity==8.2.3 +threadpoolctl==3.2.0 +tokenizers==0.15.0 +torch==2.1.1 +torchaudio==2.1.1 +torchvision==0.16.1 +tqdm==4.66.1 +transformers==4.36.1 +transformers-stream-generator==0.0.4 +triton==2.1.0 +typing-inspect==0.9.0 +typing_extensions==4.9.0 +tzdata==2023.3 +urllib3==2.1.0 +xformers==0.0.23 +xgboost==2.0.2 +xxhash==3.4.1 +yarl==1.9.4 \ No newline at end of file diff --git a/pgml-extension/requirements.txt b/pgml-extension/requirements.txt index ddc7229b6..4a6bf1626 100644 --- a/pgml-extension/requirements.txt +++ b/pgml-extension/requirements.txt @@ -1,104 +1,42 @@ -accelerate==0.25.0 -aiohttp==3.9.1 -aiosignal==1.3.1 -annotated-types==0.6.0 -anyio==4.1.0 -attrs==23.1.0 -auto-gptq==0.5.1; sys_platform == 'linux' -bitsandbytes==0.41.3.post2 -catboost==1.2.2 -certifi==2023.11.17 -charset-normalizer==3.3.2 -click==8.1.7 -colorama==0.4.6 -contourpy==1.2.0 -ctransformers==0.2.27 -cycler==0.12.1 -dataclasses-json==0.6.3 -datasets==2.15.0 -deepspeed==0.12.4 -dill==0.3.7 -einops==0.7.0 -filelock==3.13.1 -fonttools==4.46.0 -frozenlist==1.4.0 -fsspec==2023.10.0 -gekko==1.0.6 -graphviz==0.20.1 -hjson==3.1.0 -huggingface-hub==0.19.4 -idna==3.6 -InstructorEmbedding==1.0.1 -Jinja2==3.1.2 -joblib==1.3.2 -jsonpatch==1.33 -jsonpointer==2.4 -kiwisolver==1.4.5 -langchain==0.0.349 -langchain-community==0.0.1 -langchain-core==0.0.13 -langsmith==0.0.69 -lightgbm==4.1.0 -lxml==4.9.3 -MarkupSafe==2.1.3 -marshmallow==3.20.1 -matplotlib==3.8.2 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.15 -mypy-extensions==1.0.0 -networkx==3.2.1 -ninja==1.11.1.1 -nltk==3.8.1 -numpy==1.26.2 -optimum==1.13.2 -orjson==3.9.10 -packaging==23.2 -pandas==2.1.4 -peft==0.6.2 -Pillow==10.1.0 -plotly==5.18.0 -portalocker==2.8.2 -psutil==5.9.6 -py-cpuinfo==9.0.0 -pyarrow==14.0.1 -pyarrow-hotfix==0.6 -pydantic==2.5.2 -pydantic_core==2.14.5 -pynvml==11.5.0 -pyparsing==3.1.1 -python-dateutil==2.8.2 -pytz==2023.3.post1 -PyYAML==6.0.1 -regex==2023.10.3 -requests==2.31.0 -rouge==1.0.1 -sacrebleu==2.3.3 -sacremoses==0.1.1 -safetensors==0.4.1 -scikit-learn==1.3.2 -scipy==1.11.4 -sentence-transformers==2.2.2 -sentencepiece==0.1.99 -six==1.16.0 -sniffio==1.3.0 -SQLAlchemy==2.0.23 -sympy==1.12 -tabulate==0.9.0 -tenacity==8.2.3 -threadpoolctl==3.2.0 -tokenizers==0.15.0 -torch==2.0.1 -torchaudio==2.0.2 -torchvision==0.15.2 -tqdm==4.66.1 -transformers==4.36.0 -transformers-stream-generator==0.0.4 -typing-inspect==0.9.0 -typing_extensions==4.9.0 -tzdata==2023.3 -urllib3==2.1.0 -xformers==0.0.22; sys_platform == 'linux' -xgboost==2.0.2 -xxhash==3.4.1 -yarl==1.9.4 +# The immediate dependencies of PostgresML are maintained here. +# See requirements.linux.txt or requirements.macos.txt for complete and locked versions. + +# Specifically locked to avoid known bugs +pyarrow==11.0.0 # newer versions cause Postgres segfaults on OOM + +# ML +catboost +lightgbm +torch +torchaudio +torchvision +xgboost + +# Transformers +accelerate +auto-gptq; sys_platform == 'linux' +bitsandbytes +ctransformers +huggingface-hub +deepspeed +einops +optimum +peft +tokenizers +transformers +transformers-stream-generator +xformers; sys_platform == 'linux' + +# Embeddings +InstructorEmbedding +sentence-transformers + +# Ratings +rouge +sacrebleu +sacremoses + +# Utils +datasets +orjson +langchain From 0ea6207fce338aaeedb52d18cea6703c084faac8 Mon Sep 17 00:00:00 2001 From: Montana Low Date: Mon, 18 Dec 2023 17:06:14 -0800 Subject: [PATCH 2/4] tested on macos --- pgml-extension/requirements.macos.txt | 49 ++++++++------------------- pgml-extension/requirements.txt | 8 +++-- 2 files changed, 21 insertions(+), 36 deletions(-) diff --git a/pgml-extension/requirements.macos.txt b/pgml-extension/requirements.macos.txt index b08bde5e9..b41533af7 100644 --- a/pgml-extension/requirements.macos.txt +++ b/pgml-extension/requirements.macos.txt @@ -2,10 +2,8 @@ accelerate==0.25.0 aiohttp==3.9.1 aiosignal==1.3.1 annotated-types==0.6.0 -anyio==4.1.0 -async-timeout==4.0.3 +anyio==4.2.0 attrs==23.1.0 -auto-gptq==0.5.1 bitsandbytes==0.41.3.post2 catboost==1.2.2 certifi==2023.11.17 @@ -18,17 +16,14 @@ ctransformers==0.2.27 cycler==0.12.1 dataclasses-json==0.6.3 datasets==2.15.0 -deepspeed==0.12.4 +deepspeed==0.12.5 dill==0.3.7 einops==0.7.0 -exceptiongroup==1.2.0 filelock==3.13.1 -fonttools==4.46.0 -frozenlist==1.4.0 +fonttools==4.47.0 +frozenlist==1.4.1 fsspec==2023.10.0 -gekko==1.0.6 graphviz==0.20.1 -greenlet==3.0.2 hjson==3.1.0 huggingface-hub==0.19.4 humanfriendly==10.0 @@ -39,10 +34,10 @@ joblib==1.3.2 jsonpatch==1.33 jsonpointer==2.4 kiwisolver==1.4.5 -langchain==0.0.350 -langchain-community==0.0.3 +langchain==0.0.351 +langchain-community==0.0.4 langchain-core==0.1.1 -langsmith==0.0.70 +langsmith==0.0.72 lightgbm==4.1.0 lxml==4.9.3 MarkupSafe==2.1.3 @@ -56,28 +51,16 @@ networkx==3.2.1 ninja==1.11.1.1 nltk==3.8.1 numpy==1.26.2 -nvidia-cublas-cu12==12.1.3.1 -nvidia-cuda-cupti-cu12==12.1.105 -nvidia-cuda-nvrtc-cu12==12.1.105 -nvidia-cuda-runtime-cu12==12.1.105 -nvidia-cudnn-cu12==8.9.2.26 -nvidia-cufft-cu12==11.0.2.54 -nvidia-curand-cu12==10.3.2.106 -nvidia-cusolver-cu12==11.4.5.107 -nvidia-cusparse-cu12==12.1.0.106 -nvidia-nccl-cu12==2.18.1 -nvidia-nvjitlink-cu12==12.3.101 -nvidia-nvtx-cu12==12.1.105 -optimum==1.13.2 +optimum==1.16.1 orjson==3.9.10 packaging==23.2 pandas==2.1.4 -peft==0.6.2 +peft==0.7.1 Pillow==10.1.0 plotly==5.18.0 portalocker==2.8.2 protobuf==4.25.1 -psutil==5.9.6 +psutil==5.9.7 py-cpuinfo==9.0.0 pyarrow==11.0.0 pyarrow-hotfix==0.6 @@ -106,18 +89,16 @@ tabulate==0.9.0 tenacity==8.2.3 threadpoolctl==3.2.0 tokenizers==0.15.0 -torch==2.1.1 -torchaudio==2.1.1 -torchvision==0.16.1 +torch==2.1.2 +torchaudio==2.1.2 +torchvision==0.16.2 tqdm==4.66.1 -transformers==4.36.1 +transformers==4.36.2 transformers-stream-generator==0.0.4 -triton==2.1.0 typing-inspect==0.9.0 typing_extensions==4.9.0 tzdata==2023.3 urllib3==2.1.0 -xformers==0.0.23 xgboost==2.0.2 xxhash==3.4.1 -yarl==1.9.4 \ No newline at end of file +yarl==1.9.4 diff --git a/pgml-extension/requirements.txt b/pgml-extension/requirements.txt index 4a6bf1626..e2fdadb01 100644 --- a/pgml-extension/requirements.txt +++ b/pgml-extension/requirements.txt @@ -1,6 +1,10 @@ # The immediate dependencies of PostgresML are maintained here. # See requirements.linux.txt or requirements.macos.txt for complete and locked versions. +# Python 3.12 can't currently resolve all dependencies: +# catboost fails to build +# virtualenv -p python3.11 pgml-venv + # Specifically locked to avoid known bugs pyarrow==11.0.0 # newer versions cause Postgres segfaults on OOM @@ -14,7 +18,7 @@ xgboost # Transformers accelerate -auto-gptq; sys_platform == 'linux' +auto-gptq; sys_platform == 'linux' # only runs on nvidia hardware bitsandbytes ctransformers huggingface-hub @@ -25,7 +29,7 @@ peft tokenizers transformers transformers-stream-generator -xformers; sys_platform == 'linux' +xformers; sys_platform == 'linux' # only runs on nvidia hardware # Embeddings InstructorEmbedding From ed49732fd5a5744b06371e188ccdd2d107238afb Mon Sep 17 00:00:00 2001 From: Montana Low Date: Mon, 18 Dec 2023 17:13:59 -0800 Subject: [PATCH 3/4] document the reasons --- pgml-extension/requirements.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pgml-extension/requirements.txt b/pgml-extension/requirements.txt index e2fdadb01..8f37b28b3 100644 --- a/pgml-extension/requirements.txt +++ b/pgml-extension/requirements.txt @@ -1,11 +1,17 @@ -# The immediate dependencies of PostgresML are maintained here. +# If you update this file, `pip freeze` the full locked requirements as well to prevent +# future dependency incompatibility on all supported platforms. We'd use Poetry or some +# other sane lockfile resolution mechanism other than pip, except we have to maintain +# this isn't really a Python project, so Poetry conventions don't work, and we need +# different dependencies on platforms that have different hardware acceleration. + +# Only the immediate dependencies of PostgresML are maintained here. # See requirements.linux.txt or requirements.macos.txt for complete and locked versions. # Python 3.12 can't currently resolve all dependencies: # catboost fails to build # virtualenv -p python3.11 pgml-venv -# Specifically locked to avoid known bugs +# These packages are specifically locked to avoid known bugs pyarrow==11.0.0 # newer versions cause Postgres segfaults on OOM # ML From 20ef1423dbb8d1d862ddcba34f99dfdc48f269f3 Mon Sep 17 00:00:00 2001 From: Montana Low Date: Mon, 18 Dec 2023 18:27:23 -0800 Subject: [PATCH 4/4] bump for release --- .github/workflows/ubuntu-packages-and-docker-image.yml | 2 +- .github/workflows/ubuntu-postgresml-python-package.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml index 27970f03b..953c5d969 100644 --- a/.github/workflows/ubuntu-packages-and-docker-image.yml +++ b/.github/workflows/ubuntu-packages-and-docker-image.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: packageVersion: - default: "2.8.0" + default: "2.8.1" jobs: # # PostgresML extension. diff --git a/.github/workflows/ubuntu-postgresml-python-package.yaml b/.github/workflows/ubuntu-postgresml-python-package.yaml index 1af8ef614..0e4be9b21 100644 --- a/.github/workflows/ubuntu-postgresml-python-package.yaml +++ b/.github/workflows/ubuntu-postgresml-python-package.yaml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: packageVersion: - default: "2.7.13" + default: "2.8.1" jobs: postgresml-python:

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier! Saves Data!