From 39ccb685cfb9f880ec53491cddc0e50ebb21ebf8 Mon Sep 17 00:00:00 2001 From: Montana Low Date: Mon, 18 Dec 2023 16:49:43 -0800 Subject: [PATCH 1/4] love me some deps --- .github/workflows/ci.yml | 2 +- packages/postgresml-python/DEBIAN/postinst | 3 +- packages/postgresml-python/build.sh | 11 +- .../resources/developer-docs/installation.md | 5 +- pgml-extension/Cargo.lock | 48 +++--- pgml-extension/Cargo.toml | 2 +- pgml-extension/build.rs | 2 +- pgml-extension/requirements.base.txt | 41 ----- pgml-extension/requirements.linux.txt | 123 +++++++++++++++ pgml-extension/requirements.macos.txt | 123 +++++++++++++++ pgml-extension/requirements.txt | 146 +++++------------- 11 files changed, 316 insertions(+), 190 deletions(-) delete mode 100644 pgml-extension/requirements.base.txt create mode 100644 pgml-extension/requirements.linux.txt create mode 100644 pgml-extension/requirements.macos.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb5c47e82..e9b0b1412 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: python3-pip \ python3 \ lld - sudo pip3 install -r requirements.txt + sudo pip3 install -r requirements.linux.txt - name: Cache dependencies uses: buildjet/cache@v3 if: steps.pgml_extension_changed.outputs.PGML_EXTENSION_CHANGED_FILES != '0' diff --git a/packages/postgresml-python/DEBIAN/postinst b/packages/postgresml-python/DEBIAN/postinst index ae680b4ea..d62a53350 100755 --- a/packages/postgresml-python/DEBIAN/postinst +++ b/packages/postgresml-python/DEBIAN/postinst @@ -7,6 +7,5 @@ set -e # Setup virtualenv virtualenv /var/lib/postgresml-python/pgml-venv source "/var/lib/postgresml-python/pgml-venv/bin/activate" -python -m pip install -r "/etc/postgresml-python/requirements.txt" -python -m pip install -r "/etc/postgresml-python/requirements-xformers.txt" --no-dependencies +python -m pip install -r "/etc/postgresml-python/requirements.linux.txt" deactivate diff --git a/packages/postgresml-python/build.sh b/packages/postgresml-python/build.sh index f13fb241e..f559547f5 100644 --- a/packages/postgresml-python/build.sh +++ b/packages/postgresml-python/build.sh @@ -28,19 +28,12 @@ rm "$deb_dir/release.sh" (cat ${SCRIPT_DIR}/DEBIAN/prerm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/prerm" (cat ${SCRIPT_DIR}/DEBIAN/postrm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postrm" -cp ${SCRIPT_DIR}/../../pgml-extension/requirements.txt "$deb_dir/etc/postgresml-python/requirements.txt" +cp ${SCRIPT_DIR}/../../pgml-extension/requirements.linux.txt "$deb_dir/etc/postgresml-python/requirements.linux.txt" virtualenv --python="python$PYTHON_VERSION" "$deb_dir/var/lib/postgresml-python/pgml-venv" source "$deb_dir/var/lib/postgresml-python/pgml-venv/bin/activate" -python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.txt" - -# No source included, can't build on non x86 platforms -set +e -python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements-autogptq.txt" -set -e - -python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements-xformers.txt" --no-dependencies +python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.linux.txt" deactivate diff --git a/pgml-cms/docs/resources/developer-docs/installation.md b/pgml-cms/docs/resources/developer-docs/installation.md index 990cec5a8..40bd4ab57 100644 --- a/pgml-cms/docs/resources/developer-docs/installation.md +++ b/pgml-cms/docs/resources/developer-docs/installation.md @@ -63,8 +63,7 @@ To install the necessary Python packages into a virtual environment, use the `vi ```bash virtualenv pgml-venv && \ source pgml-venv/bin/activate && \ -pip install -r requirements.txt && \ -pip install -r requirements-xformers.txt --no-dependencies +pip install -r requirements.linux.txt ``` {% endtab %} @@ -72,7 +71,7 @@ pip install -r requirements-xformers.txt --no-dependencies Installing Python packages globally can cause issues with your system. If you wish to proceed nonetheless, you can do so: ```bash -pip3 install -r requirements.txt +pip3 install -r requirements.linux.txt ``` {% endtab %} {% endtabs %} diff --git a/pgml-extension/Cargo.lock b/pgml-extension/Cargo.lock index ec0c7e847..1697813d8 100644 --- a/pgml-extension/Cargo.lock +++ b/pgml-extension/Cargo.lock @@ -503,7 +503,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset 0.9.0", + "memoffset", "scopeguard", ] @@ -1070,9 +1070,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "instant" @@ -1335,15 +1335,6 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.9.0" @@ -1833,7 +1824,7 @@ dependencies = [ "clang-sys", "eyre", "libc", - "memoffset 0.9.0", + "memoffset", "once_cell", "pgrx-macros", "pgrx-pg-config", @@ -2018,14 +2009,14 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543" +checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" dependencies = [ "cfg-if", "indoc", "libc", - "memoffset 0.6.5", + "memoffset", "parking_lot", "pyo3-build-config", "pyo3-ffi", @@ -2035,9 +2026,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8" +checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" dependencies = [ "once_cell", "target-lexicon", @@ -2045,9 +2036,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc" +checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" dependencies = [ "libc", "pyo3-build-config", @@ -2055,25 +2046,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28" +checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote 1.0.33", - "syn 1.0.109", + "syn 2.0.40", ] [[package]] name = "pyo3-macros-backend" -version = "0.17.3" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f" +checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" dependencies = [ + "heck", "proc-macro2", "quote 1.0.33", - "syn 1.0.109", + "syn 2.0.40", ] [[package]] @@ -3028,9 +3020,9 @@ checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "ureq" diff --git a/pgml-extension/Cargo.toml b/pgml-extension/Cargo.toml index 1acce9f58..aaf78ff9c 100644 --- a/pgml-extension/Cargo.toml +++ b/pgml-extension/Cargo.toml @@ -26,7 +26,6 @@ blas = { version = "0.22" } blas-src = { version = "0.9", features = ["openblas"] } indexmap = { version = "1.0", features = ["serde"] } itertools = "0.11" -pyo3 = { version = "0.17", features = ["auto-initialize"], optional = true } heapless = "0.7" lightgbm = { git = "https://github.com/postgresml/lightgbm-rs", branch = "main" } linfa = { path = "deps/linfa" } @@ -42,6 +41,7 @@ ndarray-stats = "0.5.1" parking_lot = "0.12" pgrx = "=0.11.2" pgrx-pg-sys = "=0.11.2" +pyo3 = { version = "0.20.0", features = ["auto-initialize"], optional = true } rand = "0.8" rmp-serde = { version = "1.1" } signal-hook = "0.3" diff --git a/pgml-extension/build.rs b/pgml-extension/build.rs index 355c19293..ca4ab1faf 100644 --- a/pgml-extension/build.rs +++ b/pgml-extension/build.rs @@ -9,7 +9,7 @@ fn main() { // of symbols into the previous opened .so file, but the others will use a // relative offset in pgml.so, and will cause a null-pointer crash. // - // hid all symbol to avoid symbol conflicts. + // hide all symbol to avoid symbol conflicts. // // append mode (link-args) only works with clang ld (lld) println!( diff --git a/pgml-extension/requirements.base.txt b/pgml-extension/requirements.base.txt deleted file mode 100644 index aeffd168c..000000000 --- a/pgml-extension/requirements.base.txt +++ /dev/null @@ -1,41 +0,0 @@ -# The immediate dependencies of PostgresML are maintained here. - -# Locked because newer versions have bugs -transformers-stream-generator==0.0.4 -optimum==1.13.2 -peft==0.6.2 -pyarrow==11.0.0 -torch==2.0.1 # 2.1.1 breaks sentence-transformers==2.2.2 - -# ML -catboost -lightgbm -torchaudio -torchvision -xgboost - -# Transformers -accelerate -auto-gptq; sys_platform == 'linux' -bitsandbytes -ctransformers -huggingface-hub -deepspeed -einops -tokenizers -transformers -xformers; sys_platform == 'linux' - -# Embeddings -InstructorEmbedding -sentence-transformers - -# Ratings -rouge -sacrebleu -sacremoses - -# Utils -datasets -orjson -langchain diff --git a/pgml-extension/requirements.linux.txt b/pgml-extension/requirements.linux.txt new file mode 100644 index 000000000..067036d25 --- /dev/null +++ b/pgml-extension/requirements.linux.txt @@ -0,0 +1,123 @@ +accelerate==0.25.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +anyio==4.2.0 +async-timeout==4.0.3 +attrs==23.1.0 +auto-gptq==0.6.0 +bitsandbytes==0.41.3.post2 +catboost==1.2.2 +certifi==2023.11.17 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +coloredlogs==15.0.1 +contourpy==1.2.0 +ctransformers==0.2.27 +cycler==0.12.1 +dataclasses-json==0.6.3 +datasets==2.15.0 +deepspeed==0.12.5 +dill==0.3.7 +einops==0.7.0 +exceptiongroup==1.2.0 +filelock==3.13.1 +fonttools==4.47.0 +frozenlist==1.4.1 +fsspec==2023.10.0 +gekko==1.0.6 +graphviz==0.20.1 +greenlet==3.0.2 +hjson==3.1.0 +huggingface-hub==0.19.4 +humanfriendly==10.0 +idna==3.6 +InstructorEmbedding==1.0.1 +Jinja2==3.1.2 +joblib==1.3.2 +jsonpatch==1.33 +jsonpointer==2.4 +kiwisolver==1.4.5 +langchain==0.0.351 +langchain-community==0.0.4 +langchain-core==0.1.1 +langsmith==0.0.72 +lightgbm==4.1.0 +lxml==4.9.3 +MarkupSafe==2.1.3 +marshmallow==3.20.1 +matplotlib==3.8.2 +mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.15 +mypy-extensions==1.0.0 +networkx==3.2.1 +ninja==1.11.1.1 +nltk==3.8.1 +numpy==1.26.2 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.3.101 +nvidia-nvtx-cu12==12.1.105 +optimum==1.16.1 +orjson==3.9.10 +packaging==23.2 +pandas==2.1.4 +peft==0.7.1 +Pillow==10.1.0 +plotly==5.18.0 +portalocker==2.8.2 +protobuf==4.25.1 +psutil==5.9.7 +py-cpuinfo==9.0.0 +pyarrow==11.0.0 +pyarrow-hotfix==0.6 +pydantic==2.5.2 +pydantic_core==2.14.5 +pynvml==11.5.0 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.10.3 +requests==2.31.0 +rouge==1.0.1 +sacrebleu==2.4.0 +sacremoses==0.1.1 +safetensors==0.4.1 +scikit-learn==1.3.2 +scipy==1.11.4 +sentence-transformers==2.2.2 +sentencepiece==0.1.99 +six==1.16.0 +sniffio==1.3.0 +SQLAlchemy==2.0.23 +sympy==1.12 +tabulate==0.9.0 +tenacity==8.2.3 +threadpoolctl==3.2.0 +tokenizers==0.15.0 +torch==2.1.2 +torchaudio==2.1.2 +torchvision==0.16.2 +tqdm==4.66.1 +transformers==4.36.2 +transformers-stream-generator==0.0.4 +triton==2.1.0 +typing-inspect==0.9.0 +typing_extensions==4.9.0 +tzdata==2023.3 +urllib3==2.1.0 +xformers==0.0.23.post1 +xgboost==2.0.2 +xxhash==3.4.1 +yarl==1.9.4 diff --git a/pgml-extension/requirements.macos.txt b/pgml-extension/requirements.macos.txt new file mode 100644 index 000000000..b08bde5e9 --- /dev/null +++ b/pgml-extension/requirements.macos.txt @@ -0,0 +1,123 @@ +accelerate==0.25.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +anyio==4.1.0 +async-timeout==4.0.3 +attrs==23.1.0 +auto-gptq==0.5.1 +bitsandbytes==0.41.3.post2 +catboost==1.2.2 +certifi==2023.11.17 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +coloredlogs==15.0.1 +contourpy==1.2.0 +ctransformers==0.2.27 +cycler==0.12.1 +dataclasses-json==0.6.3 +datasets==2.15.0 +deepspeed==0.12.4 +dill==0.3.7 +einops==0.7.0 +exceptiongroup==1.2.0 +filelock==3.13.1 +fonttools==4.46.0 +frozenlist==1.4.0 +fsspec==2023.10.0 +gekko==1.0.6 +graphviz==0.20.1 +greenlet==3.0.2 +hjson==3.1.0 +huggingface-hub==0.19.4 +humanfriendly==10.0 +idna==3.6 +InstructorEmbedding==1.0.1 +Jinja2==3.1.2 +joblib==1.3.2 +jsonpatch==1.33 +jsonpointer==2.4 +kiwisolver==1.4.5 +langchain==0.0.350 +langchain-community==0.0.3 +langchain-core==0.1.1 +langsmith==0.0.70 +lightgbm==4.1.0 +lxml==4.9.3 +MarkupSafe==2.1.3 +marshmallow==3.20.1 +matplotlib==3.8.2 +mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.15 +mypy-extensions==1.0.0 +networkx==3.2.1 +ninja==1.11.1.1 +nltk==3.8.1 +numpy==1.26.2 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.3.101 +nvidia-nvtx-cu12==12.1.105 +optimum==1.13.2 +orjson==3.9.10 +packaging==23.2 +pandas==2.1.4 +peft==0.6.2 +Pillow==10.1.0 +plotly==5.18.0 +portalocker==2.8.2 +protobuf==4.25.1 +psutil==5.9.6 +py-cpuinfo==9.0.0 +pyarrow==11.0.0 +pyarrow-hotfix==0.6 +pydantic==2.5.2 +pydantic_core==2.14.5 +pynvml==11.5.0 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3.post1 +PyYAML==6.0.1 +regex==2023.10.3 +requests==2.31.0 +rouge==1.0.1 +sacrebleu==2.4.0 +sacremoses==0.1.1 +safetensors==0.4.1 +scikit-learn==1.3.2 +scipy==1.11.4 +sentence-transformers==2.2.2 +sentencepiece==0.1.99 +six==1.16.0 +sniffio==1.3.0 +SQLAlchemy==2.0.23 +sympy==1.12 +tabulate==0.9.0 +tenacity==8.2.3 +threadpoolctl==3.2.0 +tokenizers==0.15.0 +torch==2.1.1 +torchaudio==2.1.1 +torchvision==0.16.1 +tqdm==4.66.1 +transformers==4.36.1 +transformers-stream-generator==0.0.4 +triton==2.1.0 +typing-inspect==0.9.0 +typing_extensions==4.9.0 +tzdata==2023.3 +urllib3==2.1.0 +xformers==0.0.23 +xgboost==2.0.2 +xxhash==3.4.1 +yarl==1.9.4 \ No newline at end of file diff --git a/pgml-extension/requirements.txt b/pgml-extension/requirements.txt index ddc7229b6..4a6bf1626 100644 --- a/pgml-extension/requirements.txt +++ b/pgml-extension/requirements.txt @@ -1,104 +1,42 @@ -accelerate==0.25.0 -aiohttp==3.9.1 -aiosignal==1.3.1 -annotated-types==0.6.0 -anyio==4.1.0 -attrs==23.1.0 -auto-gptq==0.5.1; sys_platform == 'linux' -bitsandbytes==0.41.3.post2 -catboost==1.2.2 -certifi==2023.11.17 -charset-normalizer==3.3.2 -click==8.1.7 -colorama==0.4.6 -contourpy==1.2.0 -ctransformers==0.2.27 -cycler==0.12.1 -dataclasses-json==0.6.3 -datasets==2.15.0 -deepspeed==0.12.4 -dill==0.3.7 -einops==0.7.0 -filelock==3.13.1 -fonttools==4.46.0 -frozenlist==1.4.0 -fsspec==2023.10.0 -gekko==1.0.6 -graphviz==0.20.1 -hjson==3.1.0 -huggingface-hub==0.19.4 -idna==3.6 -InstructorEmbedding==1.0.1 -Jinja2==3.1.2 -joblib==1.3.2 -jsonpatch==1.33 -jsonpointer==2.4 -kiwisolver==1.4.5 -langchain==0.0.349 -langchain-community==0.0.1 -langchain-core==0.0.13 -langsmith==0.0.69 -lightgbm==4.1.0 -lxml==4.9.3 -MarkupSafe==2.1.3 -marshmallow==3.20.1 -matplotlib==3.8.2 -mpmath==1.3.0 -multidict==6.0.4 -multiprocess==0.70.15 -mypy-extensions==1.0.0 -networkx==3.2.1 -ninja==1.11.1.1 -nltk==3.8.1 -numpy==1.26.2 -optimum==1.13.2 -orjson==3.9.10 -packaging==23.2 -pandas==2.1.4 -peft==0.6.2 -Pillow==10.1.0 -plotly==5.18.0 -portalocker==2.8.2 -psutil==5.9.6 -py-cpuinfo==9.0.0 -pyarrow==14.0.1 -pyarrow-hotfix==0.6 -pydantic==2.5.2 -pydantic_core==2.14.5 -pynvml==11.5.0 -pyparsing==3.1.1 -python-dateutil==2.8.2 -pytz==2023.3.post1 -PyYAML==6.0.1 -regex==2023.10.3 -requests==2.31.0 -rouge==1.0.1 -sacrebleu==2.3.3 -sacremoses==0.1.1 -safetensors==0.4.1 -scikit-learn==1.3.2 -scipy==1.11.4 -sentence-transformers==2.2.2 -sentencepiece==0.1.99 -six==1.16.0 -sniffio==1.3.0 -SQLAlchemy==2.0.23 -sympy==1.12 -tabulate==0.9.0 -tenacity==8.2.3 -threadpoolctl==3.2.0 -tokenizers==0.15.0 -torch==2.0.1 -torchaudio==2.0.2 -torchvision==0.15.2 -tqdm==4.66.1 -transformers==4.36.0 -transformers-stream-generator==0.0.4 -typing-inspect==0.9.0 -typing_extensions==4.9.0 -tzdata==2023.3 -urllib3==2.1.0 -xformers==0.0.22; sys_platform == 'linux' -xgboost==2.0.2 -xxhash==3.4.1 -yarl==1.9.4 +# The immediate dependencies of PostgresML are maintained here. +# See requirements.linux.txt or requirements.macos.txt for complete and locked versions. + +# Specifically locked to avoid known bugs +pyarrow==11.0.0 # newer versions cause Postgres segfaults on OOM + +# ML +catboost +lightgbm +torch +torchaudio +torchvision +xgboost + +# Transformers +accelerate +auto-gptq; sys_platform == 'linux' +bitsandbytes +ctransformers +huggingface-hub +deepspeed +einops +optimum +peft +tokenizers +transformers +transformers-stream-generator +xformers; sys_platform == 'linux' + +# Embeddings +InstructorEmbedding +sentence-transformers + +# Ratings +rouge +sacrebleu +sacremoses + +# Utils +datasets +orjson +langchain From 0ea6207fce338aaeedb52d18cea6703c084faac8 Mon Sep 17 00:00:00 2001 From: Montana Low Date: Mon, 18 Dec 2023 17:06:14 -0800 Subject: [PATCH 2/4] tested on macos --- pgml-extension/requirements.macos.txt | 49 ++++++++------------------- pgml-extension/requirements.txt | 8 +++-- 2 files changed, 21 insertions(+), 36 deletions(-) diff --git a/pgml-extension/requirements.macos.txt b/pgml-extension/requirements.macos.txt index b08bde5e9..b41533af7 100644 --- a/pgml-extension/requirements.macos.txt +++ b/pgml-extension/requirements.macos.txt @@ -2,10 +2,8 @@ accelerate==0.25.0 aiohttp==3.9.1 aiosignal==1.3.1 annotated-types==0.6.0 -anyio==4.1.0 -async-timeout==4.0.3 +anyio==4.2.0 attrs==23.1.0 -auto-gptq==0.5.1 bitsandbytes==0.41.3.post2 catboost==1.2.2 certifi==2023.11.17 @@ -18,17 +16,14 @@ ctransformers==0.2.27 cycler==0.12.1 dataclasses-json==0.6.3 datasets==2.15.0 -deepspeed==0.12.4 +deepspeed==0.12.5 dill==0.3.7 einops==0.7.0 -exceptiongroup==1.2.0 filelock==3.13.1 -fonttools==4.46.0 -frozenlist==1.4.0 +fonttools==4.47.0 +frozenlist==1.4.1 fsspec==2023.10.0 -gekko==1.0.6 graphviz==0.20.1 -greenlet==3.0.2 hjson==3.1.0 huggingface-hub==0.19.4 humanfriendly==10.0 @@ -39,10 +34,10 @@ joblib==1.3.2 jsonpatch==1.33 jsonpointer==2.4 kiwisolver==1.4.5 -langchain==0.0.350 -langchain-community==0.0.3 +langchain==0.0.351 +langchain-community==0.0.4 langchain-core==0.1.1 -langsmith==0.0.70 +langsmith==0.0.72 lightgbm==4.1.0 lxml==4.9.3 MarkupSafe==2.1.3 @@ -56,28 +51,16 @@ networkx==3.2.1 ninja==1.11.1.1 nltk==3.8.1 numpy==1.26.2 -nvidia-cublas-cu12==12.1.3.1 -nvidia-cuda-cupti-cu12==12.1.105 -nvidia-cuda-nvrtc-cu12==12.1.105 -nvidia-cuda-runtime-cu12==12.1.105 -nvidia-cudnn-cu12==8.9.2.26 -nvidia-cufft-cu12==11.0.2.54 -nvidia-curand-cu12==10.3.2.106 -nvidia-cusolver-cu12==11.4.5.107 -nvidia-cusparse-cu12==12.1.0.106 -nvidia-nccl-cu12==2.18.1 -nvidia-nvjitlink-cu12==12.3.101 -nvidia-nvtx-cu12==12.1.105 -optimum==1.13.2 +optimum==1.16.1 orjson==3.9.10 packaging==23.2 pandas==2.1.4 -peft==0.6.2 +peft==0.7.1 Pillow==10.1.0 plotly==5.18.0 portalocker==2.8.2 protobuf==4.25.1 -psutil==5.9.6 +psutil==5.9.7 py-cpuinfo==9.0.0 pyarrow==11.0.0 pyarrow-hotfix==0.6 @@ -106,18 +89,16 @@ tabulate==0.9.0 tenacity==8.2.3 threadpoolctl==3.2.0 tokenizers==0.15.0 -torch==2.1.1 -torchaudio==2.1.1 -torchvision==0.16.1 +torch==2.1.2 +torchaudio==2.1.2 +torchvision==0.16.2 tqdm==4.66.1 -transformers==4.36.1 +transformers==4.36.2 transformers-stream-generator==0.0.4 -triton==2.1.0 typing-inspect==0.9.0 typing_extensions==4.9.0 tzdata==2023.3 urllib3==2.1.0 -xformers==0.0.23 xgboost==2.0.2 xxhash==3.4.1 -yarl==1.9.4 \ No newline at end of file +yarl==1.9.4 diff --git a/pgml-extension/requirements.txt b/pgml-extension/requirements.txt index 4a6bf1626..e2fdadb01 100644 --- a/pgml-extension/requirements.txt +++ b/pgml-extension/requirements.txt @@ -1,6 +1,10 @@ # The immediate dependencies of PostgresML are maintained here. # See requirements.linux.txt or requirements.macos.txt for complete and locked versions. +# Python 3.12 can't currently resolve all dependencies: +# catboost fails to build +# virtualenv -p python3.11 pgml-venv + # Specifically locked to avoid known bugs pyarrow==11.0.0 # newer versions cause Postgres segfaults on OOM @@ -14,7 +18,7 @@ xgboost # Transformers accelerate -auto-gptq; sys_platform == 'linux' +auto-gptq; sys_platform == 'linux' # only runs on nvidia hardware bitsandbytes ctransformers huggingface-hub @@ -25,7 +29,7 @@ peft tokenizers transformers transformers-stream-generator -xformers; sys_platform == 'linux' +xformers; sys_platform == 'linux' # only runs on nvidia hardware # Embeddings InstructorEmbedding From ed49732fd5a5744b06371e188ccdd2d107238afb Mon Sep 17 00:00:00 2001 From: Montana Low Date: Mon, 18 Dec 2023 17:13:59 -0800 Subject: [PATCH 3/4] document the reasons --- pgml-extension/requirements.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pgml-extension/requirements.txt b/pgml-extension/requirements.txt index e2fdadb01..8f37b28b3 100644 --- a/pgml-extension/requirements.txt +++ b/pgml-extension/requirements.txt @@ -1,11 +1,17 @@ -# The immediate dependencies of PostgresML are maintained here. +# If you update this file, `pip freeze` the full locked requirements as well to prevent +# future dependency incompatibility on all supported platforms. We'd use Poetry or some +# other sane lockfile resolution mechanism other than pip, except we have to maintain +# this isn't really a Python project, so Poetry conventions don't work, and we need +# different dependencies on platforms that have different hardware acceleration. + +# Only the immediate dependencies of PostgresML are maintained here. # See requirements.linux.txt or requirements.macos.txt for complete and locked versions. # Python 3.12 can't currently resolve all dependencies: # catboost fails to build # virtualenv -p python3.11 pgml-venv -# Specifically locked to avoid known bugs +# These packages are specifically locked to avoid known bugs pyarrow==11.0.0 # newer versions cause Postgres segfaults on OOM # ML From 20ef1423dbb8d1d862ddcba34f99dfdc48f269f3 Mon Sep 17 00:00:00 2001 From: Montana Low Date: Mon, 18 Dec 2023 18:27:23 -0800 Subject: [PATCH 4/4] bump for release --- .github/workflows/ubuntu-packages-and-docker-image.yml | 2 +- .github/workflows/ubuntu-postgresml-python-package.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml index 27970f03b..953c5d969 100644 --- a/.github/workflows/ubuntu-packages-and-docker-image.yml +++ b/.github/workflows/ubuntu-packages-and-docker-image.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: packageVersion: - default: "2.8.0" + default: "2.8.1" jobs: # # PostgresML extension. diff --git a/.github/workflows/ubuntu-postgresml-python-package.yaml b/.github/workflows/ubuntu-postgresml-python-package.yaml index 1af8ef614..0e4be9b21 100644 --- a/.github/workflows/ubuntu-postgresml-python-package.yaml +++ b/.github/workflows/ubuntu-postgresml-python-package.yaml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: packageVersion: - default: "2.7.13" + default: "2.8.1" jobs: postgresml-python: pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy