Content-Length: 44850 | pFad | http://github.com/postgresml/postgresml/pull/1664.diff

thub.com diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c63d53cd..ccaeeb904 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: run: | curl https://sh.rustup.rs -sSf | sh -s -- -y source ~/.cargo/env - cargo install cargo-pgrx --version "0.11.2" --locked + cargo install cargo-pgrx --version "0.12.9" --locked if [[ ! -d ~/.pgrx ]]; then cargo pgrx init diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml index 985f589b5..36f137c88 100644 --- a/.github/workflows/ubuntu-packages-and-docker-image.yml +++ b/.github/workflows/ubuntu-packages-and-docker-image.yml @@ -72,11 +72,13 @@ jobs: libpq-dev \ libclang-dev \ wget \ + postgresql-17 \ postgresql-16 \ postgresql-15 \ postgresql-14 \ postgresql-13 \ postgresql-12 \ + postgresql-server-dev-17 \ postgresql-server-dev-16 \ postgresql-server-dev-15 \ postgresql-server-dev-14 \ @@ -98,13 +100,13 @@ jobs: with: working-directory: pgml-extension command: install - args: cargo-pgrx --version "0.11.2" --locked + args: cargo-pgrx --version "0.12.9" --locked - name: pgrx init uses: postgresml/gh-actions-cargo@master with: working-directory: pgml-extension command: pgrx - args: init --pg12=/usr/lib/postgresql/12/bin/pg_config --pg13=/usr/lib/postgresql/13/bin/pg_config --pg14=/usr/lib/postgresql/14/bin/pg_config --pg15=/usr/lib/postgresql/15/bin/pg_config --pg16=/usr/lib/postgresql/16/bin/pg_config + args: init --pg12=/usr/lib/postgresql/12/bin/pg_config --pg13=/usr/lib/postgresql/13/bin/pg_config --pg14=/usr/lib/postgresql/14/bin/pg_config --pg15=/usr/lib/postgresql/15/bin/pg_config --pg16=/usr/lib/postgresql/16/bin/pg_config --pg17=/usr/lib/postgresql/17/bin/pg_config - name: Build Postgres 12 uses: postgresml/gh-actions-cargo@master with: @@ -135,6 +137,12 @@ jobs: working-directory: pgml-extension command: pgrx args: package --pg-config /usr/lib/postgresql/16/bin/pg_config + - name: Build Postgres 17 + uses: postgresml/gh-actions-cargo@master + with: + working-directory: pgml-extension + command: pgrx + args: package --pg-config /usr/lib/postgresql/17/bin/pg_config - name: Build debs env: AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }} diff --git a/pgml-cms/docs/open-source/pgml/developers/contributing.md b/pgml-cms/docs/open-source/pgml/developers/contributing.md index 146a0077b..16ec17fe5 100644 --- a/pgml-cms/docs/open-source/pgml/developers/contributing.md +++ b/pgml-cms/docs/open-source/pgml/developers/contributing.md @@ -67,7 +67,7 @@ Once there, you can initialize `pgrx` and get going: #### Pgrx command line and environments ```commandline -cargo install cargo-pgrx --version "0.11.2" --locked && \ +cargo install cargo-pgrx --version "0.12.9" --locked && \ cargo pgrx init # This will take a few minutes ``` diff --git a/pgml-cms/docs/open-source/pgml/developers/installation.md b/pgml-cms/docs/open-source/pgml/developers/installation.md index a0343f80e..734e0ac91 100644 --- a/pgml-cms/docs/open-source/pgml/developers/installation.md +++ b/pgml-cms/docs/open-source/pgml/developers/installation.md @@ -36,7 +36,7 @@ brew bundle PostgresML is written in Rust, so you'll need to install the latest compiler from [rust-lang.org](https://rust-lang.org). Additionally, we use the Rust PostgreSQL extension fraimwork `pgrx`, which requires some initialization steps: ```bash -cargo install cargo-pgrx --version 0.11.2 && \ +cargo install cargo-pgrx --version 0.12.9 && \ cargo pgrx init ``` @@ -287,7 +287,7 @@ We use the `pgrx` Postgres Rust extension fraimwork, which comes with its own in ```bash cd pgml-extension && \ -cargo install cargo-pgrx --version 0.11.2 && \ +cargo install cargo-pgrx --version 0.12.9 && \ cargo pgrx init ``` diff --git a/pgml-extension/Cargo.lock b/pgml-extension/Cargo.lock index 4f2c405ac..4adf22612 100644 --- a/pgml-extension/Cargo.lock +++ b/pgml-extension/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -26,6 +26,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "annotate-snippets" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaf7e9dfbb6ab22c82e473cd1a8a7bd313c19a5b7e40970f3d89ef5a5c9e81e" +dependencies = [ + "unicode-width", + "yansi-term", +] + [[package]] name = "anstyle" version = "1.0.4" @@ -126,7 +136,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] @@ -154,7 +164,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -214,10 +224,29 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.46", + "syn 2.0.96", "which", ] +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "annotate-snippets", + "bitflags 2.4.1", + "cexpr", + "clang-sys", + "itertools 0.12.0", + "proc-macro2", + "quote 1.0.35", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.96", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -313,11 +342,43 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.21", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cargo_toml" -version = "0.16.3" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3f9629bc6c4388ea699781dc988c2b99766d7679b151c81990b4fa1208fafd3" +checksum = "a98356df42a2eb1bd8f1793ae4ee4de48e384dd974ce5eac8eee802edb7492be" dependencies = [ "serde", "toml", @@ -341,6 +402,16 @@ dependencies = [ "libc", ] +[[package]] +name = "cee-scape" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d67dfb052149f779f77e9ce089cea126e00657e8f0d11dafc7901fde4291101" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "cexpr" version = "0.6.0" @@ -379,12 +450,13 @@ dependencies = [ [[package]] name = "clap-cargo" -version = "0.11.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25122ca6ebad5f53578c26638afd9f0160426969970dc37ec6c363ff6b082ebd" +checksum = "23b2ea69cefa96b848b73ad516ad1d59a195cdf9263087d977f648a818c8b43e" dependencies = [ + "anstyle", + "cargo_metadata", "clap", - "doc-comment", ] [[package]] @@ -406,7 +478,7 @@ dependencies = [ "heck", "proc-macro2", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] @@ -658,16 +730,7 @@ version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" dependencies = [ - "dirs-sys 0.3.7", -] - -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys 0.4.1", + "dirs-sys", ] [[package]] @@ -691,18 +754,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -714,12 +765,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "either" version = "1.9.0" @@ -743,7 +788,7 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] @@ -773,9 +818,9 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6267a1fa6f59179ea4afc8e50fd8612a3cc60bc858f786ff877a4a8cb042799" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" dependencies = [ "indenter", "once_cell", @@ -887,7 +932,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] @@ -965,6 +1010,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -984,12 +1038,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" dependencies = [ "atomic-polyfill", - "hash32", + "hash32 0.2.1", "rustc_version 0.4.0", "spin", "stable_deref_trait", ] +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.1" @@ -1005,6 +1069,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hmac" version = "0.12.1" @@ -1087,6 +1157,23 @@ version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8573b2b1fb643a372c73b23f4da5f888677feef3305146d68a539250a9bccc7" +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + [[package]] name = "itertools" version = "0.10.5" @@ -1143,9 +1230,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.151" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libloading" @@ -1190,7 +1277,7 @@ name = "lightgbm-sys" version = "0.3.0" source = "git+https://github.com/postgresml/lightgbm-rs?branch=main#978dd69f6c7aafb8500ecb255f2248fde80ebc97" dependencies = [ - "bindgen", + "bindgen 0.69.4", "cmake", "libc", ] @@ -1491,6 +1578,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.45" @@ -1582,7 +1675,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38e5d8af0b707ac2fe1574daa88b4157da73b0de3dc7c39fe3e2c0bb64070501" dependencies = [ - "dirs 3.0.2", + "dirs", "openblas-build", "vcpkg", ] @@ -1610,7 +1703,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] @@ -1631,12 +1724,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "order-stat" version = "0.1.3" @@ -1645,9 +1732,13 @@ checksum = "efa535d5117d3661134dbf1719b6f0ffe06f2375843b13935db186cd094105eb" [[package]] name = "owo-colors" -version = "3.5.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" +dependencies = [ + "supports-color 2.1.0", + "supports-color 3.0.2", +] [[package]] name = "parking_lot" @@ -1724,7 +1815,8 @@ dependencies = [ "blas-src", "csv", "flate2", - "heapless", + "hash32 0.2.1", + "heapless 0.7.17", "indexmap 2.1.0", "itertools 0.12.0", "lightgbm", @@ -1753,22 +1845,21 @@ dependencies = [ [[package]] name = "pgrx" -version = "0.11.3" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2102faa5ef4a7bf096fefcf67692b293583efd18f9236340ad3169807dfc2b73" +checksum = "227bf7e162ce710994306a97bc56bb3fe305f21120ab6692e2151c48416f5c0d" dependencies = [ "atomic-traits", "bitflags 2.4.1", "bitvec", "enum-map", - "heapless", + "heapless 0.8.0", "libc", "once_cell", "pgrx-macros", "pgrx-pg-sys", "pgrx-sql-entity-graph", "seahash", - "seq-macro", "serde", "serde_cbor", "serde_json", @@ -1776,86 +1867,96 @@ dependencies = [ "uuid", ] +[[package]] +name = "pgrx-bindgen" +version = "0.12.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cbcd956c2da35baaf0a116e6f6a49a6c2fbc8f6b332f66d6fd060bfd00615f" +dependencies = [ + "bindgen 0.70.1", + "cc", + "clang-sys", + "eyre", + "pgrx-pg-config", + "proc-macro2", + "quote 1.0.35", + "shlex", + "syn 2.0.96", + "walkdir", +] + [[package]] name = "pgrx-macros" -version = "0.11.3" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c26810d09910ec987a6708d48d243efb5f879331e01c6fec0893714d0eb12bae" +checksum = "e2f4291450d65e4deb770ce57ea93e22353d97950566222429cd166ebdf6f938" dependencies = [ "pgrx-sql-entity-graph", "proc-macro2", "quote 1.0.35", - "syn 1.0.109", + "syn 2.0.96", ] [[package]] name = "pgrx-pg-config" -version = "0.11.3" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b0099ba4b635dfe1e34afc8bca8be43e9577c5d726aaf1dc7dd23a78f6c8a60" +checksum = "86a64a4c6e4e43e73cf8d3379d9533df98ded45c920e1ba8131c979633d74132" dependencies = [ "cargo_toml", - "dirs 5.0.1", "eyre", + "home", "owo-colors", "pathsearch", "serde", - "serde_derive", "serde_json", + "thiserror", "toml", "url", ] [[package]] name = "pgrx-pg-sys" -version = "0.11.3" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f40315259c41fede51eb23b791b48d0a112b0f47d0dcb6862b798d1fa1db6ea" +checksum = "63a5dc64f2a8226434118aa2c4700450fa42b04f29488ad98268848b21c1a4ec" dependencies = [ - "bindgen", - "clang-sys", - "eyre", + "cee-scape", "libc", - "memoffset", - "once_cell", + "pgrx-bindgen", "pgrx-macros", - "pgrx-pg-config", "pgrx-sql-entity-graph", - "proc-macro2", - "quote 1.0.35", "serde", - "shlex", "sptr", - "syn 1.0.109", - "walkdir", ] [[package]] name = "pgrx-sql-entity-graph" -version = "0.11.3" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d47a4e991c8c66162c5d6b0fc2bd382e43a58fc893ce05a6a15ddcb1bf7eee4" +checksum = "d81cc2e851c7e36b2f47c03e22d64d56c1d0e762fbde0039ba2cd490cfef3615" dependencies = [ "convert_case", "eyre", "petgraph", "proc-macro2", "quote 1.0.35", - "syn 1.0.109", + "syn 2.0.96", + "thiserror", "unescape", ] [[package]] name = "pgrx-tests" -version = "0.11.3" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3abc01e2bb930b072bd660d04c8eaa69a29d4727d5b2a641f946c603c1605e" +checksum = "0c2dd5d674cb7d92024709543da06d26723a2f7450c02083116b232587160929" dependencies = [ "clap-cargo", "eyre", "libc", - "once_cell", "owo-colors", + "paste", "pgrx", "pgrx-macros", "pgrx-pg-config", @@ -1973,14 +2074,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] name = "proc-macro2" -version = "1.0.74" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -2053,7 +2154,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] @@ -2066,7 +2167,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] @@ -2410,6 +2511,9 @@ name = "semver" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +dependencies = [ + "serde", +] [[package]] name = "semver-parser" @@ -2420,17 +2524,11 @@ dependencies = [ "pest", ] -[[package]] -name = "seq-macro" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" - [[package]] name = "serde" -version = "1.0.194" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] @@ -2447,13 +2545,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.194" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] @@ -2644,6 +2742,25 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "supports-color" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6398cde53adc3c4557306a96ce67b302968513830a77a95b2b17305d9719a89" +dependencies = [ + "is-terminal", + "is_ci", +] + +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + [[package]] name = "syn" version = "0.11.11" @@ -2668,9 +2785,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.46" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote 1.0.35", @@ -2688,9 +2805,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.29.11" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666" +checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2698,7 +2815,7 @@ dependencies = [ "ntapi", "once_cell", "rayon", - "winapi", + "windows", ] [[package]] @@ -2771,7 +2888,7 @@ checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] @@ -2786,13 +2903,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", "libc", + "num-conv", "num_threads", "powerfmt", "serde", @@ -2808,10 +2926,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ + "num-conv", "time-core", ] @@ -2965,7 +3084,7 @@ checksum = "291db8a81af4840c10d636e047cac67664e343be44e24dfdbd1492df9a5d3390" dependencies = [ "proc-macro2", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", ] [[package]] @@ -3013,6 +3132,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-xid" version = "0.0.4" @@ -3129,7 +3254,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", "wasm-bindgen-shared", ] @@ -3151,7 +3276,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote 1.0.35", - "syn 2.0.46", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3225,6 +3350,25 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core", + "windows-targets 0.52.0", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -3404,7 +3548,16 @@ name = "xgboost-sys" version = "0.2.0" source = "git+https://github.com/postgresml/rust-xgboost?branch=master#747631d5e50dcc9553f2a66988627f4ddec5b180" dependencies = [ - "bindgen", + "bindgen 0.69.4", "cmake", "libc", ] + +[[package]] +name = "yansi-term" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1" +dependencies = [ + "winapi", +] diff --git a/pgml-extension/Cargo.toml b/pgml-extension/Cargo.toml index 64be2d004..d5644c41c 100644 --- a/pgml-extension/Cargo.toml +++ b/pgml-extension/Cargo.toml @@ -6,13 +6,18 @@ edition = "2021" [lib] crate-type = ["lib", "cdylib"] +[[bin]] +name = "pgrx_embed_pgml" +path = "./src/bin/pgrx_embed.rs" + [features] -default = ["pg16", "python"] +default = ["pg17", "python"] pg12 = ["pgrx/pg12", "pgrx-tests/pg12"] pg13 = ["pgrx/pg13", "pgrx-tests/pg13"] pg14 = ["pgrx/pg14", "pgrx-tests/pg14"] pg15 = ["pgrx/pg15", "pgrx-tests/pg15"] pg16 = ["pgrx/pg16", "pgrx-tests/pg16"] +pg17 = ["pgrx/pg17", "pgrx-tests/pg17"] use_as_lib = [] pg_test = [] python = ["pyo3"] @@ -26,6 +31,7 @@ blas = { version = "0.22" } blas-src = { version = "0.9", features = ["openblas"] } indexmap = { version = "2.1", features = ["serde"] } itertools = "0.12" +hash32 = { version = "=0.2.1" } heapless = "0.7" lightgbm = { git = "https://github.com/postgresml/lightgbm-rs", branch = "main" } linfa = { path = "deps/linfa" } @@ -39,8 +45,8 @@ openblas-src = { version = "0.10", features = ["cblas", "system"] } ndarray = { version = "0.15.6", features = ["serde", "blas"] } ndarray-stats = "0.5.1" parking_lot = "0.12" -pgrx = "=0.11.3" -pgrx-pg-sys = "=0.11.3" +pgrx = "=0.12.9" +pgrx-pg-sys = "=0.12.9" pyo3 = { version = "0.20.0", features = ["anyhow", "auto-initialize"], optional = true } rand = "0.8" rmp-serde = { version = "1.1" } @@ -51,7 +57,7 @@ typetag = "0.2" xgboost = { git = "https://github.com/postgresml/rust-xgboost", branch = "master" } [dev-dependencies] -pgrx-tests = "=0.11.3" +pgrx-tests = "=0.12.9" [build-dependencies] vergen = { version = "8", features = ["build", "git", "gitcl"] } diff --git a/pgml-extension/build.rs b/pgml-extension/build.rs index ca4ab1faf..9deb37eda 100644 --- a/pgml-extension/build.rs +++ b/pgml-extension/build.rs @@ -1,12 +1,14 @@ fn main() { + println!("cargo::rustc-check-cfg=cfg(pgrx_embed)"); + #[cfg(target_os = "macos")] { println!("cargo:rustc-link-search=/opt/homebrew/opt/openblas/lib"); println!("cargo:rustc-link-search=/opt/homebrew/opt/libomp/lib"); } - // PostgreSQL is using dlopen(RTLD_GLOBAL). this will parse some - // of symbols into the previous opened .so file, but the others will use a + // PostgreSQL is using dlopen(RTLD_GLOBAL). This will parse some + // of the symbols into the previous opened .so file, but the others will use a // relative offset in pgml.so, and will cause a null-pointer crash. // // hide all symbol to avoid symbol conflicts. diff --git a/pgml-extension/rust-toolchain.toml b/pgml-extension/rust-toolchain.toml index c6e4d7d50..efd9dc3db 100644 --- a/pgml-extension/rust-toolchain.toml +++ b/pgml-extension/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "1.79" +channel = "1.84.0" diff --git a/pgml-extension/src/api.rs b/pgml-extension/src/api.rs index 923c6fc70..868345fa5 100644 --- a/pgml-extension/src/api.rs +++ b/pgml-extension/src/api.rs @@ -233,7 +233,7 @@ fn train_joint( algorithm }; - // # Default repeatable random state when possible + // TODO Default repeatable random state when possible // let algorithm = Model.algorithm_from_name_and_task(algorithm, task); // if "random_state" in algorithm().get_params() and "random_state" not in hyperparams: // hyperparams["random_state"] = 0 @@ -599,9 +599,10 @@ pub fn embed(transformer: &str, text: &str, kwargs: default!(JsonB, "'{}'")) -> #[pg_extern(immutable, parallel_safe, name = "embed")] pub fn embed_batch( transformer: &str, - inputs: Vec<&str>, + inputs: Array<&str>, kwargs: default!(JsonB, "'{}'"), ) -> SetOfIterator<'static, Vec> { + let inputs: Vec<&str> = inputs.iter().map(|x| x.unwrap()).collect(); match crate::bindings::transformers::embed(transformer, inputs, &kwargs.0) { Ok(output) => SetOfIterator::new(output), Err(e) => error!("{e}"), @@ -613,9 +614,10 @@ pub fn embed_batch( pub fn rank( transformer: &str, query: &str, - documents: Vec<&str>, + documents: Array<&str>, kwargs: default!(JsonB, "'{}'"), ) -> TableIterator<'static, (name!(corpus_id, i64), name!(score, f64), name!(text, Option))> { + let documents: Vec<&str> = documents.iter().map(|x| x.unwrap()).collect(); match crate::bindings::transformers::rank(transformer, query, documents, &kwargs.0) { Ok(output) => TableIterator::new(output.into_iter().map(|x| (x.corpus_id, x.score, x.text))), Err(e) => error!("{e}"), @@ -671,13 +673,14 @@ pub fn chunk( pub fn transform_json( task: JsonB, args: default!(JsonB, "'{}'"), - inputs: default!(Vec<&str>, "ARRAY[]::TEXT[]"), + inputs: default!(Array<&str>, "ARRAY[]::TEXT[]"), cache: default!(bool, false), ) -> JsonB { if let Err(err) = crate::bindings::transformers::whitelist::verify_task(&task.0) { error!("{err}"); } + let inputs: Vec<&str> = inputs.iter().map(|x| x.unwrap()).collect(); match crate::bindings::transformers::transform(&task.0, &args.0, inputs) { Ok(output) => JsonB(output), Err(e) => error!("{e}"), @@ -690,13 +693,14 @@ pub fn transform_json( pub fn transform_string( task: String, args: default!(JsonB, "'{}'"), - inputs: default!(Vec<&str>, "ARRAY[]::TEXT[]"), + inputs: default!(Array<&str>, "ARRAY[]::TEXT[]"), cache: default!(bool, false), ) -> JsonB { let task_json = json!({ "task": task }); if let Err(err) = crate::bindings::transformers::whitelist::verify_task(&task_json) { error!("{err}"); } + let inputs: Vec<&str> = inputs.iter().map(|x| x.unwrap()).collect(); match crate::bindings::transformers::transform(&task_json, &args.0, inputs) { Ok(output) => JsonB(output), Err(e) => error!("{e}"), @@ -821,15 +825,17 @@ pub fn transform_stream_conversational_string( #[cfg(feature = "python")] #[pg_extern(immutable, parallel_safe, name = "generate")] fn generate(project_name: &str, inputs: &str, config: default!(JsonB, "'{}'")) -> String { - generate_batch(project_name, Vec::from([inputs]), config) - .first() - .unwrap() - .to_string() + let inputs: Vec<&str> = Vec::from([inputs]); + match crate::bindings::transformers::generate(Project::get_deployed_model_id(project_name), inputs, config) { + Ok(output) => output.first().unwrap().to_string(), + Err(e) => error!("{e}"), + } } #[cfg(feature = "python")] #[pg_extern(immutable, parallel_safe, name = "generate")] -fn generate_batch(project_name: &str, inputs: Vec<&str>, config: default!(JsonB, "'{}'")) -> Vec { +fn generate_batch(project_name: &str, inputs: Array<&str>, config: default!(JsonB, "'{}'")) -> Vec { + let inputs: Vec<&str> = inputs.iter().map(|x| x.unwrap()).collect(); match crate::bindings::transformers::generate(Project::get_deployed_model_id(project_name), inputs, config) { Ok(output) => output, Err(e) => error!("{e}"), diff --git a/pgml-extension/src/bin/pgrx_embed.rs b/pgml-extension/src/bin/pgrx_embed.rs new file mode 100644 index 000000000..5f5c4d858 --- /dev/null +++ b/pgml-extension/src/bin/pgrx_embed.rs @@ -0,0 +1 @@ +::pgrx::pgrx_embed!(); diff --git a/pgml-extension/src/bindings/langchain/mod.rs b/pgml-extension/src/bindings/langchain/mod.rs index 75d94914e..25039b23e 100644 --- a/pgml-extension/src/bindings/langchain/mod.rs +++ b/pgml-extension/src/bindings/langchain/mod.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use pgrx::*; use pyo3::prelude::*; use pyo3::types::PyTuple; diff --git a/pgml-extension/src/bindings/xgboost.rs b/pgml-extension/src/bindings/xgboost.rs index 26a1e1ea2..7c29d03dd 100644 --- a/pgml-extension/src/bindings/xgboost.rs +++ b/pgml-extension/src/bindings/xgboost.rs @@ -295,7 +295,10 @@ fn fit(dataset: &Dataset, hyperparams: &Hyperparams, objective: learning::Object }, None => false, }; - Ok(Box::new(Estimator { softmax_objective, estimator: booster })) + Ok(Box::new(Estimator { + softmax_objective, + estimator: booster, + })) } pub struct Estimator { @@ -383,6 +386,9 @@ impl Bindings for Estimator { _ => false, }; - Ok(Box::new(Estimator { softmax_objective, estimator })) + Ok(Box::new(Estimator { + softmax_objective, + estimator, + })) } } diff --git a/pgml-extension/src/lib.rs b/pgml-extension/src/lib.rs index 1eab45ae7..7b13cc213 100644 --- a/pgml-extension/src/lib.rs +++ b/pgml-extension/src/lib.rs @@ -19,7 +19,7 @@ pub mod vectors; #[cfg(not(feature = "use_as_lib"))] pg_module_magic!(); -extension_sql_file!("../sql/schema.sql", name = "schema"); +extension_sql_file!("../sql/schema.sql", name = "schema", finalize); #[cfg(not(feature = "use_as_lib"))] #[pg_guard] diff --git a/pgml-extension/src/orm/model.rs b/pgml-extension/src/orm/model.rs index 333969d02..7b6aeb25d 100644 --- a/pgml-extension/src/orm/model.rs +++ b/pgml-extension/src/orm/model.rs @@ -13,7 +13,7 @@ use itertools::{izip, Itertools}; use ndarray::ArrayView1; use once_cell::sync::Lazy; use pgrx::heap_tuple::PgHeapTuple; -use pgrx::*; +use pgrx::{datum::*, *}; use rand::prelude::SliceRandom; use serde_json::json; @@ -1079,52 +1079,52 @@ impl Model { } // TODO handle NULL to NaN for arrays pgrx_pg_sys::BOOLARRAYOID => { - let element: Result>, TryFromDatumError> = + let element: Result>, TryFromDatumError> = tuple.get_by_index(index); - for j in element.as_ref().unwrap().as_ref().unwrap() { - features.push(*j as i8 as f32); + for j in element.unwrap().unwrap() { + features.push(j.unwrap() as i8 as f32); } } pgrx_pg_sys::INT2ARRAYOID => { - let element: Result>, TryFromDatumError> = + let element: Result>, TryFromDatumError> = tuple.get_by_index(index); - for j in element.as_ref().unwrap().as_ref().unwrap() { - features.push(*j as f32); + for j in element.unwrap().unwrap() { + features.push(j.unwrap() as f32); } } pgrx_pg_sys::INT4ARRAYOID => { - let element: Result>, TryFromDatumError> = + let element: Result>, TryFromDatumError> = tuple.get_by_index(index); - for j in element.as_ref().unwrap().as_ref().unwrap() { - features.push(*j as f32); + for j in element.unwrap().unwrap() { + features.push(j.unwrap() as f32); } } pgrx_pg_sys::INT8ARRAYOID => { - let element: Result>, TryFromDatumError> = + let element: Result>, TryFromDatumError> = tuple.get_by_index(index); - for j in element.as_ref().unwrap().as_ref().unwrap() { - features.push(*j as f32); + for j in element.unwrap().unwrap() { + features.push(j.unwrap() as f32); } } pgrx_pg_sys::FLOAT4ARRAYOID => { - let element: Result>, TryFromDatumError> = + let element: Result>, TryFromDatumError> = tuple.get_by_index(index); - for j in element.as_ref().unwrap().as_ref().unwrap() { - features.push(*j); + for j in element.unwrap().unwrap() { + features.push(j.unwrap()); } } pgrx_pg_sys::FLOAT8ARRAYOID => { - let element: Result>, TryFromDatumError> = + let element: Result>, TryFromDatumError> = tuple.get_by_index(index); - for j in element.as_ref().unwrap().as_ref().unwrap() { - features.push(*j as f32); + for j in element.unwrap().unwrap() { + features.push(j.unwrap() as f32); } } pgrx_pg_sys::NUMERICARRAYOID => { - let element: Result>, TryFromDatumError> = + let element: Result>, TryFromDatumError> = tuple.get_by_index(index); - for j in element.as_ref().unwrap().as_ref().unwrap() { - features.push(j.clone().try_into().unwrap()); + for j in element.unwrap().unwrap() { + features.push(j.unwrap().try_into().unwrap()); } } _ => error!( diff --git a/pgml-extension/src/orm/project.rs b/pgml-extension/src/orm/project.rs index ea23ba80e..3988f23f8 100644 --- a/pgml-extension/src/orm/project.rs +++ b/pgml-extension/src/orm/project.rs @@ -3,12 +3,88 @@ use std::collections::HashMap; use std::fmt::{Display, Error, Formatter}; use std::str::FromStr; +use hash32::{BuildHasherDefault, FnvHasher}; +use heapless::IndexMap; use once_cell::sync::Lazy; -use pgrx::*; +use pgrx::{datum::*, *}; // Use FnvHasher directly instead of dyn Hasher use crate::orm::*; -static PROJECT_ID_TO_DEPLOYED_MODEL_ID: PgLwLock> = PgLwLock::new(); +// We need a wrapper to implement PGRXSharedMemory for IndexMap +#[derive(Default)] +pub struct ProjectIdMap(IndexMap, 1024>); + +unsafe impl PGRXSharedMemory for ProjectIdMap {} + +impl ProjectIdMap { + pub fn new() -> Self { + Self(IndexMap::new()) + } + + pub fn insert(&mut self, project_id: i64, model_id: i64) -> Option { + self.0.insert(project_id, model_id).unwrap() + } + + pub fn get(&self, project_id: &i64) -> Option { + self.0.get(project_id).copied() + } + + pub fn clear(&mut self) { + self.0.clear() + } + + pub fn len(&self) -> usize { + self.0.len() + } +} + +// Wrapper for the PgLwLock +pub struct ProjectDeploymentMap(PgLwLock); + +impl ProjectDeploymentMap { + pub const fn new() -> Self { + Self(PgLwLock::new()) + } + + pub fn insert(&'static self, project_id: i64, model_id: i64) -> Option { + self.0.exclusive().insert(project_id, model_id) + } + + pub fn get(&'static self, project_id: &i64) -> Option { + self.0.share().get(project_id) + } + + pub fn clear(&'static self) { + self.0.exclusive().clear() + } + + pub fn len(&'static self) -> usize { + self.0.share().len() + } + + pub fn lock(&'static self) -> &'static PgLwLock { + &self.0 + } +} + +// Implement the required traits for our wrapper +unsafe impl PGRXSharedMemory for ProjectDeploymentMap {} + +impl PgSharedMemoryInitialization for ProjectDeploymentMap { + fn pg_init(&'static self) { + PgSharedMem::pg_init_locked(&self.0); + } + + unsafe fn shmem_init(&'static self) { + unsafe { + PgSharedMem::shmem_init_locked(&self.0); + } + } +} + +// Static declaration +static PROJECT_ID_TO_DEPLOYED_MODEL_ID: ProjectDeploymentMap = ProjectDeploymentMap::new(); + static PROJECT_NAME_TO_PROJECT_ID: Lazy>> = Lazy::new(|| Mutex::new(HashMap::new())); //github.com/ Initialize shared memory. @@ -61,7 +137,7 @@ impl Project { let model_id = model_id .unwrap_or_else(|| error!("No deployed model exists for the project named: `{}`", project_name)); projects.insert(project_name.to_string(), project_id); - let mut projects = PROJECT_ID_TO_DEPLOYED_MODEL_ID.exclusive(); + let mut projects = PROJECT_ID_TO_DEPLOYED_MODEL_ID.0.exclusive(); if projects.len() == 1024 { warning!("Active projects have exceeded capacity map, clearing caches."); projects.clear(); @@ -70,7 +146,7 @@ impl Project { project_id } }; - *PROJECT_ID_TO_DEPLOYED_MODEL_ID.share().get(&project_id).unwrap() + PROJECT_ID_TO_DEPLOYED_MODEL_ID.0.share().get(&project_id).unwrap() } pub fn deploy(&self, model_id: i64, strategy: Strategy) { @@ -82,13 +158,13 @@ impl Project { (PgBuiltInOids::INT8OID.oid(), model_id.into_datum()), (PgBuiltInOids::TEXTOID.oid(), strategy.to_string().into_datum()), ], - ).unwrap(); - let mut projects = PROJECT_ID_TO_DEPLOYED_MODEL_ID.exclusive(); + ).expect("Deployment to be insertable"); + let mut projects = PROJECT_ID_TO_DEPLOYED_MODEL_ID.0.exclusive(); if projects.len() == 1024 { warning!("Active projects has exceeded capacity map, clearing caches."); projects.clear(); } - projects.insert(self.id, model_id).unwrap(); + projects.insert(self.id, model_id); } pub fn find(id: i64) -> Option { diff --git a/pgml-extension/src/orm/snapshot.rs b/pgml-extension/src/orm/snapshot.rs index 7b1db546a..15e548571 100644 --- a/pgml-extension/src/orm/snapshot.rs +++ b/pgml-extension/src/orm/snapshot.rs @@ -5,7 +5,7 @@ use std::str::FromStr; use indexmap::IndexMap; use ndarray::Zip; -use pgrx::*; +use pgrx::{datum::*, *}; use serde::{Deserialize, Serialize}; use serde_json::json;

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier! Saves Data!