From 16d32d82dc9350e3125261185293fe79fd7fc211 Mon Sep 17 00:00:00 2001 From: Sasasu Date: Fri, 18 Aug 2023 17:17:30 +0800 Subject: [PATCH 1/3] fix pgml conflicts with plpython pgml is not compatible with plpython, if using both pgml and plpython in the same session, postgresql will crash. minimum reproducible code: ```sql SELECT pgml.embed('intfloat/e5-small', 'hi mom'); create or replace function pyudf() returns int as $$ return 0 $$ language 'plpython3u'; ``` the call stack: ``` Stack trace of thread 161970: #0 0x00007efc1429edb8 PyImport_Import (libpython3.9.so.1.0 + 0x9edb8) #1 0x00007efc1429f125 PyImport_ImportModule (libpython3.9.so.1.0 + 0x9f125) #2 0x00007efb04b0f496 n/a (plpython3.so + 0x10496) #3 0x00007efb04b1039d plpython3_validator (plpython3.so + 0x1139d) #4 0x0000559d0cdbc5c2 OidFunctionCall1Coll (postgres + 0x6465c2) #5 0x0000559d0c9d68bb ProcedureCreate (postgres + 0x2608bb) #6 0x0000559d0ca5030c CreateFunction (postgres + 0x2da30c) #7 0x0000559d0ce1c730 n/a (postgres + 0x6a6730) #8 0x0000559d0cc5a030 standard_ProcessUtility (postgres + 0x4e4030) #9 0x0000559d0cc545ed n/a (postgres + 0x4de5ed) #10 0x0000559d0cc546e7 n/a (postgres + 0x4de6e7) #11 0x0000559d0cc54beb PortalRun (postgres + 0x4debeb) #12 0x0000559d0cc55249 n/a (postgres + 0x4df249) #13 0x0000559d0cc576f0 PostgresMain (postgres + 0x4e16f0) #14 0x0000559d0cbc3e9c n/a (postgres + 0x44de9c) #15 0x0000559d0cbc50aa PostmasterMain (postgres + 0x44f0aa) #16 0x0000559d0c8ce7d2 main (postgres + 0x1587d2) #17 0x00007efc18427cd0 n/a (libc.so.6 + 0x27cd0) #18 0x00007efc18427d8a __libc_start_main (libc.so.6 + 0x27d8a) #19 0x0000559d0c8cee15 _start (postgres + 0x158e15) ``` this is because PostgreSQL is using dlopen(RTLD_GLOBAL). this will parse some of symbols into the previous opened .so file, but the others will use a relative offset in pgml.so, and will cause a null-pointer crash. this commit hide all symbols except the UDF symbols (ends with `_wrapper`) and the magic symbols (`_PG_init` `Pg_magic_func`). so dlopen(RTLD_GLOBAL) will parse the symbols to the correct position. --- pgml-extension/.cargo/config | 6 +++--- pgml-extension/build.rs | 12 ++++++++++++ pgml-extension/ld.map | 8 ++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 pgml-extension/ld.map diff --git a/pgml-extension/.cargo/config b/pgml-extension/.cargo/config index 4739f795d..256675cdc 100644 --- a/pgml-extension/.cargo/config +++ b/pgml-extension/.cargo/config @@ -1,9 +1,9 @@ [build] # Postgres symbols won't be available until runtime -rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup"] +rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup,-fuse-ld=lld"] [target.x86_64-unknown-linux-gnu] -rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup,-fuse-ld=/usr/bin/mold"] +rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup,-fuse-ld=lld"] [target.aarch64-unknown-linux-gnu] -rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup,-fuse-ld=/usr/bin/mold"] +rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup,-fuse-ld=lld"] diff --git a/pgml-extension/build.rs b/pgml-extension/build.rs index eb180a21c..ae6761a6a 100644 --- a/pgml-extension/build.rs +++ b/pgml-extension/build.rs @@ -4,4 +4,16 @@ fn main() { println!("cargo:rustc-link-search=/opt/homebrew/opt/openblas/lib"); println!("cargo:rustc-link-search=/opt/homebrew/opt/libomp/lib"); } + + // PostgreSQL is using dlopen(RTLD_GLOBAL). this will parse some + // of symbols into the previous opened .so file, but the others will use a + // relative offset in pgml.so, and will cause a null-pointer crash. + // + // hid all symbol to avoid symbol conflicts. + // + // append mode (link-args) only works with clang ld (lld) + println!( + "cargo:link-args=-Wl,--version-script={}/ld.map", + std::env::current_dir().unwrap().to_string_lossy(), + ); } diff --git a/pgml-extension/ld.map b/pgml-extension/ld.map new file mode 100644 index 000000000..930b96e33 --- /dev/null +++ b/pgml-extension/ld.map @@ -0,0 +1,8 @@ +{ +global: + Pg_magic_func; + _PG_init; + *_wrapper; +local: + *; +}; From 441bd2a246815a553b855b89ec6e280b28d18916 Mon Sep 17 00:00:00 2001 From: Sasasu Date: Sun, 20 Aug 2023 21:00:57 +0800 Subject: [PATCH 2/3] only use lld to link postgresml --- pgml-extension/.cargo/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgml-extension/.cargo/config b/pgml-extension/.cargo/config index 256675cdc..4eb992743 100644 --- a/pgml-extension/.cargo/config +++ b/pgml-extension/.cargo/config @@ -1,6 +1,6 @@ [build] # Postgres symbols won't be available until runtime -rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup,-fuse-ld=lld"] +rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup"] [target.x86_64-unknown-linux-gnu] rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup,-fuse-ld=lld"] From 306a0a21d4fa0966171537623d720eaeae852424 Mon Sep 17 00:00:00 2001 From: Sasasu Date: Mon, 28 Aug 2023 10:14:35 +0800 Subject: [PATCH 3/3] also change CI and document --- .github/workflows/ubuntu-packages-and-docker-image.yml | 2 +- pgml-dashboard/content/docs/guides/setup/v2/installation.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml index 8d8e05f39..a60657e44 100644 --- a/.github/workflows/ubuntu-packages-and-docker-image.yml +++ b/.github/workflows/ubuntu-packages-and-docker-image.yml @@ -88,7 +88,7 @@ jobs: libpython3.10-dev \ python3.10-dev \ ruby \ - mold + lld curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem sudo gem install deb-s3-0.11.4.gem diff --git a/pgml-dashboard/content/docs/guides/setup/v2/installation.md b/pgml-dashboard/content/docs/guides/setup/v2/installation.md index 9fc753db0..dec066ed7 100644 --- a/pgml-dashboard/content/docs/guides/setup/v2/installation.md +++ b/pgml-dashboard/content/docs/guides/setup/v2/installation.md @@ -278,7 +278,7 @@ postgresql-server-dev-${POSTGRES_VERSION} python3 python3-pip libpython3 -mold +lld ``` ##### Rust pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy