Skip to content

Commit 9dc95ae

Browse files
martijnekenmartijneken
authored
Full Abseil + RE2 support (#173)
Add RE2 support + full Abseil Related cleanups: - Simpler Docker container - Single Makefile - Support for -flto - SDK proto upgrade to v26.1 - Updated build docs - More/independent examples - Remove protobuf CI check Signed-off-by: Martijn Stevenson <mstevenson@google.com>
1 parent 6b3dc93 commit 9dc95ae

20 files changed

+17981
-17848
lines changed

‎.github/workflows/cpp.yml

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -60,29 +60,6 @@ jobs:
6060
export PATH=$PATH:$(go env GOPATH)/bin
6161
addlicense -ignore="**/BUILD" -check .
6262
63-
protobuf:
64-
runs-on: ubuntu-22.04
65-
66-
steps:
67-
- uses: actions/checkout@v2
68-
69-
- name: Install protobuf v3.9.1
70-
run: |
71-
git clone https://github.com/protocolbuffers/protobuf
72-
cd protobuf
73-
git checkout v3.9.1
74-
./autogen.sh
75-
./configure
76-
make
77-
sudo make install
78-
sudo ldconfig
79-
80-
- name: Re-generate and verify protobuf artifacts
81-
run: |
82-
rm *.pb.{cc,h}
83-
make protobuf
84-
git diff --exit-code -G "(^[^ /])|(^\s+[^\*])" *.pb.{cc,h}
85-
8663
build:
8764
runs-on: ubuntu-22.04
8865

‎Dockerfile-sdk

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
FROM ubuntu:noble
22

3-
COPY ./sdk_container.sh /
4-
COPY ./build_wasm.sh /
3+
COPY *.sh /
54
COPY *.cc *.h *.js *.proto Makefile* /sdk/
65

76
RUN ./sdk_container.sh

‎Makefile

Lines changed: 50 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,57 @@
1-
CPP_API ?= .
1+
ifdef NO_CONTEXT
2+
CPP_CONTEXT_LIB =
3+
else
4+
CPP_CONTEXT_LIB = ${PROXY_WASM_CPP_SDK}/proxy_wasm_intrinsics.cc
5+
endif
26

3-
all: proxy_wasm_intrinsics.pb.h proxy_wasm_intrinsics_lite.pb.h struct_lite.pb.h ${CPP_API}/libprotobuf.a ${CPP_API}/libprotobuf-lite.a
7+
PROTOBUF ?= none
8+
ifeq ($(PROTOBUF), full)
9+
PROTO_DEPS := protobuf
10+
PROTO_OPTS := -DPROXY_WASM_PROTOBUF_FULL=1 \
11+
${PROXY_WASM_CPP_SDK}/proxy_wasm_intrinsics.pb.cc
12+
else ifeq ($(PROTOBUF), lite)
13+
PROTO_DEPS := protobuf-lite
14+
PROTO_OPTS := -DPROXY_WASM_PROTOBUF_LITE=1 \
15+
${PROXY_WASM_CPP_SDK}/proxy_wasm_intrinsics_lite.pb.cc \
16+
${PROXY_WASM_CPP_SDK}/struct_lite.pb.cc
17+
else
18+
PROTO_DEPS :=
19+
PROTO_OPTS :=
20+
endif
421

5-
protobuf: proxy_wasm_intrinsics.pb.h proxy_wasm_intrinsics_lite.pb.h struct_lite.pb.h
22+
# Provide a list of libraries that the wasm depends on (absl_*, re2, etc).
23+
WASM_DEPS ?=
624

7-
proxy_wasm_intrinsics.pb.h: proxy_wasm_intrinsics.proto
8-
protoc --cpp_out=. proxy_wasm_intrinsics.proto
25+
# Determine dependency link options.
26+
# NOTE: Strip out -pthread which RE2 claims to need...
27+
PKG_CONFIG ?= pkg-config
28+
PKG_CONFIG_PATH = ${EMSDK}/upstream/emscripten/cache/sysroot/lib/pkgconfig
29+
WASM_LIBS = $(shell $(PKG_CONFIG) $(WASM_DEPS) $(PROTO_DEPS) \
30+
--with-path=$(PKG_CONFIG_PATH) --libs | sed -e 's/-pthread //g')
931

10-
proxy_wasm_intrinsics_lite.pb.h struct_lite.pb.h: proxy_wasm_intrinsics_lite.proto
11-
protoc --cpp_out=. -I. proxy_wasm_intrinsics_lite.proto
12-
protoc --cpp_out=. struct_lite.proto
32+
debug-deps:
33+
# WASM_DEPS : ${WASM_DEPS}
34+
# WASM_LIBS : ${WASM_LIBS}
35+
# PROTO_DEPS: ${PROTO_DEPS}
36+
# PROTO_OPTS: ${PROTO_OPTS}
1337

14-
${CPP_API}/libprotobuf.a ${CPP_API}/libprotobuf-lite.a:
15-
rm -rf protobuf-wasm \
16-
&& git clone https://github.com/protocolbuffers/protobuf protobuf-wasm \
17-
&& cd protobuf-wasm \
18-
&& git checkout v3.9.1 \
19-
&& ./autogen.sh \
20-
&& emconfigure ./configure --disable-shared CXXFLAGS="-O3 -flto" \
21-
&& emmake make \
22-
&& cd .. \
23-
&& cp protobuf-wasm/src/.libs/libprotobuf-lite.a ${CPP_API}/libprotobuf-lite.a \
24-
&& cp protobuf-wasm/src/.libs/libprotobuf.a ${CPP_API}/libprotobuf.a
38+
# TODO(mpwarres): Add Emscripten stack/heap size params in PR#174.
39+
%.wasm %.wat: %.cc
40+
em++ --no-entry -sSTANDALONE_WASM -sEXPORTED_FUNCTIONS=_malloc \
41+
--std=c++17 -O3 -flto \
42+
--js-library ${PROXY_WASM_CPP_SDK}/proxy_wasm_intrinsics.js \
43+
-I${PROXY_WASM_CPP_SDK} \
44+
${CPP_CONTEXT_LIB} \
45+
${PROTO_OPTS} \
46+
${WASM_LIBS} \
47+
$*.cc -o $*.wasm
2548

2649
clean:
27-
rm -f proxy_wasm_intrinsics.pb.h proxy_wasm_intrinsics_lite.pb.h struct_lite.pb.h ${CPP_API}/libprotobuf.a ${CPP_API}/libprotobuf-lite.a
50+
rm *.wasm
51+
52+
# NOTE: How to regenerate .pb.h and .pb.cc files for a protobuf update:
53+
# - download + extract protobuf release (currently v26.1)
54+
# - regenerate:
55+
# ./bin/protoc --cpp_out=../ -I../ -Iinclude/google/protobuf/ ../struct_lite.proto
56+
# ./bin/protoc --cpp_out=../ -I../ -Iinclude/google/protobuf/ ../proxy_wasm_intrinsics_lite.proto
57+
# ./bin/protoc --cpp_out=../ -I../ -Iinclude/google/protobuf/ ../proxy_wasm_intrinsics.proto

‎Makefile.base

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎Makefile.base_lite

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎bazel/defs.bzl

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,11 @@ load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary")
1616
load("@rules_cc//cc:defs.bzl", "cc_binary")
1717

1818
def _optimized_wasm_cc_binary_transition_impl(settings, attr):
19-
# TODO(PiotrSikora): Add -flto to copts/linkopts when fixed in emsdk.
20-
# See: https://github.com/emscripten-core/emsdk/issues/971
19+
# Define STANDALONE_WASM at compile time as well as link time (below).
20+
# This fixes Abseil by not including Emscripten JS stacktraces + symbolization.
21+
# TODO(martijneken): Remove after Abseil stops using this define.
2122
return {
22-
"//command_line_option:copt": ["-O3"],
23+
"//command_line_option:copt": ["-O3", "-flto", "-DSTANDALONE_WASM"],
2324
"//command_line_option:cxxopt": [],
2425
"//command_line_option:linkopt": [],
2526
"//command_line_option:collect_code_coverage": False,
@@ -102,6 +103,8 @@ def proxy_wasm_cc_binary(
102103
)
103104

104105
wasm_cc_binary(
106+
standalone = True,
107+
threads = "off",
105108
name = "wasm_" + name,
106109
cc_target = ":proxy_wasm_" + name.rstrip(".wasm"),
107110
tags = tags + [

‎bazel/repositories.bzl

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,19 @@ def proxy_wasm_cpp_sdk_repositories():
3131
strip_prefix = "protobuf-3.17.3",
3232
url = "https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-all-3.17.3.tar.gz",
3333
)
34+
35+
maybe(
36+
http_archive,
37+
name = "com_google_absl",
38+
sha256 = "95e90be7c3643e658670e0dd3c1b27092349c34b632c6e795686355f67eca89f",
39+
strip_prefix = "abseil-cpp-20240722.0",
40+
urls = ["https://github.com/abseil/abseil-cpp/archive/20240722.0.zip"],
41+
)
42+
43+
maybe(
44+
http_archive,
45+
name = "com_google_re2",
46+
sha256 = "18cf85922e27fad3ed9c96a27733037da445f35eb1a2744c306a37c6d11e95c4",
47+
strip_prefix = "re2-2023-07-01",
48+
url = "https://github.com/google/re2/archive/2023-07-01.tar.gz",
49+
)

‎build_wasm.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@
1616

1717
source /root/emsdk/emsdk_env.sh
1818
export PATH=/usr/local/bin:$PATH
19-
make
19+
make "$@"

‎docs/building.md

Lines changed: 30 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# Proxy-Wasm C++ SDK Build Instructions
22

33
The C++ SDK has dependencies on specific versions of the C++ WebAssembly
4-
toolchain [Emscripten](https://emscripten.org) and the protobuf library,
5-
therefore use of a Docker image is recommended.
4+
toolchain [Emscripten](https://emscripten.org). Use of a Docker image is
5+
recommended to achieve repeatable builds and save work.
66

77
## Docker
88

@@ -11,7 +11,7 @@ A Dockerfile for the C++ SDK is provided in [Dockerfile-sdk](../Dockerfile-sdk).
1111
It can built in this repository's root directory by:
1212

1313
```bash
14-
docker build -t wasmsdk:v2 -f Dockerfile-sdk .
14+
docker build -t wasmsdk:v3 -f Dockerfile-sdk .
1515
```
1616

1717
The docker image can be used for compiling C++ plugin code into Wasm modules.
@@ -23,12 +23,10 @@ Create a directory with your source files and a Makefile:
2323
```makefile
2424
PROXY_WASM_CPP_SDK=/sdk
2525

26-
all: myproject.wasm
27-
28-
include ${PROXY_WASM_CPP_SDK}/Makefile.base_lite
26+
include ${PROXY_WASM_CPP_SDK}/Makefile
2927
```
3028

31-
Source file (myproject.cc):
29+
Create a C++ source file (myproject.cc):
3230

3331
```c++
3432
#include <string>
@@ -57,62 +55,40 @@ void ExampleContext::onDone() { logInfo("onDone " + std::to_string(id())); }
5755

5856
### Compiling with the Docker build image
5957

60-
Run docker:
61-
62-
```bash
63-
docker run -v $PWD:/work -w /work wasmsdk:v2 /build_wasm.sh
64-
```
65-
66-
### Caching the standard libraries
67-
68-
The first time that emscripten runs it will generate the standard libraries. To
69-
cache these in the docker image, after the first successful compilation (e.g
70-
myproject.cc above), commit the image with the standard libraries:
58+
Run docker to build wasm, using a target with a .wasm suffix:
7159

7260
```bash
73-
docker commit `docker ps -l | grep wasmsdk:v2 | awk '{print $1}'` wasmsdk:v2
61+
docker run -v $PWD:/work -w /work wasmsdk:v3 /build_wasm.sh myproject.wasm
7462
```
7563

76-
This will save time on subsequent compiles.
64+
You can specify wasm dependencies via these Makefile variables:
7765

78-
### Using Abseil from the Docker image
66+
- PROTOBUF = {full, lite, none}
67+
- WASM_DEPS = list of libraries
7968

80-
Abseil (optionally) is built in /root/abseil and can be used. Note that the
81-
Abseil containers (e.g. `absl::flat_hash_set`) exercise many syscalls which are
82-
not supported. Consequentially individual files should be pulled in which are
83-
relatively self contained (e.g. `strings`). Example customized Makefile:
69+
For example:
8470

8571
```makefile
8672
PROXY_WASM_CPP_SDK=/sdk
87-
CPP_API:=${PROXY_WASM_CPP_SDK}
88-
CPP_CONTEXT_LIB = ${CPP_API}/proxy_wasm_intrinsics.cc
89-
ABSL = /root/abseil-cpp
90-
ABSL_CPP = ${ABSL}/absl/strings/str_cat.cc ${ABSL}/absl/strings/str_split.cc ${ABSL}/absl/strings/numbers.cc ${ABSL}/absl/strings/ascii.cc
9173

92-
all: plugin.wasm
74+
PROTOBUF=lite
75+
WASM_DEPS=re2 absl_strings
9376

94-
%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
95-
ls /root
96-
em++ --no-entry -s EXPORTED_FUNCTIONS=['_malloc'] --std=c++17 -O3 -flto -I${CPP_API} -I${CPP_API}/google/protobuf -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js ${ABSL_CPP} $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a -o $*.wasm
77+
include ${PROXY_WASM_CPP_SDK}/Makefile
9778
```
9879

99-
Precompiled Abseil libraries are also available, so the above can also be done
100-
as:
101-
102-
```makefile
103-
PROXY_WASM_CPP_SDK=/sdk
104-
CPP_API:=${PROXY_WASM_CPP_SDK}
105-
CPP_CONTEXT_LIB = ${CPP_API}/proxy_wasm_intrinsics.cc
106-
ABSL = /root/abseil-cpp
107-
ABSL_LIBS = ${ABSL}/absl/strings/libabsl_strings.a ${ABSL}/absl/strings/libabsl_strings_internal.a ${ABSL}/absl/strings/libabsl_str_format_internal.a
80+
### Caching the standard libraries
10881

109-
all: plugin.wasm
82+
The first time that emscripten runs it will generate the standard libraries. To
83+
cache these in the docker image, after the first successful compilation (e.g
84+
myproject.cc above), commit the image with the standard libraries:
11085

111-
%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
112-
ls /root
113-
em++ --no-entry -s EXPORTED_FUNCTIONS=['_malloc'] --std=c++17 -O3 -flto -I${CPP_API} -I${CPP_API}/google/protobuf -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a ${ABSL_LIBS} -o $*.wasm
86+
```bash
87+
docker commit `docker ps -l | grep wasmsdk:v3 | awk '{print $1}'` wasmsdk:v3
11488
```
11589

90+
This will save time on subsequent compiles.
91+
11692
### Ownership of the resulting .wasm files
11793

11894
The compiled files may be owned by root. To chown them, add the follow lines to
@@ -124,48 +100,32 @@ PROXY_WASM_CPP_SDK=/sdk
124100
all: myproject.wasm
125101
chown ${uid}.${gid} $^
126102

127-
include ${PROXY_WASM_CPP_SDK}/Makefile.base_lite
103+
include ${PROXY_WASM_CPP_SDK}/Makefile
128104
```
129105

130106
Invocation file (e.g. build.sh):
131107

132108
```bash
133109
#!/bin/bash
134-
docker run -e uid="$(id -u)" -e gid="$(id -g)" -v $PWD:/work -w /work wasmsdk:v2 /build_wasm.sh
110+
docker run -e uid="$(id -u)" -e gid="$(id -g)" -v $PWD:/work -w /work wasmsdk:v3 /build_wasm.sh
135111
```
136112

137113
## Dependencies for building Wasm modules:
138114

139115
If you do not wish to use the Docker image, the dependencies can be installed by
140-
script (sdk\_container.sh), or by hand.
141-
142-
### protobuf v3.9.1
143-
144-
You must install the version of protobuf on your build system that matches the
145-
libprotobuf.a files (without any patches) so that the generated code matches the
146-
.a library. Currently this is based on tag v3.9.1 of
147-
https://github.com/protocolbuffers/protobuf.
148-
149-
```bash
150-
git clone https://github.com/protocolbuffers/protobuf
151-
cd protobuf
152-
git checkout v3.9.1
153-
git submodule update --init --recursive
154-
./autogen.sh
155-
./configure
156-
make
157-
make check
158-
sudo make install
159-
```
116+
script (sdk\_container.sh), or by hand. First you need Emscripten to build
117+
everything else. Then you can build other wasm-compatible libraries, such as
118+
protobuf, abseil, and RE2.
160119

161120
### Emscripten
162121

122+
Version 3.1.67 is known to work:
123+
163124
```bash
164125
git clone https://github.com/emscripten-core/emsdk.git
165126
cd emsdk
166127
git checkout 3.1.67
167-
168-
./emsdk install 3.1.67
128+
./emsdk install --shallow 3.1.67
169129
./emsdk activate 3.1.67
170130

171131
source ./emsdk_env.sh
@@ -178,26 +138,3 @@ It is possible later versions will work, e.g.
178138
./emsdk install latest
179139
./emsdk activate latest
180140
```
181-
182-
However 3.1.67 is known to work.
183-
184-
### Rebuilding the libprotobuf.a files
185-
186-
To build the protobuf static libraries for use in your proxy-wasm wasm module,
187-
if you need them, you may use Emscripten in the same way sdk\_container.sh does
188-
it:
189-
190-
```bash
191-
git clone https://github.com/protocolbuffers/protobuf protobuf-wasm
192-
cd protobuf-wasm
193-
git checkout v3.9.1
194-
git submodule update --init --recursive
195-
./autogen.sh
196-
emconfigure ./configure --disable-shared CXXFLAGS="-O3 -flto"
197-
emmake make
198-
cd ..
199-
cp protobuf-wasm/src/.libs/libprotobuf-lite.a ${CPP_API}/libprotobuf-lite.a
200-
cp protobuf-wasm/src/.libs/libprotobuf.a ${CPP_API}/libprotobuf.a
201-
```
202-
203-
Note: ensure /usr/local/bin is in your path.

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy