Skip to content

Commit 543873f

Browse files
edenhilledenhill
committed
Travis: restructure wheel building
1 parent 9764f82 commit 543873f

File tree

5 files changed

+210
-31
lines changed

5 files changed

+210
-31
lines changed

‎.travis.yml

Lines changed: 62 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
env:
22
global:
3-
- LIBRDKAFKA_VERSION=v1.6.0-RC1
3+
# Note: nuget package version has no v-prefix
4+
- LIBRDKAFKA_VERSION=v1.6.0-RC2
5+
46
jobs:
57
include:
68
- name: "Source package verification with Python 2.7 (Linux)"
@@ -10,65 +12,94 @@ jobs:
1012
python: "2.7"
1113
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
1214
services: docker
15+
1316
- name: "Source package verification with Python 3.6 (Linux)"
1417
os: linux
1518
language: python
1619
dist: xenial
1720
python: "3.6"
1821
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
1922
services: docker
23+
2024
- name: "Source package verification with Python 2.7 (OSX)"
2125
os: osx
2226
python: "2.7"
2327
env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" INTERPRETER_VERSION="2.7.17"
28+
2429
- name: "Source package verification with Python 3.6 (OSX) +docs"
2530
os: osx
2631
python: "3.6"
2732
env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" MK_DOCS="y" INTERPRETER_VERSION="3.6.5"
28-
- name: "cibuildwheel (OSX)"
29-
os: osx
30-
env: CIBW_BEFORE_BUILD="tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp" CFLAGS="-Itmp/include" LDFLAGS="-Ltmp/lib" INTERPRETER_VERSION="2.7.17"
31-
- name: "cibuildwheel (manylinux)"
32-
os: linux
33-
dist: xenial
34-
env:
35-
- CIBW_BEFORE_BUILD="tools/prepare-cibuildwheel-linux.sh ${LIBRDKAFKA_VERSION}"
36-
- PYTHON_CONFIGURE_OPTS="--enable-unicode=ucs4 --with-wide-unicode"
37-
- CIBW_MANYLINUX_X86_64_IMAGE="manylinux2010"
38-
- CIBW_MANYLINUX_I686_IMAGE="manylinux2010"
39-
language: python
40-
python: "2.7"
33+
34+
- name: "Wheels: Windows x64"
35+
if: tag is present
36+
os: windows
37+
language: shell
38+
env: BUILD_WHEELS=1
39+
before_install:
40+
- choco install python --version 3.8.0
41+
- export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
42+
# make sure it's on PATH as 'python3'
43+
- ln -s /c/Python38/python.exe /c/Python38/python3.exe
44+
install:
45+
- bash tools/wheels/install-librdkafka.sh $LIBRDKAFKA_VERSION dest
46+
script:
47+
- tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
48+
49+
- name: "Wheels: Windows x86"
50+
if: tag is present
51+
os: windows
52+
language: shell
53+
env: BUILD_WHEELS=1
54+
before_install:
55+
- choco install python --version 3.8.0
56+
- export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
57+
# make sure it's on PATH as 'python3'
58+
- ln -s /c/Python38/python.exe /c/Python38/python3.exe
59+
install:
60+
- bash tools/wheels/install-librdkafka.sh $LIBRDKAFKA_VERSION dest
61+
script:
62+
- tools/wheels/build-wheels.bat x86 win32 dest wheelhouse
63+
64+
- name: "Wheels: Linux x64"
65+
if: tag is present
4166
services: docker
67+
env: BUILD_WHEELS=1
68+
script: tools/wheels/build-wheels.sh $LIBRDKAFKA_VERSION wheelhouse
69+
70+
- name: "MacOSX x64"
71+
if: tag is present
72+
os: osx
73+
language: shell
74+
env: BUILD_WHEELS=1
75+
script: tools/wheels/build-wheels.sh $LIBRDKAFKA_VERSION wheelhouse
4276

43-
# See https://cibuildwheel.readthedocs.io/en/latest/options/ for CIBW* vars
4477

4578
# Install test dependencies unconditionally
4679
# Travis OSX envs requires some setup; see tools/prepare-osx.sh
4780
# Install cibuildwheel if this is a tagged PR
4881
before_install:
49-
- if [[ $TRAVIS_OS_NAME == "osx" ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
50-
- if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then pip install -U -r tools/requirements-manylinux.txt --force-reinstall ; fi
82+
- if [[ $TRAVIS_OS_NAME == "osx" && $BUILD_WHEELS != 1 ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
5183

52-
# Install interceptors
53-
# Install librdkafka if not CIBW_BEFORE_BUILD
54-
# Install confluent_kafka[avro] if not CIBW_BEFORE_BUILD
5584
install:
85+
# Install interceptors
5686
- tools/install-interceptors.sh
57-
- pip install -r tests/requirements.txt
58-
- flake8
87+
- if [[ $BUILD_WHEELS != 1 ]]; then pip install -r tests/requirements.txt ; fi
5988
- if [[ $MK_DOCS == y ]]; then pip install -r docs/requirements.txt; fi
60-
- if [[ -z $CIBW_BEFORE_BUILD ]]; then pip install -U protobuf && tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build && pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf] ; fi
89+
# Install librdkafka and confluent_kafka[avro] if not building wheels
90+
- if [[ $BUILD_WHEELS != 1 ]]; then pip install -U protobuf && tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build ; fi
91+
6192

62-
# Build wheels
63-
# Make plugins available for tests
64-
# Execute tests if not CIBW_BEFORE_BUILD [osx, linux]
65-
# Execute integration tests if CIBW_BEFORE_BUILD
66-
# Build docs if MK_DOCS
93+
94+
# Note: Will not be run for wheel builds.
6795
script:
68-
- if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then cibuildwheel --output-dir wheelhouse1 && tools/fixup-wheels.sh wheelhouse1 wheelhouse; fi
96+
- flake8
97+
# Build package
98+
- pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf] ; fi
6999
- ldd staging/libs/* || otool -L staging/libs/* || true
70-
- if [[ -z $CIBW_BEFORE_BUILD && $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi
71-
- if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == osx && -n $CIBW_BEFORE_BUILD ]]; then tools/test-wheels.sh wheelhouse; fi
100+
# Run tests
101+
- if [[ $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi
102+
# Build docs
72103
- if [[ $MK_DOCS == y ]]; then make docs; fi
73104

74105
deploy:

‎tools/wheels/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Wheel building scripts

‎tools/wheels/build-wheels.bat

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
@echo off
2+
SETLOCAL ENABLEEXTENSIONS
3+
4+
rem x86 or x64
5+
set ARCH=%1
6+
rem win32 or win_amd64
7+
set BW_ARCH=%2
8+
rem librdkafka install destdir (relative path)
9+
set DEST=%3
10+
rem wheelhouse output dir
11+
set WHEELHOUSE=%4
12+
13+
if [%WHEELHOUSE%]==[] goto usage
14+
echo on
15+
16+
set CIBW_BUILD=cp27-%BW_ARCH% cp36-%BW_ARCH% cp37-%BW_ARCH% cp38-%BW_ARCH% cp39-%BW_WARCH%
17+
set CIBW_BEFORE_BUILD=python -m pip install delvewheel==0.0.6
18+
set CIBW_TEST_COMMAND=python {project}\test.py
19+
rem set CIBW_BUILD_VERBOSITY=3
20+
set INCLUDE_DIRS=%cd%\%DEST%\build\native\include
21+
set DLL_DIR=%cd%\%DEST%\runtimes\win-%ARCH%\native
22+
set LIB_DIRS=%cd%\%DEST%\build\native\lib\win\%ARCH%\win-%ARCH%-Release\v120
23+
set CIBW_REPAIR_WHEEL_COMMAND=python -m delvewheel repair --add-path %DLL_DIR% -w {dest_dir} {wheel}
24+
25+
set PATH=%PATH%;c:\Program Files\Git\bin\
26+
27+
python -m pip install cibuildwheel==1.7.4
28+
29+
python -m cibuildwheel --output-dir %WHEELHOUSE% --platform windows
30+
31+
dir %WHEELHOUSE%
32+
33+
goto :eof
34+
35+
:usage
36+
@echo "Usage: %0 x86|x64 win32|win_amd64 wheelhouse-dir"
37+
exit /B 1

‎tools/wheels/build-wheels.sh

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
#
3+
#
4+
# Build wheels (on Linux or OSX) using cibuildwheel.
5+
#
6+
7+
this_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
8+
9+
10+
# Skip PyPy, old Python3 versions, and x86 builds.
11+
export CIBW_SKIP="pp* cp35-* *i686"
12+
export CIBW_TEST_COMMAND="python {project}/test.py"
13+
14+
15+
librdkafka_version=$1
16+
wheeldir=$2
17+
18+
if [[ -z $wheeldir ]]; then
19+
echo "Usage: $0 <librdkafka-nuget-version> <wheeldir>"
20+
exit 1
21+
fi
22+
23+
set -ex
24+
25+
[[ -d $wheeldir ]] || mkdir -p "$wheeldir"
26+
27+
28+
case $OSTYPE in
29+
linux*)
30+
os=linux
31+
# Need to set up env vars (in docker) so that setup.py
32+
# finds librdkafka.
33+
lib_dir=dest/runtimes/linux-x64/native
34+
export CIBW_ENVIRONMENT="INCLUDE_DIRS=dest/build/native/include LIB_DIRS=$lib_dir LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$PWD/$lib_dir"
35+
;;
36+
darwin*)
37+
os=macos
38+
# Need to set up env vars so that setup.py finds librdkafka.
39+
lib_dir=dest/runtimes/osx-x64/native
40+
export INCLUDE_DIRS="${PWD}/dest/build/native/include"
41+
export LIB_DIRS="${PWD}/$lib_dir"
42+
;;
43+
*)
44+
echo "$0: Unsupported OSTYPE $OSTYPE"
45+
exit 1
46+
;;
47+
esac
48+
49+
50+
$this_dir/install-librdkafka.sh $librdkafka_version dest
51+
52+
python3 -m pip install cibuildwheel==1.7.4
53+
54+
if [[ -z $TRAVIS ]]; then
55+
cibw_args="--platform $os"
56+
fi
57+
58+
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/$lib_dir python3 -m cibuildwheel --output-dir $wheeldir $cibw_args
59+
60+
ls $wheeldir
61+
62+
for f in $wheeldir/*whl ; do
63+
echo $f
64+
unzip -l $f
65+
done
66+

‎tools/wheels/install-librdkafka.sh

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
VER="$1"
6+
DEST="$2"
7+
8+
if [[ -z $DEST ]]; then
9+
echo "Usage: $0 <librdkafka-redist-version> <destdir>"
10+
exit 1
11+
fi
12+
13+
if [[ -f $DEST/build/native/include/librdkafka/rdkafka.h ]]; then
14+
echo "$0: librdkafka already installed in $DEST"
15+
exit 0
16+
fi
17+
18+
echo "$0: Installing lbirdkafka $VER to $DEST"
19+
[[ -d "$DEST" ]] || mkdir -p "$DEST"
20+
pushd "$DEST"
21+
22+
curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
23+
24+
unzip lrk$VER.zip
25+
26+
27+
if [[ $OSTYPE == linux* ]]; then
28+
# Linux
29+
30+
# Copy the librdkafka build with least dependencies to librdkafka.so.1
31+
cp -v runtimes/linux-x64/native/{centos6-librdkafka.so,librdkafka.so.1}
32+
ldd runtimes/linux-x64/native/librdkafka.so.1
33+
34+
elif [[ $OSTYPE == darwin* ]]; then
35+
# MacOS X
36+
37+
# Change the library's self-referencing name from
38+
# /Users/travis/.....somelocation/librdkafka.1.dylib to its local path.
39+
install_name_tool -id $PWD/runtimes/osx-x64/native/librdkafka.dylib runtimes/osx-x64/native/librdkafka.dylib
40+
41+
otool -L runtimes/osx-x64/native/librdkafka.dylib
42+
fi
43+
44+
popd

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy