Skip to content

Commit 8a35678

Browse files
edenhilledenhill
committed
Travis: restructure wheel building
1 parent 9764f82 commit 8a35678

File tree

5 files changed

+209
-31
lines changed

5 files changed

+209
-31
lines changed

‎.travis.yml

Lines changed: 61 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
env:
22
global:
3-
- LIBRDKAFKA_VERSION=v1.6.0-RC1
3+
- LIBRDKAFKA_VERSION=v1.6.0-RC2
4+
45
jobs:
56
include:
67
- name: "Source package verification with Python 2.7 (Linux)"
@@ -10,65 +11,94 @@ jobs:
1011
python: "2.7"
1112
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
1213
services: docker
14+
1315
- name: "Source package verification with Python 3.6 (Linux)"
1416
os: linux
1517
language: python
1618
dist: xenial
1719
python: "3.6"
1820
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
1921
services: docker
22+
2023
- name: "Source package verification with Python 2.7 (OSX)"
2124
os: osx
2225
python: "2.7"
2326
env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" INTERPRETER_VERSION="2.7.17"
27+
2428
- name: "Source package verification with Python 3.6 (OSX) +docs"
2529
os: osx
2630
python: "3.6"
2731
env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" MK_DOCS="y" INTERPRETER_VERSION="3.6.5"
28-
- name: "cibuildwheel (OSX)"
29-
os: osx
30-
env: CIBW_BEFORE_BUILD="tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp" CFLAGS="-Itmp/include" LDFLAGS="-Ltmp/lib" INTERPRETER_VERSION="2.7.17"
31-
- name: "cibuildwheel (manylinux)"
32-
os: linux
33-
dist: xenial
34-
env:
35-
- CIBW_BEFORE_BUILD="tools/prepare-cibuildwheel-linux.sh ${LIBRDKAFKA_VERSION}"
36-
- PYTHON_CONFIGURE_OPTS="--enable-unicode=ucs4 --with-wide-unicode"
37-
- CIBW_MANYLINUX_X86_64_IMAGE="manylinux2010"
38-
- CIBW_MANYLINUX_I686_IMAGE="manylinux2010"
39-
language: python
40-
python: "2.7"
32+
33+
- name: "Wheels: Windows x64"
34+
if: tag is present
35+
os: windows
36+
language: shell
37+
env: BUILD_WHEELS=1
38+
before_install:
39+
- choco install python --version 3.8.0
40+
- export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
41+
# make sure it's on PATH as 'python3'
42+
- ln -s /c/Python38/python.exe /c/Python38/python3.exe
43+
install:
44+
- bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
45+
script:
46+
- tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
47+
48+
- name: "Wheels: Windows x86"
49+
if: tag is present
50+
os: windows
51+
language: shell
52+
env: BUILD_WHEELS=1
53+
before_install:
54+
- choco install python --version 3.8.0
55+
- export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
56+
# make sure it's on PATH as 'python3'
57+
- ln -s /c/Python38/python.exe /c/Python38/python3.exe
58+
install:
59+
- bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
60+
script:
61+
- tools/wheels/build-wheels.bat x86 win32 dest wheelhouse
62+
63+
- name: "Wheels: Linux x64"
64+
if: tag is present
4165
services: docker
66+
env: BUILD_WHEELS=1
67+
script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
68+
69+
- name: "Wheels: MacOSX x64"
70+
if: tag is present
71+
os: osx
72+
language: shell
73+
env: BUILD_WHEELS=1
74+
script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
4275

43-
# See https://cibuildwheel.readthedocs.io/en/latest/options/ for CIBW* vars
4476

4577
# Install test dependencies unconditionally
4678
# Travis OSX envs requires some setup; see tools/prepare-osx.sh
4779
# Install cibuildwheel if this is a tagged PR
4880
before_install:
49-
- if [[ $TRAVIS_OS_NAME == "osx" ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
50-
- if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then pip install -U -r tools/requirements-manylinux.txt --force-reinstall ; fi
81+
- if [[ $TRAVIS_OS_NAME == "osx" && $BUILD_WHEELS != 1 ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
5182

52-
# Install interceptors
53-
# Install librdkafka if not CIBW_BEFORE_BUILD
54-
# Install confluent_kafka[avro] if not CIBW_BEFORE_BUILD
5583
install:
84+
# Install interceptors
5685
- tools/install-interceptors.sh
57-
- pip install -r tests/requirements.txt
58-
- flake8
86+
- if [[ $BUILD_WHEELS != 1 ]]; then pip install -r tests/requirements.txt ; fi
5987
- if [[ $MK_DOCS == y ]]; then pip install -r docs/requirements.txt; fi
60-
- if [[ -z $CIBW_BEFORE_BUILD ]]; then pip install -U protobuf && tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build && pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf] ; fi
88+
# Install librdkafka and confluent_kafka[avro] if not building wheels
89+
- if [[ $BUILD_WHEELS != 1 ]]; then pip install -U protobuf && tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build ; fi
90+
6191

62-
# Build wheels
63-
# Make plugins available for tests
64-
# Execute tests if not CIBW_BEFORE_BUILD [osx, linux]
65-
# Execute integration tests if CIBW_BEFORE_BUILD
66-
# Build docs if MK_DOCS
92+
93+
# Note: Will not be run for wheel builds.
6794
script:
68-
- if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then cibuildwheel --output-dir wheelhouse1 && tools/fixup-wheels.sh wheelhouse1 wheelhouse; fi
95+
- flake8
96+
# Build package
97+
- pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf]
6998
- ldd staging/libs/* || otool -L staging/libs/* || true
70-
- if [[ -z $CIBW_BEFORE_BUILD && $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi
71-
- if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == osx && -n $CIBW_BEFORE_BUILD ]]; then tools/test-wheels.sh wheelhouse; fi
99+
# Run tests
100+
- if [[ $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi
101+
# Build docs
72102
- if [[ $MK_DOCS == y ]]; then make docs; fi
73103

74104
deploy:

‎tools/wheels/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Wheel building scripts

‎tools/wheels/build-wheels.bat

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
@echo off
2+
SETLOCAL ENABLEEXTENSIONS
3+
4+
rem x86 or x64
5+
set ARCH=%1
6+
rem win32 or win_amd64
7+
set BW_ARCH=%2
8+
rem librdkafka install destdir (relative path)
9+
set DEST=%3
10+
rem wheelhouse output dir
11+
set WHEELHOUSE=%4
12+
13+
if [%WHEELHOUSE%]==[] goto usage
14+
echo on
15+
16+
set CIBW_BUILD=cp27-%BW_ARCH% cp36-%BW_ARCH% cp37-%BW_ARCH% cp38-%BW_ARCH% cp39-%BW_WARCH%
17+
set CIBW_BEFORE_BUILD=python -m pip install delvewheel==0.0.6
18+
set CIBW_TEST_COMMAND=python {project}\test.py
19+
rem set CIBW_BUILD_VERBOSITY=3
20+
set INCLUDE_DIRS=%cd%\%DEST%\build\native\include
21+
set DLL_DIR=%cd%\%DEST%\runtimes\win-%ARCH%\native
22+
set LIB_DIRS=%cd%\%DEST%\build\native\lib\win\%ARCH%\win-%ARCH%-Release\v120
23+
set CIBW_REPAIR_WHEEL_COMMAND=python -m delvewheel repair --add-path %DLL_DIR% -w {dest_dir} {wheel}
24+
25+
set PATH=%PATH%;c:\Program Files\Git\bin\
26+
27+
python -m pip install cibuildwheel==1.7.4
28+
29+
python -m cibuildwheel --output-dir %WHEELHOUSE% --platform windows
30+
31+
dir %WHEELHOUSE%
32+
33+
goto :eof
34+
35+
:usage
36+
@echo "Usage: %0 x86|x64 win32|win_amd64 wheelhouse-dir"
37+
exit /B 1

‎tools/wheels/build-wheels.sh

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
#
3+
#
4+
# Build wheels (on Linux or OSX) using cibuildwheel.
5+
#
6+
7+
this_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
8+
9+
10+
# Skip PyPy, old Python3 versions, and x86 builds.
11+
export CIBW_SKIP="pp* cp35-* *i686"
12+
export CIBW_TEST_COMMAND="python {project}/test.py"
13+
14+
15+
librdkafka_version=$1
16+
wheeldir=$2
17+
18+
if [[ -z $wheeldir ]]; then
19+
echo "Usage: $0 <librdkafka-nuget-version> <wheeldir>"
20+
exit 1
21+
fi
22+
23+
set -ex
24+
25+
[[ -d $wheeldir ]] || mkdir -p "$wheeldir"
26+
27+
28+
case $OSTYPE in
29+
linux*)
30+
os=linux
31+
# Need to set up env vars (in docker) so that setup.py
32+
# finds librdkafka.
33+
lib_dir=dest/runtimes/linux-x64/native
34+
export CIBW_ENVIRONMENT="INCLUDE_DIRS=dest/build/native/include LIB_DIRS=$lib_dir LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$PWD/$lib_dir"
35+
;;
36+
darwin*)
37+
os=macos
38+
# Need to set up env vars so that setup.py finds librdkafka.
39+
lib_dir=dest/runtimes/osx-x64/native
40+
export INCLUDE_DIRS="${PWD}/dest/build/native/include"
41+
export LIB_DIRS="${PWD}/$lib_dir"
42+
;;
43+
*)
44+
echo "$0: Unsupported OSTYPE $OSTYPE"
45+
exit 1
46+
;;
47+
esac
48+
49+
50+
$this_dir/install-librdkafka.sh $librdkafka_version dest
51+
52+
python3 -m pip install cibuildwheel==1.7.4
53+
54+
if [[ -z $TRAVIS ]]; then
55+
cibw_args="--platform $os"
56+
fi
57+
58+
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/$lib_dir python3 -m cibuildwheel --output-dir $wheeldir $cibw_args
59+
60+
ls $wheeldir
61+
62+
for f in $wheeldir/*whl ; do
63+
echo $f
64+
unzip -l $f
65+
done
66+

‎tools/wheels/install-librdkafka.sh

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
VER="$1"
6+
DEST="$2"
7+
8+
if [[ -z $DEST ]]; then
9+
echo "Usage: $0 <librdkafka-redist-version> <destdir>"
10+
exit 1
11+
fi
12+
13+
if [[ -f $DEST/build/native/include/librdkafka/rdkafka.h ]]; then
14+
echo "$0: librdkafka already installed in $DEST"
15+
exit 0
16+
fi
17+
18+
echo "$0: Installing librdkafka $VER to $DEST"
19+
[[ -d "$DEST" ]] || mkdir -p "$DEST"
20+
pushd "$DEST"
21+
22+
curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
23+
24+
unzip lrk$VER.zip
25+
26+
27+
if [[ $OSTYPE == linux* ]]; then
28+
# Linux
29+
30+
# Copy the librdkafka build with least dependencies to librdkafka.so.1
31+
cp -v runtimes/linux-x64/native/{centos6-librdkafka.so,librdkafka.so.1}
32+
ldd runtimes/linux-x64/native/librdkafka.so.1
33+
34+
elif [[ $OSTYPE == darwin* ]]; then
35+
# MacOS X
36+
37+
# Change the library's self-referencing name from
38+
# /Users/travis/.....somelocation/librdkafka.1.dylib to its local path.
39+
install_name_tool -id $PWD/runtimes/osx-x64/native/librdkafka.dylib runtimes/osx-x64/native/librdkafka.dylib
40+
41+
otool -L runtimes/osx-x64/native/librdkafka.dylib
42+
fi
43+
44+
popd

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy