Skip to content

Commit 0622c47

Browse files
edenhilledenhill
committed
Travis: restructure wheel building
1 parent 9764f82 commit 0622c47

File tree

5 files changed

+221
-31
lines changed

5 files changed

+221
-31
lines changed

‎.travis.yml

Lines changed: 63 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
env:
22
global:
3-
- LIBRDKAFKA_VERSION=v1.6.0-RC1
3+
- LIBRDKAFKA_VERSION=v1.6.0-RC2
4+
45
jobs:
56
include:
67
- name: "Source package verification with Python 2.7 (Linux)"
@@ -10,65 +11,96 @@ jobs:
1011
python: "2.7"
1112
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
1213
services: docker
14+
1315
- name: "Source package verification with Python 3.6 (Linux)"
1416
os: linux
1517
language: python
1618
dist: xenial
1719
python: "3.6"
1820
env: LD_LIBRARY_PATH="$PWD/tmp-build/lib"
1921
services: docker
22+
2023
- name: "Source package verification with Python 2.7 (OSX)"
2124
os: osx
2225
python: "2.7"
2326
env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" INTERPRETER_VERSION="2.7.17"
27+
2428
- name: "Source package verification with Python 3.6 (OSX) +docs"
2529
os: osx
2630
python: "3.6"
2731
env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" MK_DOCS="y" INTERPRETER_VERSION="3.6.5"
28-
- name: "cibuildwheel (OSX)"
29-
os: osx
30-
env: CIBW_BEFORE_BUILD="tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp" CFLAGS="-Itmp/include" LDFLAGS="-Ltmp/lib" INTERPRETER_VERSION="2.7.17"
31-
- name: "cibuildwheel (manylinux)"
32-
os: linux
33-
dist: xenial
34-
env:
35-
- CIBW_BEFORE_BUILD="tools/prepare-cibuildwheel-linux.sh ${LIBRDKAFKA_VERSION}"
36-
- PYTHON_CONFIGURE_OPTS="--enable-unicode=ucs4 --with-wide-unicode"
37-
- CIBW_MANYLINUX_X86_64_IMAGE="manylinux2010"
38-
- CIBW_MANYLINUX_I686_IMAGE="manylinux2010"
32+
33+
- name: "Wheels: Windows x64"
34+
if: tag is present
35+
os: windows
36+
language: shell
37+
env: BUILD_WHEELS=1
38+
before_install:
39+
- choco install python --version 3.8.0
40+
- export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
41+
# make sure it's on PATH as 'python3'
42+
- ln -s /c/Python38/python.exe /c/Python38/python3.exe
43+
install:
44+
- bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
45+
script:
46+
- tools/wheels/build-wheels.bat x64 win_amd64 dest wheelhouse
47+
48+
- name: "Wheels: Windows x86"
49+
if: tag is present
50+
os: windows
51+
language: shell
52+
env: BUILD_WHEELS=1
53+
before_install:
54+
- choco install python --version 3.8.0
55+
- export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
56+
# make sure it's on PATH as 'python3'
57+
- ln -s /c/Python38/python.exe /c/Python38/python3.exe
58+
install:
59+
- bash tools/wheels/install-librdkafka.sh ${LIBRDKAFKA_VERSION#v} dest
60+
script:
61+
- tools/wheels/build-wheels.bat x86 win32 dest wheelhouse
62+
63+
- name: "Wheels: Linux x64"
64+
if: tag is present
3965
language: python
40-
python: "2.7"
66+
python: "3.8"
4167
services: docker
68+
env: BUILD_WHEELS=1
69+
script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
70+
71+
- name: "Wheels: MacOSX x64"
72+
if: tag is present
73+
os: osx
74+
language: shell
75+
env: BUILD_WHEELS=1
76+
script: tools/wheels/build-wheels.sh ${LIBRDKAFKA_VERSION#v} wheelhouse
4277

43-
# See https://cibuildwheel.readthedocs.io/en/latest/options/ for CIBW* vars
4478

4579
# Install test dependencies unconditionally
4680
# Travis OSX envs requires some setup; see tools/prepare-osx.sh
4781
# Install cibuildwheel if this is a tagged PR
4882
before_install:
49-
- if [[ $TRAVIS_OS_NAME == "osx" ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
50-
- if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then pip install -U -r tools/requirements-manylinux.txt --force-reinstall ; fi
83+
- if [[ $TRAVIS_OS_NAME == "osx" && $BUILD_WHEELS != 1 ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
5184

52-
# Install interceptors
53-
# Install librdkafka if not CIBW_BEFORE_BUILD
54-
# Install confluent_kafka[avro] if not CIBW_BEFORE_BUILD
5585
install:
86+
# Install interceptors
5687
- tools/install-interceptors.sh
57-
- pip install -r tests/requirements.txt
58-
- flake8
88+
- if [[ $BUILD_WHEELS != 1 ]]; then pip install -r tests/requirements.txt ; fi
5989
- if [[ $MK_DOCS == y ]]; then pip install -r docs/requirements.txt; fi
60-
- if [[ -z $CIBW_BEFORE_BUILD ]]; then pip install -U protobuf && tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build && pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf] ; fi
90+
# Install librdkafka and confluent_kafka[avro] if not building wheels
91+
- if [[ $BUILD_WHEELS != 1 ]]; then pip install -U protobuf && tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build ; fi
92+
6193

62-
# Build wheels
63-
# Make plugins available for tests
64-
# Execute tests if not CIBW_BEFORE_BUILD [osx, linux]
65-
# Execute integration tests if CIBW_BEFORE_BUILD
66-
# Build docs if MK_DOCS
94+
95+
# Note: Will not be run for wheel builds.
6796
script:
68-
- if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then cibuildwheel --output-dir wheelhouse1 && tools/fixup-wheels.sh wheelhouse1 wheelhouse; fi
97+
- flake8
98+
# Build package
99+
- pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] .[schema-registry] .[json] .[protobuf]
69100
- ldd staging/libs/* || otool -L staging/libs/* || true
70-
- if [[ -z $CIBW_BEFORE_BUILD && $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi
71-
- if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == osx && -n $CIBW_BEFORE_BUILD ]]; then tools/test-wheels.sh wheelhouse; fi
101+
# Run tests
102+
- if [[ $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs python -m pytest --timeout 600 --ignore=tmp-build || travis_terminate 1; fi
103+
# Build docs
72104
- if [[ $MK_DOCS == y ]]; then make docs; fi
73105

74106
deploy:
@@ -87,4 +119,4 @@ deploy:
87119
on:
88120
repo: confluentinc/confluent-kafka-python
89121
tags: true
90-
condition: "-n $CIBW_BEFORE_BUILD"
122+
condition: "BUILD_WHEEL == 1"

‎tools/wheels/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Wheel building scripts

‎tools/wheels/build-wheels.bat

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
@echo off
2+
SETLOCAL ENABLEEXTENSIONS
3+
4+
rem x86 or x64
5+
set ARCH=%1
6+
rem win32 or win_amd64
7+
set BW_ARCH=%2
8+
rem librdkafka install destdir (relative path)
9+
set DEST=%3
10+
rem wheelhouse output dir
11+
set WHEELHOUSE=%4
12+
13+
if [%WHEELHOUSE%]==[] goto usage
14+
echo on
15+
16+
set CIBW_BUILD=cp27-%BW_ARCH% cp36-%BW_ARCH% cp37-%BW_ARCH% cp38-%BW_ARCH% cp39-%BW_WARCH%
17+
set CIBW_BEFORE_BUILD=python -m pip install delvewheel==0.0.6
18+
set CIBW_TEST_REQUIRES=-r tests/requirements.txt
19+
set CIBW_TEST_COMMAND=pytest {project}\tests\test_Producer.py
20+
rem set CIBW_BUILD_VERBOSITY=3
21+
set include=%cd%\%DEST%\build\native\include
22+
set lib=%cd%\%DEST%\build\native\lib\win\%ARCH%\win-%ARCH%-Release\v120
23+
set DLL_DIR=%cd%\%DEST%\runtimes\win-%ARCH%\native
24+
set CIBW_REPAIR_WHEEL_COMMAND=python -m delvewheel repair --add-path %DLL_DIR% -w {dest_dir} {wheel}
25+
26+
set PATH=%PATH%;c:\Program Files\Git\bin\
27+
28+
python -m pip install cibuildwheel==1.7.4 || goto :error
29+
30+
python -m cibuildwheel --output-dir %WHEELHOUSE% --platform windows || goto :error
31+
32+
dir %WHEELHOUSE%
33+
34+
goto :eof
35+
36+
:usage
37+
@echo "Usage: %0 x86|x64 win32|win_amd64 wheelhouse-dir"
38+
exit /B 1
39+
40+
:error
41+
echo Failed with error #%errorlevel%.
42+
exit /b %errorlevel%

‎tools/wheels/build-wheels.sh

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/bin/bash
2+
#
3+
#
4+
# Build wheels (on Linux or OSX) using cibuildwheel.
5+
#
6+
7+
this_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
8+
9+
10+
# Skip PyPy, old Python3 versions, and x86 builds.
11+
export CIBW_SKIP="pp* cp35-* *i686"
12+
# Run a simple test suite
13+
export CIBW_TEST_REQUIRES="-r tests/requirements.txt"
14+
export CIBW_TEST_COMMAND="pytest {project}/tests/test_Producer.py"
15+
16+
17+
librdkafka_version=$1
18+
wheeldir=$2
19+
20+
if [[ -z $wheeldir ]]; then
21+
echo "Usage: $0 <librdkafka-nuget-version> <wheeldir>"
22+
exit 1
23+
fi
24+
25+
set -ex
26+
27+
[[ -d $wheeldir ]] || mkdir -p "$wheeldir"
28+
29+
30+
case $OSTYPE in
31+
linux*)
32+
os=linux
33+
# Need to set up env vars (in docker) so that setup.py
34+
# finds librdkafka.
35+
lib_dir=dest/runtimes/linux-x64/native
36+
export CIBW_ENVIRONMENT="CFLAGS=-I\$PWD/dest/build/native/include LDFLAGS=-L\$PWD/$lib_dir LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:\$PWD/$lib_dir"
37+
;;
38+
darwin*)
39+
os=macos
40+
# Need to set up env vars so that setup.py finds librdkafka.
41+
lib_dir=dest/runtimes/osx-x64/native
42+
export CFLAGS="-I${PWD}/dest/build/native/include"
43+
export LDFLAGS="-L${PWD}/$lib_dir"
44+
;;
45+
*)
46+
echo "$0: Unsupported OSTYPE $OSTYPE"
47+
exit 1
48+
;;
49+
esac
50+
51+
52+
$this_dir/install-librdkafka.sh $librdkafka_version dest
53+
54+
install_pkgs=cibuildwheel==1.7.4
55+
56+
python3 -m pip install $install_pkgs ||
57+
pip3 install $install_pkgs
58+
59+
if [[ -z $TRAVIS ]]; then
60+
cibw_args="--platform $os"
61+
fi
62+
63+
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/$lib_dir python3 -m cibuildwheel --output-dir $wheeldir $cibw_args
64+
65+
ls $wheeldir
66+
67+
for f in $wheeldir/*whl ; do
68+
echo $f
69+
unzip -l $f
70+
done
71+

‎tools/wheels/install-librdkafka.sh

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
VER="$1"
6+
DEST="$2"
7+
8+
if [[ -z $DEST ]]; then
9+
echo "Usage: $0 <librdkafka-redist-version> <destdir>"
10+
exit 1
11+
fi
12+
13+
if [[ -f $DEST/build/native/include/librdkafka/rdkafka.h ]]; then
14+
echo "$0: librdkafka already installed in $DEST"
15+
exit 0
16+
fi
17+
18+
echo "$0: Installing librdkafka $VER to $DEST"
19+
[[ -d "$DEST" ]] || mkdir -p "$DEST"
20+
pushd "$DEST"
21+
22+
curl -L -o lrk$VER.zip https://www.nuget.org/api/v2/package/librdkafka.redist/$VER
23+
24+
unzip lrk$VER.zip
25+
26+
27+
if [[ $OSTYPE == linux* ]]; then
28+
# Linux
29+
30+
# Copy the librdkafka build with least dependencies to librdkafka.so.1
31+
cp -v runtimes/linux-x64/native/{centos6-librdkafka.so,librdkafka.so.1}
32+
ldd runtimes/linux-x64/native/librdkafka.so.1
33+
34+
elif [[ $OSTYPE == darwin* ]]; then
35+
# MacOS X
36+
37+
# Change the library's self-referencing name from
38+
# /Users/travis/.....somelocation/librdkafka.1.dylib to its local path.
39+
install_name_tool -id $PWD/runtimes/osx-x64/native/librdkafka.dylib runtimes/osx-x64/native/librdkafka.dylib
40+
41+
otool -L runtimes/osx-x64/native/librdkafka.dylib
42+
fi
43+
44+
popd

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy