Move depenndencies to requirements files, update build config

rnpridgeon · Ryan P · commit f97a9c204184 · 2020-03-30T19:03:09.000-04:00
diff --git a/.gitignore b/.gitignore
@@ -22,5 +22,7 @@ dl-*
 .pytest_cache
 staging
 tests/docker/conf/tls/*
-.idea
 .python-version
+.DS_Store
+.idea
+tmp-KafkaCluster
diff --git a/.travis.yml b/.travis.yml
@@ -18,28 +18,14 @@ matrix:
   # Source package verification with Python 2.7
   - os: osx
     python: "2.7"
-    env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib"
-    before_install:
-        - pip install -U pip && pip install virtualenv
-        - pyenv install -f 2.7.15
-        - pip install virtualenv
-        - virtualenv -p ~/.pyenv/versions/2.7.15/bin/python ./env
-        - source env/bin/activate
+    env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" INTERPRETER_VERSION="2.7.17"
   # Source package verification with Python 3.6
   - os: osx
     python: "3.6"
-    env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" MK_DOCS="y" EXTRA_PKGS="sphinx sphinx_rtd_theme avro-python3"
-    before_install:
-        - pip install -U pip && pip install virtualenv
-        - pyenv install -f 3.6.5
-        - pip install virtualenv
-        - virtualenv -p ~/.pyenv/versions/3.6.5/bin/python ./env
-        - source env/bin/activate
+    env: DYLD_LIBRARY_PATH="$PWD/tmp-build/lib" MK_DOCS="y" INTERPRETER_VERSION="3.6.5"
   # cibuildwheel for osx
   - os: osx
-    env: CIBW_BEFORE_BUILD="tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp" CFLAGS="-Itmp/include" LDFLAGS="-Ltmp/lib"
-    before_install:
-        - pip install virtualenv
+    env: CIBW_BEFORE_BUILD="tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp" CFLAGS="-Itmp/include" LDFLAGS="-Ltmp/lib" INTERPRETER_VERSION="2.7.17"
   # cibuildwheel for manylinux
   - os: linux
     dist: trusty
@@ -49,30 +35,45 @@ matrix:
       - PYTHON_CONFIGURE_OPTS="--enable-unicode=ucs4 --with-wide-unicode"
       - CIBW_MANYLINUX_X86_64_IMAGE="manylinux1"
       - CIBW_MANYLINUX_I686_IMAGE="manylinux1
+      -  INTERPRETER_VERSION="2.7.17"
     language: python
     python: "2.7"
     services: docker
 
+# See https://cibuildwheel.readthedocs.io/en/latest/options/ for CIBW* vars
+
+# Install test dependencies unconditionally
+# Travis OSX envs requires some setup; see tools/prepare-osx.sh
+# Install cibuildwheel if this is a tagged PR
+before_install:
+  - if [[ $TRAVIS_OS_NAME == "osx" ]]; then tools/prepare-osx.sh ${INTERPRETER_VERSION} /tmp/venv && source /tmp/venv/bin/activate; fi
+  - if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then pip install cibuildwheel --force-reinstall; fi
+
+# Install interceptors
+# Install librdkafka if not CIBW_BEFORE_BUILD
+# Install confluent_kafka[avro] if not CIBW_BEFORE_BUILD
 install:
  - tools/install-interceptors.sh
- - pip install -U pip && pip install virtualenv
- - if [[ $TRAVIS_OS_NAME == "osx" ]]; then python -m ensurepip && virtualenv /tmp/venv && source /tmp/venv/bin/activate ; fi
- - if [[ -z $CIBW_BEFORE_BUILD ]]; then pip install pytest-timeout flake8 trivup; fi
- - if [[ -z $CIBW_BEFORE_BUILD ]]; then rm -rf tmp-build ; tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build ; fi
- - if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then pip install cibuildwheel --force-reinstall; fi
- - if [[ ! -z $EXTRA_PKGS ]]; then pip install $(echo $EXTRA_PKGS) ; fi
+ - pip install -r tests/requirements.txt
+ - flake8
+ - if [[ $MK_DOCS == y ]]; then pip install -r docs/requirements.txt; fi
+ - if [[ -z $CIBW_BEFORE_BUILD ]]; then tools/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} tmp-build && pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro]; fi
 
+# Build wheels
 script:
- - if [[ -z $CIBW_BEFORE_BUILD ]]; then pip install --global-option=build_ext --global-option="-Itmp-build/include/" --global-option="-Ltmp-build/lib" . .[avro] ; fi
- - if [[ -z $CIBW_BEFORE_BUILD ]]; then flake8 ; fi
- # Make plugins available for tests
+ - if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then cibuildwheel --output-dir wheelhouse1 && tools/fixup-wheels.sh wheelhouse1 wheelhouse; fi
+
+# Make plugins available for tests
+# Execute tests if not CIBW_BEFORE_BUILD [osx, linux]
+# Execute integration tests if CIBW_BEFORE_BUILD
+# Build docs if MK_DOCS
+after_script:
  - ldd staging/libs/* || otool -L staging/libs/* || true
- - if [[ -z $CIBW_BEFORE_BUILD && $TRAVIS_OS_NAME == "osx" ]]; then DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs py.test -v --timeout=60 --ignore=tmp-build --import-mode append ; fi
- - if [[ -z $CIBW_BEFORE_BUILD && $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs py.test -v --timeout=60 --ignore=tmp-build --import-mode append ; fi
- - if [[ -n $TRAVIS_TAG && -n $CIBW_BEFORE_BUILD ]]; then cibuildwheel --output-dir wheelhouse1 && tools/fixup-wheels.sh wheelhouse1 wheelhouse ; fi
- - if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == linux && -n $CIBW_BEFORE_BUILD ]]; then tools/test-manylinux.sh ; fi
+ - if [[ -z $CIBW_BEFORE_BUILD && $TRAVIS_OS_NAME == "osx" ]]; then DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:staging/libs py.test --timeout=60 --ignore=tmp-build --import-mode append; fi
+ - if [[ -z $CIBW_BEFORE_BUILD && $TRAVIS_OS_NAME == "linux" ]]; then LD_LIBRARY_PATH=$LD_LIBRARY_PATH:staging/libs py.test --timeout=60 --ignore=tmp-build --import-mode append; fi
+ - if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == linux && -n $CIBW_BEFORE_BUILD ]];then tools/test-manylinux.sh; fi
  - if [[ -n $TRAVIS_TAG && $TRAVIS_OS_NAME == osx && -n $CIBW_BEFORE_BUILD ]]; then tools/test-osx.sh; fi
- - if [[ $MK_DOCS == y ]]; then make docs ; fi
+ - if [[ $MK_DOCS == y ]]; then make docs; fi
 
 deploy:
  provider: s3
diff --git a/confluent_kafka/avro/requirements.txt b/confluent_kafka/avro/requirements.txt
@@ -0,0 +1,4 @@
+fastavro
+requests
+avro==1.9.2;python_version<='3.0'
+avro-python3==1.9.2.1;python_version>='3.0'
diff --git a/confluent_kafka/requirements.txt b/confluent_kafka/requirements.txt
@@ -0,0 +1,2 @@
+enum34;python_version<"3.4"
+futures;python_version<"3.2"
diff --git a/confluent_kafka/schema_registry/requirements.txt b/confluent_kafka/schema_registry/requirements.txt
@@ -0,0 +1,4 @@
+fastavro>=0.23.0
+jsonschema
+protobuf
+requests
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -0,0 +1,2 @@
+sphinx
+sphinx-rtd-theme
diff --git a/examples/requirements.txt b/examples/requirements.txt
@@ -0,0 +1,5 @@
+confluent-kafka[examples]
+fastapi
+pydantic
+uvicorn
+six
diff --git a/setup.py b/setup.py
@@ -5,25 +5,24 @@
 from distutils.core import Extension
 import platform
 
-INSTALL_REQUIRES = [
-    'futures;python_version<"3.2"',
-    'enum34;python_version<"3.4"',
-]
+work_dir = os.path.dirname(os.path.realpath(__file__))
+mod_dir = os.path.join(work_dir, 'confluent_kafka')
+ext_dir = os.path.join(mod_dir, 'src')
 
-AVRO_REQUIRES = [
-    'fastavro',
-    'requests',
-    'avro==1.9.2;python_version<"3.0"',
-    'avro-python3==1.9.2.1;python_version>"3.0"'
-]
+with open(os.path.join(mod_dir, 'requirements.txt')) as f:
+    INSTALL_REQUIRES = f.read().split()
 
-TEST_REQUIRES = [
-    'pytest==4.6.4;python_version<"3.0"',
-    'pytest;python_version>="3.0"',
-    'pytest-timeout',
-    'trivup',
-    'flake8'
-]
+with open(os.path.join(mod_dir, 'schema_registry', 'requirements.txt')) as f:
+    INSTALL_REQUIRES += f.read().split()
+
+with open(os.path.join(mod_dir, 'avro', 'requirements.txt')) as f:
+    AVRO_REQUIRES = f.read().splitlines()
+
+with open(os.path.join(work_dir, 'tests', 'requirements.txt')) as f:
+    TEST_REQUIRES = f.read().splitlines()
+
+with open(os.path.join(work_dir, 'docs', 'requirements.txt')) as f:
+    DOC_REQUIRES = f.read().splitlines()
 
 # On Un*x the library is linked as -lrdkafka,
 # while on windows we need the full librdkafka name.
@@ -34,12 +33,12 @@
 
 module = Extension('confluent_kafka.cimpl',
                    libraries=[librdkafka_libname],
-                   sources=['confluent_kafka/src/confluent_kafka.c',
-                            'confluent_kafka/src/Producer.c',
-                            'confluent_kafka/src/Consumer.c',
-                            'confluent_kafka/src/Metadata.c',
-                            'confluent_kafka/src/AdminTypes.c',
-                            'confluent_kafka/src/Admin.c'])
+                   sources=[os.path.join(ext_dir, 'confluent_kafka.c'),
+                            os.path.join(ext_dir, 'Producer.c'),
+                            os.path.join(ext_dir, 'Consumer.c'),
+                            os.path.join(ext_dir, 'Metadata.c'),
+                            os.path.join(ext_dir, 'AdminTypes.c'),
+                            os.path.join(ext_dir, 'Admin.c')])
 
 
 def get_install_requirements(path):
@@ -61,9 +60,10 @@ def get_install_requirements(path):
       url='https://github.com/confluentinc/confluent-kafka-python',
       ext_modules=[module],
       packages=find_packages(exclude=("tests", "tests.*")),
-      data_files=[('', ['LICENSE.txt'])],
+      data_files=[('', [os.path.join(work_dir, 'LICENSE.txt')])],
       install_requires=INSTALL_REQUIRES,
       extras_require={
           'avro': AVRO_REQUIRES,
-          'dev': TEST_REQUIRES + AVRO_REQUIRES
+          'dev': TEST_REQUIRES + AVRO_REQUIRES,
+          'doc': DOC_REQUIRES + AVRO_REQUIRES
       })
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -0,0 +1,6 @@
+ flake8
+ pytest==4.6.9;python_version<="3.0"
+ pytest;python_version>="3.0"
+ pytest-timeout
+ requests-mock
+ trivup
diff --git a/tools/fixup-wheels.sh b/tools/fixup-wheels.sh
@@ -128,7 +128,7 @@ fixup_wheel () {
 build_patchelf () {
     # Download, build and install patchelf from source.
 
-    curl -l https://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.gz | tar xzf -
+    curl -l https://releases.nixos.org/patchelf/patchelf-0.9/patchelf-0.9.tar.gz | tar xzf -
     pushd patchelf-0.9
     ./configure
     make -j
diff --git a/tools/prepare-osx.sh b/tools/prepare-osx.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# This script prepares the Travis OSX env with a particular interpreter
+# https://docs.travis-ci.com/user/languages/python/
+#
+# Default OSX environment
+# https://docs.travis-ci.com/user/reference/osx/#compilers-and-build-toolchain
+#
+PY_INTERPRETER=$1
+VENV_HOME=$2
+
+set -ev
+
+if [[ -z ${PY_INTERPRETER} ]] || [[  -z ${VENV_HOME} ]]; then
+    echo "Usage: $0 <Python interpreter version> <destination>"
+    exit 1
+fi
+
+# Update virtualenv and install requested interpreter
+echo "# Updating basic dependencies" 
+pip install -U pip
+pip install virtualenv
+pyenv install -f ${PY_INTERPRETER}
+
+# Create virtualenv
+echo "# Constructing virtualenv for interpreter ${PY_INTERPRETER}"
+virtualenv -p ~/.pyenv/versions/${PY_INTERPRETER}/bin/python ${VENV_HOME}
diff --git a/tox.ini b/tox.ini
@@ -51,9 +51,9 @@ commands = flake8
 [pytest]
 python_files = test_*
 testpaths = tests
-ignore = tests/integration
+norecursedirs = tests/integration/*/java
 
 [flake8]
-exclude = venv*,env,.env,.tox,.toxenv,.git,build,docs,tmp-build
+exclude = venv*,env,.env,.tox,.toxenv,.git,build,docs,tools,tmp-build,tests/integration/schema_registry/data,tests/integration/schema_registry/gen
 max-line-length = 119
 accept-encodings = utf-8

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+enum34;python_version<"3.4"`
	`2`	`+futures;python_version<"3.2"`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +confluent-kafka[examples]
 +fastapi
 +pydantic
 +uvicorn
 +six