From 2da87e3109bfd6526a788330173daf31649de096 Mon Sep 17 00:00:00 2001 From: Andrey Senyaev <76472231+asenyaev@users.noreply.github.com> Date: Fri, 26 Aug 2022 15:27:15 +0300 Subject: [PATCH] Build zlib from the source on Linux with the latest version (#715) * Added zlib build from the source on Linux, handling auditwheel issues * Removed unused variable * Removed bzip2 from dependencies for OpenCV Python packages --- .github/workflows/build_wheels_linux.yml | 10 +---- .github/workflows/build_wheels_linux_arm.yml | 12 +----- .github/workflows/build_wheels_macos.yml | 4 -- .github/workflows/build_wheels_macos_m1.yml | 10 ----- docker/manylinux2014/Dockerfile_aarch64 | 29 +++++++++----- docker/manylinux2014/Dockerfile_x86_64 | 29 +++++++++----- patch_auditwheel_whitelist.py | 40 ++++++++++++++++---- 7 files changed, 75 insertions(+), 59 deletions(-) diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index a8cdd34e..ab377325 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -35,14 +35,13 @@ jobs: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true REPO_DIR: . - BUILD_COMMIT: ${{ github.ref }} PROJECT_SPEC: opencv-python MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} MB_ML_VER: 2014 TRAVIS_BUILD_DIR: ${{ github.workspace }} CONFIG_PATH: travis_config.sh - DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64:20220628 + DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64:20220824 USE_CCACHE: 0 UNICODE_WIDTH: 32 PLAT: x86_64 @@ -55,9 +54,6 @@ jobs: working-directory: ${{ github.workspace }} - name: Setup environment run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "ENABLE_ROLLING=1" >> $GITHUB_ENV fi @@ -135,7 +131,6 @@ jobs: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true REPO_DIR: . - BUILD_COMMIT: ${{ github.ref }} PROJECT_SPEC: opencv-python PLAT: x86_64 MB_PYTHON_VERSION: ${{ matrix.python-version }} @@ -156,9 +151,6 @@ jobs: working-directory: ${{ github.workspace }} - name: Setup environment run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "ENABLE_ROLLING=1" >> $GITHUB_ENV fi diff --git a/.github/workflows/build_wheels_linux_arm.yml b/.github/workflows/build_wheels_linux_arm.yml index c6b9b4c5..079efbe1 100644 --- a/.github/workflows/build_wheels_linux_arm.yml +++ b/.github/workflows/build_wheels_linux_arm.yml @@ -35,7 +35,6 @@ jobs: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true REPO_DIR: . - BUILD_COMMIT: ${{ github.ref }} PROJECT_SPEC: opencv-python MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} @@ -43,7 +42,7 @@ jobs: MB_ML_VER: 2014 TRAVIS_BUILD_DIR: ${{ github.workspace }} CONFIG_PATH: travis_config.sh - DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64:20220628 + DOCKER_IMAGE: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64:20220824 USE_CCACHE: 0 UNICODE_WIDTH: 32 SDIST: ${{ matrix.build_sdist || 0 }} @@ -55,9 +54,6 @@ jobs: working-directory: ${{ github.workspace }} - name: Setup environment run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "ENABLE_ROLLING=1" >> $GITHUB_ENV fi @@ -97,7 +93,6 @@ jobs: ACTIONS_ALLOW_UNSECURE_COMMANDS: true MB_PYTHON_VERSION: ${{ matrix.python-version }} PLAT: aarch64 - BUILD_COMMIT: ${{ github.ref }} NP_TEST_DEP: numpy==1.19.4 NP_TEST_DEP_LATEST: numpy==1.21.4 CONFIG_PATH: travis_config.sh @@ -107,11 +102,6 @@ jobs: - name: Cleanup run: find . -mindepth 1 -delete working-directory: ${{ github.workspace }} - - name: Setup environment - run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi - name: Checkout uses: actions/checkout@v3 with: diff --git a/.github/workflows/build_wheels_macos.yml b/.github/workflows/build_wheels_macos.yml index 1ac459cd..1c2a1614 100644 --- a/.github/workflows/build_wheels_macos.yml +++ b/.github/workflows/build_wheels_macos.yml @@ -35,7 +35,6 @@ jobs: env: ACTIONS_ALLOW_UNSECURE_COMMANDS: true REPO_DIR: . - BUILD_COMMIT: ${{ github.ref }} PROJECT_SPEC: opencv-python MB_PYTHON_VERSION: ${{ matrix.python-version }} TRAVIS_PYTHON_VERSION: ${{ matrix.python-version }} @@ -57,9 +56,6 @@ jobs: working-directory: ${{ github.workspace }} - name: Setup environment run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "ENABLE_ROLLING=1" >> $GITHUB_ENV fi diff --git a/.github/workflows/build_wheels_macos_m1.yml b/.github/workflows/build_wheels_macos_m1.yml index 3d8f9b62..b51801ab 100644 --- a/.github/workflows/build_wheels_macos_m1.yml +++ b/.github/workflows/build_wheels_macos_m1.yml @@ -30,7 +30,6 @@ jobs: without_gui: [0, 1] build_sdist: [0] env: - BUILD_COMMIT: ${{ github.ref }} CI_BUILD: 1 SDIST: ${{ matrix.build_sdist || 0 }} ENABLE_HEADLESS: ${{ matrix.without_gui }} @@ -41,9 +40,6 @@ jobs: working-directory: ${{ github.workspace }} - name: Setup environment run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "ENABLE_ROLLING=1" >> $GITHUB_ENV fi @@ -84,18 +80,12 @@ jobs: without_gui: [0, 1] build_sdist: [0] env: - BUILD_COMMIT: ${{ github.ref }} OPENCV_TEST_DATA_PATH: ${{ github.workspace }}/opencv_extra/testdata PYLINT_TEST_FILE: ${{ github.workspace }}/opencv/samples/python/squares.py steps: - name: Cleanup run: find . -mindepth 1 -delete working-directory: ${{ github.workspace }} - - name: Setup environment - run: | - if [[ "${{ github.event.pull_request }}" == "true" ]]; then - echo "BUILD_COMMIT=${{ github.base_ref }}" >> $GITHUB_ENV - fi - name: Checkout uses: actions/checkout@v3 with: diff --git a/docker/manylinux2014/Dockerfile_aarch64 b/docker/manylinux2014/Dockerfile_aarch64 index 2679cbbf..48ad3f98 100644 --- a/docker/manylinux2014/Dockerfile_aarch64 +++ b/docker/manylinux2014/Dockerfile_aarch64 @@ -1,4 +1,4 @@ -# Version: 20220628 +# Version: 20220824 # Image name: quay.io/opencv-ci/opencv-python-manylinux2014-aarch64 FROM quay.io/pypa/manylinux2014_aarch64:latest @@ -11,16 +11,28 @@ ARG NASM_VERSION=2.15.04 ARG OPENSSL_VERSION=1_1_1o ARG QT_VERSION=5.15.0 ARG YASM_VERSION=1.3.0 +ARG ZLIB_VERSION=1.2.12 ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH # epel-release need for aarch64 to get openblas packages -RUN yum install bzip2-devel curl-devel zlib-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ +RUN yum install bzip2-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ yum install openblas-devel -y && \ cp /usr/include/lapacke/lapacke*.h /usr/include/ && \ curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \ # libpng will be built from source - yum remove libpng -y + yum remove libpng zlib-devel bzip2-devel -y + +RUN mkdir ~/zlib_sources && \ + cd ~/zlib_sources && \ + curl -O -L https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz && \ + tar -xf zlib-${ZLIB_VERSION}.tar.gz && \ + cd zlib-${ZLIB_VERSION} && \ + ./configure --prefix=/usr/local && \ + make && \ + make install && \ + cd .. && \ + rm -rf ~/zlib_sources RUN mkdir ~/libpng_sources && \ cd ~/libpng_sources && \ @@ -71,8 +83,8 @@ RUN mkdir ~/openssl_sources && \ RUN mkdir ~/nasm_sources && \ cd ~/nasm_sources && \ - curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.bz2 && \ - tar -xf nasm-${NASM_VERSION}.tar.bz2 && cd nasm-${NASM_VERSION} && ./autogen.sh && \ + curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.gz && \ + tar -xf nasm-${NASM_VERSION}.tar.gz && cd nasm-${NASM_VERSION} && ./autogen.sh && \ ./configure --prefix="/ffmpeg_build" --bindir="$HOME/bin" && \ make -j$(getconf _NPROCESSORS_ONLN) && \ make install && \ @@ -102,8 +114,8 @@ RUN mkdir ~/libvpx_sources && \ RUN mkdir ~/ffmpeg_sources && \ cd ~/ffmpeg_sources && \ - curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ - tar -xf ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ + curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz && \ + tar -xf ffmpeg-${FFMPEG_VERSION}.tar.gz && \ cd ffmpeg-${FFMPEG_VERSION} && \ PATH=~/bin:$PATH && \ PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --extra-cflags="-I/ffmpeg_build/include" --extra-ldflags="-L/ffmpeg_build/lib" --enable-openssl --enable-libvpx --enable-shared --enable-pic --bindir="$HOME/bin" && \ @@ -111,8 +123,7 @@ RUN mkdir ~/ffmpeg_sources && \ make install && \ echo "/ffmpeg_build/lib/" >> /etc/ld.so.conf && \ ldconfig && \ - rm -rf ~/ffmpeg_sources && \ - yum remove bzip2-devel -y + rm -rf ~/ffmpeg_sources RUN curl -O -L https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \ tar -xf ccache-${CCACHE_VERSION}.tar.gz && \ diff --git a/docker/manylinux2014/Dockerfile_x86_64 b/docker/manylinux2014/Dockerfile_x86_64 index 8bb77341..85fc22fd 100644 --- a/docker/manylinux2014/Dockerfile_x86_64 +++ b/docker/manylinux2014/Dockerfile_x86_64 @@ -1,4 +1,4 @@ -# Version: 20220628 +# Version: 20220824 # Image name: quay.io/opencv-ci/opencv-python-manylinux2014-x86-64 FROM quay.io/pypa/manylinux2014_x86_64:latest @@ -11,16 +11,28 @@ ARG NASM_VERSION=2.15.04 ARG OPENSSL_VERSION=1_1_1o ARG QT_VERSION=5.15.0 ARG YASM_VERSION=1.3.0 +ARG ZLIB_VERSION=1.2.12 ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH # epel-release need for aarch64 to get openblas packages -RUN yum install bzip2-devel curl-devel zlib-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ +RUN yum install bzip2-devel curl-devel xcb-util-renderutil-devel xcb-util-devel xcb-util-image-devel xcb-util-keysyms-devel xcb-util-wm-devel mesa-libGL-devel libxkbcommon-devel libxkbcommon-x11-devel libXi-devel lapack-devel epel-release -y && \ yum install openblas-devel -y && \ cp /usr/include/lapacke/lapacke*.h /usr/include/ && \ curl https://raw.githubusercontent.com/xianyi/OpenBLAS/v0.3.3/cblas.h -o /usr/include/cblas.h && \ # libpng will be built from source - yum remove libpng -y + yum remove libpng zlib-devel bzip2-devel -y + +RUN mkdir ~/zlib_sources && \ + cd ~/zlib_sources && \ + curl -O -L https://www.zlib.net/zlib-${ZLIB_VERSION}.tar.gz && \ + tar -xf zlib-${ZLIB_VERSION}.tar.gz && \ + cd zlib-${ZLIB_VERSION} && \ + ./configure --prefix=/usr/local && \ + make && \ + make install && \ + cd .. && \ + rm -rf ~/zlib_sources RUN mkdir ~/libpng_sources && \ cd ~/libpng_sources && \ @@ -71,8 +83,8 @@ RUN mkdir ~/openssl_sources && \ RUN mkdir ~/nasm_sources && \ cd ~/nasm_sources && \ - curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.bz2 && \ - tar -xf nasm-${NASM_VERSION}.tar.bz2 && cd nasm-${NASM_VERSION} && ./autogen.sh && \ + curl -O -L http://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.gz && \ + tar -xf nasm-${NASM_VERSION}.tar.gz && cd nasm-${NASM_VERSION} && ./autogen.sh && \ ./configure --prefix="/ffmpeg_build" --bindir="$HOME/bin" && \ make -j$(getconf _NPROCESSORS_ONLN) && \ make install && \ @@ -102,8 +114,8 @@ RUN mkdir ~/libvpx_sources && \ RUN mkdir ~/ffmpeg_sources && \ cd ~/ffmpeg_sources && \ - curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ - tar -xf ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ + curl -O -L https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.gz && \ + tar -xf ffmpeg-${FFMPEG_VERSION}.tar.gz && \ cd ffmpeg-${FFMPEG_VERSION} && \ PATH=~/bin:$PATH && \ PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --extra-cflags="-I/ffmpeg_build/include" --extra-ldflags="-L/ffmpeg_build/lib" --enable-openssl --enable-libvpx --enable-shared --enable-pic --bindir="$HOME/bin" && \ @@ -111,8 +123,7 @@ RUN mkdir ~/ffmpeg_sources && \ make install && \ echo "/ffmpeg_build/lib/" >> /etc/ld.so.conf && \ ldconfig && \ - rm -rf ~/ffmpeg_sources && \ - yum remove bzip2-devel -y + rm -rf ~/ffmpeg_sources RUN curl -O -L https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz && \ tar -xf ccache-${CCACHE_VERSION}.tar.gz && \ diff --git a/patch_auditwheel_whitelist.py b/patch_auditwheel_whitelist.py index b32da17a..150f97ca 100644 --- a/patch_auditwheel_whitelist.py +++ b/patch_auditwheel_whitelist.py @@ -3,13 +3,39 @@ from auditwheel import policy -policies = None -with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json")) as f: - policies = json.load(f) +def add_zlib_versions(): + with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), 'r') as manylinux_policy: + manylinux_policy_lines = manylinux_policy.readlines() -for p in policies: - p["lib_whitelist"].append("libxcb.so.1") + manylinux_policy_data = "" + for line in manylinux_policy_lines: + if "ZLIB" in line: + if len(line) > 22: + updated_line = line[:-2] + ', "1.2.9", "1.2.12"]' + else: + updated_line = line[:-2] + '"1.2.9", "1.2.12"]' + manylinux_policy_replacement = line.replace(line, updated_line) + else: + manylinux_policy_replacement = line + manylinux_policy_data = manylinux_policy_data + manylinux_policy_replacement -with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), "w") as f: - f.write(json.dumps(policies)) + with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), 'w') as manylinux_final_policy: + manylinux_final_policy.write(manylinux_policy_data) + +def add_whitelisted_libs(): + policies = None + + with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json")) as f: + policies = json.load(f) + + for p in policies: + p["lib_whitelist"].append("libxcb.so.1") + + with open(join(dirname(abspath(policy.__file__)), "manylinux-policy.json"), "w") as f: + f.write(json.dumps(policies)) + + +if __name__ == '__main__': + add_zlib_versions() + add_whitelisted_libs() pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy