Improve Python package build process

montanalow · montanalow · commit 686942a01e2d · 2025-04-15T13:16:31.000-07:00
- Update ubuntu-postgresml-python-package workflow with better Python setup
- Add workflow_call support to make it callable from main workflow
- Change main workflow to use reusable workflow for Python package build
- Addresses dependency issues with Python build for Ubuntu 24.04
diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml
@@ -10,41 +10,10 @@ jobs:
   # PostgresML Python package.
   #
   postgresml-python:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"]
-        ubuntu_version: ["20.04", "22.04", "24.04"]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v3
-    - name: Build and release Python package
-      env:
-        AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
-        UBUNTU_VERSION: ${{ matrix.ubuntu_version }}
-      run: |
-        sudo apt update
-        sudo apt install -y python3-dev python3-pip python3-virtualenv software-properties-common python3-wheel-whl python3-pip-whl python3-setuptools-whl
-        
-        # Add deadsnakes PPA for all Python versions
-        sudo add-apt-repository -y ppa:deadsnakes/ppa
-        sudo apt update
-        
-        # Install specific Python versions based on Ubuntu target
-        if [[ "$UBUNTU_VERSION" == "20.04" ]]; then
-          sudo apt install -y python3.8 python3.8-dev python3.8-venv
-        elif [[ "$UBUNTU_VERSION" == "22.04" ]]; then
-          sudo apt install -y python3.10 python3.10-dev python3.10-venv
-        elif [[ "$UBUNTU_VERSION" == "24.04" ]]; then
-          sudo apt install -y python3.12 python3.12-dev python3.12-venv
-        fi
-        
-        # Install PyTorch before running the build script to satisfy auto_gptq's requirements
-        pip install torch --user
-        
-        bash packages/postgresml-python/release.sh ${{ inputs.packageVersion }} ${{ matrix.ubuntu_version }}
+    uses: ./.github/workflows/ubuntu-postgresml-python-package.yaml
+    with:
+      packageVersion: ${{ inputs.packageVersion }}
+    secrets: inherit
 
   #
   # PostgresML extension.
diff --git a/.github/workflows/ubuntu-postgresml-python-package.yaml b/.github/workflows/ubuntu-postgresml-python-package.yaml
@@ -4,7 +4,13 @@ on:
   workflow_dispatch:
     inputs:
       packageVersion:
-        default: "2.8.4"
+        default: "2.10.0"
+  workflow_call:
+    inputs:
+      packageVersion:
+        type: string
+        required: true
+        default: "2.10.0"
 
 jobs:
   postgresml-python:
@@ -21,5 +27,28 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
+        UBUNTU_VERSION: ${{ matrix.ubuntu_version }}
       run: |
+        sudo apt update
+        sudo apt install -y python3-dev python3-pip python3-virtualenv software-properties-common python3-wheel-whl python3-pip-whl python3-setuptools-whl
+        
+        # Add deadsnakes PPA for all Python versions
+        sudo add-apt-repository -y ppa:deadsnakes/ppa
+        sudo apt update
+        
+        # Install specific Python versions based on Ubuntu target
+        if [[ "$UBUNTU_VERSION" == "20.04" ]]; then
+          sudo apt install -y python3.8 python3.8-dev python3.8-venv
+        elif [[ "$UBUNTU_VERSION" == "22.04" ]]; then
+          sudo apt install -y python3.10 python3.10-dev python3.10-venv
+        elif [[ "$UBUNTU_VERSION" == "24.04" ]]; then
+          sudo apt install -y python3.12 python3.12-dev python3.12-venv
+        fi
+        
+        # Ensure pip is updated
+        python3 -m pip install --upgrade pip setuptools wheel
+        
+        # Install PyTorch globally before running the build script
+        sudo python3 -m pip install torch
+        
         bash packages/postgresml-python/release.sh ${{ inputs.packageVersion }} ${{ matrix.ubuntu_version }}
