GH actions sanitizer

RedisJSON · DvirDukhan · May 30, 2024 · May 30, 2024 · May 30, 2024 · Jun 5, 2024
commit b8cd2fc6f2e93a7827f0b028a63a59581a011c8c
diff --git a/.github/workflows/event-pull_request.yml b/.github/workflows/event-pull_request.yml
@@ -0,0 +1,53 @@
+name: Pull Request Flow
+
+# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review] # Defaults + ready_for_review
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  get-latest-redis-tag:
+    uses: ./.github/workflows/task-get-latest-tag.yml
+    with:
+      repo: redis/redis
+      prefix: '7.4'
+
+
+
+  basic-test:
+    needs: [get-latest-redis-tag]
+    uses: ./.github/workflows/task-test.yml
+    with:
+      env: ubuntu-latest
+      get-redis: ${{ needs.get-latest-redis-tag.outputs.tag }}
+    secrets: inherit
+
+  coverage:
+    if: ${{ !github.event.pull_request.draft }}
+    uses: ./.github/workflows/flow-coverage.yml
+    secrets: inherit
+
+  sanitize:
+    needs: [get-latest-redis-tag]
+    if: ${{ !github.event.pull_request.draft }}
+    uses: ./.github/workflows/flow-sanitizer.yml
+    secrets: inherit
+    with:
+      get-redis: ${{ needs.get-latest-redis-tag.outputs.tag }}
+
+  pr-validation:
+    needs:
+      - get-latest-redis-tag
+      - basic-test
+      - coverage
+      - sanitize
+    runs-on: ubuntu-latest
+    if: ${{ !cancelled() }}
+    steps:
+      - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
+        run: exit 1
diff --git a/.github/workflows/flow-sanitizer.yml b/.github/workflows/flow-sanitizer.yml
@@ -0,0 +1,23 @@
+name: CLang Sanitizer
+
+# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor
+
+on:
+  workflow_call:
+    inputs:
+      flow-config:
+        type: string
+        required: true
+      get-redis:
+        type: string
+
+
+jobs:
+
+  clang-sanitizer:
+    uses: ./.github/workflows/task-test.yml
+    with:
+      test-config: ${{ inputs.flow-config }}
+      san: address
+      env: ubuntu-latest
+    secrets: inherit
diff --git a/.github/workflows/task-get-latest-tag.yml b/.github/workflows/task-get-latest-tag.yml
@@ -0,0 +1,63 @@
+name: Get Latest Release Tag of a GitHub Repository
+
+on:
+  workflow_call:
+    inputs:
+      repo:
+        description: "Repository name in the format of owner/repo"
+        type: string
+        required: true
+      prefix:
+        description: "Prefix to filter tags, for getting latest release of a specific version"
+        type: string
+    outputs:
+      tag: # Latest release tag
+        description: "Latest release tag"
+        value: ${{ jobs.get-tag.outputs.tag }}
+
+env:
+  RETRY_COUNT: 10
+  RETRY_MAX_TIME: 60 # seconds
+
+jobs:
+  get-tag: # Following best practices: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28
+    name: Get Latest Release Tag for ${{ inputs.repo }} ${{ inputs.prefix }}
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.latest.outputs.tag || steps.with-prefix.outputs.tag }}
+    defaults:
+      run:
+        shell: bash -l -eo pipefail {0}
+    steps:
+      - name: Get Latest Release Tag
+        id: latest
+        if: ${{ !inputs.prefix }}
+        # Get the `tag_name` of the latest release (latest patch of latest minor of latest major)
+        run: |
+          TAG=$(curl -sL --retry-all-errors \
+                      --retry ${{ env.RETRY_COUNT }} \
+                      --retry-max-time ${{ env.RETRY_MAX_TIME }} \
+                      -H "Accept: application/vnd.github+json" \
+                      -H "X-GitHub-Api-Version: 2022-11-28" \
+                      https://api.github.com/repos/${{ inputs.repo }}/releases/latest | \
+                jq -e -r '.tag_name') && \
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+      - name: Get Latest Release Tag with Prefix
+        id: with-prefix
+        if: ${{ inputs.prefix }}
+        # Get the `tag_name` of the latest release with prefix:
+        # Get 30 latest releases (by date), filter by prefix, sort by version, get the last one
+        run: |
+          TAG=$(curl -sL --retry-all-errors \
+                      --retry ${{ env.RETRY_COUNT }} \
+                      --retry-max-time ${{ env.RETRY_MAX_TIME }} \
+                      -H "Accept: application/vnd.github+json" \
+                      -H "X-GitHub-Api-Version: 2022-11-28" \
+                      https://api.github.com/repos/${{ inputs.repo }}/releases | \
+                jq -e -r '.[].tag_name | select(startswith("${{ inputs.prefix }}"))' | \
+                sort -V | tail -1) && \
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+
+      - name: Validate Tag
+        if: ${{ !steps.latest.outputs.tag && !steps.with-prefix.outputs.tag }}
+        run: exit 1
diff --git a/.github/workflows/task-test.yml b/.github/workflows/task-test.yml
@@ -0,0 +1,103 @@
+name: Common Flow for Tests
+
+# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor
+
+on:
+  workflow_call:
+    inputs:
+      env:
+        default: "ubuntu-latest"
+        type: string
+      container:
+        type: string
+      san:
+        type: string
+      get-redis:
+        type: string
+      test-config:
+        description: 'Test configuration environment variable (e.g. "CONFIG=tls" or "QUICK=1")'
+        required: true
+        type: string
+      pre-steps-script:
+        type: string
+        description: 'Script to run before any other steps (extremely basic dependency needs only)'
+      llvm_version:
+        type: string
+        description: 'Version of llvm to install'
+        default: '17'
+
+
+jobs:
+  common-flow:
+    name: Test ${{ inputs.container || inputs.env }}, Redis ${{ inputs.get-redis || 'unstable' }}
+    runs-on: ${{ inputs.env }}
+    container: ${{ inputs.container || null }}
+    # Nothing to do if both are `false`, skip
+    defaults:
+      run:
+        shell: bash -l -eo pipefail {0}
+    steps:
+      - name: Pre-steps Dependencies
+        if: inputs.pre-steps-script
+        run: ${{ inputs.pre-steps-script }}
+      - name: Get Installation Mode
+        id: mode
+        run: |
+          [[ -z "${{ inputs.container }}" ]] && echo "mode=sudo" >> $GITHUB_OUTPUT || echo "mode=" >> $GITHUB_OUTPUT
+      - name: Deps checkout
+        uses: actions/checkout@v3
+        with:
+          path: setup
+          sparse-checkout-cone-mode: false
+          sparse-checkout: |
+            .install
+            tests/pytests/requirements.*
+      # - name: Setup specific
+      #   working-directory: setup/.install
+      #   run: ./install_script.sh ${{ steps.mode.outputs.mode }}
+      - name: Full checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      # - name: Setup common
+      #   run: .install/common_installations.sh ${{ steps.mode.outputs.mode }}
+
+
+      - name: Install llvm & rust
+        if: inputs.san == 'address'
+        run: |
+          .install/install_llvm.sh ${{ steps.mode.outputs.mode }} ${{ inputs.llvm_version }}
+          .install/install_rust.sh 
+          echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
+          echo "CC=$(which clang-${{inputs.llvm_version}})" >> $GITHUB_ENV
+          echo "CXX=$(which clang++-${{inputs.llvm_version}})" >> $GITHUB_ENV
+
+      - name: verify CC and CXX
+        run: |
+          echo "CC=$CC"
+          echo "CXX=$CXX"
+
+      - name: Get Redis
+        uses: actions/checkout@v3
+        with:
+          repository: redis/redis
+          ref: ${{ inputs.get-redis }}
+          path: redis
+      - name: Build Redis
+        working-directory: redis
+        run: ${{ steps.mode.outputs.mode }} make install BUILD_TLS=yes SANIIZER=${{ inputs.san }}
+
+      - name: Build
+        if: ${{ inputs.standalone }}
+        run: make SAN=${{ inputs.san }}
+
+
+      - name: Tests
+        id: test1
+        if: ${{ inputs.standalone }}
+        continue-on-error: true
+        run: make test LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }}
+
+      - name: Fail flow if tests failed
+        if: steps.test1.outcome == 'failure'
+        run: exit 1
diff --git a/.install/install_llvm.sh b/.install/install_llvm.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+OS_TYPE=$(uname -s)
+
+MODE=$1 
+VERSION=$2
+
+
+
+if [[ $OS_TYPE == Darwin ]]; then
+    brew install llvm@$VERSION
+else
+    $MODE apt install lsb-release wget software-properties-common gnupg
+    wget https://apt.llvm.org/llvm.sh
+    chmod +x llvm.sh
+    $MODE ./llvm.sh $VERSION
+fi
diff --git a/.install/install_rust.sh b/.install/install_rust.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+processor=$(uname -m)
+OS_TYPE=$(uname -s)
+
+
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+. "$HOME/.cargo/env"  
+rustup update 
+rustup update nightly
+# for RedisJSON build with addess santizer
+rustup component add rust-src --toolchain nightly
+# if [[ $OS_TYPE = 'Darwin' ]]
+# then
+#     rustup component add rust-src --toolchain nightly-aarch64-apple-darwin
+# else
+#     if [[ $processor = 'x86_64' ]]
+#     then
+#         rustup component add rust-src --toolchain nightly-x86_64-unknown-linux-gnu
+#     else
+#         rustup component add rust-src --toolchain nightly-aarch64-unknown-linux-gnu
+#     fi
+# fi
+
diff --git a/tests/pytest/tests.sh b/tests/pytest/tests.sh
@@ -183,12 +183,12 @@ setup_clang_sanitizer() {
 	RLTEST_SAN_ARGS="--sanitizer $SAN"
 
 	if [[ $SAN == addr || $SAN == address ]]; then
-		REDIS_SERVER=${REDIS_SERVER:-redis-server-asan-$SAN_REDIS_SUFFIX}
-		if ! command -v $REDIS_SERVER > /dev/null; then
-			echo "Building Redis $SAN_REDIS_VER for clang-asan ..."
-			runn $READIES/bin/getredis --force -v $SAN_REDIS_VER --own-openssl --no-run \
-				--suffix asan-${SAN_REDIS_SUFFIX} --clang-asan --clang-san-blacklist $ignorelist
-		fi
+		# REDIS_SERVER=${REDIS_SERVER:-redis-server-asan-$SAN_REDIS_SUFFIX}
+		# if ! command -v $REDIS_SERVER > /dev/null; then
+		# 	echo "Building Redis $SAN_REDIS_VER for clang-asan ..."
+		# 	runn $READIES/bin/getredis --force -v $SAN_REDIS_VER --own-openssl --no-run \
+		# 		--suffix asan-${SAN_REDIS_SUFFIX} --clang-asan --clang-san-blacklist $ignorelist
+		# fi
 
 		# RLTest places log file details in ASAN_OPTIONS
 		export ASAN_OPTIONS="detect_odr_violation=0:halt_on_error=0:detect_leaks=1"