From 6a199f719b1fa8ac4b38c6a516d7e4994c2a9caa Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Tue, 8 Jul 2025 12:30:49 +0000 Subject: [PATCH 1/3] revert to previous nightly gauntlet --- .github/workflows/nightly-gauntlet.yaml | 183 ++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 .github/workflows/nightly-gauntlet.yaml diff --git a/.github/workflows/nightly-gauntlet.yaml b/.github/workflows/nightly-gauntlet.yaml new file mode 100644 index 0000000000000..64b520d07ba6e --- /dev/null +++ b/.github/workflows/nightly-gauntlet.yaml @@ -0,0 +1,183 @@ +# The nightly-gauntlet runs tests that are either too flaky or too slow to block +# every PR. +name: nightly-gauntlet +on: + schedule: + # Every day at 4AM + - cron: "0 4 * * 1-5" + workflow_dispatch: + +permissions: + contents: read + +jobs: + test-go-pg: + # make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below + # when changing runner sizes + runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }} + # This timeout must be greater than the timeout set by `go test` in + # `make test-postgres` to ensure we receive a trace of running + # goroutines. Setting this to the timeout +5m should work quite well + # even if some of the preceding steps are slow. + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - windows-2022 + steps: + - name: Harden Runner + uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 + with: + egress-policy: audit + + # macOS indexes all new files in the background. Our Postgres tests + # create and destroy thousands of databases on disk, and Spotlight + # tries to index all of them, seriously slowing down the tests. + - name: Disable Spotlight Indexing + if: runner.os == 'macOS' + run: | + sudo mdutil -a -i off + sudo mdutil -X / + sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist + + # Set up RAM disks to speed up the rest of the job. This action is in + # a separate repository to allow its use before actions/checkout. + - name: Setup RAM Disks + if: runner.os == 'Windows' + uses: coder/setup-ramdisk-action@79dacfe70c47ad6d6c0dd7f45412368802641439 + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 1 + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + # Runners have Go baked-in and Go will automatically + # download the toolchain configured in go.mod, so we don't + # need to reinstall it. It's faster on Windows runners. + use-preinstalled-go: ${{ runner.os == 'Windows' }} + use-temp-cache-dirs: ${{ runner.os == 'Windows' }} + + - name: Setup Terraform + uses: ./.github/actions/setup-tf + + - name: Test with PostgreSQL Database + env: + POSTGRES_VERSION: "13" + TS_DEBUG_DISCO: "true" + LC_CTYPE: "en_US.UTF-8" + LC_ALL: "en_US.UTF-8" + shell: bash + run: | + if [ "${{ runner.os }}" == "Windows" ]; then + # Create a temp dir on the R: ramdisk drive for Windows. The default + # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755 + mkdir -p "R:/temp/embedded-pg" + go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg" + fi + if [ "${{ runner.os }}" == "macOS" ]; then + # Postgres runs faster on a ramdisk on macOS too + mkdir -p /tmp/tmpfs + sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs + go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg + fi + + # if macOS, install google-chrome for scaletests + # As another concern, should we really have this kind of external dependency + # requirement on standard CI? + if [ "${{ matrix.os }}" == "macos-latest" ]; then + brew install google-chrome + fi + + # By default Go will use the number of logical CPUs, which + # is a fine default. + PARALLEL_FLAG="" + + # macOS will output "The default interactive shell is now zsh" + # intermittently in CI... + if [ "${{ matrix.os }}" == "macos-latest" ]; then + touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile + fi + + # Golang's default for these 2 variables is the number of logical CPUs. + # Our Windows and Linux runners have 16 cores, so they match up there. + NUM_PARALLEL_PACKAGES=16 + NUM_PARALLEL_TESTS=16 + if [ "${{ runner.os }}" == "Windows" ]; then + # On Windows Postgres chokes up when we have 16x16=256 tests + # running in parallel, and dbtestutil.NewDB starts to take more than + # 10s to complete sometimes causing test timeouts. With 16x8=128 tests + # Postgres tends not to choke. + NUM_PARALLEL_PACKAGES=8 + fi + if [ "${{ runner.os }}" == "macOS" ]; then + # Our macOS runners have 8 cores. We leave NUM_PARALLEL_TESTS at 16 + # because the tests complete faster and Postgres doesn't choke. It seems + # that macOS's tmpfs is faster than the one on Windows. + NUM_PARALLEL_PACKAGES=8 + fi + + # We rerun failing tests to counteract flakiness coming from Postgres + # choking on macOS and Windows sometimes. + DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=1000 \ + --format standard-quiet --packages "./..." \ + -- -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS -count=1 + + - name: Upload test stats to Datadog + timeout-minutes: 1 + continue-on-error: true + uses: ./.github/actions/upload-datadog + if: success() || failure() + with: + api-key: ${{ secrets.DATADOG_API_KEY }} + + notify-slack-on-failure: + needs: + - test-go-pg + runs-on: ubuntu-latest + if: failure() && github.ref == 'refs/heads/main' + + steps: + - name: Send Slack notification + run: | + curl -X POST -H 'Content-type: application/json' \ + --data '{ + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "❌ Nightly gauntlet failed", + "emoji": true + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Workflow:*\n${{ github.workflow }}" + }, + { + "type": "mrkdwn", + "text": "*Committer:*\n${{ github.actor }}" + }, + { + "type": "mrkdwn", + "text": "*Commit:*\n${{ github.sha }}" + } + ] + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*View failure:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Click here>" + } + } + ] + }' ${{ secrets.CI_FAILURE_SLACK_WEBHOOK }} From b3b94d77d5e29a5ac834cc6775d2f3c28884bd66 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Tue, 8 Jul 2025 12:34:15 +0000 Subject: [PATCH 2/3] update to current test-go-pg from main --- .github/workflows/nightly-gauntlet.yaml | 127 +++++++++++++++++++----- 1 file changed, 103 insertions(+), 24 deletions(-) diff --git a/.github/workflows/nightly-gauntlet.yaml b/.github/workflows/nightly-gauntlet.yaml index 64b520d07ba6e..f18ed511b796a 100644 --- a/.github/workflows/nightly-gauntlet.yaml +++ b/.github/workflows/nightly-gauntlet.yaml @@ -14,21 +14,23 @@ jobs: test-go-pg: # make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below # when changing runner sizes - runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }} + runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || matrix.os && matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }} + needs: changes + if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main' # This timeout must be greater than the timeout set by `go test` in # `make test-postgres` to ensure we receive a trace of running # goroutines. Setting this to the timeout +5m should work quite well # even if some of the preceding steps are slow. timeout-minutes: 25 strategy: - fail-fast: false matrix: os: + - ubuntu-latest - macos-latest - windows-2022 steps: - name: Harden Runner - uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0 + uses: step-security/harden-runner@6c439dc8bdf85cadbbce9ed30d1c7b959517bc49 # v2.12.2 with: egress-policy: audit @@ -46,13 +48,24 @@ jobs: # a separate repository to allow its use before actions/checkout. - name: Setup RAM Disks if: runner.os == 'Windows' - uses: coder/setup-ramdisk-action@79dacfe70c47ad6d6c0dd7f45412368802641439 + uses: coder/setup-ramdisk-action@e1100847ab2d7bcd9d14bcda8f2d1b0f07b36f1b - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 1 + - name: Setup Go Paths + id: go-paths + uses: ./.github/actions/setup-go-paths + + - name: Download Go Build Cache + id: download-go-build-cache + uses: ./.github/actions/test-cache/download + with: + key-prefix: test-go-build-${{ runner.os }}-${{ runner.arch }} + cache-path: ${{ steps.go-paths.outputs.cached-dirs }} + - name: Setup Go uses: ./.github/actions/setup-go with: @@ -60,11 +73,38 @@ jobs: # download the toolchain configured in go.mod, so we don't # need to reinstall it. It's faster on Windows runners. use-preinstalled-go: ${{ runner.os == 'Windows' }} - use-temp-cache-dirs: ${{ runner.os == 'Windows' }} + # Cache is already downloaded above + use-cache: false - name: Setup Terraform uses: ./.github/actions/setup-tf + - name: Download Test Cache + id: download-cache + uses: ./.github/actions/test-cache/download + with: + key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }} + + - name: Setup Embedded Postgres Cache Paths + id: embedded-pg-cache + uses: ./.github/actions/setup-embedded-pg-cache-paths + + - name: Download Embedded Postgres Cache + id: download-embedded-pg-cache + uses: ./.github/actions/embedded-pg-cache/download + with: + key-prefix: embedded-pg-${{ runner.os }}-${{ runner.arch }} + cache-path: ${{ steps.embedded-pg-cache.outputs.cached-dirs }} + + - name: Normalize File and Directory Timestamps + shell: bash + # Normalize file modification timestamps so that go test can use the + # cache from the previous CI run. See https://github.com/golang/go/issues/58571 + # for more details. + run: | + find . -type f ! -path ./.git/\*\* | mtimehash + find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} + + - name: Test with PostgreSQL Database env: POSTGRES_VERSION: "13" @@ -73,17 +113,21 @@ jobs: LC_ALL: "en_US.UTF-8" shell: bash run: | + set -o errexit + set -o pipefail + if [ "${{ runner.os }}" == "Windows" ]; then # Create a temp dir on the R: ramdisk drive for Windows. The default # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755 mkdir -p "R:/temp/embedded-pg" - go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg" - fi - if [ "${{ runner.os }}" == "macOS" ]; then + go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg" -cache "${EMBEDDED_PG_CACHE_DIR}" + elif [ "${{ runner.os }}" == "macOS" ]; then # Postgres runs faster on a ramdisk on macOS too mkdir -p /tmp/tmpfs sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs - go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg + go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg -cache "${EMBEDDED_PG_CACHE_DIR}" + elif [ "${{ runner.os }}" == "Linux" ]; then + make test-postgres-docker fi # if macOS, install google-chrome for scaletests @@ -93,39 +137,74 @@ jobs: brew install google-chrome fi - # By default Go will use the number of logical CPUs, which - # is a fine default. - PARALLEL_FLAG="" - # macOS will output "The default interactive shell is now zsh" # intermittently in CI... if [ "${{ matrix.os }}" == "macos-latest" ]; then touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile fi - # Golang's default for these 2 variables is the number of logical CPUs. - # Our Windows and Linux runners have 16 cores, so they match up there. - NUM_PARALLEL_PACKAGES=16 - NUM_PARALLEL_TESTS=16 if [ "${{ runner.os }}" == "Windows" ]; then + # Our Windows runners have 16 cores. # On Windows Postgres chokes up when we have 16x16=256 tests # running in parallel, and dbtestutil.NewDB starts to take more than # 10s to complete sometimes causing test timeouts. With 16x8=128 tests # Postgres tends not to choke. NUM_PARALLEL_PACKAGES=8 - fi - if [ "${{ runner.os }}" == "macOS" ]; then - # Our macOS runners have 8 cores. We leave NUM_PARALLEL_TESTS at 16 + NUM_PARALLEL_TESTS=16 + elif [ "${{ runner.os }}" == "macOS" ]; then + # Our macOS runners have 8 cores. We set NUM_PARALLEL_TESTS to 16 # because the tests complete faster and Postgres doesn't choke. It seems # that macOS's tmpfs is faster than the one on Windows. NUM_PARALLEL_PACKAGES=8 + NUM_PARALLEL_TESTS=16 + elif [ "${{ runner.os }}" == "Linux" ]; then + # Our Linux runners have 8 cores. + NUM_PARALLEL_PACKAGES=8 + NUM_PARALLEL_TESTS=8 + fi + + # by default, run tests with cache + TESTCOUNT="" + if [ "${{ github.ref }}" == "refs/heads/main" ]; then + # on main, run tests without cache + TESTCOUNT="-count=1" fi - # We rerun failing tests to counteract flakiness coming from Postgres - # choking on macOS and Windows sometimes. - DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=1000 \ + TEST_RETRIES="--rerun-fails=2 --rerun-fails-max-failures=50" + if [ "${{ github.ref }}" == "refs/heads/hugodutka/disable-test-retries-on-main" ]; then + # on main, run tests without retries + TEST_RETRIES="" + fi + + mkdir -p "$RUNNER_TEMP/sym" + source scripts/normalize_path.sh + # terraform gets installed in a random directory, so we need to normalize + # the path to the terraform binary or a bunch of cached tests will be + # invalidated. See scripts/normalize_path.sh for more details. + normalize_path_with_symlinks "$RUNNER_TEMP/sym" "$(dirname $(which terraform))" + + DB=ci gotestsum $TEST_RETRIES \ --format standard-quiet --packages "./..." \ - -- -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS -count=1 + -- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT + + - name: Upload Go Build Cache + uses: ./.github/actions/test-cache/upload + with: + cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }} + cache-path: ${{ steps.go-paths.outputs.cached-dirs }} + + - name: Upload Test Cache + uses: ./.github/actions/test-cache/upload + with: + cache-key: ${{ steps.download-cache.outputs.cache-key }} + + - name: Upload Embedded Postgres Cache + uses: ./.github/actions/embedded-pg-cache/upload + # We only use the embedded Postgres cache on macOS and Windows runners. + if: runner.OS == 'macOS' || runner.OS == 'Windows' + with: + cache-key: ${{ steps.download-embedded-pg-cache.outputs.cache-key }} + cache-path: "${{ steps.embedded-pg-cache.outputs.embedded-pg-cache }}" - name: Upload test stats to Datadog timeout-minutes: 1 From 52971030f0147bd162d5c5f19e71bb2fa0bc0cb0 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Tue, 8 Jul 2025 12:35:47 +0000 Subject: [PATCH 3/3] adapt for the nightly gauntlet --- .github/workflows/nightly-gauntlet.yaml | 67 ++----------------------- 1 file changed, 4 insertions(+), 63 deletions(-) diff --git a/.github/workflows/nightly-gauntlet.yaml b/.github/workflows/nightly-gauntlet.yaml index f18ed511b796a..a8e8fc957ee37 100644 --- a/.github/workflows/nightly-gauntlet.yaml +++ b/.github/workflows/nightly-gauntlet.yaml @@ -14,9 +14,7 @@ jobs: test-go-pg: # make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below # when changing runner sizes - runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || matrix.os && matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }} - needs: changes - if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main' + runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }} # This timeout must be greater than the timeout set by `go test` in # `make test-postgres` to ensure we receive a trace of running # goroutines. Setting this to the timeout +5m should work quite well @@ -25,7 +23,6 @@ jobs: strategy: matrix: os: - - ubuntu-latest - macos-latest - windows-2022 steps: @@ -55,17 +52,6 @@ jobs: with: fetch-depth: 1 - - name: Setup Go Paths - id: go-paths - uses: ./.github/actions/setup-go-paths - - - name: Download Go Build Cache - id: download-go-build-cache - uses: ./.github/actions/test-cache/download - with: - key-prefix: test-go-build-${{ runner.os }}-${{ runner.arch }} - cache-path: ${{ steps.go-paths.outputs.cached-dirs }} - - name: Setup Go uses: ./.github/actions/setup-go with: @@ -73,18 +59,10 @@ jobs: # download the toolchain configured in go.mod, so we don't # need to reinstall it. It's faster on Windows runners. use-preinstalled-go: ${{ runner.os == 'Windows' }} - # Cache is already downloaded above - use-cache: false - name: Setup Terraform uses: ./.github/actions/setup-tf - - name: Download Test Cache - id: download-cache - uses: ./.github/actions/test-cache/download - with: - key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }} - - name: Setup Embedded Postgres Cache Paths id: embedded-pg-cache uses: ./.github/actions/setup-embedded-pg-cache-paths @@ -96,15 +74,6 @@ jobs: key-prefix: embedded-pg-${{ runner.os }}-${{ runner.arch }} cache-path: ${{ steps.embedded-pg-cache.outputs.cached-dirs }} - - name: Normalize File and Directory Timestamps - shell: bash - # Normalize file modification timestamps so that go test can use the - # cache from the previous CI run. See https://github.com/golang/go/issues/58571 - # for more details. - run: | - find . -type f ! -path ./.git/\*\* | mtimehash - find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} + - - name: Test with PostgreSQL Database env: POSTGRES_VERSION: "13" @@ -163,41 +132,13 @@ jobs: NUM_PARALLEL_TESTS=8 fi - # by default, run tests with cache - TESTCOUNT="" - if [ "${{ github.ref }}" == "refs/heads/main" ]; then - # on main, run tests without cache - TESTCOUNT="-count=1" - fi - - TEST_RETRIES="--rerun-fails=2 --rerun-fails-max-failures=50" - if [ "${{ github.ref }}" == "refs/heads/hugodutka/disable-test-retries-on-main" ]; then - # on main, run tests without retries - TEST_RETRIES="" - fi - - mkdir -p "$RUNNER_TEMP/sym" - source scripts/normalize_path.sh - # terraform gets installed in a random directory, so we need to normalize - # the path to the terraform binary or a bunch of cached tests will be - # invalidated. See scripts/normalize_path.sh for more details. - normalize_path_with_symlinks "$RUNNER_TEMP/sym" "$(dirname $(which terraform))" + # run tests without cache + TESTCOUNT="-count=1" - DB=ci gotestsum $TEST_RETRIES \ + DB=ci gotestsum \ --format standard-quiet --packages "./..." \ -- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT - - name: Upload Go Build Cache - uses: ./.github/actions/test-cache/upload - with: - cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }} - cache-path: ${{ steps.go-paths.outputs.cached-dirs }} - - - name: Upload Test Cache - uses: ./.github/actions/test-cache/upload - with: - cache-key: ${{ steps.download-cache.outputs.cache-key }} - - name: Upload Embedded Postgres Cache uses: ./.github/actions/embedded-pg-cache/upload # We only use the embedded Postgres cache on macOS and Windows runners. pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy