Skip to content

Commit 2d8d57f

Browse files
hugodutkahugodutka
committed
enable go test cache in test-go-pg
1 parent 5e2fd9d commit 2d8d57f

File tree

5 files changed

+180
-41
lines changed

5 files changed

+180
-41
lines changed

‎.github/actions/setup-go-paths/action.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: "Setup Go Paths"
2+
description: Overrides Go paths like GOCACHE and GOMODCACHE to use temporary directories.
3+
outputs:
4+
gocache:
5+
description: "Value of GOCACHE"
6+
value: ${{ steps.paths.outputs.gocache }}
7+
gomodcache:
8+
description: "Value of GOMODCACHE"
9+
value: ${{ steps.paths.outputs.gomodcache }}
10+
gopath:
11+
description: "Value of GOPATH"
12+
value: ${{ steps.paths.outputs.gopath }}
13+
gotmp:
14+
description: "Value of GOTMPDIR"
15+
value: ${{ steps.paths.outputs.gotmp }}
16+
cached-dirs:
17+
description: "Go directories that should be cached between CI runs"
18+
value: ${{ steps.paths.outputs.cached-dirs }}
19+
runs:
20+
using: "composite"
21+
steps:
22+
- name: Override Go paths
23+
id: paths
24+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
25+
with:
26+
script: |
27+
const path = require('path');
28+
29+
// RUNNER_TEMP should be backed by a RAM disk on Windows if
30+
// coder/setup-ramdisk-action was used
31+
const runnerTemp = process.env.RUNNER_TEMP;
32+
const gocacheDir = path.join(runnerTemp, 'go-cache');
33+
const gomodcacheDir = path.join(runnerTemp, 'go-mod-cache');
34+
const gopathDir = path.join(runnerTemp, 'go-path');
35+
const gotmpDir = path.join(runnerTemp, 'go-tmp');
36+
37+
core.exportVariable('GOCACHE', gocacheDir);
38+
core.exportVariable('GOMODCACHE', gomodcacheDir);
39+
core.exportVariable('GOPATH', gopathDir);
40+
core.exportVariable('GOTMPDIR', gotmpDir);
41+
42+
core.setOutput('gocache', gocacheDir);
43+
core.setOutput('gomodcache', gomodcacheDir);
44+
core.setOutput('gopath', gopathDir);
45+
core.setOutput('gotmp', gotmpDir);
46+
47+
const cachedDirs = `${gocacheDir}\n${gomodcacheDir}`;
48+
core.setOutput('cached-dirs', cachedDirs);
49+
50+
- name: Create directories
51+
shell: bash
52+
run: |
53+
set -e
54+
mkdir -p "$GOCACHE"
55+
mkdir -p "$GOMODCACHE"
56+
mkdir -p "$GOPATH"
57+
mkdir -p "$GOTMPDIR"

‎.github/actions/setup-go/action.yaml

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,42 +8,26 @@ inputs:
88
use-preinstalled-go:
99
description: "Whether to use preinstalled Go."
1010
default: "false"
11-
use-temp-cache-dirs:
12-
description: "Whether to use temporary GOCACHE and GOMODCACHE directories."
13-
default: "false"
11+
use-cache:
12+
description: "Whether to use the cache."
13+
default: "true"
1414
runs:
1515
using: "composite"
1616
steps:
17-
- name: Override GOCACHE and GOMODCACHE
18-
shell: bash
19-
if: inputs.use-temp-cache-dirs == 'true'
20-
run: |
21-
# cd to another directory to ensure we're not inside a Go project.
22-
# That'd trigger Go to download the toolchain for that project.
23-
cd "$RUNNER_TEMP"
24-
# RUNNER_TEMP should be backed by a RAM disk on Windows if
25-
# coder/setup-ramdisk-action was used
26-
export GOCACHE_DIR="$RUNNER_TEMP""\go-cache"
27-
export GOMODCACHE_DIR="$RUNNER_TEMP""\go-mod-cache"
28-
export GOPATH_DIR="$RUNNER_TEMP""\go-path"
29-
export GOTMP_DIR="$RUNNER_TEMP""\go-tmp"
30-
mkdir -p "$GOCACHE_DIR"
31-
mkdir -p "$GOMODCACHE_DIR"
32-
mkdir -p "$GOPATH_DIR"
33-
mkdir -p "$GOTMP_DIR"
34-
go env -w GOCACHE="$GOCACHE_DIR"
35-
go env -w GOMODCACHE="$GOMODCACHE_DIR"
36-
go env -w GOPATH="$GOPATH_DIR"
37-
go env -w GOTMPDIR="$GOTMP_DIR"
3817
- name: Setup Go
3918
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
4019
with:
4120
go-version: ${{ inputs.use-preinstalled-go == 'false' && inputs.version || '' }}
21+
cache: ${{ inputs.use-cache }}
4222

4323
- name: Install gotestsum
4424
shell: bash
4525
run: go install gotest.tools/gotestsum@0d9599e513d70e5792bb9334869f82f6e8b53d4d # main as of 2025-05-15
4626

27+
- name: Install mtimehash
28+
shell: bash
29+
run: go install github.com/slsyy/mtimehash/cmd/mtimehash@a6b5da4ed2c4a40e7b805534b004e9fde7b53ce0 # v1.0.0
30+
4731
# It isn't necessary that we ever do this, but it helps
4832
# separate the "setup" from the "run" times.
4933
- name: go mod download

‎.github/workflows/ci.yaml

Lines changed: 57 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -336,21 +336,23 @@ jobs:
336336
# a separate repository to allow its use before actions/checkout.
337337
- name: Setup RAM Disks
338338
if: runner.os == 'Windows'
339-
uses: coder/setup-ramdisk-action@81c5c441bda00c6c3d6bcee2e5a33ed4aadbbcc1
339+
uses: coder/setup-ramdisk-action@a4b59caa8be2e88c348abeef042d7c1a33d8743e
340340

341341
- name: Checkout
342342
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
343343
with:
344344
fetch-depth: 1
345345

346+
- name: Setup Go Paths
347+
uses: ./.github/actions/setup-go-paths
348+
346349
- name: Setup Go
347350
uses: ./.github/actions/setup-go
348351
with:
349352
# Runners have Go baked-in and Go will automatically
350353
# download the toolchain configured in go.mod, so we don't
351354
# need to reinstall it. It's faster on Windows runners.
352355
use-preinstalled-go: ${{ runner.os == 'Windows' }}
353-
use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
354356

355357
- name: Setup Terraform
356358
uses: ./.github/actions/setup-tf
@@ -490,21 +492,33 @@ jobs:
490492
# a separate repository to allow its use before actions/checkout.
491493
- name: Setup RAM Disks
492494
if: runner.os == 'Windows'
493-
uses: coder/setup-ramdisk-action@79dacfe70c47ad6d6c0dd7f45412368802641439
495+
uses: coder/setup-ramdisk-action@a4b59caa8be2e88c348abeef042d7c1a33d8743e
494496

495497
- name: Checkout
496498
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
497499
with:
498500
fetch-depth: 1
499501

502+
- name: Setup Go Paths
503+
id: go-paths
504+
uses: ./.github/actions/setup-go-paths
505+
506+
- name: Download Go Build Cache
507+
id: download-go-build-cache
508+
uses: ./.github/actions/test-cache/download
509+
with:
510+
key-prefix: test-go-build-pr5-${{ runner.os }}-${{ runner.arch }}
511+
cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
512+
500513
- name: Setup Go
501514
uses: ./.github/actions/setup-go
502515
with:
503516
# Runners have Go baked-in and Go will automatically
504517
# download the toolchain configured in go.mod, so we don't
505518
# need to reinstall it. It's faster on Windows runners.
506519
use-preinstalled-go: ${{ runner.os == 'Windows' }}
507-
use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
520+
# Cache is already downloaded above
521+
use-cache: false
508522

509523
- name: Setup Terraform
510524
uses: ./.github/actions/setup-tf
@@ -515,14 +529,27 @@ jobs:
515529
with:
516530
key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
517531

532+
- name: Normalize File and Directory Timestamps
533+
shell: bash
534+
# Normalize file modification timestamps so that go test can use the
535+
# cache from the previous CI run. See https://github.com/golang/go/issues/58571
536+
# for more details.
537+
run: |
538+
find . -type f ! -path ./.git/\*\* | mtimehash
539+
find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} +
540+
518541
- name: Test with PostgreSQL Database
519542
env:
520543
POSTGRES_VERSION: "13"
521544
TS_DEBUG_DISCO: "true"
522545
LC_CTYPE: "en_US.UTF-8"
523546
LC_ALL: "en_US.UTF-8"
547+
524548
shell: bash
525549
run: |
550+
set -o errexit
551+
set -o pipefail
552+
526553
if [ "${{ runner.os }}" == "Windows" ]; then
527554
# Create a temp dir on the R: ramdisk drive for Windows. The default
528555
# C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
@@ -533,6 +560,8 @@ jobs:
533560
mkdir -p /tmp/tmpfs
534561
sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
535562
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
563+
elif [ "${{ runner.os }}" == "Linux" ]; then
564+
make test-postgres-docker
536565
fi
537566
538567
# if macOS, install google-chrome for scaletests
@@ -542,10 +571,6 @@ jobs:
542571
brew install google-chrome
543572
fi
544573
545-
# By default Go will use the number of logical CPUs, which
546-
# is a fine default.
547-
PARALLEL_FLAG=""
548-
549574
# macOS will output "The default interactive shell is now zsh"
550575
# intermittently in CI...
551576
if [ "${{ matrix.os }}" == "macos-latest" ]; then
@@ -572,16 +597,32 @@ jobs:
572597
NUM_PARALLEL_TESTS=8
573598
fi
574599
575-
if [ "${{ runner.os }}" == "Linux" ]; then
576-
make test-postgres
577-
else
578-
# We rerun failing tests to counteract flakiness coming from Postgres
579-
# choking on macOS and Windows sometimes.
580-
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
581-
--format standard-quiet --packages "./..." \
582-
-- -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS -count=1
600+
# by default, run tests with cache
601+
TESTCOUNT=""
602+
if [ "${{ github.ref }}" == "refs/heads/main" ]; then
603+
# on main, run tests without cache
604+
TESTCOUNT="-count=1"
583605
fi
584606
607+
mkdir -p "$RUNNER_TEMP/sym"
608+
source scripts/normalize_path.sh
609+
# terraform gets installed in a random directory, so we need to normalize
610+
# the path to the terraform binary or a bunch of cached tests will be
611+
# invalidated. See scripts/normalize_path.sh for more details.
612+
normalize_path_with_symlinks "$RUNNER_TEMP/sym" "$(dirname $(which terraform))"
613+
614+
# We rerun failing tests to counteract flakiness coming from Postgres
615+
# choking on macOS and Windows sometimes.
616+
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
617+
--format standard-quiet --packages "./..." \
618+
-- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
619+
620+
- name: Upload Go Build Cache
621+
uses: ./.github/actions/test-cache/upload
622+
with:
623+
cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}
624+
cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
625+
585626
- name: Upload Test Cache
586627
uses: ./.github/actions/test-cache/upload
587628
with:

‎.github/workflows/nightly-gauntlet.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,16 @@ jobs:
5353
with:
5454
fetch-depth: 1
5555

56+
- name: Setup Go Paths
57+
uses: ./.github/actions/setup-go-paths
58+
5659
- name: Setup Go
5760
uses: ./.github/actions/setup-go
5861
with:
5962
# Runners have Go baked-in and Go will automatically
6063
# download the toolchain configured in go.mod, so we don't
6164
# need to reinstall it. It's faster on Windows runners.
6265
use-preinstalled-go: ${{ runner.os == 'Windows' }}
63-
use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
6466

6567
- name: Setup Terraform
6668
uses: ./.github/actions/setup-tf

‎scripts/normalize_path.sh

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
3+
# Call: normalize_path_with_symlinks [target_dir] [dir_prefix]
4+
#
5+
# Normalizes the PATH environment variable by replacing each directory that
6+
# begins with dir_prefix with a symbolic link in target_dir. For example, if
7+
# PATH is "/usr/bin:/bin", target_dir is /tmp, and dir_prefix is /usr, then
8+
# PATH will become "/tmp/0:/bin", where /tmp/0 links to /usr/bin.
9+
#
10+
# This is useful for ensuring that PATH is consistent across CI runs and helps
11+
# with reusing the same cache across them. Many of our go tests read the PATH
12+
# variable, and if it changes between runs, the cache gets invalidated.
13+
normalize_path_with_symlinks() {
14+
local target_dir="${1:-}"
15+
local dir_prefix="${2:-}"
16+
17+
if [[ -z "$target_dir" || -z "$dir_prefix" ]]; then
18+
echo "Usage: normalize_path_with_symlinks <target_dir> <dir_prefix>"
19+
return 1
20+
fi
21+
22+
local old_path="$PATH"
23+
local -a new_parts=()
24+
local i=0
25+
26+
IFS=':' read -ra _parts <<<"$old_path"
27+
for dir in "${_parts[@]}"; do
28+
# Skip empty components that can arise from "::"
29+
[[ -z $dir ]] && continue
30+
31+
# Skip directories that don't start with $dir_prefix
32+
if [[ "$dir" != "$dir_prefix"* ]]; then
33+
new_parts+=("$dir")
34+
continue
35+
fi
36+
37+
local link="$target_dir/$i"
38+
39+
# Replace any pre-existing file or link at $target_dir/$i
40+
if [[ -e $link || -L $link ]]; then
41+
rm -rf -- "$link"
42+
fi
43+
44+
# without MSYS ln will deepcopy the directory on Windows
45+
MSYS=winsymlinks:nativestrict ln -s -- "$dir" "$link"
46+
new_parts+=("$link")
47+
i=$((i + 1))
48+
done
49+
50+
export PATH
51+
PATH="$(
52+
IFS=':'
53+
echo "${new_parts[*]}"
54+
)"
55+
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy