Skip to content

Commit a1e6946

Browse files
hugodutkahugodutka
committed
enable go test cache in test-go-pg
1 parent 4ba317c commit a1e6946

File tree

4 files changed

+163
-38
lines changed

4 files changed

+163
-38
lines changed

‎.github/actions/setup-go-paths/action.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: "Setup Go Paths"
2+
description: Overrides Go paths like GOCACHE and GOMODCACHE to use temporary directories.
3+
outputs:
4+
gocache:
5+
description: "Value of GOCACHE"
6+
value: ${{ steps.paths.outputs.gocache }}
7+
gomodcache:
8+
description: "Value of GOMODCACHE"
9+
value: ${{ steps.paths.outputs.gomodcache }}
10+
gopath:
11+
description: "Value of GOPATH"
12+
value: ${{ steps.paths.outputs.gopath }}
13+
gotmp:
14+
description: "Value of GOTMPDIR"
15+
value: ${{ steps.paths.outputs.gotmp }}
16+
cached-dirs:
17+
description: "Go directories that should be cached between CI runs"
18+
value: ${{ steps.paths.outputs.cached-dirs }}
19+
runs:
20+
using: "composite"
21+
steps:
22+
- name: Override Go paths
23+
id: paths
24+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
25+
with:
26+
script: |
27+
const path = require('path');
28+
29+
// RUNNER_TEMP should be backed by a RAM disk on Windows if
30+
// coder/setup-ramdisk-action was used
31+
const runnerTemp = process.env.RUNNER_TEMP;
32+
const gocacheDir = path.join(runnerTemp, 'go-cache');
33+
const gomodcacheDir = path.join(runnerTemp, 'go-mod-cache');
34+
const gopathDir = path.join(runnerTemp, 'go-path');
35+
const gotmpDir = path.join(runnerTemp, 'go-tmp');
36+
37+
core.exportVariable('GOCACHE', gocacheDir);
38+
core.exportVariable('GOMODCACHE', gomodcacheDir);
39+
core.exportVariable('GOPATH', gopathDir);
40+
core.exportVariable('GOTMPDIR', gotmpDir);
41+
42+
core.setOutput('gocache', gocacheDir);
43+
core.setOutput('gomodcache', gomodcacheDir);
44+
core.setOutput('gopath', gopathDir);
45+
core.setOutput('gotmp', gotmpDir);
46+
47+
const cachedDirs = `${gocacheDir}\n${gomodcacheDir}`;
48+
core.setOutput('cached-dirs', cachedDirs);
49+
50+
- name: Create directories
51+
shell: bash
52+
run: |
53+
set -e
54+
mkdir -p "$GOCACHE"
55+
mkdir -p "$GOMODCACHE"
56+
mkdir -p "$GOPATH"
57+
mkdir -p "$GOTMPDIR"

‎.github/actions/setup-go/action.yaml

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,42 +8,26 @@ inputs:
88
use-preinstalled-go:
99
description: "Whether to use preinstalled Go."
1010
default: "false"
11-
use-temp-cache-dirs:
12-
description: "Whether to use temporary GOCACHE and GOMODCACHE directories."
13-
default: "false"
11+
use-cache:
12+
description: "Whether to use the cache."
13+
default: "true"
1414
runs:
1515
using: "composite"
1616
steps:
17-
- name: Override GOCACHE and GOMODCACHE
18-
shell: bash
19-
if: inputs.use-temp-cache-dirs == 'true'
20-
run: |
21-
# cd to another directory to ensure we're not inside a Go project.
22-
# That'd trigger Go to download the toolchain for that project.
23-
cd "$RUNNER_TEMP"
24-
# RUNNER_TEMP should be backed by a RAM disk on Windows if
25-
# coder/setup-ramdisk-action was used
26-
export GOCACHE_DIR="$RUNNER_TEMP""\go-cache"
27-
export GOMODCACHE_DIR="$RUNNER_TEMP""\go-mod-cache"
28-
export GOPATH_DIR="$RUNNER_TEMP""\go-path"
29-
export GOTMP_DIR="$RUNNER_TEMP""\go-tmp"
30-
mkdir -p "$GOCACHE_DIR"
31-
mkdir -p "$GOMODCACHE_DIR"
32-
mkdir -p "$GOPATH_DIR"
33-
mkdir -p "$GOTMP_DIR"
34-
go env -w GOCACHE="$GOCACHE_DIR"
35-
go env -w GOMODCACHE="$GOMODCACHE_DIR"
36-
go env -w GOPATH="$GOPATH_DIR"
37-
go env -w GOTMPDIR="$GOTMP_DIR"
3817
- name: Setup Go
3918
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
4019
with:
4120
go-version: ${{ inputs.use-preinstalled-go == 'false' && inputs.version || '' }}
21+
cache: ${{ inputs.use-cache }}
4222

4323
- name: Install gotestsum
4424
shell: bash
4525
run: go install gotest.tools/gotestsum@0d9599e513d70e5792bb9334869f82f6e8b53d4d # main as of 2025-05-15
4626

27+
- name: Install mtimehash
28+
shell: bash
29+
run: go install github.com/slsyy/mtimehash/cmd/mtimehash@a6b5da4ed2c4a40e7b805534b004e9fde7b53ce0 # v1.0.0
30+
4731
# It isn't necessary that we ever do this, but it helps
4832
# separate the "setup" from the "run" times.
4933
- name: go mod download

‎.github/workflows/ci.yaml

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -490,21 +490,33 @@ jobs:
490490
# a separate repository to allow its use before actions/checkout.
491491
- name: Setup RAM Disks
492492
if: runner.os == 'Windows'
493-
uses: coder/setup-ramdisk-action@79dacfe70c47ad6d6c0dd7f45412368802641439
493+
uses: coder/setup-ramdisk-action@417abac44189f3f234f4fd6dd8c1ddcc8c3acd5e
494494

495495
- name: Checkout
496496
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
497497
with:
498498
fetch-depth: 1
499499

500+
- name: Setup Go Paths
501+
id: go-paths
502+
uses: ./.github/actions/setup-go-paths
503+
504+
- name: Download Go Build Cache
505+
id: download-go-build-cache
506+
uses: ./.github/actions/test-cache/download
507+
with:
508+
key-prefix: test-go-build-pr4-${{ runner.os }}-${{ runner.arch }}
509+
cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
510+
500511
- name: Setup Go
512+
id: setup-go
501513
uses: ./.github/actions/setup-go
502514
with:
503515
# Runners have Go baked-in and Go will automatically
504516
# download the toolchain configured in go.mod, so we don't
505517
# need to reinstall it. It's faster on Windows runners.
506518
use-preinstalled-go: ${{ runner.os == 'Windows' }}
507-
use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
519+
use-cache: false
508520

509521
- name: Setup Terraform
510522
uses: ./.github/actions/setup-tf
@@ -515,14 +527,24 @@ jobs:
515527
with:
516528
key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
517529

530+
- name: Normalize File and Directory Timestamps
531+
shell: bash
532+
run: |
533+
find . -type f ! -path ./.git/\*\* | mtimehash
534+
find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} +
535+
518536
- name: Test with PostgreSQL Database
519537
env:
520538
POSTGRES_VERSION: "13"
521539
TS_DEBUG_DISCO: "true"
522540
LC_CTYPE: "en_US.UTF-8"
523541
LC_ALL: "en_US.UTF-8"
542+
524543
shell: bash
525544
run: |
545+
set -o errexit
546+
set -o pipefail
547+
526548
if [ "${{ runner.os }}" == "Windows" ]; then
527549
# Create a temp dir on the R: ramdisk drive for Windows. The default
528550
# C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
@@ -533,6 +555,8 @@ jobs:
533555
mkdir -p /tmp/tmpfs
534556
sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
535557
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
558+
elif [ "${{ runner.os }}" == "Linux" ]; then
559+
make test-postgres-docker
536560
fi
537561
538562
# if macOS, install google-chrome for scaletests
@@ -542,10 +566,6 @@ jobs:
542566
brew install google-chrome
543567
fi
544568
545-
# By default Go will use the number of logical CPUs, which
546-
# is a fine default.
547-
PARALLEL_FLAG=""
548-
549569
# macOS will output "The default interactive shell is now zsh"
550570
# intermittently in CI...
551571
if [ "${{ matrix.os }}" == "macos-latest" ]; then
@@ -572,16 +592,33 @@ jobs:
572592
NUM_PARALLEL_TESTS=8
573593
fi
574594
575-
if [ "${{ runner.os }}" == "Linux" ]; then
576-
make test-postgres
577-
else
578-
# We rerun failing tests to counteract flakiness coming from Postgres
579-
# choking on macOS and Windows sometimes.
580-
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
581-
--format standard-quiet --packages "./..." \
582-
-- -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS -count=1
595+
# by default, run tests with cache
596+
TESTCOUNT=""
597+
if [ "${{ github.ref }}" == "refs/heads/main" ]; then
598+
# on main, run tests without cache
599+
TESTCOUNT="-count=1"
583600
fi
584601
602+
# see scripts/normalize_path.sh for why we need this
603+
mkdir -p "$RUNNER_TEMP/sym"
604+
source scripts/normalize_path.sh
605+
export DIR_PREFIX="$(dirname $(which terraform))"
606+
normalize_path_with_symlinks "$RUNNER_TEMP/sym"
607+
608+
# We rerun failing tests to counteract flakiness coming from Postgres
609+
# choking on macOS and Windows sometimes.
610+
# TODO: for testing purposes we're only running the agent tests.
611+
# THIS MUST BE REVERTED BEFORE MERGING.
612+
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
613+
--format standard-quiet --packages "./agent/..." \
614+
-- -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
615+
616+
- name: Upload Go Build Cache
617+
uses: ./.github/actions/test-cache/upload
618+
with:
619+
cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}
620+
cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
621+
585622
- name: Upload Test Cache
586623
uses: ./.github/actions/test-cache/upload
587624
with:

‎scripts/normalize_path.sh

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
3+
# Call: normalize_path_with_symlinks [target_dir]
4+
# Normalizes the PATH environment variable by replacing each directory
5+
# with a symbolic link in target_dir. For example, if PATH is "/usr/bin:/bin",
6+
# and target_dir is /tmp, PATH will become "/tmp/0:/tmp/1", where /tmp/0 links
7+
# to /usr/bin and /tmp/1 links to /bin.
8+
#
9+
# This is useful for ensuring that PATH is consistent across CI runs and helps
10+
# with reusing the same cache across them. Many of our go tests read the PATH
11+
# variable, and if it changes between runs, the cache gets invalidated.
12+
normalize_path_with_symlinks() {
13+
local target_dir="${1:-/tmp}" # Use first argument or default to /tmp
14+
local old_path="$PATH"
15+
local -a new_parts=()
16+
local i=0
17+
18+
IFS=':' read -ra _parts <<<"$old_path"
19+
for dir in "${_parts[@]}"; do
20+
# Skip empty components that can arise from "::"
21+
[[ -z $dir ]] && continue
22+
23+
# Skip directories that don't start with $DIR_PREFIX, if $DIR_PREFIX is set
24+
if [[ -n "$DIR_PREFIX" && "$dir" != "$DIR_PREFIX"* ]]; then
25+
new_parts+=("$dir")
26+
continue
27+
fi
28+
29+
local link="$target_dir/$i"
30+
31+
# Replace any pre-existing file or link at $target_dir/$i
32+
if [[ -e $link || -L $link ]]; then
33+
rm -rf -- "$link"
34+
fi
35+
36+
# without MSYS ln will deepcopy the directory on Windows
37+
MSYS=winsymlinks:nativestrict ln -s -- "$dir" "$link"
38+
new_parts+=("$link")
39+
i=$((i + 1))
40+
done
41+
42+
export PATH
43+
PATH="$(
44+
IFS=':'
45+
echo "${new_parts[*]}"
46+
)"
47+
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy