Skip to content

Commit 42fd1c1

Browse files
johnstcnjohnstcn
authored
ci: cache embedded postgres downloaded binaries (#18477)
Updates CI job definitions to cache downloaded binaries for embedded-postgres.
1 parent c4e4fe8 commit 42fd1c1

File tree

5 files changed

+158
-10
lines changed

5 files changed

+158
-10
lines changed

‎.github/actions/embedded-pg-cache/download/action.yml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: "Download Embedded Postgres Cache"
2+
description: |
3+
Downloads the embedded postgres cache and outputs today's cache key.
4+
A PR job can use a cache if it was created by its base branch, its current
5+
branch, or the default branch.
6+
https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
7+
outputs:
8+
cache-key:
9+
description: "Today's cache key"
10+
value: ${{ steps.vars.outputs.cache-key }}
11+
inputs:
12+
key-prefix:
13+
description: "Prefix for the cache key"
14+
required: true
15+
cache-path:
16+
description: "Path to the cache directory"
17+
required: true
18+
runs:
19+
using: "composite"
20+
steps:
21+
- name: Get date values and cache key
22+
id: vars
23+
shell: bash
24+
run: |
25+
export YEAR_MONTH=$(date +'%Y-%m')
26+
export PREV_YEAR_MONTH=$(date -d 'last month' +'%Y-%m')
27+
export DAY=$(date +'%d')
28+
echo "year-month=$YEAR_MONTH" >> $GITHUB_OUTPUT
29+
echo "prev-year-month=$PREV_YEAR_MONTH" >> $GITHUB_OUTPUT
30+
echo "cache-key=${{ inputs.key-prefix }}-${YEAR_MONTH}-${DAY}" >> $GITHUB_OUTPUT
31+
32+
# By default, depot keeps caches for 14 days. This is plenty for embedded
33+
# postgres, which changes infrequently.
34+
# https://depot.dev/docs/github-actions/overview#cache-retention-policy
35+
- name: Download embedded Postgres cache
36+
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
37+
with:
38+
path: ${{ inputs.cache-path }}
39+
key: ${{ steps.vars.outputs.cache-key }}
40+
# > If there are multiple partial matches for a restore key, the action returns the most recently created cache.
41+
# https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
42+
# The second restore key allows non-main branches to use the cache from the previous month.
43+
# This prevents PRs from rebuilding the cache on the first day of the month.
44+
# It also makes sure that once a month, the cache is fully reset.
45+
restore-keys: |
46+
${{ inputs.key-prefix }}-${{ steps.vars.outputs.year-month }}-
47+
${{ github.ref != 'refs/heads/main' && format('{0}-{1}-', inputs.key-prefix, steps.vars.outputs.prev-year-month) || '' }}

‎.github/actions/embedded-pg-cache/upload/action.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: "Upload Embedded Postgres Cache"
2+
description: Uploads the embedded Postgres cache. This only runs on the main branch.
3+
inputs:
4+
cache-key:
5+
description: "Cache key"
6+
required: true
7+
cache-path:
8+
description: "Path to the cache directory"
9+
required: true
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Upload Embedded Postgres cache
14+
if: ${{ github.ref == 'refs/heads/main' }}
15+
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
16+
with:
17+
path: ${{ inputs.cache-path }}
18+
key: ${{ inputs.cache-key }}

‎.github/actions/setup-embedded-pg-cache-paths/action.yml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: "Setup Embedded Postgres Cache Paths"
2+
description: Sets up a path for cached embedded postgres binaries.
3+
outputs:
4+
embedded-pg-cache:
5+
description: "Value of EMBEDDED_PG_CACHE_DIR"
6+
value: ${{ steps.paths.outputs.embedded-pg-cache }}
7+
cached-dirs:
8+
description: "directories that should be cached between CI runs"
9+
value: ${{ steps.paths.outputs.cached-dirs }}
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Override Go paths
14+
id: paths
15+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
16+
with:
17+
script: |
18+
const path = require('path');
19+
20+
// RUNNER_TEMP should be backed by a RAM disk on Windows if
21+
// coder/setup-ramdisk-action was used
22+
const runnerTemp = process.env.RUNNER_TEMP;
23+
const embeddedPgCacheDir = path.join(runnerTemp, 'embedded-pg-cache');
24+
core.exportVariable('EMBEDDED_PG_CACHE_DIR', embeddedPgCacheDir);
25+
core.setOutput('embedded-pg-cache', embeddedPgCacheDir);
26+
const cachedDirs = `${embeddedPgCacheDir}`;
27+
core.setOutput('cached-dirs', cachedDirs);
28+
29+
- name: Create directories
30+
shell: bash
31+
run: |
32+
set -e
33+
mkdir -p "$EMBEDDED_PG_CACHE_DIR"

‎.github/workflows/ci.yaml

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,17 @@ jobs:
473473
with:
474474
key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
475475

476+
- name: Setup Embedded Postgres Cache Paths
477+
id: embedded-pg-cache
478+
uses: ./.github/actions/setup-embedded-pg-cache-paths
479+
480+
- name: Download Embedded Postgres Cache
481+
id: download-embedded-pg-cache
482+
uses: ./.github/actions/embedded-pg-cache/download
483+
with:
484+
key-prefix: embedded-pg-${{ runner.os }}-${{ runner.arch }}
485+
cache-path: ${{ steps.embedded-pg-cache.outputs.cached-dirs }}
486+
476487
- name: Normalize File and Directory Timestamps
477488
shell: bash
478489
# Normalize file modification timestamps so that go test can use the
@@ -497,12 +508,12 @@ jobs:
497508
# Create a temp dir on the R: ramdisk drive for Windows. The default
498509
# C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
499510
mkdir -p "R:/temp/embedded-pg"
500-
go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
511+
go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg" -cache "${EMBEDDED_PG_CACHE_DIR}"
501512
elif [ "${{ runner.os }}" == "macOS" ]; then
502513
# Postgres runs faster on a ramdisk on macOS too
503514
mkdir -p /tmp/tmpfs
504515
sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
505-
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
516+
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg -cache "${EMBEDDED_PG_CACHE_DIR}"
506517
elif [ "${{ runner.os }}" == "Linux" ]; then
507518
make test-postgres-docker
508519
fi
@@ -571,6 +582,14 @@ jobs:
571582
with:
572583
cache-key: ${{ steps.download-cache.outputs.cache-key }}
573584

585+
- name: Upload Embedded Postgres Cache
586+
uses: ./.github/actions/embedded-pg-cache/upload
587+
# We only use the embedded Postgres cache on macOS and Windows runners.
588+
if: runner.OS == 'macOS' || runner.OS == 'Windows'
589+
with:
590+
cache-key: ${{ steps.download-embedded-pg-cache.outputs.cache-key }}
591+
cache-path: "${{ steps.embedded-pg-cache.outputs.embedded-pg-cache }}"
592+
574593
- name: Upload test stats to Datadog
575594
timeout-minutes: 1
576595
continue-on-error: true

‎scripts/embedded-pg/main.go

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,31 +4,43 @@ package main
44
import (
55
"database/sql"
66
"flag"
7+
"log"
78
"os"
89
"path/filepath"
10+
"time"
911

1012
embeddedpostgres "github.com/fergusstrange/embedded-postgres"
1113
)
1214

1315
func main() {
1416
var customPath string
17+
var cachePath string
1518
flag.StringVar(&customPath, "path", "", "Optional custom path for postgres data directory")
19+
flag.StringVar(&cachePath, "cache", "", "Optional custom path for embedded postgres binaries")
1620
flag.Parse()
1721

1822
postgresPath := filepath.Join(os.TempDir(), "coder-test-postgres")
1923
if customPath != "" {
2024
postgresPath = customPath
2125
}
26+
if err := os.MkdirAll(postgresPath, os.ModePerm); err != nil {
27+
log.Fatalf("Failed to create directory %s: %v", postgresPath, err)
28+
}
29+
if cachePath == "" {
30+
cachePath = filepath.Join(postgresPath, "cache")
31+
}
32+
if err := os.MkdirAll(cachePath, os.ModePerm); err != nil {
33+
log.Fatalf("Failed to create directory %s: %v", cachePath, err)
34+
}
2235

2336
ep := embeddedpostgres.NewDatabase(
2437
embeddedpostgres.DefaultConfig().
2538
Version(embeddedpostgres.V16).
2639
BinariesPath(filepath.Join(postgresPath, "bin")).
27-
// Default BinaryRepositoryURL repo1.maven.org is flaky.
2840
BinaryRepositoryURL("https://repo.maven.apache.org/maven2").
2941
DataPath(filepath.Join(postgresPath, "data")).
3042
RuntimePath(filepath.Join(postgresPath, "runtime")).
31-
CachePath(filepath.Join(postgresPath, "cache")).
43+
CachePath(cachePath).
3244
Username("postgres").
3345
Password("postgres").
3446
Database("postgres").
@@ -38,8 +50,27 @@ func main() {
3850
)
3951
err := ep.Start()
4052
if err != nil {
41-
panic(err)
53+
log.Fatalf("Failed to start embedded postgres: %v", err)
54+
}
55+
56+
// Troubleshooting: list files in cachePath
57+
if err := filepath.Walk(cachePath, func(path string, info os.FileInfo, err error) error {
58+
if err != nil {
59+
return err
60+
}
61+
switch {
62+
case info.IsDir():
63+
log.Printf("D: %s", path)
64+
case info.Mode().IsRegular():
65+
log.Printf("F: %s [%s] (%d bytes) %s", path, info.Mode().String(), info.Size(), info.ModTime().Format(time.RFC3339))
66+
default:
67+
log.Printf("Other: %s [%s] %s", path, info.Mode(), info.ModTime().Format(time.RFC3339))
68+
}
69+
return nil
70+
}); err != nil {
71+
log.Printf("Failed to list files in cachePath %s: %v", cachePath, err)
4272
}
73+
4374
// We execute these queries instead of using the embeddedpostgres
4475
// StartParams because it doesn't work on Windows. The library
4576
// seems to have a bug where it sends malformed parameters to
@@ -58,21 +89,21 @@ func main() {
5889
}
5990
db, err := sql.Open("postgres", "postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable")
6091
if err != nil {
61-
panic(err)
92+
log.Fatalf("Failed to connect to embedded postgres: %v", err)
6293
}
6394
for _, query := range paramQueries {
6495
if _, err := db.Exec(query); err != nil {
65-
panic(err)
96+
log.Fatalf("Failed to execute setup query %q: %v", query, err)
6697
}
6798
}
6899
if err := db.Close(); err != nil {
69-
panic(err)
100+
log.Fatalf("Failed to close database connection: %v", err)
70101
}
71102
// We restart the database to apply all the parameters.
72103
if err := ep.Stop(); err != nil {
73-
panic(err)
104+
log.Fatalf("Failed to stop embedded postgres after applying parameters: %v", err)
74105
}
75106
if err := ep.Start(); err != nil {
76-
panic(err)
107+
log.Fatalf("Failed to start embedded postgres after applying parameters: %v", err)
77108
}
78109
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy