chore: run macOS, windows, and race tests with Postgres in CI (#15520)

hugodutka · web-flow · commit c7c35ef4d751 · 2024-12-03T13:33:17.000+01:00
This PR is the second in a series aimed at closing #15109. ## Changes - adds `scripts/embedded-pg/main.go`, which can start a native Postgres database. This is used to set up PG on Windows and macOS, as these platforms don't support Docker in Github Actions. - runs the `test-go-pg` job on macOS and Windows too - adds the `test-go-race-go` job, which runs race tests with Postgres on Linux
diff --git a/.github/actions/setup-imdisk/action.yaml b/.github/actions/setup-imdisk/action.yaml
@@ -0,0 +1,27 @@
+name: "Setup ImDisk"
+if: runner.os == 'Windows'
+description: |
+  Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
+runs:
+  using: "composite"
+  steps:
+    - name: Download ImDisk
+      if: runner.os == 'Windows'
+      shell: bash
+      run: |
+        mkdir imdisk
+        cd imdisk
+        curl -L -o files.cab https://github.com/coder/imdisk-artifacts/raw/92a17839ebc0ee3e69be019f66b3e9b5d2de4482/files.cab
+        curl -L -o install.bat https://github.com/coder/imdisk-artifacts/raw/92a17839ebc0ee3e69be019f66b3e9b5d2de4482/install.bat
+        cd ..
+
+    - name: Install ImDisk
+      shell: cmd
+      run: |
+        cd imdisk
+        install.bat /silent
+
+    - name: Create RAM Disk
+      shell: cmd
+      run: |
+        imdisk -a -s 4096M -m R: -p "/fs:ntfs /q /y"
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -370,15 +370,20 @@ jobs:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
   test-go-pg:
-    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
-    needs:
-      - changes
+    runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'macos-latest-xlarge' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
+    needs: changes
     if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
     # This timeout must be greater than the timeout set by `go test` in
     # `make test-postgres` to ensure we receive a trace of running
     # goroutines. Setting this to the timeout +5m should work quite well
     # even if some of the preceding steps are slow.
     timeout-minutes: 25
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-2022
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
@@ -396,12 +401,46 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      # Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
+      - name: Setup ImDisk
+        if: runner.os == 'Windows'
+        uses: ./.github/actions/setup-imdisk
+
       - name: Test with PostgreSQL Database
         env:
           POSTGRES_VERSION: "13"
           TS_DEBUG_DISCO: "true"
+        shell: bash
         run: |
-          make test-postgres
+          # if macOS, install google-chrome for scaletests
+          # As another concern, should we really have this kind of external dependency
+          # requirement on standard CI?
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            brew install google-chrome
+          fi
+
+          # By default Go will use the number of logical CPUs, which
+          # is a fine default.
+          PARALLEL_FLAG=""
+
+          # macOS will output "The default interactive shell is now zsh"
+          # intermittently in CI...
+          if [ "${{ matrix.os }}" == "macos-latest" ]; then
+            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
+          fi
+
+          if [ "${{ runner.os }}" == "Linux" ]; then
+            make test-postgres
+          elif [ "${{ runner.os }}" == "Windows" ]; then
+            # Create a temp dir on the R: ramdisk drive for Windows. The default
+            # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
+            mkdir -p "R:/temp/embedded-pg"
+            go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
+            DB=ci gotestsum --format standard-quiet -- -v -short -count=1 ./...
+          else
+            go run scripts/embedded-pg/main.go
+            DB=ci gotestsum --format standard-quiet -- -v -short -count=1 ./...
+          fi
 
       - name: Upload test stats to Datadog
         timeout-minutes: 1
@@ -494,6 +533,47 @@ jobs:
         with:
           api-key: ${{ secrets.DATADOG_API_KEY }}
 
+  test-go-race-pg:
+    runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }}
+    needs: changes
+    if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
+    timeout-minutes: 25
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+        with:
+          egress-policy: audit
+
+      - name: Checkout
+        uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+        with:
+          fetch-depth: 1
+
+      - name: Setup Go
+        uses: ./.github/actions/setup-go
+
+      - name: Setup Terraform
+        uses: ./.github/actions/setup-tf
+
+      # We run race tests with reduced parallelism because they use more CPU and we were finding
+      # instances where tests appear to hang for multiple seconds, resulting in flaky tests when
+      # short timeouts are used.
+      # c.f. discussion on https://github.com/coder/coder/pull/15106
+      - name: Run Tests
+        env:
+          POSTGRES_VERSION: "16"
+        run: |
+          make test-postgres-docker
+          DB=ci gotestsum --junitfile="gotests.xml" -- -race -parallel 4 -p 4 ./...
+
+      - name: Upload test stats to Datadog
+        timeout-minutes: 1
+        continue-on-error: true
+        uses: ./.github/actions/upload-datadog
+        if: always()
+        with:
+          api-key: ${{ secrets.DATADOG_API_KEY }}
+
   # Tailnet integration tests only run when the `tailnet` directory or `go.sum`
   # and `go.mod` are changed. These tests are to ensure we don't add regressions
   # to tailnet, either due to our code or due to updating dependencies.
@@ -771,6 +851,7 @@ jobs:
       - test-go
       - test-go-pg
       - test-go-race
+      - test-go-race-pg
       - test-js
       - test-e2e
       - offlinedocs
@@ -793,6 +874,7 @@ jobs:
           echo "- test-go: ${{ needs.test-go.result }}"
           echo "- test-go-pg: ${{ needs.test-go-pg.result }}"
           echo "- test-go-race: ${{ needs.test-go-race.result }}"
+          echo "- test-go-race-pg: ${{ needs.test-go-race-pg.result }}"
           echo "- test-js: ${{ needs.test-js.result }}"
           echo "- test-e2e: ${{ needs.test-e2e.result }}"
           echo "- offlinedocs: ${{ needs.offlinedocs.result }}"
diff --git a/scripts/embedded-pg/main.go b/scripts/embedded-pg/main.go
@@ -0,0 +1,76 @@
+// Start an embedded postgres database on port 5432. Used in CI on macOS and Windows.
+package main
+
+import (
+	"database/sql"
+	"flag"
+	"os"
+	"path/filepath"
+
+	embeddedpostgres "github.com/fergusstrange/embedded-postgres"
+)
+
+func main() {
+	var customPath string
+	flag.StringVar(&customPath, "path", "", "Optional custom path for postgres data directory")
+	flag.Parse()
+
+	postgresPath := filepath.Join(os.TempDir(), "coder-test-postgres")
+	if customPath != "" {
+		postgresPath = customPath
+	}
+
+	ep := embeddedpostgres.NewDatabase(
+		embeddedpostgres.DefaultConfig().
+			Version(embeddedpostgres.V16).
+			BinariesPath(filepath.Join(postgresPath, "bin")).
+			DataPath(filepath.Join(postgresPath, "data")).
+			RuntimePath(filepath.Join(postgresPath, "runtime")).
+			CachePath(filepath.Join(postgresPath, "cache")).
+			Username("postgres").
+			Password("postgres").
+			Database("postgres").
+			Encoding("UTF8").
+			Port(uint32(5432)).
+			Logger(os.Stdout),
+	)
+	err := ep.Start()
+	if err != nil {
+		panic(err)
+	}
+	// We execute these queries instead of using the embeddedpostgres
+	// StartParams because it doesn't work on Windows. The library
+	// seems to have a bug where it sends malformed parameters to
+	// pg_ctl. It encloses each parameter in single quotes, which
+	// Windows can't handle.
+	// Related issue:
+	// https://github.com/fergusstrange/embedded-postgres/issues/145
+	paramQueries := []string{
+		`ALTER SYSTEM SET effective_cache_size = '1GB';`,
+		`ALTER SYSTEM SET fsync = 'off';`,
+		`ALTER SYSTEM SET full_page_writes = 'off';`,
+		`ALTER SYSTEM SET max_connections = '1000';`,
+		`ALTER SYSTEM SET shared_buffers = '1GB';`,
+		`ALTER SYSTEM SET synchronous_commit = 'off';`,
+		`ALTER SYSTEM SET client_encoding = 'UTF8';`,
+	}
+	db, err := sql.Open("postgres", "postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable")
+	if err != nil {
+		panic(err)
+	}
+	for _, query := range paramQueries {
+		if _, err := db.Exec(query); err != nil {
+			panic(err)
+		}
+	}
+	if err := db.Close(); err != nil {
+		panic(err)
+	}
+	// We restart the database to apply all the parameters.
+	if err := ep.Stop(); err != nil {
+		panic(err)
+	}
+	if err := ep.Start(); err != nil {
+		panic(err)
+	}
+}
