chore: add database dump and dbfake logging (#19144)

spikecurtis · web-flow · commit d4b44185da63 · 2025-08-04T13:22:04.000+04:00
relates to #778 Somehow in `TestWorkspaceAgent` the agent with the test instance identifier is not being added to the database, or is getting deleted. I'm adding some additional logging to `dbfake` and setting the affected tests to dump postgres on error, to see if we can get to the bottom of the issue.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -490,6 +490,12 @@ jobs:
           gotestsum --format standard-quiet --packages "$PACKAGES" \
             -- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
 
+      - name: Upload failed test db dumps
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: failed-test-db-dump-${{matrix.os}}
+          path: "**/*.test.sql"
+
       - name: Upload Go Build Cache
         uses: ./.github/actions/test-cache/upload
         with:
diff --git a/cli/agent_test.go b/cli/agent_test.go
@@ -21,6 +21,7 @@ import (
 	"github.com/coder/coder/v2/coderd/coderdtest"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbfake"
+	"github.com/coder/coder/v2/coderd/database/dbtestutil"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/workspacesdk"
 	"github.com/coder/coder/v2/provisionersdk/proto"
@@ -67,7 +68,12 @@ func TestWorkspaceAgent(t *testing.T) {
 		t.Parallel()
 		instanceID := "instanceidentifier"
 		certificates, metadataClient := coderdtest.NewAzureInstanceIdentity(t, instanceID)
-		client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
+		db, ps := dbtestutil.NewDB(t,
+			dbtestutil.WithDumpOnFailure(),
+		)
+		client := coderdtest.New(t, &coderdtest.Options{
+			Database:          db,
+			Pubsub:            ps,
 			AzureCertificates: certificates,
 		})
 		user := coderdtest.CreateFirstUser(t, client)
@@ -106,7 +112,12 @@ func TestWorkspaceAgent(t *testing.T) {
 		t.Parallel()
 		instanceID := "instanceidentifier"
 		certificates, metadataClient := coderdtest.NewAWSInstanceIdentity(t, instanceID)
-		client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
+		db, ps := dbtestutil.NewDB(t,
+			dbtestutil.WithDumpOnFailure(),
+		)
+		client := coderdtest.New(t, &coderdtest.Options{
+			Database:        db,
+			Pubsub:          ps,
 			AWSCertificates: certificates,
 		})
 		user := coderdtest.CreateFirstUser(t, client)
@@ -146,7 +157,12 @@ func TestWorkspaceAgent(t *testing.T) {
 		t.Parallel()
 		instanceID := "instanceidentifier"
 		validator, metadataClient := coderdtest.NewGoogleInstanceIdentity(t, instanceID, false)
-		client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
+		db, ps := dbtestutil.NewDB(t,
+			dbtestutil.WithDumpOnFailure(),
+		)
+		client := coderdtest.New(t, &coderdtest.Options{
+			Database:             db,
+			Pubsub:               ps,
 			GoogleTokenValidator: validator,
 		})
 		owner := coderdtest.CreateFirstUser(t, client)
diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go
@@ -12,6 +12,9 @@ import (
 	"github.com/sqlc-dev/pqtype"
 	"github.com/stretchr/testify/require"
 
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
+
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
 	"github.com/coder/coder/v2/coderd/database/dbgen"
@@ -43,6 +46,7 @@ type WorkspaceResponse struct {
 // resources.
 type WorkspaceBuildBuilder struct {
 	t          testing.TB
+	logger     slog.Logger
 	db         database.Store
 	ps         pubsub.Pubsub
 	ws         database.WorkspaceTable
@@ -62,7 +66,10 @@ type workspaceBuildDisposition struct {
 // Omitting the template ID on a workspace will also generate a new template
 // with a template version.
 func WorkspaceBuild(t testing.TB, db database.Store, ws database.WorkspaceTable) WorkspaceBuildBuilder {
-	return WorkspaceBuildBuilder{t: t, db: db, ws: ws}
+	return WorkspaceBuildBuilder{
+		t: t, db: db, ws: ws,
+		logger: slogtest.Make(t, &slogtest.Options{}).Named("dbfake").Leveled(slog.LevelDebug),
+	}
 }
 
 func (b WorkspaceBuildBuilder) Pubsub(ps pubsub.Pubsub) WorkspaceBuildBuilder {
@@ -131,6 +138,7 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 		AgentToken: b.agentToken,
 	}
 	if b.ws.TemplateID == uuid.Nil {
+		b.logger.Debug(context.Background(), "creating template and version")
 		resp.TemplateVersionResponse = TemplateVersion(b.t, b.db).
 			Resources(b.resources...).
 			Pubsub(b.ps).
@@ -145,6 +153,7 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 
 	// If no template version is set assume the active version.
 	if b.seed.TemplateVersionID == uuid.Nil {
+		b.logger.Debug(context.Background(), "assuming active template version")
 		template, err := b.db.GetTemplateByID(ownerCtx, b.ws.TemplateID)
 		require.NoError(b.t, err)
 		require.NotNil(b.t, template.ActiveVersionID, "active version ID unexpectedly nil")
@@ -156,6 +165,9 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 		// nolint: revive
 		b.ws = dbgen.Workspace(b.t, b.db, b.ws)
 		resp.Workspace = b.ws
+		b.logger.Debug(context.Background(), "created workspace",
+			slog.F("name", resp.Workspace.Name),
+			slog.F("workspace_id", resp.Workspace.ID))
 	}
 	b.seed.WorkspaceID = b.ws.ID
 	b.seed.InitiatorID = takeFirst(b.seed.InitiatorID, b.ws.OwnerID)
@@ -182,10 +194,12 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 		LogsOverflowed: false,
 	})
 	require.NoError(b.t, err, "insert job")
+	b.logger.Debug(context.Background(), "inserted provisioner job", slog.F("job_id", job.ID))
 
 	if b.dispo.starting {
 		// might need to do this multiple times if we got a template version
 		// import job as well
+		b.logger.Debug(context.Background(), "looping to acquire provisioner job")
 		for {
 			j, err := b.db.AcquireProvisionerJob(ownerCtx, database.AcquireProvisionerJobParams{
 				OrganizationID: job.OrganizationID,
@@ -202,10 +216,12 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 			})
 			require.NoError(b.t, err, "acquire starting job")
 			if j.ID == job.ID {
+				b.logger.Debug(context.Background(), "acquired provisioner job", slog.F("job_id", job.ID))
 				break
 			}
 		}
 	} else {
+		b.logger.Debug(context.Background(), "completing the provisioner job")
 		err = b.db.UpdateProvisionerJobWithCompleteByID(ownerCtx, database.UpdateProvisionerJobWithCompleteByIDParams{
 			ID:        job.ID,
 			UpdatedAt: dbtime.Now(),
@@ -221,18 +237,24 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 	}
 
 	resp.Build = dbgen.WorkspaceBuild(b.t, b.db, b.seed)
+	b.logger.Debug(context.Background(), "created workspace build",
+		slog.F("build_id", resp.Build.ID),
+		slog.F("workspace_id", resp.Workspace.ID),
+		slog.F("build_number", resp.Build.BuildNumber))
 
 	for i := range b.params {
 		b.params[i].WorkspaceBuildID = resp.Build.ID
 	}
-	_ = dbgen.WorkspaceBuildParameters(b.t, b.db, b.params)
+	params := dbgen.WorkspaceBuildParameters(b.t, b.db, b.params)
+	b.logger.Debug(context.Background(), "created workspace build parameters", slog.F("count", len(params)))
 
 	if b.ws.Deleted {
 		err = b.db.UpdateWorkspaceDeletedByID(ownerCtx, database.UpdateWorkspaceDeletedByIDParams{
 			ID:      b.ws.ID,
 			Deleted: true,
 		})
 		require.NoError(b.t, err)
+		b.logger.Debug(context.Background(), "deleted workspace", slog.F("workspace_id", resp.Workspace.ID))
 	}
 
 	if b.ps != nil {
@@ -243,6 +265,9 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 		require.NoError(b.t, err)
 		err = b.ps.Publish(wspubsub.WorkspaceEventChannel(resp.Workspace.OwnerID), msg)
 		require.NoError(b.t, err)
+		b.logger.Debug(context.Background(), "published workspace event",
+			slog.F("owner_id", resp.Workspace.ID),
+			slog.F("owner_id", resp.Workspace.OwnerID))
 	}
 
 	agents, err := b.db.GetWorkspaceAgentsByWorkspaceAndBuildNumber(ownerCtx, database.GetWorkspaceAgentsByWorkspaceAndBuildNumberParams{
@@ -260,7 +285,12 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 			err = b.db.DeleteWorkspaceSubAgentByID(ownerCtx, subAgent.ID)
 			require.NoError(b.t, err, "delete workspace agent subagent antagonist")
 
-			b.t.Logf("inserted deleted subagent antagonist %s (%v) for workspace agent %s (%v)", subAgent.Name, subAgent.ID, agent.Name, agent.ID)
+			b.logger.Debug(context.Background(), "inserted deleted subagent antagonist",
+				slog.F("subagent_name", subAgent.Name),
+				slog.F("subagent_id", subAgent.ID),
+				slog.F("agent_name", agent.Name),
+				slog.F("agent_id", agent.ID),
+			)
 		}
 	}
 
@@ -269,6 +299,7 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 
 type ProvisionerJobResourcesBuilder struct {
 	t          testing.TB
+	logger     slog.Logger
 	db         database.Store
 	jobID      uuid.UUID
 	transition database.WorkspaceTransition
@@ -281,6 +312,7 @@ func ProvisionerJobResources(
 ) ProvisionerJobResourcesBuilder {
 	return ProvisionerJobResourcesBuilder{
 		t:          t,
+		logger:     slogtest.Make(t, &slogtest.Options{}).Named("dbfake").Leveled(slog.LevelDebug).With(slog.F("job_id", jobID)),
 		db:         db,
 		jobID:      jobID,
 		transition: transition,
@@ -292,13 +324,17 @@ func (b ProvisionerJobResourcesBuilder) Do() {
 	b.t.Helper()
 	transition := b.transition
 	if transition == "" {
-		// Default to start!
+		b.logger.Debug(context.Background(), "setting default transition to start")
 		transition = database.WorkspaceTransitionStart
 	}
 	for _, resource := range b.resources {
 		//nolint:gocritic // This is only used by tests.
 		err := provisionerdserver.InsertWorkspaceResource(ownerCtx, b.db, b.jobID, transition, resource, &telemetry.Snapshot{})
 		require.NoError(b.t, err)
+		b.logger.Debug(context.Background(), "created workspace resource",
+			slog.F("resource_name", resource.Name),
+			slog.F("agent_count", len(resource.Agents)),
+		)
 	}
 }
 
@@ -309,6 +345,7 @@ type TemplateVersionResponse struct {
 
 type TemplateVersionBuilder struct {
 	t                  testing.TB
+	logger             slog.Logger
 	db                 database.Store
 	seed               database.TemplateVersion
 	fileID             uuid.UUID
@@ -326,6 +363,7 @@ type TemplateVersionBuilder struct {
 func TemplateVersion(t testing.TB, db database.Store) TemplateVersionBuilder {
 	return TemplateVersionBuilder{
 		t:                  t,
+		logger:             slogtest.Make(t, &slogtest.Options{}).Named("dbfake").Leveled(slog.LevelDebug),
 		db:                 db,
 		promote:            true,
 		autoCreateTemplate: true,
@@ -396,20 +434,30 @@ func (t TemplateVersionBuilder) Do() TemplateVersionResponse {
 			Valid: true,
 			UUID:  resp.Template.ID,
 		}
+		t.logger.Debug(context.Background(), "created template",
+			slog.F("organization_id", resp.Template.OrganizationID),
+			slog.F("template_id", resp.Template.CreatedBy),
+		)
 	}
 
 	version := dbgen.TemplateVersion(t.t, t.db, t.seed)
+	t.logger.Debug(context.Background(), "created template version",
+		slog.F("template_version_id", version.ID),
+	)
 	if t.promote {
 		err := t.db.UpdateTemplateActiveVersionByID(ownerCtx, database.UpdateTemplateActiveVersionByIDParams{
 			ID:              t.seed.TemplateID.UUID,
 			ActiveVersionID: t.seed.ID,
 			UpdatedAt:       dbtime.Now(),
 		})
 		require.NoError(t.t, err)
+		t.logger.Debug(context.Background(), "promoted template version",
+			slog.F("template_version_id", t.seed.ID),
+		)
 	}
 
 	for _, preset := range t.presets {
-		dbgen.Preset(t.t, t.db, database.InsertPresetParams{
+		prst := dbgen.Preset(t.t, t.db, database.InsertPresetParams{
 			ID:                  preset.ID,
 			TemplateVersionID:   version.ID,
 			Name:                preset.Name,
@@ -421,14 +469,19 @@ func (t TemplateVersionBuilder) Do() TemplateVersionResponse {
 			Description:         preset.Description,
 			Icon:                preset.Icon,
 		})
+		t.logger.Debug(context.Background(), "added preset",
+			slog.F("preset_id", prst.ID),
+			slog.F("preset_name", prst.Name),
+		)
 	}
 
 	for _, presetParam := range t.presetParams {
-		dbgen.PresetParameter(t.t, t.db, database.InsertPresetParametersParams{
+		prm := dbgen.PresetParameter(t.t, t.db, database.InsertPresetParametersParams{
 			TemplateVersionPresetID: presetParam.TemplateVersionPresetID,
 			Names:                   []string{presetParam.Name},
 			Values:                  []string{presetParam.Value},
 		})
+		t.logger.Debug(context.Background(), "added preset parameter", slog.F("param_name", prm[0].Name))
 	}
 
 	payload, err := json.Marshal(provisionerdserver.TemplateVersionImportJob{
@@ -448,6 +501,7 @@ func (t TemplateVersionBuilder) Do() TemplateVersionResponse {
 		},
 		FileID: t.fileID,
 	})
+	t.logger.Debug(context.Background(), "added template version import job", slog.F("job_id", job.ID))
 
 	t.seed.JobID = job.ID
 
diff --git a/coderd/database/dbtestutil/db.go b/coderd/database/dbtestutil/db.go
@@ -206,7 +206,7 @@ func DumpOnFailure(t testing.TB, connectionURL string) {
 	outPath := filepath.Join(cwd, snakeCaseName+"."+timeSuffix+".test.sql")
 	dump, err := PGDump(connectionURL)
 	if err != nil {
-		t.Errorf("dump on failure: failed to run pg_dump")
+		t.Errorf("dump on failure: failed to run pg_dump: %s", err.Error())
 		return
 	}
 	if err := os.WriteFile(outPath, normalizeDump(dump), 0o600); err != nil {

Original file line number	Diff line number	Diff line change
`@@ -206,7 +206,7 @@ func DumpOnFailure(t testing.TB, connectionURL string) {`
`206`	`206`	`outPath := filepath.Join(cwd, snakeCaseName+"."+timeSuffix+".test.sql")`
`207`	`207`	`dump, err := PGDump(connectionURL)`
`208`	`208`	`if err != nil {`
`209`		`- t.Errorf("dump on failure: failed to run pg_dump")`
	`209`	`+ t.Errorf("dump on failure: failed to run pg_dump: %s", err.Error())`
`210`	`210`	`return`
`211`	`211`	`}`
`212`	`212`	`if err := os.WriteFile(outPath, normalizeDump(dump), 0o600); err != nil {`