Skip to content

fix(coderd/prometheusmetrics)!: filter deleted wsbuilds to reduce db load #19197

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions coderd/database/dbauthz/dbauthz.go
Original file line number Diff line number Diff line change
Expand Up @@ -2178,17 +2178,6 @@ func (q *querier) GetLatestWorkspaceBuildByWorkspaceID(ctx context.Context, work
return q.db.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspaceID)
}

func (q *querier) GetLatestWorkspaceBuilds(ctx context.Context) ([]database.WorkspaceBuild, error) {
// This function is a system function until we implement a join for workspace builds.
// This is because we need to query for all related workspaces to the returned builds.
// This is a very inefficient method of fetching the latest workspace builds.
// We should just join the rbac properties.
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
return nil, err
}
return q.db.GetLatestWorkspaceBuilds(ctx)
}

func (q *querier) GetLatestWorkspaceBuildsByWorkspaceIDs(ctx context.Context, ids []uuid.UUID) ([]database.WorkspaceBuild, error) {
// This function is a system function until we implement a join for workspace builds.
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
Expand Down
6 changes: 0 additions & 6 deletions coderd/database/dbauthz/dbauthz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4033,12 +4033,6 @@ func (s *MethodTestSuite) TestSystemFunctions() {
LoginType: l.LoginType,
}).Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(l)
}))
s.Run("GetLatestWorkspaceBuilds", s.Subtest(func(db database.Store, check *expects) {
dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
dbgen.WorkspaceBuild(s.T(), db, database.WorkspaceBuild{})
dbgen.WorkspaceBuild(s.T(), db, database.WorkspaceBuild{})
check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
s.Run("GetActiveUserCount", s.Subtest(func(db database.Store, check *expects) {
check.Args(false).Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(int64(0))
}))
Expand Down
7 changes: 0 additions & 7 deletions coderd/database/dbmetrics/querymetrics.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 0 additions & 15 deletions coderd/database/dbmock/dbmock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion coderd/database/querier.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

59 changes: 0 additions & 59 deletions coderd/database/queries.sql.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 0 additions & 14 deletions coderd/database/queries/workspacebuilds.sql
Original file line number Diff line number Diff line change
Expand Up @@ -91,20 +91,6 @@ JOIN
workspace_build_with_user AS wb
ON m.workspace_id = wb.workspace_id AND m.max_build_number = wb.build_number;

-- name: GetLatestWorkspaceBuilds :many
SELECT wb.*
FROM (
SELECT
workspace_id, MAX(build_number) as max_build_number
FROM
workspace_build_with_user AS workspace_builds
GROUP BY
workspace_id
) m
JOIN
workspace_build_with_user AS wb
ON m.workspace_id = wb.workspace_id AND m.max_build_number = wb.build_number;

-- name: InsertWorkspaceBuild :exec
INSERT INTO
workspace_builds (
Expand Down
65 changes: 21 additions & 44 deletions coderd/prometheusmetrics/prometheusmetrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
Namespace: "coderd",
Subsystem: "api",
Name: "workspace_latest_build",
Help: "The current number of workspace builds by status.",
Help: "The current number of workspace builds by status for all non-deleted workspaces.",
}, []string{"status"})
if err := registerer.Register(workspaceLatestBuildTotals); err != nil {
return nil, err
Expand All @@ -159,7 +159,7 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
workspaceLatestBuildStatuses := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "coderd",
Name: "workspace_latest_build_status",
Help: "The current workspace statuses by template, transition, and owner.",
Help: "The current workspace statuses by template, transition, and owner for all non-deleted workspaces.",
}, []string{"status", "template_name", "template_version", "workspace_owner", "workspace_transition"})
if err := registerer.Register(workspaceLatestBuildStatuses); err != nil {
return nil, err
Expand All @@ -168,59 +168,37 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
ctx, cancelFunc := context.WithCancel(ctx)
done := make(chan struct{})

updateWorkspaceTotals := func() {
builds, err := db.GetLatestWorkspaceBuilds(ctx)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// clear all series if there are no database entries
workspaceLatestBuildTotals.Reset()
} else {
logger.Warn(ctx, "failed to load latest workspace builds", slog.Error(err))
}
return
}
jobIDs := make([]uuid.UUID, 0, len(builds))
for _, build := range builds {
jobIDs = append(jobIDs, build.JobID)
}
jobs, err := db.GetProvisionerJobsByIDs(ctx, jobIDs)
if err != nil {
ids := make([]string, 0, len(jobIDs))
for _, id := range jobIDs {
ids = append(ids, id.String())
}

logger.Warn(ctx, "failed to load provisioner jobs", slog.F("ids", ids), slog.Error(err))
return
}

workspaceLatestBuildTotals.Reset()
for _, job := range jobs {
status := codersdk.ProvisionerJobStatus(job.JobStatus)
workspaceLatestBuildTotals.WithLabelValues(string(status)).Add(1)
// TODO: deprecated: remove in the future
workspaceLatestBuildTotalsDeprecated.WithLabelValues(string(status)).Add(1)
}
}

updateWorkspaceStatuses := func() {
updateWorkspaceMetrics := func() {
ws, err := db.GetWorkspaces(ctx, database.GetWorkspacesParams{
Deleted: false,
WithSummary: false,
})
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// clear all series if there are no database entries
workspaceLatestBuildTotals.Reset()
workspaceLatestBuildStatuses.Reset()
} else {
logger.Warn(ctx, "failed to load active workspaces for metrics", slog.Error(err))
}

logger.Warn(ctx, "failed to load active workspaces", slog.Error(err))
return
}

workspaceLatestBuildTotals.Reset()
workspaceLatestBuildStatuses.Reset()

for _, w := range ws {
workspaceLatestBuildStatuses.WithLabelValues(string(w.LatestBuildStatus), w.TemplateName, w.TemplateVersionName.String, w.OwnerUsername, string(w.LatestBuildTransition)).Add(1)
status := string(w.LatestBuildStatus)
workspaceLatestBuildTotals.WithLabelValues(status).Add(1)
// TODO: deprecated: remove in the future
workspaceLatestBuildTotalsDeprecated.WithLabelValues(status).Add(1)

workspaceLatestBuildStatuses.WithLabelValues(
status,
w.TemplateName,
w.TemplateVersionName.String,
w.OwnerUsername,
string(w.LatestBuildTransition),
).Add(1)
}
}

Expand All @@ -230,8 +208,7 @@ func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.R
doTick := func() {
defer ticker.Reset(duration)

updateWorkspaceTotals()
updateWorkspaceStatuses()
updateWorkspaceMetrics()
}

go func() {
Expand Down
74 changes: 74 additions & 0 deletions coderd/prometheusmetrics/prometheusmetrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,32 @@ func TestWorkspaceLatestBuildTotals(t *testing.T) {
codersdk.ProvisionerJobSucceeded: 3,
codersdk.ProvisionerJobRunning: 1,
},
}, {
Name: "MultipleWithDeleted",
Database: func() database.Store {
db, _ := dbtestutil.NewDB(t)
u := dbgen.User(t, db, database.User{})
org := dbgen.Organization(t, db, database.Organization{})
insertCanceled(t, db, u, org)
insertFailed(t, db, u, org)
insertSuccess(t, db, u, org)
insertRunning(t, db, u, org)

// Verify that deleted workspaces/builds are NOT counted in metrics.
n, err := cryptorand.Intn(5)
require.NoError(t, err)
for range 1 + n {
insertDeleted(t, db, u, org)
}
return db
},
Total: 4, // Only non-deleted workspaces should be counted
Status: map[codersdk.ProvisionerJobStatus]int{
codersdk.ProvisionerJobCanceled: 1,
codersdk.ProvisionerJobFailed: 1,
codersdk.ProvisionerJobSucceeded: 1,
codersdk.ProvisionerJobRunning: 1,
},
}} {
t.Run(tc.Name, func(t *testing.T) {
t.Parallel()
Expand Down Expand Up @@ -323,6 +349,33 @@ func TestWorkspaceLatestBuildStatuses(t *testing.T) {
codersdk.ProvisionerJobSucceeded: 3,
codersdk.ProvisionerJobRunning: 1,
},
}, {
Name: "MultipleWithDeleted",
Database: func() database.Store {
db, _ := dbtestutil.NewDB(t)
u := dbgen.User(t, db, database.User{})
org := dbgen.Organization(t, db, database.Organization{})
insertTemplates(t, db, u, org)
insertCanceled(t, db, u, org)
insertFailed(t, db, u, org)
insertSuccess(t, db, u, org)
insertRunning(t, db, u, org)

// Verify that deleted workspaces/builds are NOT counted in metrics.
n, err := cryptorand.Intn(5)
require.NoError(t, err)
for range 1 + n {
insertDeleted(t, db, u, org)
}
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review: This fails in the old implementation, but I thought it best to formalize the assumption. Considering the old implementation did not test for this, it seems like validation for the new approach.

return db
},
ExpectedWorkspaces: 4, // Only non-deleted workspaces should be counted
ExpectedStatuses: map[codersdk.ProvisionerJobStatus]int{
codersdk.ProvisionerJobCanceled: 1,
codersdk.ProvisionerJobFailed: 1,
codersdk.ProvisionerJobSucceeded: 1,
codersdk.ProvisionerJobRunning: 1,
},
}} {
t.Run(tc.Name, func(t *testing.T) {
t.Parallel()
Expand Down Expand Up @@ -907,3 +960,24 @@ func insertSuccess(t *testing.T, db database.Store, u database.User, org databas
})
require.NoError(t, err)
}

func insertDeleted(t *testing.T, db database.Store, u database.User, org database.Organization) {
job := insertRunning(t, db, u, org)
err := db.UpdateProvisionerJobWithCompleteByID(context.Background(), database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
CompletedAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
})
require.NoError(t, err)

build, err := db.GetWorkspaceBuildByJobID(context.Background(), job.ID)
require.NoError(t, err)

err = db.UpdateWorkspaceDeletedByID(context.Background(), database.UpdateWorkspaceDeletedByIDParams{
ID: build.WorkspaceID,
Deleted: true,
})
require.NoError(t, err)
}
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy