Skip to content

Commit f2915ff

Browse files
committed
feat: add support for stopping failed workspaces
1 parent 7d0f53d commit f2915ff

File tree

13 files changed

+368
-83
lines changed

13 files changed

+368
-83
lines changed

coderd/autobuild/lifecycle_executor.go

Lines changed: 40 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@ import (
1313

1414
"cdr.dev/slog"
1515
"github.com/coder/coder/coderd/database"
16+
"github.com/coder/coder/coderd/database/db2sdk"
1617
"github.com/coder/coder/coderd/database/dbauthz"
1718
"github.com/coder/coder/coderd/schedule"
1819
"github.com/coder/coder/coderd/wsbuilder"
20+
"github.com/coder/coder/codersdk"
1921
)
2022

2123
// Executor automatically starts or stops workspaces.
@@ -108,7 +110,7 @@ func (e *Executor) runOnce(t time.Time) Stats {
108110
// NOTE: If a workspace build is created with a given TTL and then the user either
109111
// changes or unsets the TTL, the deadline for the workspace build will not
110112
// have changed. This behavior is as expected per #2229.
111-
workspaces, err := e.db.GetWorkspacesEligibleForAutoStartStop(e.ctx, t)
113+
workspaces, err := e.db.GetWorkspacesEligibleForTransition(e.ctx, t)
112114
if err != nil {
113115
e.log.Error(e.ctx, "get workspaces for autostart or autostop", slog.Error(err))
114116
return stats
@@ -198,24 +200,6 @@ func (e *Executor) runOnce(t time.Time) Stats {
198200
return stats
199201
}
200202

201-
// isEligibleForTransition returns true if the workspace meets basic criteria
202-
// for transitioning to a new state.
203-
func isEligibleForTransition(ws database.Workspace, latestBuild database.WorkspaceBuild, templateSchedule schedule.TemplateScheduleOptions) bool {
204-
if ws.Deleted {
205-
return false
206-
}
207-
if templateSchedule.UserAutostartEnabled && ws.AutostartSchedule.Valid && ws.AutostartSchedule.String != "" {
208-
return true
209-
}
210-
// Don't check the template schedule to see whether it allows autostop, this
211-
// is done during the build when determining the deadline.
212-
if latestBuild.Transition == database.WorkspaceTransitionStart && !latestBuild.Deadline.IsZero() {
213-
return true
214-
}
215-
216-
return false
217-
}
218-
219203
func getNextTransition(
220204
ws database.Workspace,
221205
latestBuild database.WorkspaceBuild,
@@ -227,32 +211,37 @@ func getNextTransition(
227211
database.BuildReason,
228212
error,
229213
) {
230-
if !isEligibleForTransition(ws, latestBuild, templateSchedule) {
231-
return "", "", xerrors.Errorf("workspace ineligible for transition")
232-
}
233-
234-
if !latestJob.CompletedAt.Valid || latestJob.Error.String != "" {
235-
return "", "", xerrors.Errorf("last workspace build did not complete successfully")
236-
}
237-
238214
switch {
239-
case isEligibleForAutostop(latestBuild, currentTick):
215+
case isEligibleForAutostop(latestBuild, latestJob, currentTick):
240216
return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
241-
case isEligibleForAutostart(ws, latestBuild, currentTick):
217+
case isEligibleForAutostart(ws, latestBuild, latestJob, templateSchedule, currentTick):
242218
return database.WorkspaceTransitionStart, database.BuildReasonAutostart, nil
219+
case isEligibleForFailedStop(latestJob, templateSchedule):
220+
return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
243221
default:
244222
return "", "", xerrors.Errorf("last transition not valid for autostart or autostop")
245223
}
246224
}
247225

248226
// isEligibleForAutostart returns true if the workspace should be autostarted.
249-
func isEligibleForAutostart(ws database.Workspace, build database.WorkspaceBuild, currentTick time.Time) bool {
227+
func isEligibleForAutostart(ws database.Workspace, build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
228+
// Don't attempt to autostart failed workspaces.
229+
if !job.CompletedAt.Valid || job.Error.String != "" {
230+
return false
231+
}
232+
250233
// If the last transition for the workspace was not 'stop' then the workspace
251234
// cannot be started.
252235
if build.Transition != database.WorkspaceTransitionStop {
253236
return false
254237
}
255238

239+
// If autostart isn't enabled, or the schedule isn't valid/populated we can't
240+
// autostart the workspace.
241+
if !templateSchedule.UserAutostartEnabled || !ws.AutostartSchedule.Valid || ws.AutostartSchedule.String == "" {
242+
return false
243+
}
244+
256245
sched, err := schedule.Weekly(ws.AutostartSchedule.String)
257246
if err != nil {
258247
return false
@@ -265,10 +254,30 @@ func isEligibleForAutostart(ws database.Workspace, build database.WorkspaceBuild
265254
}
266255

267256
// isEligibleForAutostart returns true if the workspace should be autostopped.
268-
func isEligibleForAutostop(build database.WorkspaceBuild, currentTick time.Time) bool {
257+
func isEligibleForAutostop(build database.WorkspaceBuild, job database.ProvisionerJob, currentTick time.Time) bool {
258+
// Don't attempt to autostop failed workspaces.
259+
if !job.CompletedAt.Valid || job.Error.String != "" {
260+
return false
261+
}
262+
269263
// A workspace must be started in order for it to be auto-stopped.
270264
return build.Transition == database.WorkspaceTransitionStart &&
271265
!build.Deadline.IsZero() &&
272266
// We do not want to stop a workspace prior to it breaching its deadline.
273267
!currentTick.Before(build.Deadline)
274268
}
269+
270+
// isEligibleForFailedStop returns true if the workspace is eligible to be stopped
271+
// due to a failed build.
272+
func isEligibleForFailedStop(job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions) bool {
273+
// If the template has specified a failure TLL.
274+
return templateSchedule.FailureTTL > 0 &&
275+
// And the job resulted in failure.
276+
db2sdk.ProvisionerJobStatus(job) == codersdk.ProvisionerJobFailed &&
277+
// And sufficient time has elapsed since the job has completed.
278+
(job.CompletedAt.Valid && database.Now().Sub(job.CompletedAt.Time) > templateSchedule.FailureTTL ||
279+
// Or sufficient time has elapsed since the job was canceled.
280+
job.CanceledAt.Valid && database.Now().Sub(job.CanceledAt.Time) > templateSchedule.FailureTTL ||
281+
// Or the job is stuck/abandoned.
282+
database.Now().Sub(job.UpdatedAt) > 30*time.Second)
283+
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy