From 10a877bb070c8a9e07fa5a6e1ab9d35ab17c6776 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Mon, 19 Sep 2022 19:04:33 +0000 Subject: [PATCH 1/9] feat: bump workspace deadline on user activity Resolves #2995 --- coderd/coderd.go | 2 + coderd/httpmw/workspacebump.go | 75 ++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 coderd/httpmw/workspacebump.go diff --git a/coderd/coderd.go b/coderd/coderd.go index 607f15ff7931b..d9c55715e2d9c 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -201,6 +201,7 @@ func New(options *Options) *API { httpmw.ExtractUserParam(api.Database), // Extracts the from the url httpmw.ExtractWorkspaceAndAgentParam(api.Database), + httpmw.BumpWorkspaceAutoStop(api.Logger, api.Database), ) r.HandleFunc("/*", api.workspaceAppsProxyPath) } @@ -421,6 +422,7 @@ func New(options *Options) *API { apiKeyMiddleware, httpmw.ExtractWorkspaceAgentParam(options.Database), httpmw.ExtractWorkspaceParam(options.Database), + httpmw.BumpWorkspaceAutoStop(api.Logger, options.Database), ) r.Get("/", api.workspaceAgent) r.Get("/pty", api.workspaceAgentPTY) diff --git a/coderd/httpmw/workspacebump.go b/coderd/httpmw/workspacebump.go new file mode 100644 index 0000000000000..fa8043c28b87e --- /dev/null +++ b/coderd/httpmw/workspacebump.go @@ -0,0 +1,75 @@ +package httpmw + +import ( + "database/sql" + "errors" + "net/http" + "time" + + "golang.org/x/xerrors" + + "cdr.dev/slog" + "github.com/coder/coder/coderd/database" +) + +// BumpWorkspaceAutoStop automatically bumps the workspace's auto-off timer +// if it is set to expire soon. +// It must be ran after ExtractWorkspace. +func BumpWorkspaceAutoStop(log slog.Logger, db database.Store) func(h http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + workspace := WorkspaceParam(r) + + err := db.InTx(func(s database.Store) error { + build, err := s.GetLatestWorkspaceBuildByWorkspaceID(r.Context(), workspace.ID) + if errors.Is(err, sql.ErrNoRows) { + return nil + } else if err != nil { + return xerrors.Errorf("get latest workspace build: %w", err) + } + + job, err := s.GetProvisionerJobByID(r.Context(), build.JobID) + if err != nil { + return xerrors.Errorf("get provisioner job: %w", err) + } + + if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { + return nil + } + + if build.Deadline.IsZero() { + // Workspace shutdown is manual + return nil + } + + // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. + const ( + bumpAmount = time.Hour + bumpThreshold = time.Hour - time.Minute*10 + ) + + if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { + return nil + } + + newDeadline := time.Now().Add(bumpAmount) + + if err := s.UpdateWorkspaceBuildByID(r.Context(), database.UpdateWorkspaceBuildByIDParams{ + ID: build.ID, + UpdatedAt: build.UpdatedAt, + ProvisionerState: build.ProvisionerState, + Deadline: newDeadline, + }); err != nil { + return xerrors.Errorf("update workspace build: %w", err) + } + return nil + }) + + if err != nil { + log.Error(r.Context(), "auto-bump", slog.Error(err)) + } + + next.ServeHTTP(w, r) + }) + } +} From c43c2ffc840ba7a74ab5fa82464c7ae116397db5 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 02:30:23 +0000 Subject: [PATCH 2/9] Add backend --- coderd/coderd.go | 4 +- coderd/httpmw/workspaceactivitybump.go | 89 ++++++++++++++++++++++++++ coderd/httpmw/workspacebump.go | 75 ---------------------- coderd/workspaceapps_test.go | 19 +++++- coderd/workspaces_test.go | 84 ++++++++++++++++++++++++ 5 files changed, 192 insertions(+), 79 deletions(-) create mode 100644 coderd/httpmw/workspaceactivitybump.go delete mode 100644 coderd/httpmw/workspacebump.go diff --git a/coderd/coderd.go b/coderd/coderd.go index d9c55715e2d9c..bbe67b19f446a 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -201,7 +201,7 @@ func New(options *Options) *API { httpmw.ExtractUserParam(api.Database), // Extracts the from the url httpmw.ExtractWorkspaceAndAgentParam(api.Database), - httpmw.BumpWorkspaceAutoStop(api.Logger, api.Database), + httpmw.ActivityBumpWorkspace(api.Logger, api.Database), ) r.HandleFunc("/*", api.workspaceAppsProxyPath) } @@ -422,7 +422,7 @@ func New(options *Options) *API { apiKeyMiddleware, httpmw.ExtractWorkspaceAgentParam(options.Database), httpmw.ExtractWorkspaceParam(options.Database), - httpmw.BumpWorkspaceAutoStop(api.Logger, options.Database), + httpmw.ActivityBumpWorkspace(api.Logger, options.Database), ) r.Get("/", api.workspaceAgent) r.Get("/pty", api.workspaceAgentPTY) diff --git a/coderd/httpmw/workspaceactivitybump.go b/coderd/httpmw/workspaceactivitybump.go new file mode 100644 index 0000000000000..76d3127e2fe10 --- /dev/null +++ b/coderd/httpmw/workspaceactivitybump.go @@ -0,0 +1,89 @@ +package httpmw + +import ( + "context" + "database/sql" + "errors" + "net/http" + "time" + + "golang.org/x/xerrors" + + "cdr.dev/slog" + "github.com/coder/coder/coderd/database" +) + +// ActivityBumpWorkspace automatically bumps the workspace's auto-off timer +// if it is set to expire soon. +// It must be ran after ExtractWorkspace. +func ActivityBumpWorkspace(log slog.Logger, db database.Store) func(h http.Handler) http.Handler { + log = log.Named("activity_bump") + + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + workspace := WorkspaceParam(r) + log.Debug(r.Context(), "middleware called") + // We run the bump logic asynchronously since the result doesn't + // affect the response. + go func() { + // We cannot use the Request context since the goroutine + // may be around after the request terminates. + // We set a short timeout so if the app is under load, these + // low priority operations fail first. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + err := db.InTx(func(s database.Store) error { + build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) + log.Debug(ctx, "build", slog.F("build", build)) + if errors.Is(err, sql.ErrNoRows) { + return nil + } else if err != nil { + return xerrors.Errorf("get latest workspace build: %w", err) + } + + job, err := s.GetProvisionerJobByID(ctx, build.JobID) + if err != nil { + return xerrors.Errorf("get provisioner job: %w", err) + } + + if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { + return nil + } + + if build.Deadline.IsZero() { + // Workspace shutdown is manual + return nil + } + + // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. + const ( + bumpAmount = time.Hour + bumpThreshold = time.Hour - (time.Minute * 10) + ) + + if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { + return nil + } + + newDeadline := time.Now().Add(bumpAmount) + + if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ + ID: build.ID, + UpdatedAt: build.UpdatedAt, + ProvisionerState: build.ProvisionerState, + Deadline: newDeadline, + }); err != nil { + return xerrors.Errorf("update workspace build: %w", err) + } + return nil + }) + + if err != nil { + log.Error(ctx, "bump failed", slog.Error(err)) + } + }() + next.ServeHTTP(w, r) + }) + } +} diff --git a/coderd/httpmw/workspacebump.go b/coderd/httpmw/workspacebump.go deleted file mode 100644 index fa8043c28b87e..0000000000000 --- a/coderd/httpmw/workspacebump.go +++ /dev/null @@ -1,75 +0,0 @@ -package httpmw - -import ( - "database/sql" - "errors" - "net/http" - "time" - - "golang.org/x/xerrors" - - "cdr.dev/slog" - "github.com/coder/coder/coderd/database" -) - -// BumpWorkspaceAutoStop automatically bumps the workspace's auto-off timer -// if it is set to expire soon. -// It must be ran after ExtractWorkspace. -func BumpWorkspaceAutoStop(log slog.Logger, db database.Store) func(h http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - workspace := WorkspaceParam(r) - - err := db.InTx(func(s database.Store) error { - build, err := s.GetLatestWorkspaceBuildByWorkspaceID(r.Context(), workspace.ID) - if errors.Is(err, sql.ErrNoRows) { - return nil - } else if err != nil { - return xerrors.Errorf("get latest workspace build: %w", err) - } - - job, err := s.GetProvisionerJobByID(r.Context(), build.JobID) - if err != nil { - return xerrors.Errorf("get provisioner job: %w", err) - } - - if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { - return nil - } - - if build.Deadline.IsZero() { - // Workspace shutdown is manual - return nil - } - - // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. - const ( - bumpAmount = time.Hour - bumpThreshold = time.Hour - time.Minute*10 - ) - - if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { - return nil - } - - newDeadline := time.Now().Add(bumpAmount) - - if err := s.UpdateWorkspaceBuildByID(r.Context(), database.UpdateWorkspaceBuildByIDParams{ - ID: build.ID, - UpdatedAt: build.UpdatedAt, - ProvisionerState: build.ProvisionerState, - Deadline: newDeadline, - }); err != nil { - return xerrors.Errorf("update workspace build: %w", err) - } - return nil - }) - - if err != nil { - log.Error(r.Context(), "auto-bump", slog.Error(err)) - } - - next.ServeHTTP(w, r) - }) - } -} diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index 831b3761693df..2337095e185d8 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -36,7 +36,7 @@ const ( // setupProxyTest creates a workspace with an agent and some apps. It returns a // codersdk client, the workspace, and the port number the test listener is // running on. -func setupProxyTest(t *testing.T) (*codersdk.Client, uuid.UUID, codersdk.Workspace, uint16) { +func setupProxyTest(t *testing.T, workspaceMutators ...func(*codersdk.CreateWorkspaceRequest)) (*codersdk.Client, uuid.UUID, codersdk.Workspace, uint16) { // #nosec ln, err := net.Listen("tcp", ":0") require.NoError(t, err) @@ -95,7 +95,7 @@ func setupProxyTest(t *testing.T) (*codersdk.Client, uuid.UUID, codersdk.Workspa }) template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) coderdtest.AwaitTemplateVersionJob(t, client, version.ID) - workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID) + workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID, workspaceMutators...) coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID) agentClient := codersdk.New(client.URL) @@ -208,6 +208,21 @@ func TestWorkspaceAppsProxyPath(t *testing.T) { require.Equal(t, http.StatusOK, resp.StatusCode) }) + t.Run("Proxies", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() + + resp, err := client.Request(ctx, http.MethodGet, "/@me/"+workspace.Name+"/apps/example/?"+proxyTestAppQuery, nil) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, proxyTestAppBody, string(body)) + require.Equal(t, http.StatusOK, resp.StatusCode) + }) + t.Run("ProxyError", func(t *testing.T) { t.Parallel() diff --git a/coderd/workspaces_test.go b/coderd/workspaces_test.go index 11e23ea341c60..6d921050482c1 100644 --- a/coderd/workspaces_test.go +++ b/coderd/workspaces_test.go @@ -1029,6 +1029,90 @@ func TestWorkspaceUpdateAutostart(t *testing.T) { require.Contains(t, coderSDKErr.Message, "Resource not found", "unexpected response code") }) } +func TestWorkspaceActivityBump(t *testing.T) { + t.Parallel() + + ctx := context.Background() + setupActivityTest := func(t *testing.T) (client *codersdk.Client, workspace codersdk.Workspace, assertBumped func(want bool)) { + var ttlMillis int64 = 60 * 1000 + + client, _, workspace, _ = setupProxyTest(t, func(cwr *codersdk.CreateWorkspaceRequest) { + cwr.TTLMillis = &ttlMillis + }) + + // Sanity-check that deadline is near. + workspace, err := client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + require.WithinDuration(t, + time.Now().Add(time.Duration(ttlMillis)*time.Millisecond), + workspace.LatestBuild.Deadline.Time, testutil.WaitShort, + ) + firstDeadline := workspace.LatestBuild.Deadline.Time + + return client, workspace, func(want bool) { + if !want { + time.Sleep(testutil.IntervalMedium) + workspace, err = client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + require.Equal(t, workspace.LatestBuild.Deadline.Time, firstDeadline) + return + } + + // The Deadline bump occurs asynchronously. + require.Eventuallyf(t, + func() bool { + workspace, err = client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + return workspace.LatestBuild.Deadline.Time != firstDeadline + }, + testutil.WaitShort, testutil.IntervalFast, + "deadline %v never updated", firstDeadline, + ) + + require.WithinDuration(t, time.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) + } + } + + t.Run("Apps", func(t *testing.T) { + t.Parallel() + + client, workspace, assertBumped := setupActivityTest(t) + + // A request to the /apps/ endpoint extends the deadline an hour. + // The particular app doesn't matter. The deadline is extended + // regardless of error state. + resp, err := client.Request(ctx, http.MethodGet, "/@me/"+workspace.Name+"/apps/example", nil) + require.NoError(t, err) + resp.Body.Close() + assertBumped(true) + }) + + t.Run("Dial", func(t *testing.T) { + t.Parallel() + + client, workspace, assertBumped := setupActivityTest(t) + + resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID) + conn, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil) + require.NoError(t, err) + _ = conn.Close() + + assertBumped(true) + }) + + t.Run("NoBump", func(t *testing.T) { + t.Parallel() + + client, workspace, assertBumped := setupActivityTest(t) + + // Doing some inactive operation like retrieving resources must not + // bump the deadline. + _, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID) + require.NoError(t, err) + + assertBumped(false) + }) +} func TestWorkspaceUpdateTTL(t *testing.T) { t.Parallel() From 52f68e668f65d3118b304276ffe45be7700bda8e Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 02:33:49 +0000 Subject: [PATCH 3/9] The point at which I realized I'm doing this wrong --- coderd/httpmw/workspaceactivitybump.go | 103 ++++++++++++++----------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/coderd/httpmw/workspaceactivitybump.go b/coderd/httpmw/workspaceactivitybump.go index 76d3127e2fe10..dc6280ff9296b 100644 --- a/coderd/httpmw/workspaceactivitybump.go +++ b/coderd/httpmw/workspaceactivitybump.go @@ -26,62 +26,73 @@ func ActivityBumpWorkspace(log slog.Logger, db database.Store) func(h http.Handl // We run the bump logic asynchronously since the result doesn't // affect the response. go func() { - // We cannot use the Request context since the goroutine - // may be around after the request terminates. - // We set a short timeout so if the app is under load, these - // low priority operations fail first. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() + bump := func() { + // We cannot use the Request context since the goroutine + // may be around after the request terminates. + // We set a short timeout so if the app is under load, these + // low priority operations fail first. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() - err := db.InTx(func(s database.Store) error { - build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) - log.Debug(ctx, "build", slog.F("build", build)) - if errors.Is(err, sql.ErrNoRows) { - return nil - } else if err != nil { - return xerrors.Errorf("get latest workspace build: %w", err) - } + err := db.InTx(func(s database.Store) error { + build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) + log.Debug(ctx, "build", slog.F("build", build)) + if errors.Is(err, sql.ErrNoRows) { + return nil + } else if err != nil { + return xerrors.Errorf("get latest workspace build: %w", err) + } - job, err := s.GetProvisionerJobByID(ctx, build.JobID) - if err != nil { - return xerrors.Errorf("get provisioner job: %w", err) - } + job, err := s.GetProvisionerJobByID(ctx, build.JobID) + if err != nil { + return xerrors.Errorf("get provisioner job: %w", err) + } - if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { - return nil - } + if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { + return nil + } - if build.Deadline.IsZero() { - // Workspace shutdown is manual - return nil - } + if build.Deadline.IsZero() { + // Workspace shutdown is manual + return nil + } - // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. - const ( - bumpAmount = time.Hour - bumpThreshold = time.Hour - (time.Minute * 10) - ) + // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. + const ( + bumpAmount = time.Hour + bumpThreshold = time.Hour - (time.Minute * 10) + ) - if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { - return nil - } + if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { + return nil + } - newDeadline := time.Now().Add(bumpAmount) + newDeadline := time.Now().Add(bumpAmount) - if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ - ID: build.ID, - UpdatedAt: build.UpdatedAt, - ProvisionerState: build.ProvisionerState, - Deadline: newDeadline, - }); err != nil { - return xerrors.Errorf("update workspace build: %w", err) - } - return nil - }) + if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ + ID: build.ID, + UpdatedAt: build.UpdatedAt, + ProvisionerState: build.ProvisionerState, + Deadline: newDeadline, + }); err != nil { + return xerrors.Errorf("update workspace build: %w", err) + } + return nil + }) - if err != nil { - log.Error(ctx, "bump failed", slog.Error(err)) + if err != nil { + log.Error(ctx, "bump failed", slog.Error(err)) + } else { + log.Debug( + ctx, "bumped deadline from activity", + slog.F("workspace_id", workspace.ID), + ) + } } + // For long running connections (e.g. web terminal), we need + // to bump periodically + // ticker := time.NewTicker(time.Minute) + bump() }() next.ServeHTTP(w, r) }) From 24e42e17422588e7ad36e88ee00846e9f01d23de Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 03:24:10 +0000 Subject: [PATCH 4/9] Move logic to agent + frontend --- coderd/activitybump.go | 79 ++++++++++++++ coderd/activitybump_test.go | 93 ++++++++++++++++ coderd/coderd.go | 3 +- coderd/httpmw/workspaceactivitybump.go | 100 ------------------ coderd/workspaceagents.go | 2 + coderd/workspaceapps_test.go | 5 +- coderd/workspaces_test.go | 84 --------------- codersdk/workspaceagents.go | 2 +- .../WorkspaceScheduleForm.tsx | 3 +- 9 files changed, 182 insertions(+), 189 deletions(-) create mode 100644 coderd/activitybump.go create mode 100644 coderd/activitybump_test.go delete mode 100644 coderd/httpmw/workspaceactivitybump.go diff --git a/coderd/activitybump.go b/coderd/activitybump.go new file mode 100644 index 0000000000000..35f4b725c94d9 --- /dev/null +++ b/coderd/activitybump.go @@ -0,0 +1,79 @@ +package coderd + +import ( + "context" + "database/sql" + "errors" + "time" + + "golang.org/x/xerrors" + + "cdr.dev/slog" + "github.com/coder/coder/coderd/database" +) + +// activityBumpWorkspace automatically bumps the workspace's auto-off timer +// if it is set to expire soon. +func activityBumpWorkspace(log slog.Logger, db database.Store, workspace database.Workspace) { + // We cannot use the Request context since the goroutine + // may be around after the request terminates. + // We set a short timeout so if the app is under load, these + // low priority operations fail first. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + + err := db.InTx(func(s database.Store) error { + build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) + log.Debug(ctx, "build", slog.F("build", build)) + if errors.Is(err, sql.ErrNoRows) { + return nil + } else if err != nil { + return xerrors.Errorf("get latest workspace build: %w", err) + } + + job, err := s.GetProvisionerJobByID(ctx, build.JobID) + if err != nil { + return xerrors.Errorf("get provisioner job: %w", err) + } + + if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { + return nil + } + + if build.Deadline.IsZero() { + // Workspace shutdown is manual + return nil + } + + // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. + const ( + bumpAmount = time.Hour + bumpThreshold = time.Hour - (time.Minute * 10) + ) + + if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { + return nil + } + + newDeadline := time.Now().Add(bumpAmount) + + if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ + ID: build.ID, + UpdatedAt: build.UpdatedAt, + ProvisionerState: build.ProvisionerState, + Deadline: newDeadline, + }); err != nil { + return xerrors.Errorf("update workspace build: %w", err) + } + return nil + }) + + if err != nil { + log.Error(ctx, "bump failed", slog.Error(err)) + } else { + log.Debug( + ctx, "bumped deadline from activity", + slog.F("workspace_id", workspace.ID), + ) + } +} diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go new file mode 100644 index 0000000000000..2246e3dc1bf15 --- /dev/null +++ b/coderd/activitybump_test.go @@ -0,0 +1,93 @@ +package coderd_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "cdr.dev/slog/sloggers/slogtest" + + "github.com/coder/coder/coderd/coderdtest" + "github.com/coder/coder/codersdk" + "github.com/coder/coder/testutil" +) + +func TestWorkspaceActivityBump(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + setupActivityTest := func(t *testing.T) (client *codersdk.Client, workspace codersdk.Workspace, assertBumped func(want bool)) { + var ttlMillis int64 = 60 * 1000 + + client, _, workspace, _ = setupProxyTest(t, func(cwr *codersdk.CreateWorkspaceRequest) { + cwr.TTLMillis = &ttlMillis + }) + + // Sanity-check that deadline is near. + workspace, err := client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + require.WithinDuration(t, + time.Now().Add(time.Duration(ttlMillis)*time.Millisecond), + workspace.LatestBuild.Deadline.Time, testutil.WaitShort, + ) + firstDeadline := workspace.LatestBuild.Deadline.Time + + _ = coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID) + + return client, workspace, func(want bool) { + if !want { + time.Sleep(testutil.IntervalMedium) + workspace, err = client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + require.Equal(t, workspace.LatestBuild.Deadline.Time, firstDeadline) + return + } + + // The Deadline bump occurs asynchronously. + require.Eventuallyf(t, + func() bool { + workspace, err = client.Workspace(ctx, workspace.ID) + require.NoError(t, err) + return workspace.LatestBuild.Deadline.Time != firstDeadline + }, + testutil.WaitShort, testutil.IntervalFast, + "deadline %v never updated", firstDeadline, + ) + + require.WithinDuration(t, time.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) + } + } + + t.Run("Dial", func(t *testing.T) { + t.Parallel() + + client, workspace, assertBumped := setupActivityTest(t) + + resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID) + conn, err := client.DialWorkspaceAgentTailnet(ctx, slogtest.Make(t, nil), resources[0].Agents[0].ID) + require.NoError(t, err) + defer conn.Close() + + sshConn, err := conn.SSHClient() + require.NoError(t, err) + _ = sshConn.Close() + + assertBumped(true) + }) + + t.Run("NoBump", func(t *testing.T) { + t.Parallel() + + client, workspace, assertBumped := setupActivityTest(t) + + // Doing some inactive operation like retrieving resources must not + // bump the deadline. + _, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID) + require.NoError(t, err) + + assertBumped(false) + }) +} diff --git a/coderd/coderd.go b/coderd/coderd.go index bbe67b19f446a..b49fdaf6eaba6 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -201,7 +201,7 @@ func New(options *Options) *API { httpmw.ExtractUserParam(api.Database), // Extracts the from the url httpmw.ExtractWorkspaceAndAgentParam(api.Database), - httpmw.ActivityBumpWorkspace(api.Logger, api.Database), + // httpmw.ActivityBumpWorkspace(api.Logger, api.Database), ) r.HandleFunc("/*", api.workspaceAppsProxyPath) } @@ -422,7 +422,6 @@ func New(options *Options) *API { apiKeyMiddleware, httpmw.ExtractWorkspaceAgentParam(options.Database), httpmw.ExtractWorkspaceParam(options.Database), - httpmw.ActivityBumpWorkspace(api.Logger, options.Database), ) r.Get("/", api.workspaceAgent) r.Get("/pty", api.workspaceAgentPTY) diff --git a/coderd/httpmw/workspaceactivitybump.go b/coderd/httpmw/workspaceactivitybump.go deleted file mode 100644 index dc6280ff9296b..0000000000000 --- a/coderd/httpmw/workspaceactivitybump.go +++ /dev/null @@ -1,100 +0,0 @@ -package httpmw - -import ( - "context" - "database/sql" - "errors" - "net/http" - "time" - - "golang.org/x/xerrors" - - "cdr.dev/slog" - "github.com/coder/coder/coderd/database" -) - -// ActivityBumpWorkspace automatically bumps the workspace's auto-off timer -// if it is set to expire soon. -// It must be ran after ExtractWorkspace. -func ActivityBumpWorkspace(log slog.Logger, db database.Store) func(h http.Handler) http.Handler { - log = log.Named("activity_bump") - - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - workspace := WorkspaceParam(r) - log.Debug(r.Context(), "middleware called") - // We run the bump logic asynchronously since the result doesn't - // affect the response. - go func() { - bump := func() { - // We cannot use the Request context since the goroutine - // may be around after the request terminates. - // We set a short timeout so if the app is under load, these - // low priority operations fail first. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - - err := db.InTx(func(s database.Store) error { - build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) - log.Debug(ctx, "build", slog.F("build", build)) - if errors.Is(err, sql.ErrNoRows) { - return nil - } else if err != nil { - return xerrors.Errorf("get latest workspace build: %w", err) - } - - job, err := s.GetProvisionerJobByID(ctx, build.JobID) - if err != nil { - return xerrors.Errorf("get provisioner job: %w", err) - } - - if build.Transition != database.WorkspaceTransitionStart || !job.CompletedAt.Valid { - return nil - } - - if build.Deadline.IsZero() { - // Workspace shutdown is manual - return nil - } - - // We sent bumpThreshold slightly under bumpAmount to minimize DB writes. - const ( - bumpAmount = time.Hour - bumpThreshold = time.Hour - (time.Minute * 10) - ) - - if !build.Deadline.Before(time.Now().Add(bumpThreshold)) { - return nil - } - - newDeadline := time.Now().Add(bumpAmount) - - if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ - ID: build.ID, - UpdatedAt: build.UpdatedAt, - ProvisionerState: build.ProvisionerState, - Deadline: newDeadline, - }); err != nil { - return xerrors.Errorf("update workspace build: %w", err) - } - return nil - }) - - if err != nil { - log.Error(ctx, "bump failed", slog.Error(err)) - } else { - log.Debug( - ctx, "bumped deadline from activity", - slog.F("workspace_id", workspace.ID), - ) - } - } - // For long running connections (e.g. web terminal), we need - // to bump periodically - // ticker := time.NewTicker(time.Minute) - bump() - }() - next.ServeHTTP(w, r) - }) - } -} diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 57c90c5b84479..e04046f2762bd 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -616,6 +616,8 @@ func (api *API) workspaceAgentReportStats(rw http.ResponseWriter, r *http.Reques ) if updateDB { + go activityBumpWorkspace(api.Logger.Named("activty_bump"), api.Database, workspace) + lastReport = rep _, err = api.Database.InsertAgentStat(ctx, database.InsertAgentStatParams{ diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index 2337095e185d8..c20c9c67b4330 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -58,7 +58,9 @@ func setupProxyTest(t *testing.T, workspaceMutators ...func(*codersdk.CreateWork require.True(t, ok) client := coderdtest.New(t, &coderdtest.Options{ - IncludeProvisionerDaemon: true, + IncludeProvisionerDaemon: true, + AgentStatsRefreshInterval: time.Millisecond * 100, + MetricsCacheRefreshInterval: time.Millisecond * 100, }) user := coderdtest.CreateFirstUser(t, client) authToken := uuid.NewString() @@ -104,6 +106,7 @@ func setupProxyTest(t *testing.T, workspaceMutators ...func(*codersdk.CreateWork FetchMetadata: agentClient.WorkspaceAgentMetadata, CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, Logger: slogtest.Make(t, nil).Named("agent"), + StatsReporter: agentClient.AgentReportStats, }) t.Cleanup(func() { _ = agentCloser.Close() diff --git a/coderd/workspaces_test.go b/coderd/workspaces_test.go index 6d921050482c1..11e23ea341c60 100644 --- a/coderd/workspaces_test.go +++ b/coderd/workspaces_test.go @@ -1029,90 +1029,6 @@ func TestWorkspaceUpdateAutostart(t *testing.T) { require.Contains(t, coderSDKErr.Message, "Resource not found", "unexpected response code") }) } -func TestWorkspaceActivityBump(t *testing.T) { - t.Parallel() - - ctx := context.Background() - setupActivityTest := func(t *testing.T) (client *codersdk.Client, workspace codersdk.Workspace, assertBumped func(want bool)) { - var ttlMillis int64 = 60 * 1000 - - client, _, workspace, _ = setupProxyTest(t, func(cwr *codersdk.CreateWorkspaceRequest) { - cwr.TTLMillis = &ttlMillis - }) - - // Sanity-check that deadline is near. - workspace, err := client.Workspace(ctx, workspace.ID) - require.NoError(t, err) - require.WithinDuration(t, - time.Now().Add(time.Duration(ttlMillis)*time.Millisecond), - workspace.LatestBuild.Deadline.Time, testutil.WaitShort, - ) - firstDeadline := workspace.LatestBuild.Deadline.Time - - return client, workspace, func(want bool) { - if !want { - time.Sleep(testutil.IntervalMedium) - workspace, err = client.Workspace(ctx, workspace.ID) - require.NoError(t, err) - require.Equal(t, workspace.LatestBuild.Deadline.Time, firstDeadline) - return - } - - // The Deadline bump occurs asynchronously. - require.Eventuallyf(t, - func() bool { - workspace, err = client.Workspace(ctx, workspace.ID) - require.NoError(t, err) - return workspace.LatestBuild.Deadline.Time != firstDeadline - }, - testutil.WaitShort, testutil.IntervalFast, - "deadline %v never updated", firstDeadline, - ) - - require.WithinDuration(t, time.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) - } - } - - t.Run("Apps", func(t *testing.T) { - t.Parallel() - - client, workspace, assertBumped := setupActivityTest(t) - - // A request to the /apps/ endpoint extends the deadline an hour. - // The particular app doesn't matter. The deadline is extended - // regardless of error state. - resp, err := client.Request(ctx, http.MethodGet, "/@me/"+workspace.Name+"/apps/example", nil) - require.NoError(t, err) - resp.Body.Close() - assertBumped(true) - }) - - t.Run("Dial", func(t *testing.T) { - t.Parallel() - - client, workspace, assertBumped := setupActivityTest(t) - - resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID) - conn, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil) - require.NoError(t, err) - _ = conn.Close() - - assertBumped(true) - }) - - t.Run("NoBump", func(t *testing.T) { - t.Parallel() - - client, workspace, assertBumped := setupActivityTest(t) - - // Doing some inactive operation like retrieving resources must not - // bump the deadline. - _, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID) - require.NoError(t, err) - - assertBumped(false) - }) -} func TestWorkspaceUpdateTTL(t *testing.T) { t.Parallel() diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 2117de03c6ce3..e46eec1cbdf8c 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -285,8 +285,8 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg return } if err != nil { + // WARN: closing here may lead to nhooyr websocket panicking. logger.Debug(ctx, "failed to dial", slog.Error(err)) - _ = ws.Close(websocket.StatusAbnormalClosure, "") continue } sendNode, errChan := tailnet.ServeCoordinator(websocket.NetConn(ctx, ws, websocket.MessageBinary), func(node []*tailnet.Node) error { diff --git a/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx b/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx index 1b53583ed141c..6acdbf3ab22a8 100644 --- a/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx +++ b/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx @@ -55,7 +55,8 @@ export const Language = { timezoneLabel: "Timezone", ttlLabel: "Time until shutdown (hours)", ttlCausesShutdownHelperText: "Your workspace will shut down", - ttlCausesShutdownAfterStart: "after its next start", + ttlCausesShutdownAfterStart: `after its next start. We delay shutdown by an +hour whenever we detect activity`, ttlCausesNoShutdownHelperText: "Your workspace will not automatically shut down.", formTitle: "Workspace schedule", startSection: "Start", From bd152432f8007a9581293562d8dade2caa04a1de Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 03:27:34 +0000 Subject: [PATCH 5/9] fixup! Move logic to agent + frontend --- coderd/activitybump.go | 2 -- coderd/activitybump_test.go | 2 +- coderd/coderd.go | 1 - coderd/workspaceagents.go | 2 +- coderd/workspaceapps_test.go | 15 --------------- 5 files changed, 2 insertions(+), 20 deletions(-) diff --git a/coderd/activitybump.go b/coderd/activitybump.go index 35f4b725c94d9..1f22175f20952 100644 --- a/coderd/activitybump.go +++ b/coderd/activitybump.go @@ -15,8 +15,6 @@ import ( // activityBumpWorkspace automatically bumps the workspace's auto-off timer // if it is set to expire soon. func activityBumpWorkspace(log slog.Logger, db database.Store, workspace database.Workspace) { - // We cannot use the Request context since the goroutine - // may be around after the request terminates. // We set a short timeout so if the app is under load, these // low priority operations fail first. ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go index 2246e3dc1bf15..705b8312a99bb 100644 --- a/coderd/activitybump_test.go +++ b/coderd/activitybump_test.go @@ -83,7 +83,7 @@ func TestWorkspaceActivityBump(t *testing.T) { client, workspace, assertBumped := setupActivityTest(t) - // Doing some inactive operation like retrieving resources must not + // Benign operations like retrieving resources must not // bump the deadline. _, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID) require.NoError(t, err) diff --git a/coderd/coderd.go b/coderd/coderd.go index b49fdaf6eaba6..607f15ff7931b 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -201,7 +201,6 @@ func New(options *Options) *API { httpmw.ExtractUserParam(api.Database), // Extracts the from the url httpmw.ExtractWorkspaceAndAgentParam(api.Database), - // httpmw.ActivityBumpWorkspace(api.Logger, api.Database), ) r.HandleFunc("/*", api.workspaceAppsProxyPath) } diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index e04046f2762bd..219fbb78b7d00 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -616,7 +616,7 @@ func (api *API) workspaceAgentReportStats(rw http.ResponseWriter, r *http.Reques ) if updateDB { - go activityBumpWorkspace(api.Logger.Named("activty_bump"), api.Database, workspace) + go activityBumpWorkspace(api.Logger.Named("activity_bump"), api.Database, workspace) lastReport = rep diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index c20c9c67b4330..164ceb2b15de6 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -211,21 +211,6 @@ func TestWorkspaceAppsProxyPath(t *testing.T) { require.Equal(t, http.StatusOK, resp.StatusCode) }) - t.Run("Proxies", func(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() - - resp, err := client.Request(ctx, http.MethodGet, "/@me/"+workspace.Name+"/apps/example/?"+proxyTestAppQuery, nil) - require.NoError(t, err) - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - require.Equal(t, proxyTestAppBody, string(body)) - require.Equal(t, http.StatusOK, resp.StatusCode) - }) - t.Run("ProxyError", func(t *testing.T) { t.Parallel() From ef8e29db63933e85be6802f479312f3b9e7f7f9c Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 16:52:02 +0000 Subject: [PATCH 6/9] Address review comments --- coderd/activitybump.go | 7 +++---- codersdk/workspaceagents.go | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/coderd/activitybump.go b/coderd/activitybump.go index 1f22175f20952..91a5ede3bf6f1 100644 --- a/coderd/activitybump.go +++ b/coderd/activitybump.go @@ -17,7 +17,7 @@ import ( func activityBumpWorkspace(log slog.Logger, db database.Store, workspace database.Workspace) { // We set a short timeout so if the app is under load, these // low priority operations fail first. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() err := db.InTx(func(s database.Store) error { @@ -53,11 +53,11 @@ func activityBumpWorkspace(log slog.Logger, db database.Store, workspace databas return nil } - newDeadline := time.Now().Add(bumpAmount) + newDeadline := database.Now().Add(bumpAmount) if err := s.UpdateWorkspaceBuildByID(ctx, database.UpdateWorkspaceBuildByIDParams{ ID: build.ID, - UpdatedAt: build.UpdatedAt, + UpdatedAt: database.Now(), ProvisionerState: build.ProvisionerState, Deadline: newDeadline, }); err != nil { @@ -65,7 +65,6 @@ func activityBumpWorkspace(log slog.Logger, db database.Store, workspace databas } return nil }) - if err != nil { log.Error(ctx, "bump failed", slog.Error(err)) } else { diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index e46eec1cbdf8c..bf1f58eceafac 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -281,11 +281,9 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg CompressionMode: websocket.CompressionDisabled, }) if errors.Is(err, context.Canceled) { - _ = ws.Close(websocket.StatusAbnormalClosure, "") return } if err != nil { - // WARN: closing here may lead to nhooyr websocket panicking. logger.Debug(ctx, "failed to dial", slog.Error(err)) continue } From f6c555cbdc661e21fec38633351335361141746d Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 16:53:19 +0000 Subject: [PATCH 7/9] fixup! Address review comments --- coderd/activitybump.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/coderd/activitybump.go b/coderd/activitybump.go index 91a5ede3bf6f1..7277affe0971c 100644 --- a/coderd/activitybump.go +++ b/coderd/activitybump.go @@ -66,7 +66,11 @@ func activityBumpWorkspace(log slog.Logger, db database.Store, workspace databas return nil }) if err != nil { - log.Error(ctx, "bump failed", slog.Error(err)) + log.Error( + ctx, "bump failed", + slog.Error(err), + slog.F("workspace_id", workspace.ID), + ) } else { log.Debug( ctx, "bumped deadline from activity", From 60a271571507f5fe5da7c96b1b91210294a81dcf Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 18:22:51 +0000 Subject: [PATCH 8/9] Address PR comments (round 2) --- coderd/activitybump.go | 1 - coderd/activitybump_test.go | 3 ++- .../WorkspaceScheduleForm/WorkspaceScheduleForm.tsx | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coderd/activitybump.go b/coderd/activitybump.go index 7277affe0971c..fa90f7d275daf 100644 --- a/coderd/activitybump.go +++ b/coderd/activitybump.go @@ -22,7 +22,6 @@ func activityBumpWorkspace(log slog.Logger, db database.Store, workspace databas err := db.InTx(func(s database.Store) error { build, err := s.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) - log.Debug(ctx, "build", slog.F("build", build)) if errors.Is(err, sql.ErrNoRows) { return nil } else if err != nil { diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go index 705b8312a99bb..167c85eb11287 100644 --- a/coderd/activitybump_test.go +++ b/coderd/activitybump_test.go @@ -10,6 +10,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/coderd/coderdtest" + "github.com/coder/coder/coderd/database" "github.com/coder/coder/codersdk" "github.com/coder/coder/testutil" ) @@ -57,7 +58,7 @@ func TestWorkspaceActivityBump(t *testing.T) { "deadline %v never updated", firstDeadline, ) - require.WithinDuration(t, time.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) + require.WithinDuration(t, database.Now().Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Second) } } diff --git a/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx b/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx index 6acdbf3ab22a8..49f9aec931975 100644 --- a/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx +++ b/site/src/components/WorkspaceScheduleForm/WorkspaceScheduleForm.tsx @@ -55,8 +55,8 @@ export const Language = { timezoneLabel: "Timezone", ttlLabel: "Time until shutdown (hours)", ttlCausesShutdownHelperText: "Your workspace will shut down", - ttlCausesShutdownAfterStart: `after its next start. We delay shutdown by an -hour whenever we detect activity`, + ttlCausesShutdownAfterStart: + "after its next start. We delay shutdown by an hour whenever we detect activity", ttlCausesNoShutdownHelperText: "Your workspace will not automatically shut down.", formTitle: "Workspace schedule", startSection: "Start", From 0c48a6719106b2d9b2e1ede0e131a4e5b83c2fb8 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 20 Sep 2022 18:28:25 +0000 Subject: [PATCH 9/9] fixup! Address PR comments (round 2) --- coderd/activitybump_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go index 167c85eb11287..39c2f5fc6c5c8 100644 --- a/coderd/activitybump_test.go +++ b/coderd/activitybump_test.go @@ -40,6 +40,12 @@ func TestWorkspaceActivityBump(t *testing.T) { return client, workspace, func(want bool) { if !want { + // It is difficult to test the absence of a call in a non-racey + // way. In general, it is difficult for the API to generate + // false positive activity since Agent networking event + // is required. The Activity Bump behavior is also coupled with + // Last Used, so it would be obvious to the user if we + // are falsely recognizing activity. time.Sleep(testutil.IntervalMedium) workspace, err = client.Workspace(ctx, workspace.ID) require.NoError(t, err) pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy