From 80ad7fbefaf33b3127ba96357d73d3c5cc6d372f Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 25 Nov 2024 12:23:54 +0000 Subject: [PATCH 1/2] fix(coderd): extract provisionerdserver.StaleHeartbeats, reduce from 3 to 2 --- coderd/coderd.go | 3 ++- coderd/healthcheck/provisioner.go | 2 +- coderd/provisionerdserver/provisionerdserver.go | 4 ++++ coderd/templateversions.go | 4 ++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index b5450b1387dfe..1a1b0f4889c9f 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -628,7 +628,8 @@ func New(options *Options) *API { CurrentVersion: buildinfo.Version(), CurrentAPIMajorVersion: proto.CurrentMajor, Store: options.Database, - // TimeNow and StaleInterval set to defaults, see healthcheck/provisioner.go + StaleInterval: options.DeploymentValues.Provisioner.DaemonPollInterval.Value() * provisionerdserver.StaleHeartbeats, + // TimeNow set to default, see healthcheck/provisioner.go }, }) } diff --git a/coderd/healthcheck/provisioner.go b/coderd/healthcheck/provisioner.go index 370a5ad04de86..cfdb99f554805 100644 --- a/coderd/healthcheck/provisioner.go +++ b/coderd/healthcheck/provisioner.go @@ -50,7 +50,7 @@ func (r *ProvisionerDaemonsReport) Run(ctx context.Context, opts *ProvisionerDae now := opts.TimeNow() if opts.StaleInterval == 0 { - opts.StaleInterval = provisionerdserver.DefaultHeartbeatInterval * 3 + opts.StaleInterval = provisionerdserver.DefaultHeartbeatInterval * provisionerdserver.StaleHeartbeats } if opts.CurrentVersion == "" { diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 8387f97ea21cb..9ac1bbb65bc9f 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -56,6 +56,10 @@ const ( // DefaultHeartbeatInterval is the interval at which the provisioner daemon // will update its last seen at timestamp in the database. DefaultHeartbeatInterval = time.Minute + + // StaleHeartbeats is the number of heartbeats a provisioner can miss before + // being reported as 'stale'. + StaleHeartbeats = 2 ) type Options struct { diff --git a/coderd/templateversions.go b/coderd/templateversions.go index a0609c42c33f9..1c9027e70af61 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -1831,7 +1831,7 @@ func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUI return codersdk.MatchedProvisioners{}, xerrors.Errorf("provisioner daemons by organization: %w", err) } - threePollsAgo := time.Now().Add(-3 * pollInterval) + staleInterval := time.Now().Add(provisionerdserver.StaleHeartbeats * pollInterval) mostRecentlySeen := codersdk.NullTime{} var matched codersdk.MatchedProvisioners for _, provisioner := range eligibleProvisioners { @@ -1839,7 +1839,7 @@ func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUI continue } matched.Count++ - if provisioner.LastSeenAt.Time.After(threePollsAgo) { + if provisioner.LastSeenAt.Time.After(staleInterval) { matched.Available++ } if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) { From e8ee254d4796c2a17296ed7df376469c48271472 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 25 Nov 2024 18:03:35 +0000 Subject: [PATCH 2/2] replace multiples of poll interval with provisionerdaemon.StaleInterval --- coderd/coderd.go | 2 +- coderd/healthcheck/provisioner.go | 2 +- coderd/healthcheck/provisioner_test.go | 106 +++++++++++++----- .../provisionerdserver/provisionerdserver.go | 6 +- coderd/templateversions.go | 6 +- 5 files changed, 88 insertions(+), 34 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index 09ecbb43fa1e1..4c7ff4fe081ff 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -628,7 +628,7 @@ func New(options *Options) *API { CurrentVersion: buildinfo.Version(), CurrentAPIMajorVersion: proto.CurrentMajor, Store: options.Database, - StaleInterval: options.DeploymentValues.Provisioner.DaemonPollInterval.Value() * provisionerdserver.StaleHeartbeats, + StaleInterval: provisionerdserver.StaleInterval, // TimeNow set to default, see healthcheck/provisioner.go }, }) diff --git a/coderd/healthcheck/provisioner.go b/coderd/healthcheck/provisioner.go index cfdb99f554805..ae3220170dd69 100644 --- a/coderd/healthcheck/provisioner.go +++ b/coderd/healthcheck/provisioner.go @@ -50,7 +50,7 @@ func (r *ProvisionerDaemonsReport) Run(ctx context.Context, opts *ProvisionerDae now := opts.TimeNow() if opts.StaleInterval == 0 { - opts.StaleInterval = provisionerdserver.DefaultHeartbeatInterval * provisionerdserver.StaleHeartbeats + opts.StaleInterval = provisionerdserver.StaleInterval } if opts.CurrentVersion == "" { diff --git a/coderd/healthcheck/provisioner_test.go b/coderd/healthcheck/provisioner_test.go index 37530f9f8c747..93871f4a709ad 100644 --- a/coderd/healthcheck/provisioner_test.go +++ b/coderd/healthcheck/provisioner_test.go @@ -15,15 +15,21 @@ import ( "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/healthcheck" "github.com/coder/coder/v2/coderd/healthcheck/health" + "github.com/coder/coder/v2/coderd/provisionerdserver" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/codersdk/healthsdk" "github.com/coder/coder/v2/provisionerd/proto" + "github.com/coder/coder/v2/testutil" ) func TestProvisionerDaemonReport(t *testing.T) { t.Parallel() - now := dbtime.Now() + var ( + now = dbtime.Now() + oneHourAgo = now.Add(-time.Hour) + staleThreshold = now.Add(-provisionerdserver.StaleInterval).Add(-time.Second) + ) for _, tt := range []struct { name string @@ -65,7 +71,9 @@ func TestProvisionerDaemonReport(t *testing.T) { currentVersion: "v1.2.3", currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityOK, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -88,7 +96,9 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityWarning, expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-old"), withVersion("v1.1.2"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -116,7 +126,9 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityError, expectedWarningCode: health.CodeUnknown, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-version", "invalid", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-invalid-version"), withVersion("invalid"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -144,7 +156,9 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityError, expectedWarningCode: health.CodeUnknown, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-api", "v1.2.3", "invalid", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-invalid-api"), withVersion("v1.2.3"), withAPIVersion("invalid"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -172,7 +186,9 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: 2, expectedSeverity: health.SeverityWarning, expectedWarningCode: health.CodeProvisionerDaemonAPIMajorVersionDeprecated, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old-api", "v2.3.4", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-old-api"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -200,7 +216,10 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityWarning, expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now), fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + fakeProvisionerDaemon(t, withName("pd-old"), withVersion("v1.1.2"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -241,7 +260,10 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityWarning, expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0", now), fakeProvisionerDaemon(t, "pd-new", "v2.3.4", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v1.2.3"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + fakeProvisionerDaemon(t, withName("pd-new"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -281,7 +303,10 @@ func TestProvisionerDaemonReport(t *testing.T) { currentVersion: "v2.3.4", currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityOK, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-stale", "v1.2.3", "0.9", now.Add(-5*time.Minute), now), fakeProvisionerDaemon(t, "pd-ok", "v2.3.4", "1.0", now)}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-stale"), withVersion("v1.2.3"), withAPIVersion("0.9"), withCreatedAt(oneHourAgo), withLastSeenAt(staleThreshold)), + fakeProvisionerDaemon(t, withName("pd-ok"), withVersion("v2.3.4"), withAPIVersion("1.0"), withCreatedAt(now), withLastSeenAt(now)), + }, expectedItems: []healthsdk.ProvisionerDaemonsReportItem{ { ProvisionerDaemon: codersdk.ProvisionerDaemon{ @@ -304,8 +329,10 @@ func TestProvisionerDaemonReport(t *testing.T) { currentAPIMajorVersion: proto.CurrentMajor, expectedSeverity: health.SeverityError, expectedWarningCode: health.CodeProvisionerDaemonsNoProvisionerDaemons, - provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-ok", "v1.2.3", "0.9", now.Add(-5*time.Minute), now)}, - expectedItems: []healthsdk.ProvisionerDaemonsReportItem{}, + provisionerDaemons: []database.ProvisionerDaemon{ + fakeProvisionerDaemon(t, withName("pd-stale"), withVersion("v1.2.3"), withAPIVersion("0.9"), withCreatedAt(oneHourAgo), withLastSeenAt(staleThreshold)), + }, + expectedItems: []healthsdk.ProvisionerDaemonsReportItem{}, }, } { tt := tt @@ -353,25 +380,52 @@ func TestProvisionerDaemonReport(t *testing.T) { } } -func fakeProvisionerDaemon(t *testing.T, name, version, apiVersion string, now time.Time) database.ProvisionerDaemon { +func withName(s string) func(*database.ProvisionerDaemon) { + return func(pd *database.ProvisionerDaemon) { + pd.Name = s + } +} + +func withCreatedAt(at time.Time) func(*database.ProvisionerDaemon) { + return func(pd *database.ProvisionerDaemon) { + pd.CreatedAt = at + } +} + +func withLastSeenAt(at time.Time) func(*database.ProvisionerDaemon) { + return func(pd *database.ProvisionerDaemon) { + pd.LastSeenAt.Valid = true + pd.LastSeenAt.Time = at + } +} + +func withVersion(v string) func(*database.ProvisionerDaemon) { + return func(pd *database.ProvisionerDaemon) { + pd.Version = v + } +} + +func withAPIVersion(v string) func(*database.ProvisionerDaemon) { + return func(pd *database.ProvisionerDaemon) { + pd.APIVersion = v + } +} + +func fakeProvisionerDaemon(t *testing.T, opts ...func(*database.ProvisionerDaemon)) database.ProvisionerDaemon { t.Helper() - return database.ProvisionerDaemon{ + pd := database.ProvisionerDaemon{ ID: uuid.Nil, - Name: name, - CreatedAt: now, - LastSeenAt: sql.NullTime{Time: now, Valid: true}, + Name: testutil.GetRandomName(t), + CreatedAt: time.Time{}, + LastSeenAt: sql.NullTime{}, Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform}, ReplicaID: uuid.NullUUID{}, Tags: map[string]string{}, - Version: version, - APIVersion: apiVersion, + Version: "", + APIVersion: "", } -} - -func fakeProvisionerDaemonStale(t *testing.T, name, version, apiVersion string, lastSeenAt, now time.Time) database.ProvisionerDaemon { - t.Helper() - d := fakeProvisionerDaemon(t, name, version, apiVersion, now) - d.LastSeenAt.Valid = true - d.LastSeenAt.Time = lastSeenAt - return d + for _, o := range opts { + o(&pd) + } + return pd } diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 696b802546854..0e9892b892172 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -58,9 +58,9 @@ const ( // will update its last seen at timestamp in the database. DefaultHeartbeatInterval = time.Minute - // StaleHeartbeats is the number of heartbeats a provisioner can miss before - // being reported as 'stale'. - StaleHeartbeats = 2 + // StaleInterval is the amount of time after the last heartbeat for which + // the provisioner will be reported as 'stale'. + StaleInterval = 90 * time.Second ) type Options struct { diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 1c9027e70af61..55f3191cc4c57 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -1515,7 +1515,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht // Check for eligible provisioners. This allows us to log a message warning deployment administrators // of users submitting jobs for which no provisioners are available. - matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags, api.DeploymentValues.Provisioner.DaemonPollInterval.Value()) + matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags) if err != nil { api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err)) } else if matchedProvisioners.Count == 0 { @@ -1819,7 +1819,7 @@ func (api *API) publishTemplateUpdate(ctx context.Context, templateID uuid.UUID) } } -func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string, pollInterval time.Duration) (codersdk.MatchedProvisioners, error) { +func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string) (codersdk.MatchedProvisioners, error) { // Check for eligible provisioners. This allows us to return a warning to the user if they // submit a job for which no provisioner is available. eligibleProvisioners, err := store.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{ @@ -1831,7 +1831,7 @@ func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUI return codersdk.MatchedProvisioners{}, xerrors.Errorf("provisioner daemons by organization: %w", err) } - staleInterval := time.Now().Add(provisionerdserver.StaleHeartbeats * pollInterval) + staleInterval := time.Now().Add(-provisionerdserver.StaleInterval) mostRecentlySeen := codersdk.NullTime{} var matched codersdk.MatchedProvisioners for _, provisioner := range eligibleProvisioners { pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy