fix: reimplement reporting of preset-hard-limited metric

evgeniy-scherbina · evgeniy-scherbina · commit f7d363406f03 · 2025-05-27T14:49:52.000Z
diff --git a/coderd/prebuilds/global_snapshot.go b/coderd/prebuilds/global_snapshot.go
@@ -80,6 +80,14 @@ func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, err
 	}, nil
 }
 
+func (s GlobalSnapshot) IsHardLimited(presetID uuid.UUID) bool {
+	_, isHardLimited := slice.Find(s.HardLimitedPresets, func(row database.GetPresetsAtFailureLimitRow) bool {
+		return row.PresetID == presetID
+	})
+
+	return isHardLimited
+}
+
 // filterExpiredWorkspaces splits running workspaces into expired and non-expired
 // based on the preset's TTL.
 // If TTL is missing or zero, all workspaces are considered non-expired.
diff --git a/enterprise/coderd/prebuilds/metricscollector.go b/enterprise/coderd/prebuilds/metricscollector.go
@@ -280,16 +280,9 @@ func (k hardLimitedPresetKey) String() string {
 	return fmt.Sprintf("%s:%s:%s", k.orgName, k.templateName, k.presetName)
 }
 
-// nolint:revive // isHardLimited determines if the preset should be reported as hard-limited in Prometheus.
-func (mc *MetricsCollector) trackHardLimitedStatus(orgName, templateName, presetName string, isHardLimited bool) {
+func (mc *MetricsCollector) registerHardLimitedPresets(isPresetHardLimited map[hardLimitedPresetKey]bool) {
 	mc.isPresetHardLimitedMu.Lock()
 	defer mc.isPresetHardLimitedMu.Unlock()
 
-	key := hardLimitedPresetKey{orgName: orgName, templateName: templateName, presetName: presetName}
-
-	if isHardLimited {
-		mc.isPresetHardLimited[key] = true
-	} else {
-		delete(mc.isPresetHardLimited, key)
-	}
+	mc.isPresetHardLimited = isPresetHardLimited
 }
diff --git a/enterprise/coderd/prebuilds/reconcile.go b/enterprise/coderd/prebuilds/reconcile.go
@@ -256,6 +256,9 @@ func (c *StoreReconciler) ReconcileAll(ctx context.Context) error {
 		if err != nil {
 			return xerrors.Errorf("determine current snapshot: %w", err)
 		}
+
+		c.reportHardLimitedPresets(snapshot)
+
 		if len(snapshot.Presets) == 0 {
 			logger.Debug(ctx, "no templates found with prebuilds configured")
 			return nil
@@ -296,6 +299,56 @@ func (c *StoreReconciler) ReconcileAll(ctx context.Context) error {
 	return err
 }
 
+// Report a metric only if the preset uses the latest version of the template and the template is not deleted.
+// This avoids conflicts between metrics from old and new template versions.
+//
+// NOTE: Multiple versions of a preset can exist with the same orgName, templateName, and presetName,
+// because templates can have multiple versions — or deleted templates can share the same name.
+//
+// The safest approach is to report the metric only for the latest version of the preset.
+// When a new template version is released, the metric for the new preset should overwrite
+// the old value in Prometheus.
+//
+// However, there’s one edge case: if an admin creates a template, it becomes hard-limited,
+// then deletes the template and never creates another with the same name,
+// the old preset will continue to be reported as hard-limited —
+// even though it’s deleted. This will persist until `coderd` is restarted.
+
+func (c *StoreReconciler) reportHardLimitedPresets(snapshot *prebuilds.GlobalSnapshot) {
+	// presetsMap is a map from key (orgName:templateName:presetName) to list of corresponding presets.
+	// Multiple versions of a preset can exist with the same orgName, templateName, and presetName,
+	// because templates can have multiple versions — or deleted templates can share the same name.
+	presetsMap := make(map[hardLimitedPresetKey][]database.GetTemplatePresetsWithPrebuildsRow)
+	for _, preset := range snapshot.Presets {
+		key := hardLimitedPresetKey{
+			orgName:      preset.OrganizationName,
+			templateName: preset.TemplateName,
+			presetName:   preset.Name,
+		}
+
+		presetsMap[key] = append(presetsMap[key], preset)
+	}
+
+	// Report a preset as hard-limited only if all the following conditions are met:
+	// - The preset is marked as hard-limited
+	// - The preset is using the active version of its template, and the template has not been deleted
+	//
+	// The second condition is important because a hard-limited preset that has become outdated is no longer relevant.
+	// Its associated prebuilt workspaces were likely deleted, and it's not meaningful to continue reporting it
+	// as hard-limited to the admin.
+	isPresetHardLimited := make(map[hardLimitedPresetKey]bool)
+	for key, presets := range presetsMap {
+		for _, preset := range presets {
+			if preset.UsingActiveVersion && !preset.Deleted && snapshot.IsHardLimited(preset.ID) {
+				isPresetHardLimited[key] = true
+				break
+			}
+		}
+	}
+
+	c.metrics.registerHardLimitedPresets(isPresetHardLimited)
+}
+
 // SnapshotState captures the current state of all prebuilds across templates.
 func (c *StoreReconciler) SnapshotState(ctx context.Context, store database.Store) (*prebuilds.GlobalSnapshot, error) {
 	if err := ctx.Err(); err != nil {
@@ -375,9 +428,9 @@ func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.Pres
 	// then deletes the template and never creates another with the same name,
 	// the old preset will continue to be reported as hard-limited —
 	// even though it’s deleted. This will persist until `coderd` is restarted.
-	if ps.Preset.UsingActiveVersion && !ps.Preset.Deleted {
-		c.metrics.trackHardLimitedStatus(ps.Preset.OrganizationName, ps.Preset.TemplateName, ps.Preset.Name, ps.IsHardLimited)
-	}
+	//if ps.Preset.UsingActiveVersion && !ps.Preset.Deleted {
+	//	c.metrics.trackHardLimitedStatus(ps.Preset.OrganizationName, ps.Preset.TemplateName, ps.Preset.Name, ps.IsHardLimited)
+	//}
 
 	// If the preset reached the hard failure limit for the first time during this iteration:
 	// - Mark it as hard-limited in the database
diff --git a/enterprise/coderd/prebuilds/reconcile_test.go b/enterprise/coderd/prebuilds/reconcile_test.go
@@ -1043,9 +1043,7 @@ func TestHardLimitedPresetShouldNotBlockDeletion(t *testing.T) {
 				"preset_name":   preset.Name,
 				"org_name":      org.Name,
 			})
-			require.NotNil(t, metric)
-			require.NotNil(t, metric.GetGauge())
-			require.EqualValues(t, 1, metric.GetGauge().GetValue())
+			require.Nil(t, metric)
 		})
 	}
 }

Original file line number	Diff line number	Diff line change
`@@ -1043,9 +1043,7 @@ func TestHardLimitedPresetShouldNotBlockDeletion(t *testing.T) {`
`1043`	`1043`	`"preset_name": preset.Name,`
`1044`	`1044`	`"org_name": org.Name,`
`1045`	`1045`	`})`
`1046`		`- require.NotNil(t, metric)`
`1047`		`- require.NotNil(t, metric.GetGauge())`
`1048`		`- require.EqualValues(t, 1, metric.GetGauge().GetValue())`
	`1046`	`+ require.Nil(t, metric)`
`1049`	`1047`	`})`
`1050`	`1048`	`}`
`1051`	`1049`	`}`