chore: add profiling labels for pprof analysis

Emyrk · Emyrk · commit 74cf7aef501e · 2025-08-07T10:17:19.000-05:00
PProf labels segment the code into groups for determing the source
of cpu/memory profiles. Since the web server and background jobs
share a lot of the same code (eg wsbuilder), it helps to know
if the load is user induced, or background job based.
diff --git a/cli/server.go b/cli/server.go
@@ -55,6 +55,7 @@ import (
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
+	"github.com/coder/coder/v2/coderd/pproflabel"
 	"github.com/coder/pretty"
 	"github.com/coder/quartz"
 	"github.com/coder/retry"
@@ -1459,14 +1460,14 @@ func newProvisionerDaemon(
 			tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
 			terraformClient, terraformServer := drpcsdk.MemTransportPipe()
 			wg.Add(1)
-			go func() {
+			pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceTerraformProvisioner), func(ctx context.Context) {
 				defer wg.Done()
 				<-ctx.Done()
 				_ = terraformClient.Close()
 				_ = terraformServer.Close()
-			}()
+			})
 			wg.Add(1)
-			go func() {
+			pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceTerraformProvisioner), func(ctx context.Context) {
 				defer wg.Done()
 				defer cancel()
 
@@ -1485,7 +1486,7 @@ func newProvisionerDaemon(
 					default:
 					}
 				}
-			}()
+			})
 
 			connector[string(database.ProvisionerTypeTerraform)] = sdkproto.NewDRPCProvisionerClient(terraformClient)
 		default:
diff --git a/coderd/autobuild/lifecycle_executor.go b/coderd/autobuild/lifecycle_executor.go
@@ -20,6 +20,7 @@ import (
 
 	"cdr.dev/slog"
 	"github.com/coder/coder/v2/coderd/files"
+	"github.com/coder/coder/v2/coderd/pproflabel"
 
 	"github.com/coder/coder/v2/coderd/audit"
 	"github.com/coder/coder/v2/coderd/database"
@@ -107,7 +108,7 @@ func (e *Executor) WithStatsChannel(ch chan<- Stats) *Executor {
 // tick from its channel. It will stop when its context is Done, or when
 // its channel is closed.
 func (e *Executor) Run() {
-	go func() {
+	pproflabel.Go(e.ctx, pproflabel.Service(pproflabel.ServiceLifecycles), func(ctx context.Context) {
 		for {
 			select {
 			case <-e.ctx.Done():
@@ -128,7 +129,7 @@ func (e *Executor) Run() {
 				e.log.Debug(e.ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions))
 			}
 		}
-	}()
+	})
 }
 
 func (e *Executor) runOnce(t time.Time) Stats {
diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -852,6 +852,7 @@ func New(options *Options) *API {
 
 	r.Use(
 		httpmw.Recover(api.Logger),
+		httpmw.WithProfilingLabels,
 		tracing.StatusWriterMiddleware,
 		tracing.Middleware(api.TracerProvider),
 		httpmw.AttachRequestID,
diff --git a/coderd/httpmw/pprof.go b/coderd/httpmw/pprof.go
@@ -0,0 +1,22 @@
+package httpmw
+
+import (
+	"context"
+	"net/http"
+	"runtime/pprof"
+
+	"github.com/coder/coder/v2/coderd/pproflabel"
+)
+
+// WithProfilingLabels adds a pprof label to all http request handlers. This is
+// primarily used to determine if load is coming from background jobs, or from
+// http traffic.
+func WithProfilingLabels(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		pprof.Do(ctx, pproflabel.Service(pproflabel.ServiceHTTPServer), func(ctx context.Context) {
+			r = r.WithContext(ctx)
+			next.ServeHTTP(rw, r)
+		})
+	})
+}
diff --git a/coderd/pproflabel/pproflabel.go b/coderd/pproflabel/pproflabel.go
@@ -0,0 +1,22 @@
+package pproflabel
+
+import (
+	"context"
+	"runtime/pprof"
+)
+
+// Go is just a convince wrapper to set off a labeled goroutine.
+func Go(ctx context.Context, labels pprof.LabelSet, f func(context.Context)) {
+	go pprof.Do(ctx, labels, f)
+}
+
+const (
+	ServiceTerraformProvisioner = "terraform-provisioner"
+	ServiceMetricCollector      = "metric-collector"
+	ServiceLifecycles           = "lifecycles"
+	ServiceHTTPServer           = "http-server"
+)
+
+func Service(name string) pprof.LabelSet {
+	return pprof.Labels("service", name)
+}
diff --git a/coderd/prometheusmetrics/insights/metricscollector.go b/coderd/prometheusmetrics/insights/metricscollector.go
@@ -14,6 +14,7 @@ import (
 	"cdr.dev/slog"
 
 	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/pproflabel"
 	"github.com/coder/coder/v2/coderd/util/slice"
 	"github.com/coder/coder/v2/codersdk"
 )
@@ -158,7 +159,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) {
 		})
 	}
 
-	go func() {
+	pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceMetricCollector), func(ctx context.Context) {
 		defer close(done)
 		defer ticker.Stop()
 		for {
@@ -170,7 +171,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) {
 				doTick()
 			}
 		}
-	}()
+	})
 	return func() {
 		closeFunc()
 		<-done
diff --git a/enterprise/wsproxy/wsproxy.go b/enterprise/wsproxy/wsproxy.go
@@ -333,6 +333,7 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
 	r.Use(
 		// TODO: @emyrk Should we standardize these in some other package?
 		httpmw.Recover(s.Logger),
+		httpmw.WithProfilingLabels,
 		tracing.StatusWriterMiddleware,
 		tracing.Middleware(s.TracerProvider),
 		httpmw.AttachRequestID,

Original file line number	Diff line number	Diff line change
`@@ -14,6 +14,7 @@ import (`
`14`	`14`	`"cdr.dev/slog"`
`15`	`15`
`16`	`16`	`"github.com/coder/coder/v2/coderd/database"`
	`17`	`+ "github.com/coder/coder/v2/coderd/pproflabel"`
`17`	`18`	`"github.com/coder/coder/v2/coderd/util/slice"`
`18`	`19`	`"github.com/coder/coder/v2/codersdk"`
`19`	`20`	`)`
`@@ -158,7 +159,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) {`
`158`	`159`	`})`
`159`	`160`	`}`
`160`	`161`
`161`		`- go func() {`
	`162`	`+ pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceMetricCollector), func(ctx context.Context) {`
`162`	`163`	`defer close(done)`
`163`	`164`	`defer ticker.Stop()`
`164`	`165`	`for {`
`@@ -170,7 +171,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) {`
`170`	`171`	`doTick()`
`171`	`172`	`}`
`172`	`173`	`}`
`173`		`- }()`
	`174`	`+ })`
`174`	`175`	`return func() {`
`175`	`176`	`closeFunc()`
`176`	`177`	`<-done`