diff --git a/cli/server.go b/cli/server.go index 26d0c8f110403..f9e744761b22e 100644 --- a/cli/server.go +++ b/cli/server.go @@ -55,6 +55,7 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/sloghuman" + "github.com/coder/coder/v2/coderd/pproflabel" "github.com/coder/pretty" "github.com/coder/quartz" "github.com/coder/retry" @@ -1459,14 +1460,14 @@ func newProvisionerDaemon( tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName) terraformClient, terraformServer := drpcsdk.MemTransportPipe() wg.Add(1) - go func() { + pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceTerraformProvisioner), func(ctx context.Context) { defer wg.Done() <-ctx.Done() _ = terraformClient.Close() _ = terraformServer.Close() - }() + }) wg.Add(1) - go func() { + pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceTerraformProvisioner), func(ctx context.Context) { defer wg.Done() defer cancel() @@ -1485,7 +1486,7 @@ func newProvisionerDaemon( default: } } - }() + }) connector[string(database.ProvisionerTypeTerraform)] = sdkproto.NewDRPCProvisionerClient(terraformClient) default: diff --git a/coderd/autobuild/lifecycle_executor.go b/coderd/autobuild/lifecycle_executor.go index 234a72de04c50..16072e6517125 100644 --- a/coderd/autobuild/lifecycle_executor.go +++ b/coderd/autobuild/lifecycle_executor.go @@ -20,6 +20,7 @@ import ( "cdr.dev/slog" "github.com/coder/coder/v2/coderd/files" + "github.com/coder/coder/v2/coderd/pproflabel" "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/database" @@ -107,10 +108,10 @@ func (e *Executor) WithStatsChannel(ch chan<- Stats) *Executor { // tick from its channel. It will stop when its context is Done, or when // its channel is closed. func (e *Executor) Run() { - go func() { + pproflabel.Go(e.ctx, pproflabel.Service(pproflabel.ServiceLifecycles), func(ctx context.Context) { for { select { - case <-e.ctx.Done(): + case <-ctx.Done(): return case t, ok := <-e.tick: if !ok { @@ -120,15 +121,15 @@ func (e *Executor) Run() { e.metrics.autobuildExecutionDuration.Observe(stats.Elapsed.Seconds()) if e.statsCh != nil { select { - case <-e.ctx.Done(): + case <-ctx.Done(): return case e.statsCh <- stats: } } - e.log.Debug(e.ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions)) + e.log.Debug(ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions)) } } - }() + }) } func (e *Executor) runOnce(t time.Time) Stats { diff --git a/coderd/coderd.go b/coderd/coderd.go index 26bf4a7bf9b63..9b0e35b21032b 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -852,6 +852,7 @@ func New(options *Options) *API { r.Use( httpmw.Recover(api.Logger), + httpmw.WithProfilingLabels, tracing.StatusWriterMiddleware, tracing.Middleware(api.TracerProvider), httpmw.AttachRequestID, diff --git a/coderd/httpmw/pprof.go b/coderd/httpmw/pprof.go new file mode 100644 index 0000000000000..eee3e9c9fdbe1 --- /dev/null +++ b/coderd/httpmw/pprof.go @@ -0,0 +1,30 @@ +package httpmw + +import ( + "context" + "net/http" + "runtime/pprof" + + "github.com/coder/coder/v2/coderd/pproflabel" +) + +// WithProfilingLabels adds a pprof label to all http request handlers. This is +// primarily used to determine if load is coming from background jobs, or from +// http traffic. +func WithProfilingLabels(next http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Label to differentiate between http and websocket requests. Websocket requests + // are assumed to be long-lived and more resource consuming. + requestType := "http" + if r.Header.Get("Upgrade") == "websocket" { + requestType = "websocket" + } + + pprof.Do(ctx, pproflabel.Service(pproflabel.ServiceHTTPServer, "request_type", requestType), func(ctx context.Context) { + r = r.WithContext(ctx) + next.ServeHTTP(rw, r) + }) + }) +} diff --git a/coderd/pproflabel/pproflabel.go b/coderd/pproflabel/pproflabel.go new file mode 100644 index 0000000000000..cd803b0f1baea --- /dev/null +++ b/coderd/pproflabel/pproflabel.go @@ -0,0 +1,25 @@ +package pproflabel + +import ( + "context" + "runtime/pprof" +) + +// Go is just a convince wrapper to set off a labeled goroutine. +func Go(ctx context.Context, labels pprof.LabelSet, f func(context.Context)) { + go pprof.Do(ctx, labels, f) +} + +const ( + ServiceTag = "service" + + ServiceHTTPServer = "http-api" + ServiceLifecycles = "lifecycle-executor" + ServiceMetricCollector = "metrics-collector" + ServicePrebuildReconciler = "prebuilds-reconciler" + ServiceTerraformProvisioner = "terraform-provisioner" +) + +func Service(name string, pairs ...string) pprof.LabelSet { + return pprof.Labels(append([]string{ServiceTag, name}, pairs...)...) +} diff --git a/coderd/prometheusmetrics/insights/metricscollector.go b/coderd/prometheusmetrics/insights/metricscollector.go index 41d3a0220f391..a095968526ca8 100644 --- a/coderd/prometheusmetrics/insights/metricscollector.go +++ b/coderd/prometheusmetrics/insights/metricscollector.go @@ -14,6 +14,7 @@ import ( "cdr.dev/slog" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/pproflabel" "github.com/coder/coder/v2/coderd/util/slice" "github.com/coder/coder/v2/codersdk" ) @@ -158,7 +159,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) { }) } - go func() { + pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceMetricCollector), func(ctx context.Context) { defer close(done) defer ticker.Stop() for { @@ -170,7 +171,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) { doTick() } } - }() + }) return func() { closeFunc() <-done diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 9583e14cd7fd3..40569ead70658 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -12,20 +12,20 @@ import ( "sync" "time" - "github.com/coder/quartz" - "github.com/coder/coder/v2/buildinfo" "github.com/coder/coder/v2/coderd/appearance" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/entitlements" "github.com/coder/coder/v2/coderd/idpsync" agplportsharing "github.com/coder/coder/v2/coderd/portsharing" + "github.com/coder/coder/v2/coderd/pproflabel" agplprebuilds "github.com/coder/coder/v2/coderd/prebuilds" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/coder/coder/v2/coderd/wsbuilder" "github.com/coder/coder/v2/enterprise/coderd/connectionlog" "github.com/coder/coder/v2/enterprise/coderd/enidpsync" "github.com/coder/coder/v2/enterprise/coderd/portsharing" + "github.com/coder/quartz" "golang.org/x/xerrors" "tailscale.com/tailcfg" @@ -903,7 +903,9 @@ func (api *API) updateEntitlements(ctx context.Context) error { } api.AGPL.PrebuildsReconciler.Store(&reconciler) - go reconciler.Run(context.Background()) + // TODO: Should this context be the api.ctx context? To cancel when + // the API (and entire app) is closed via shutdown? + pproflabel.Go(context.Background(), pproflabel.Service(pproflabel.ServicePrebuildReconciler), reconciler.Run) api.AGPL.PrebuildsClaimer.Store(&claimer) } diff --git a/enterprise/wsproxy/wsproxy.go b/enterprise/wsproxy/wsproxy.go index 69241d8aa1c17..c2ac1baf2db4e 100644 --- a/enterprise/wsproxy/wsproxy.go +++ b/enterprise/wsproxy/wsproxy.go @@ -333,6 +333,7 @@ func New(ctx context.Context, opts *Options) (*Server, error) { r.Use( // TODO: @emyrk Should we standardize these in some other package? httpmw.Recover(s.Logger), + httpmw.WithProfilingLabels, tracing.StatusWriterMiddleware, tracing.Middleware(s.TracerProvider), httpmw.AttachRequestID,
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies: