Skip to content

chore: add profiling labels for pprof analysis #19232

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 7, 2025
Merged

chore: add profiling labels for pprof analysis #19232

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
74cf7ae
chore: add profiling labels for pprof analysis
Emyrk Aug 7, 2025
c92903c
add prebuild reconciler
Emyrk Aug 7, 2025
cb1ec21
linting
Emyrk Aug 7, 2025
56f67a7
add label for websocket vs http
Emyrk Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions cli/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ import (

"cdr.dev/slog"
"cdr.dev/slog/sloggers/sloghuman"
"github.com/coder/coder/v2/coderd/pproflabel"
"github.com/coder/pretty"
"github.com/coder/quartz"
"github.com/coder/retry"
Expand Down Expand Up @@ -1459,14 +1460,14 @@ func newProvisionerDaemon(
tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
terraformClient, terraformServer := drpcsdk.MemTransportPipe()
wg.Add(1)
go func() {
pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceTerraformProvisioner), func(ctx context.Context) {
defer wg.Done()
<-ctx.Done()
_ = terraformClient.Close()
_ = terraformServer.Close()
}()
})
wg.Add(1)
go func() {
pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceTerraformProvisioner), func(ctx context.Context) {
defer wg.Done()
defer cancel()

Expand All @@ -1485,7 +1486,7 @@ func newProvisionerDaemon(
default:
}
}
}()
})

connector[string(database.ProvisionerTypeTerraform)] = sdkproto.NewDRPCProvisionerClient(terraformClient)
default:
Expand Down
11 changes: 6 additions & 5 deletions coderd/autobuild/lifecycle_executor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (

"cdr.dev/slog"
"github.com/coder/coder/v2/coderd/files"
"github.com/coder/coder/v2/coderd/pproflabel"

"github.com/coder/coder/v2/coderd/audit"
"github.com/coder/coder/v2/coderd/database"
Expand Down Expand Up @@ -107,10 +108,10 @@ func (e *Executor) WithStatsChannel(ch chan<- Stats) *Executor {
// tick from its channel. It will stop when its context is Done, or when
// its channel is closed.
func (e *Executor) Run() {
go func() {
pproflabel.Go(e.ctx, pproflabel.Service(pproflabel.ServiceLifecycles), func(ctx context.Context) {
for {
select {
case <-e.ctx.Done():
case <-ctx.Done():
return
case t, ok := <-e.tick:
if !ok {
Expand All @@ -120,15 +121,15 @@ func (e *Executor) Run() {
e.metrics.autobuildExecutionDuration.Observe(stats.Elapsed.Seconds())
if e.statsCh != nil {
select {
case <-e.ctx.Done():
case <-ctx.Done():
return
case e.statsCh <- stats:
}
}
e.log.Debug(e.ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions))
e.log.Debug(ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions))
}
}
}()
})
}

func (e *Executor) runOnce(t time.Time) Stats {
Expand Down
1 change: 1 addition & 0 deletions coderd/coderd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -852,6 +852,7 @@ func New(options *Options) *API {

r.Use(
httpmw.Recover(api.Logger),
httpmw.WithProfilingLabels,
tracing.StatusWriterMiddleware,
tracing.Middleware(api.TracerProvider),
httpmw.AttachRequestID,
Expand Down
30 changes: 30 additions & 0 deletions coderd/httpmw/pprof.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package httpmw

import (
"context"
"net/http"
"runtime/pprof"

"github.com/coder/coder/v2/coderd/pproflabel"
)

// WithProfilingLabels adds a pprof label to all http request handlers. This is
// primarily used to determine if load is coming from background jobs, or from
// http traffic.
func WithProfilingLabels(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()

// Label to differentiate between http and websocket requests. Websocket requests
// are assumed to be long-lived and more resource consuming.
requestType := "http"
if r.Header.Get("Upgrade") == "websocket" {
requestType = "websocket"
}

pprof.Do(ctx, pproflabel.Service(pproflabel.ServiceHTTPServer, "request_type", requestType), func(ctx context.Context) {
r = r.WithContext(ctx)
next.ServeHTTP(rw, r)
})
})
}
25 changes: 25 additions & 0 deletions coderd/pproflabel/pproflabel.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package pproflabel

import (
"context"
"runtime/pprof"
)

// Go is just a convince wrapper to set off a labeled goroutine.
func Go(ctx context.Context, labels pprof.LabelSet, f func(context.Context)) {
go pprof.Do(ctx, labels, f)
}

const (
ServiceTag = "service"

ServiceHTTPServer = "http-api"
ServiceLifecycles = "lifecycle-executor"
ServiceMetricCollector = "metrics-collector"
ServicePrebuildReconciler = "prebuilds-reconciler"
ServiceTerraformProvisioner = "terraform-provisioner"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Follow-up: would be cool to label pubsub as well.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Which pubsub? The database pubsub listeners?

coder/coderd/database/pubsub/pubsub.go

Line 246 in cb1ec21

func (p *PGPubsub) subscribeQueue(event string, newQ *msgQueue) (cancel func(), err error) {

)

func Service(name string, pairs ...string) pprof.LabelSet {
return pprof.Labels(append([]string{ServiceTag, name}, pairs...)...)
}
5 changes: 3 additions & 2 deletions coderd/prometheusmetrics/insights/metricscollector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"cdr.dev/slog"

"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/pproflabel"
"github.com/coder/coder/v2/coderd/util/slice"
"github.com/coder/coder/v2/codersdk"
)
Expand Down Expand Up @@ -158,7 +159,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) {
})
}

go func() {
pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceMetricCollector), func(ctx context.Context) {
defer close(done)
defer ticker.Stop()
for {
Expand All @@ -170,7 +171,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) {
doTick()
}
}
}()
})
return func() {
closeFunc()
<-done
Expand Down
8 changes: 5 additions & 3 deletions enterprise/coderd/coderd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,20 @@ import (
"sync"
"time"

"github.com/coder/quartz"

"github.com/coder/coder/v2/buildinfo"
"github.com/coder/coder/v2/coderd/appearance"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/entitlements"
"github.com/coder/coder/v2/coderd/idpsync"
agplportsharing "github.com/coder/coder/v2/coderd/portsharing"
"github.com/coder/coder/v2/coderd/pproflabel"
agplprebuilds "github.com/coder/coder/v2/coderd/prebuilds"
"github.com/coder/coder/v2/coderd/rbac/policy"
"github.com/coder/coder/v2/coderd/wsbuilder"
"github.com/coder/coder/v2/enterprise/coderd/connectionlog"
"github.com/coder/coder/v2/enterprise/coderd/enidpsync"
"github.com/coder/coder/v2/enterprise/coderd/portsharing"
"github.com/coder/quartz"

"golang.org/x/xerrors"
"tailscale.com/tailcfg"
Expand Down Expand Up @@ -903,7 +903,9 @@ func (api *API) updateEntitlements(ctx context.Context) error {
}

api.AGPL.PrebuildsReconciler.Store(&reconciler)
go reconciler.Run(context.Background())
// TODO: Should this context be the api.ctx context? To cancel when
// the API (and entire app) is closed via shutdown?
pproflabel.Go(context.Background(), pproflabel.Service(pproflabel.ServicePrebuildReconciler), reconciler.Run)

api.AGPL.PrebuildsClaimer.Store(&claimer)
}
Expand Down
1 change: 1 addition & 0 deletions enterprise/wsproxy/wsproxy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
r.Use(
// TODO: @emyrk Should we standardize these in some other package?
httpmw.Recover(s.Logger),
httpmw.WithProfilingLabels,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Worth discriminating on ws vs https traffic?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh that is a really good idea. They should have that upgrade header iirc. Will add

Copy link
Member Author

@Emyrk Emyrk Aug 7, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dannykopping

request_type: Total 730.0ms
    550.0ms (75.34%): http
    180.0ms (24.66%): websocket

service: Total 810.0ms
    730.0ms (90.12%): http-api
     80.0ms ( 9.88%): terraform-provisioner

tracing.StatusWriterMiddleware,
tracing.Middleware(s.TracerProvider),
httpmw.AttachRequestID,
Expand Down
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy