Skip to content

Commit 3f9da67

Browse files
EmyrkEmyrk
authored
chore: instrument github oauth2 limits (#11532)
* chore: instrument github oauth2 limits Rate limit information for github oauth2 providers instrumented in prometheus
1 parent 50b78e3 commit 3f9da67

File tree

6 files changed

+421
-10
lines changed

6 files changed

+421
-10
lines changed

‎cli/server.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1802,7 +1802,7 @@ func configureGithubOAuth2(instrument *promoauth.Factory, accessURL *url.URL, cl
18021802
}
18031803

18041804
return &coderd.GithubOAuth2Config{
1805-
OAuth2Config: instrument.New("github-login", &oauth2.Config{
1805+
OAuth2Config: instrument.NewGithub("github-login", &oauth2.Config{
18061806
ClientID: clientID,
18071807
ClientSecret: clientSecret,
18081808
Endpoint: endpoint,

‎coderd/coderdtest/oidctest/idp.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ type FakeIDP struct {
8585
// to test something like PKI auth vs a client_secret.
8686
hookAuthenticateClient func(t testing.TB, req *http.Request) (url.Values, error)
8787
serve bool
88+
// optional middlewares
89+
middlewares chi.Middlewares
8890
}
8991

9092
func StatusError(code int, err error) error {
@@ -115,6 +117,12 @@ func WithAuthorizedRedirectURL(hook func(redirectURL string) error) func(*FakeID
115117
}
116118
}
117119

120+
func WithMiddlewares(mws ...func(http.Handler) http.Handler) func(*FakeIDP) {
121+
return func(f *FakeIDP) {
122+
f.middlewares = append(f.middlewares, mws...)
123+
}
124+
}
125+
118126
// WithRefresh is called when a refresh token is used. The email is
119127
// the email of the user that is being refreshed assuming the claims are correct.
120128
func WithRefresh(hook func(email string) error) func(*FakeIDP) {
@@ -570,6 +578,7 @@ func (f *FakeIDP) httpHandler(t testing.TB) http.Handler {
570578
t.Helper()
571579

572580
mux := chi.NewMux()
581+
mux.Use(f.middlewares...)
573582
// This endpoint is required to initialize the OIDC provider.
574583
// It is used to get the OIDC configuration.
575584
mux.Get("/.well-known/openid-configuration", func(rw http.ResponseWriter, r *http.Request) {

‎coderd/externalauth/externalauth.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,8 +464,13 @@ func ConvertConfig(instrument *promoauth.Factory, entries []codersdk.ExternalAut
464464
oauthConfig = &exchangeWithClientSecret{oc}
465465
}
466466

467+
instrumented := instrument.New(entry.ID, oauthConfig)
468+
if strings.EqualFold(entry.Type, string(codersdk.EnhancedExternalAuthProviderGitHub)) {
469+
instrumented = instrument.NewGithub(entry.ID, oauthConfig)
470+
}
471+
467472
cfg := &Config{
468-
InstrumentedOAuth2Config: instrument.New(entry.ID, oauthConfig),
473+
InstrumentedOAuth2Config: instrumented,
469474
ID: entry.ID,
470475
Regex: regex,
471476
Type: entry.Type,

‎coderd/promoauth/github.go

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package promoauth
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
"strconv"
7+
"time"
8+
)
9+
10+
type rateLimits struct {
11+
Limit int
12+
Remaining int
13+
Used int
14+
Reset time.Time
15+
Resource string
16+
}
17+
18+
// githubRateLimits checks the returned response headers and
19+
func githubRateLimits(resp *http.Response, err error) (rateLimits, bool) {
20+
if err != nil || resp == nil {
21+
return rateLimits{}, false
22+
}
23+
24+
p := headerParser{header: resp.Header}
25+
// See
26+
// https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#checking-the-status-of-your-rate-limit
27+
limits := rateLimits{
28+
Limit: p.int("x-ratelimit-limit"),
29+
Remaining: p.int("x-ratelimit-remaining"),
30+
Used: p.int("x-ratelimit-used"),
31+
Resource: p.string("x-ratelimit-resource"),
32+
}
33+
34+
if limits.Limit == 0 &&
35+
limits.Remaining == 0 &&
36+
limits.Used == 0 {
37+
// For some requests, github has no rate limit. In which case,
38+
// it returns all 0s. We can just omit these.
39+
return limits, false
40+
}
41+
42+
// Reset is when the rate limit "used" will be reset to 0.
43+
// If it's unix 0, then we do not know when it will reset.
44+
// Change it to a zero time as that is easier to handle in golang.
45+
unix := p.int("x-ratelimit-reset")
46+
resetAt := time.Unix(int64(unix), 0)
47+
if unix == 0 {
48+
resetAt = time.Time{}
49+
}
50+
limits.Reset = resetAt
51+
52+
// Unauthorized requests have their own rate limit, so we should
53+
// track them separately.
54+
if resp.StatusCode == http.StatusUnauthorized {
55+
limits.Resource += "-unauthorized"
56+
}
57+
58+
// A 401 or 429 means too many requests. This might mess up the
59+
// "resource" string because we could hit the unauthorized limit,
60+
// and we do not want that to override the authorized one.
61+
// However, in testing, it seems a 401 is always a 401, even if
62+
// the limit is hit.
63+
64+
if len(p.errors) > 0 {
65+
// If we are missing any headers, then do not try and guess
66+
// what the rate limits are.
67+
return limits, false
68+
}
69+
return limits, true
70+
}
71+
72+
type headerParser struct {
73+
errors map[string]error
74+
header http.Header
75+
}
76+
77+
func (p *headerParser) string(key string) string {
78+
if p.errors == nil {
79+
p.errors = make(map[string]error)
80+
}
81+
82+
v := p.header.Get(key)
83+
if v == "" {
84+
p.errors[key] = fmt.Errorf("missing header %q", key)
85+
}
86+
return v
87+
}
88+
89+
func (p *headerParser) int(key string) int {
90+
v := p.string(key)
91+
if v == "" {
92+
return -1
93+
}
94+
95+
i, err := strconv.Atoi(v)
96+
if err != nil {
97+
p.errors[key] = err
98+
}
99+
return i
100+
}

‎coderd/promoauth/oauth2.go

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"net/http"
7+
"time"
78

89
"github.com/prometheus/client_golang/prometheus"
910
"github.com/prometheus/client_golang/prometheus/promauto"
@@ -46,11 +47,25 @@ var _ OAuth2Config = (*Config)(nil)
4647
// Primarily to avoid any prometheus errors registering duplicate metrics.
4748
type Factory struct {
4849
metrics *metrics
50+
// optional replace now func
51+
Now func() time.Time
4952
}
5053

5154
// metrics is the reusable metrics for all oauth2 providers.
5255
type metrics struct {
5356
externalRequestCount *prometheus.CounterVec
57+
58+
// if the oauth supports it, rate limit metrics.
59+
// rateLimit is the defined limit per interval
60+
rateLimit *prometheus.GaugeVec
61+
rateLimitRemaining *prometheus.GaugeVec
62+
rateLimitUsed *prometheus.GaugeVec
63+
// rateLimitReset is unix time of the next interval (when the rate limit resets).
64+
rateLimitReset *prometheus.GaugeVec
65+
// rateLimitResetIn is the time in seconds until the rate limit resets.
66+
// This is included because it is sometimes more helpful to know the limit
67+
// will reset in 600seconds, rather than at 1704000000 unix time.
68+
rateLimitResetIn *prometheus.GaugeVec
5469
}
5570

5671
func NewFactory(registry prometheus.Registerer) *Factory {
@@ -68,6 +83,53 @@ func NewFactory(registry prometheus.Registerer) *Factory {
6883
"source",
6984
"status_code",
7085
}),
86+
rateLimit: factory.NewGaugeVec(prometheus.GaugeOpts{
87+
Namespace: "coderd",
88+
Subsystem: "oauth2",
89+
Name: "external_requests_rate_limit_total",
90+
Help: "The total number of allowed requests per interval.",
91+
}, []string{
92+
"name",
93+
// Resource allows different rate limits for the same oauth2 provider.
94+
// Some IDPs have different buckets for different rate limits.
95+
"resource",
96+
}),
97+
rateLimitRemaining: factory.NewGaugeVec(prometheus.GaugeOpts{
98+
Namespace: "coderd",
99+
Subsystem: "oauth2",
100+
Name: "external_requests_rate_limit_remaining",
101+
Help: "The remaining number of allowed requests in this interval.",
102+
}, []string{
103+
"name",
104+
"resource",
105+
}),
106+
rateLimitUsed: factory.NewGaugeVec(prometheus.GaugeOpts{
107+
Namespace: "coderd",
108+
Subsystem: "oauth2",
109+
Name: "external_requests_rate_limit_used",
110+
Help: "The number of requests made in this interval.",
111+
}, []string{
112+
"name",
113+
"resource",
114+
}),
115+
rateLimitReset: factory.NewGaugeVec(prometheus.GaugeOpts{
116+
Namespace: "coderd",
117+
Subsystem: "oauth2",
118+
Name: "external_requests_rate_limit_next_reset_unix",
119+
Help: "Unix timestamp for when the next interval starts",
120+
}, []string{
121+
"name",
122+
"resource",
123+
}),
124+
rateLimitResetIn: factory.NewGaugeVec(prometheus.GaugeOpts{
125+
Namespace: "coderd",
126+
Subsystem: "oauth2",
127+
Name: "external_requests_rate_limit_reset_in_seconds",
128+
Help: "Seconds until the next interval",
129+
}, []string{
130+
"name",
131+
"resource",
132+
}),
71133
},
72134
}
73135
}
@@ -80,13 +142,53 @@ func (f *Factory) New(name string, under OAuth2Config) *Config {
80142
}
81143
}
82144

145+
// NewGithub returns a new instrumented oauth2 config for github. It tracks
146+
// rate limits as well as just the external request counts.
147+
//
148+
//nolint:bodyclose
149+
func (f *Factory) NewGithub(name string, under OAuth2Config) *Config {
150+
cfg := f.New(name, under)
151+
cfg.interceptors = append(cfg.interceptors, func(resp *http.Response, err error) {
152+
limits, ok := githubRateLimits(resp, err)
153+
if !ok {
154+
return
155+
}
156+
labels := prometheus.Labels{
157+
"name": cfg.name,
158+
"resource": limits.Resource,
159+
}
160+
// Default to -1 for "do not know"
161+
resetIn := float64(-1)
162+
if !limits.Reset.IsZero() {
163+
now := time.Now()
164+
if f.Now != nil {
165+
now = f.Now()
166+
}
167+
resetIn = limits.Reset.Sub(now).Seconds()
168+
if resetIn < 0 {
169+
// If it just reset, just make it 0.
170+
resetIn = 0
171+
}
172+
}
173+
174+
f.metrics.rateLimit.With(labels).Set(float64(limits.Limit))
175+
f.metrics.rateLimitRemaining.With(labels).Set(float64(limits.Remaining))
176+
f.metrics.rateLimitUsed.With(labels).Set(float64(limits.Used))
177+
f.metrics.rateLimitReset.With(labels).Set(float64(limits.Reset.Unix()))
178+
f.metrics.rateLimitResetIn.With(labels).Set(resetIn)
179+
})
180+
return cfg
181+
}
182+
83183
type Config struct {
84184
// Name is a human friendly name to identify the oauth2 provider. This should be
85185
// deterministic from restart to restart, as it is going to be used as a label in
86186
// prometheus metrics.
87187
name string
88188
underlying OAuth2Config
89189
metrics *metrics
190+
// interceptors are called after every request made by the oauth2 client.
191+
interceptors []func(resp *http.Response, err error)
90192
}
91193

92194
func (c *Config) Do(ctx context.Context, source Oauth2Source, req *http.Request) (*http.Response, error) {
@@ -169,5 +271,10 @@ func (i *instrumentedTripper) RoundTrip(r *http.Request) (*http.Response, error)
169271
"source": string(i.source),
170272
"status_code": fmt.Sprintf("%d", statusCode),
171273
}).Inc()
274+
275+
// Handle any extra interceptors.
276+
for _, interceptor := range i.c.interceptors {
277+
interceptor(resp, err)
278+
}
172279
return resp, err
173280
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy