Skip to content

Commit 068f9a0

Browse files
cstyanjoobisb
cstyan
and
joobisb
authored
feat: include read/write byte stats in scaletests JSON report (#17777)
PR to fix #12157 --------- Signed-off-by: Callum Styan <callumstyan@gmail.com> Co-authored-by: joobisb <joobisb@gmail.com>
1 parent 4bd5609 commit 068f9a0

File tree

7 files changed

+136
-49
lines changed

7 files changed

+136
-49
lines changed

‎scaletest/harness/results.go

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,16 @@ type Results struct {
2727

2828
// RunResult is the result of a single test run.
2929
type RunResult struct {
30-
FullID string `json:"full_id"`
31-
TestName string `json:"test_name"`
32-
ID string `json:"id"`
33-
Logs string `json:"logs"`
34-
Error error `json:"error"`
35-
StartedAt time.Time `json:"started_at"`
36-
Duration httpapi.Duration `json:"duration"`
37-
DurationMS int64 `json:"duration_ms"`
30+
FullID string `json:"full_id"`
31+
TestName string `json:"test_name"`
32+
ID string `json:"id"`
33+
Logs string `json:"logs"`
34+
Error error `json:"error"`
35+
StartedAt time.Time `json:"started_at"`
36+
Duration httpapi.Duration `json:"duration"`
37+
DurationMS int64 `json:"duration_ms"`
38+
TotalBytesRead int64 `json:"total_bytes_read"`
39+
TotalBytesWritten int64 `json:"total_bytes_written"`
3840
}
3941

4042
// MarshalJSON implements json.Marhshaler for RunResult.
@@ -59,14 +61,16 @@ func (r *TestRun) Result() RunResult {
5961
}
6062

6163
return RunResult{
62-
FullID: r.FullID(),
63-
TestName: r.testName,
64-
ID: r.id,
65-
Logs: r.logs.String(),
66-
Error: r.err,
67-
StartedAt: r.started,
68-
Duration: httpapi.Duration(r.duration),
69-
DurationMS: r.duration.Milliseconds(),
64+
FullID: r.FullID(),
65+
TestName: r.testName,
66+
ID: r.id,
67+
Logs: r.logs.String(),
68+
Error: r.err,
69+
StartedAt: r.started,
70+
Duration: httpapi.Duration(r.duration),
71+
DurationMS: r.duration.Milliseconds(),
72+
TotalBytesRead: r.bytesRead,
73+
TotalBytesWritten: r.bytesWritten,
7074
}
7175
}
7276

‎scaletest/harness/results_test.go

Lines changed: 36 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -36,34 +36,40 @@ func Test_Results(t *testing.T) {
3636
TotalFail: 2,
3737
Runs: map[string]harness.RunResult{
3838
"test-0/0": {
39-
FullID: "test-0/0",
40-
TestName: "test-0",
41-
ID: "0",
42-
Logs: "test-0/0 log line 1\ntest-0/0 log line 2",
43-
Error: xerrors.New("test-0/0 error"),
44-
StartedAt: now,
45-
Duration: httpapi.Duration(time.Second),
46-
DurationMS: 1000,
39+
FullID: "test-0/0",
40+
TestName: "test-0",
41+
ID: "0",
42+
Logs: "test-0/0 log line 1\ntest-0/0 log line 2",
43+
Error: xerrors.New("test-0/0 error"),
44+
StartedAt: now,
45+
Duration: httpapi.Duration(time.Second),
46+
DurationMS: 1000,
47+
TotalBytesRead: 1024,
48+
TotalBytesWritten: 2048,
4749
},
4850
"test-0/1": {
49-
FullID: "test-0/1",
50-
TestName: "test-0",
51-
ID: "1",
52-
Logs: "test-0/1 log line 1\ntest-0/1 log line 2",
53-
Error: nil,
54-
StartedAt: now.Add(333 * time.Millisecond),
55-
Duration: httpapi.Duration(time.Second),
56-
DurationMS: 1000,
51+
FullID: "test-0/1",
52+
TestName: "test-0",
53+
ID: "1",
54+
Logs: "test-0/1 log line 1\ntest-0/1 log line 2",
55+
Error: nil,
56+
StartedAt: now.Add(333 * time.Millisecond),
57+
Duration: httpapi.Duration(time.Second),
58+
DurationMS: 1000,
59+
TotalBytesRead: 512,
60+
TotalBytesWritten: 1024,
5761
},
5862
"test-0/2": {
59-
FullID: "test-0/2",
60-
TestName: "test-0",
61-
ID: "2",
62-
Logs: "test-0/2 log line 1\ntest-0/2 log line 2",
63-
Error: testError{hidden: xerrors.New("test-0/2 error")},
64-
StartedAt: now.Add(666 * time.Millisecond),
65-
Duration: httpapi.Duration(time.Second),
66-
DurationMS: 1000,
63+
FullID: "test-0/2",
64+
TestName: "test-0",
65+
ID: "2",
66+
Logs: "test-0/2 log line 1\ntest-0/2 log line 2",
67+
Error: testError{hidden: xerrors.New("test-0/2 error")},
68+
StartedAt: now.Add(666 * time.Millisecond),
69+
Duration: httpapi.Duration(time.Second),
70+
DurationMS: 1000,
71+
TotalBytesRead: 2048,
72+
TotalBytesWritten: 4096,
6773
},
6874
},
6975
Elapsed: httpapi.Duration(time.Second),
@@ -109,6 +115,8 @@ Test results:
109115
"started_at": "2023-10-05T12:03:56.395813665Z",
110116
"duration": "1s",
111117
"duration_ms": 1000,
118+
"total_bytes_read": 1024,
119+
"total_bytes_written": 2048,
112120
"error": "test-0/0 error:\n github.com/coder/coder/v2/scaletest/harness_test.Test_Results\n [working_directory]/results_test.go:43"
113121
},
114122
"test-0/1": {
@@ -119,6 +127,8 @@ Test results:
119127
"started_at": "2023-10-05T12:03:56.728813665Z",
120128
"duration": "1s",
121129
"duration_ms": 1000,
130+
"total_bytes_read": 512,
131+
"total_bytes_written": 1024,
122132
"error": "\u003cnil\u003e"
123133
},
124134
"test-0/2": {
@@ -129,6 +139,8 @@ Test results:
129139
"started_at": "2023-10-05T12:03:57.061813665Z",
130140
"duration": "1s",
131141
"duration_ms": 1000,
142+
"total_bytes_read": 2048,
143+
"total_bytes_written": 4096,
132144
"error": "test-0/2 error"
133145
}
134146
}

‎scaletest/harness/run.go

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,13 @@ type Cleanable interface {
3131
Cleanup(ctx context.Context, id string, logs io.Writer) error
3232
}
3333

34+
// Collectable is an optional extension to Runnable that allows to get metrics from the runner.
35+
type Collectable interface {
36+
Runnable
37+
// Gets the bytes transferred
38+
GetBytesTransferred() (int64, int64)
39+
}
40+
3441
// AddRun creates a new *TestRun with the given name, ID and Runnable, adds it
3542
// to the harness and returns it. Panics if the harness has been started, or a
3643
// test with the given run.FullID() is already registered.
@@ -66,11 +73,13 @@ type TestRun struct {
6673
id string
6774
runner Runnable
6875

69-
logs *syncBuffer
70-
done chan struct{}
71-
started time.Time
72-
duration time.Duration
73-
err error
76+
logs *syncBuffer
77+
done chan struct{}
78+
started time.Time
79+
duration time.Duration
80+
err error
81+
bytesRead int64
82+
bytesWritten int64
7483
}
7584

7685
func NewTestRun(testName string, id string, runner Runnable) *TestRun {
@@ -98,6 +107,11 @@ func (r *TestRun) Run(ctx context.Context) (err error) {
98107
defer func() {
99108
r.duration = time.Since(r.started)
100109
r.err = err
110+
c, ok := r.runner.(Collectable)
111+
if !ok {
112+
return
113+
}
114+
r.bytesRead, r.bytesWritten = c.GetBytesTransferred()
101115
}()
102116
defer func() {
103117
e := recover()
@@ -107,6 +121,7 @@ func (r *TestRun) Run(ctx context.Context) (err error) {
107121
}()
108122

109123
err = r.runner.Run(ctx, r.id, r.logs)
124+
110125
//nolint:revive // we use named returns because we mutate it in a defer
111126
return
112127
}

‎scaletest/harness/run_test.go

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,24 @@ type testFns struct {
1717
RunFn func(ctx context.Context, id string, logs io.Writer) error
1818
// CleanupFn is optional if no cleanup is required.
1919
CleanupFn func(ctx context.Context, id string, logs io.Writer) error
20+
// getBytesTransferred is optional if byte transfer tracking is required.
21+
getBytesTransferred func() (int64, int64)
2022
}
2123

2224
// Run implements Runnable.
2325
func (fns testFns) Run(ctx context.Context, id string, logs io.Writer) error {
2426
return fns.RunFn(ctx, id, logs)
2527
}
2628

29+
// GetBytesTransferred implements Collectable.
30+
func (fns testFns) GetBytesTransferred() (bytesRead int64, bytesWritten int64) {
31+
if fns.getBytesTransferred == nil {
32+
return 0, 0
33+
}
34+
35+
return fns.getBytesTransferred()
36+
}
37+
2738
// Cleanup implements Cleanable.
2839
func (fns testFns) Cleanup(ctx context.Context, id string, logs io.Writer) error {
2940
if fns.CleanupFn == nil {
@@ -40,9 +51,10 @@ func Test_TestRun(t *testing.T) {
4051
t.Parallel()
4152

4253
var (
43-
name, id = "test", "1"
44-
runCalled int64
45-
cleanupCalled int64
54+
name, id = "test", "1"
55+
runCalled int64
56+
cleanupCalled int64
57+
collectableCalled int64
4658

4759
testFns = testFns{
4860
RunFn: func(ctx context.Context, id string, logs io.Writer) error {
@@ -53,6 +65,10 @@ func Test_TestRun(t *testing.T) {
5365
atomic.AddInt64(&cleanupCalled, 1)
5466
return nil
5567
},
68+
getBytesTransferred: func() (int64, int64) {
69+
atomic.AddInt64(&collectableCalled, 1)
70+
return 0, 0
71+
},
5672
}
5773
)
5874

@@ -62,6 +78,7 @@ func Test_TestRun(t *testing.T) {
6278
err := run.Run(context.Background())
6379
require.NoError(t, err)
6480
require.EqualValues(t, 1, atomic.LoadInt64(&runCalled))
81+
require.EqualValues(t, 1, atomic.LoadInt64(&collectableCalled))
6582

6683
err = run.Cleanup(context.Background())
6784
require.NoError(t, err)
@@ -105,6 +122,24 @@ func Test_TestRun(t *testing.T) {
105122
})
106123
})
107124

125+
t.Run("Collectable", func(t *testing.T) {
126+
t.Parallel()
127+
128+
t.Run("NoFn", func(t *testing.T) {
129+
t.Parallel()
130+
131+
run := harness.NewTestRun("test", "1", testFns{
132+
RunFn: func(ctx context.Context, id string, logs io.Writer) error {
133+
return nil
134+
},
135+
getBytesTransferred: nil,
136+
})
137+
138+
err := run.Run(context.Background())
139+
require.NoError(t, err)
140+
})
141+
})
142+
108143
t.Run("CatchesRunPanic", func(t *testing.T) {
109144
t.Parallel()
110145

‎scaletest/workspacetraffic/metrics.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package workspacetraffic
22

3-
import "github.com/prometheus/client_golang/prometheus"
3+
import (
4+
"sync/atomic"
5+
6+
"github.com/prometheus/client_golang/prometheus"
7+
)
48

59
type Metrics struct {
610
BytesReadTotal prometheus.CounterVec
@@ -75,12 +79,14 @@ type ConnMetrics interface {
7579
AddError(float64)
7680
ObserveLatency(float64)
7781
AddTotal(float64)
82+
GetTotalBytes() int64
7883
}
7984

8085
type connMetrics struct {
8186
addError func(float64)
8287
observeLatency func(float64)
8388
addTotal func(float64)
89+
total int64
8490
}
8591

8692
func (c *connMetrics) AddError(f float64) {
@@ -92,5 +98,10 @@ func (c *connMetrics) ObserveLatency(f float64) {
9298
}
9399

94100
func (c *connMetrics) AddTotal(f float64) {
101+
atomic.AddInt64(&c.total, int64(f))
95102
c.addTotal(f)
96103
}
104+
105+
func (c *connMetrics) GetTotalBytes() int64 {
106+
return c.total
107+
}

‎scaletest/workspacetraffic/run.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,12 @@ func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) (err error)
210210
}
211211
}
212212

213+
func (r *Runner) GetBytesTransferred() (bytesRead, bytesWritten int64) {
214+
bytesRead = r.cfg.ReadMetrics.GetTotalBytes()
215+
bytesWritten = r.cfg.WriteMetrics.GetTotalBytes()
216+
return bytesRead, bytesWritten
217+
}
218+
213219
// Cleanup does nothing, successfully.
214220
func (*Runner) Cleanup(context.Context, string, io.Writer) error {
215221
return nil

‎scaletest/workspacetraffic/run_test.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -422,3 +422,7 @@ func (m *testMetrics) Latencies() []float64 {
422422
defer m.Unlock()
423423
return m.latencies
424424
}
425+
426+
func (m *testMetrics) GetTotalBytes() int64 {
427+
return int64(m.total)
428+
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy