From 4ed40699cfd4eb2b18d534d1e3d49f61237a6c4a Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 30 Aug 2023 21:46:15 +0000 Subject: [PATCH 01/21] feat: implement agent process management - An opt-in feature has been added to the agent to allow deprioritizing non coder-related processes for both CPU and memory. Non coder processes have their niceness set to 10 and their oom_score_adj set to 100 --- agent/agent.go | 85 +++++++++++++++++++++++++++++++ agent/agentproc/doc.go | 3 ++ agent/agentproc/proc.go | 99 ++++++++++++++++++++++++++++++++++++ agent/agentproc/proc_test.go | 12 +++++ 4 files changed, 199 insertions(+) create mode 100644 agent/agentproc/doc.go create mode 100644 agent/agentproc/proc.go create mode 100644 agent/agentproc/proc_test.go diff --git a/agent/agent.go b/agent/agent.go index 44f55fcedc83d..59f9710723aa5 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -15,6 +15,7 @@ import ( "os/exec" "os/user" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -34,6 +35,7 @@ import ( "tailscale.com/types/netlogtype" "cdr.dev/slog" + "github.com/coder/coder/v2/agent/agentproc" "github.com/coder/coder/v2/agent/agentssh" "github.com/coder/coder/v2/agent/reconnectingpty" "github.com/coder/coder/v2/buildinfo" @@ -51,6 +53,8 @@ const ( ProtocolDial = "dial" ) +const EnvProcMemNice = "CODER_PROC_MEMNICE_ENABLE" + type Options struct { Filesystem afero.Fs LogDir string @@ -68,6 +72,7 @@ type Options struct { PrometheusRegistry *prometheus.Registry ReportMetadataInterval time.Duration ServiceBannerRefreshInterval time.Duration + Syscaller agentproc.Syscaller } type Client interface { @@ -197,6 +202,7 @@ type agent struct { prometheusRegistry *prometheus.Registry metrics *agentMetrics + syscaller agentproc.Syscaller } func (a *agent) TailnetConn() *tailnet.Conn { @@ -225,6 +231,7 @@ func (a *agent) runLoop(ctx context.Context) { go a.reportLifecycleLoop(ctx) go a.reportMetadataLoop(ctx) go a.fetchServiceBannerLoop(ctx) + go a.manageProcessPriorityLoop(ctx) for retrier := retry.New(100*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { a.logger.Info(ctx, "connecting to coderd") @@ -1253,6 +1260,84 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { } } +var exemptProcesses = []string{"coder"} + +func (a *agent) manageProcessPriorityLoop(ctx context.Context) { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + + const ( + procDir = agentproc.DefaultProcDir + niceness = 10 + oomScoreAdj = -1000 + ) + + if val := a.envVars[EnvProcMemNice]; val == "" || runtime.GOOS != "linux" { + a.logger.Info(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", + slog.F("env_var", EnvProcMemNice), + slog.F("value", val), + slog.F("goos", runtime.GOOS), + ) + return + } + + for { + select { + case <-ticker.C: + procs, err := agentproc.List(a.filesystem, agentproc.DefaultProcDir) + if err != nil { + a.logger.Error(ctx, "failed to list procs", + slog.F("dir", agentproc.DefaultProcDir), + slog.Error(err), + ) + continue + } + for _, proc := range procs { + // Trim off the path e.g. "./coder" -> "coder" + name := filepath.Base(proc.Name()) + if slices.Contains(exemptProcesses, name) { + a.logger.Debug(ctx, "skipping exempt process", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + ) + continue + } + + err := proc.SetNiceness(a.syscaller, niceness) + if err != nil { + a.logger.Error(ctx, "unable to set proc niceness", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", niceness), + slog.Error(err), + ) + continue + } + + err = proc.SetOOMAdj(oomScoreAdj) + if err != nil { + a.logger.Error(ctx, "unable to set proc oom_score_adj", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("oom_score_adj", oomScoreAdj), + slog.Error(err), + ) + continue + } + + a.logger.Debug(ctx, "deprioritized process", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", niceness), + slog.F("oom_score_adj", oomScoreAdj), + ) + } + case <-ctx.Done(): + return + } + } +} + // isClosed returns whether the API is closed or not. func (a *agent) isClosed() bool { select { diff --git a/agent/agentproc/doc.go b/agent/agentproc/doc.go new file mode 100644 index 0000000000000..8b15c52c5f9fb --- /dev/null +++ b/agent/agentproc/doc.go @@ -0,0 +1,3 @@ +// Package agentproc contains logic for interfacing with local +// processes running in the same context as the agent. +package agentproc diff --git a/agent/agentproc/proc.go b/agent/agentproc/proc.go new file mode 100644 index 0000000000000..f8b03a8ae4960 --- /dev/null +++ b/agent/agentproc/proc.go @@ -0,0 +1,99 @@ +package agentproc + +import ( + "path/filepath" + "strconv" + "strings" + "syscall" + + "github.com/spf13/afero" + "golang.org/x/sys/unix" + "golang.org/x/xerrors" +) + +const DefaultProcDir = "/proc" + +type Syscaller interface { + SetPriority(pid int32, priority int) error +} + +type UnixSyscaller struct{} + +func (UnixSyscaller) SetPriority(pid int32, nice int) error { + err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice) + if err != nil { + return xerrors.Errorf("set priority: %w", err) + } + return nil +} + +type Process struct { + Dir string + CmdLine string + PID int32 + fs afero.Fs +} + +func (p *Process) SetOOMAdj(score int) error { + path := filepath.Join(p.Dir, "oom_score_adj") + err := afero.WriteFile(p.fs, + path, + []byte(strconv.Itoa(score)), + 0644, + ) + if err != nil { + return xerrors.Errorf("write %q: %w", path, err) + } + + return nil +} + +func (p *Process) SetNiceness(sc Syscaller, score int) error { + err := sc.SetPriority(p.PID, score) + if err != nil { + return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err) + } + return nil +} + +func (p *Process) Name() string { + args := strings.Split(p.CmdLine, "\x00") + // Split will always return at least one element. + return args[0] +} + +func List(fs afero.Fs, dir string) ([]*Process, error) { + d, err := fs.Open(dir) + if err != nil { + return nil, xerrors.Errorf("open dir %q: %w", dir, err) + } + + entries, err := d.Readdirnames(0) + if err != nil { + return nil, xerrors.Errorf("readdirnames: %w", err) + } + + processes := make([]*Process, 0, len(entries)) + for _, entry := range entries { + pid, err := strconv.ParseInt(entry, 10, 32) + if err != nil { + continue + } + cmdline, err := afero.ReadFile(fs, filepath.Join(dir, entry, "cmdline")) + if err != nil { + var errNo syscall.Errno + if xerrors.As(err, &errNo) && errNo == syscall.EPERM { + continue + } + return nil, xerrors.Errorf("read cmdline: %w", err) + } + processes = append(processes, &Process{ + PID: int32(pid), + CmdLine: string(cmdline), + Dir: filepath.Join(dir, entry), + fs: fs, + }) + } + + return processes, nil +} diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go new file mode 100644 index 0000000000000..42ee8cd7fa140 --- /dev/null +++ b/agent/agentproc/proc_test.go @@ -0,0 +1,12 @@ +package agentproc_test + +type mockSyscaller struct { + SetPriorityFn func(int32, int) error +} + +func (f mockSyscaller) SetPriority(pid int32, nice int) error { + if f.SetPriorityFn == nil { + return nil + } + return f.SetPriorityFn(pid, nice) +} From 7e59db6ed7af22500be7dcc9a4f243cbabe43858 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 31 Aug 2023 00:10:34 +0000 Subject: [PATCH 02/21] improve process detection --- agent/agent.go | 44 +++++++++++++++++++++-------- agent/agentproc/proc.go | 62 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 59f9710723aa5..8f74ea8c2d4ce 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1260,7 +1260,7 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { } } -var exemptProcesses = []string{"coder"} +var prioritizedProcs = []string{"coder"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { ticker := time.NewTicker(time.Minute) @@ -1269,7 +1269,7 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { const ( procDir = agentproc.DefaultProcDir niceness = 10 - oomScoreAdj = -1000 + oomScoreAdj = 100 ) if val := a.envVars[EnvProcMemNice]; val == "" || runtime.GOOS != "linux" { @@ -1284,7 +1284,7 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { for { select { case <-ticker.C: - procs, err := agentproc.List(a.filesystem, agentproc.DefaultProcDir) + procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) if err != nil { a.logger.Error(ctx, "failed to list procs", slog.F("dir", agentproc.DefaultProcDir), @@ -1295,31 +1295,52 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { for _, proc := range procs { // Trim off the path e.g. "./coder" -> "coder" name := filepath.Base(proc.Name()) - if slices.Contains(exemptProcesses, name) { - a.logger.Debug(ctx, "skipping exempt process", + // If the process is prioritized we should adjust + // it's oom_score_adj and avoid lowering its niceness. + if slices.Contains(prioritizedProcs, name) { + err = proc.SetOOMAdj(oomScoreAdj) + if err != nil { + a.logger.Error(ctx, "unable to set proc oom_score_adj", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("oom_score_adj", oomScoreAdj), + slog.Error(err), + ) + continue + } + + a.logger.Debug(ctx, "decreased process oom_score", slog.F("name", proc.Name()), slog.F("pid", proc.PID), + slog.F("oom_score_adj", oomScoreAdj), ) continue } - err := proc.SetNiceness(a.syscaller, niceness) + score, err := proc.Nice(a.syscaller) if err != nil { - a.logger.Error(ctx, "unable to set proc niceness", + a.logger.Error(ctx, "unable to get proc niceness", slog.F("name", proc.Name()), slog.F("pid", proc.PID), - slog.F("niceness", niceness), slog.Error(err), ) continue } + if score != 20 { + a.logger.Error(ctx, "skipping process due to custom niceness", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", score), + ) + continue + } - err = proc.SetOOMAdj(oomScoreAdj) + err = proc.SetNiceness(a.syscaller, niceness) if err != nil { - a.logger.Error(ctx, "unable to set proc oom_score_adj", + a.logger.Error(ctx, "unable to set proc niceness", slog.F("name", proc.Name()), slog.F("pid", proc.PID), - slog.F("oom_score_adj", oomScoreAdj), + slog.F("niceness", niceness), slog.Error(err), ) continue @@ -1329,7 +1350,6 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { slog.F("name", proc.Name()), slog.F("pid", proc.PID), slog.F("niceness", niceness), - slog.F("oom_score_adj", oomScoreAdj), ) } case <-ctx.Done(): diff --git a/agent/agentproc/proc.go b/agent/agentproc/proc.go index f8b03a8ae4960..de367db7a4f90 100644 --- a/agent/agentproc/proc.go +++ b/agent/agentproc/proc.go @@ -15,6 +15,8 @@ const DefaultProcDir = "/proc" type Syscaller interface { SetPriority(pid int32, priority int) error + GetPriority(pid int32) (int, error) + Kill(pid int32, sig syscall.Signal) error } type UnixSyscaller struct{} @@ -27,6 +29,23 @@ func (UnixSyscaller) SetPriority(pid int32, nice int) error { return nil } +func (UnixSyscaller) GetPriority(pid int32) (int, error) { + nice, err := unix.Getpriority(0, int(pid)) + if err != nil { + return 0, xerrors.Errorf("get priority: %w", err) + } + return nice, nil +} + +func (UnixSyscaller) Kill(pid int, sig syscall.Signal) error { + err := syscall.Kill(pid, sig) + if err != nil { + return xerrors.Errorf("kill: %w", err) + } + + return nil +} + type Process struct { Dir string CmdLine string @@ -56,13 +75,21 @@ func (p *Process) SetNiceness(sc Syscaller, score int) error { return nil } +func (p *Process) Nice(sc Syscaller) (int, error) { + nice, err := sc.GetPriority(p.PID) + if err != nil { + return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err) + } + return nice, nil +} + func (p *Process) Name() string { args := strings.Split(p.CmdLine, "\x00") // Split will always return at least one element. return args[0] } -func List(fs afero.Fs, dir string) ([]*Process, error) { +func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { d, err := fs.Open(dir) if err != nil { return nil, xerrors.Errorf("open dir %q: %w", dir, err) @@ -79,6 +106,16 @@ func List(fs afero.Fs, dir string) ([]*Process, error) { if err != nil { continue } + + // Check that the process still exists. + exists, err := isProcessExist(syscaller, int32(pid), syscall.Signal(0)) + if err != nil { + return nil, xerrors.Errorf("check process exists: %w", err) + } + if !exists { + continue + } + cmdline, err := afero.ReadFile(fs, filepath.Join(dir, entry, "cmdline")) if err != nil { var errNo syscall.Errno @@ -97,3 +134,26 @@ func List(fs afero.Fs, dir string) ([]*Process, error) { return processes, nil } + +func isProcessExist(syscaller Syscaller, pid int32, sig syscall.Signal) (bool, error) { + err := syscaller.Kill(pid, sig) + if err == nil { + return true, nil + } + if err.Error() == "os: process already finished" { + return false, nil + } + + errno, ok := err.(syscall.Errno) + if !ok { + return false, err + } + switch errno { + case syscall.ESRCH: + return false, nil + case syscall.EPERM: + return true, nil + } + + return false, xerrors.Errorf("kill: %w", err) +} From 8c652162f3c30ecbb71cae6607242b6b0388ab49 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 31 Aug 2023 21:47:04 +0000 Subject: [PATCH 03/21] add agentproc tests --- agent/agent.go | 2 +- agent/agentproc/agentproctest/proc.go | 44 +++++++ agent/agentproc/doc.go | 2 + agent/agentproc/proc.go | 68 +++------- agent/agentproc/proc_test.go | 182 +++++++++++++++++++++++++- agent/agentproc/syscaller.go | 41 ++++++ agent/agentproc/syscallermock_test.go | 78 +++++++++++ 7 files changed, 359 insertions(+), 58 deletions(-) create mode 100644 agent/agentproc/agentproctest/proc.go create mode 100644 agent/agentproc/syscaller.go create mode 100644 agent/agentproc/syscallermock_test.go diff --git a/agent/agent.go b/agent/agent.go index 8f74ea8c2d4ce..1f270c4123f0c 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1317,7 +1317,7 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { continue } - score, err := proc.Nice(a.syscaller) + score, err := proc.Niceness(a.syscaller) if err != nil { a.logger.Error(ctx, "unable to get proc niceness", slog.F("name", proc.Name()), diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go new file mode 100644 index 0000000000000..1ef599c2bf85b --- /dev/null +++ b/agent/agentproc/agentproctest/proc.go @@ -0,0 +1,44 @@ +package agentproctest + +import ( + "fmt" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/agent/agentproc" + "github.com/coder/coder/v2/cryptorand" +) + +func GenerateProcess(t *testing.T, fs afero.Fs, dir string) agentproc.Process { + t.Helper() + + pid, err := cryptorand.Intn(1<<31 - 1) + require.NoError(t, err) + + err = fs.MkdirAll(fmt.Sprintf("/%s/%d", dir, pid), 0555) + require.NoError(t, err) + + arg1, err := cryptorand.String(5) + require.NoError(t, err) + + arg2, err := cryptorand.String(5) + require.NoError(t, err) + + arg3, err := cryptorand.String(5) + require.NoError(t, err) + + cmdline := fmt.Sprintf("%s\x00%s\x00%s", arg1, arg2, arg3) + + err = afero.WriteFile(fs, fmt.Sprintf("/%s/%d/cmdline", dir, pid), []byte(cmdline), 0444) + require.NoError(t, err) + + return agentproc.Process{ + PID: int32(pid), + CmdLine: cmdline, + Dir: fmt.Sprintf("%s/%d", dir, pid), + FS: fs, + } + +} diff --git a/agent/agentproc/doc.go b/agent/agentproc/doc.go index 8b15c52c5f9fb..6f65c6ba99d53 100644 --- a/agent/agentproc/doc.go +++ b/agent/agentproc/doc.go @@ -1,3 +1,5 @@ // Package agentproc contains logic for interfacing with local // processes running in the same context as the agent. package agentproc + +//go:generate mockgen -destination ./syscallermock_test.go -package agentproc_test github.com/coder/coder/v2/agent/agentproc Syscaller diff --git a/agent/agentproc/proc.go b/agent/agentproc/proc.go index de367db7a4f90..eaf213eb88faa 100644 --- a/agent/agentproc/proc.go +++ b/agent/agentproc/proc.go @@ -1,61 +1,28 @@ package agentproc import ( + "errors" "path/filepath" "strconv" "strings" "syscall" "github.com/spf13/afero" - "golang.org/x/sys/unix" "golang.org/x/xerrors" ) const DefaultProcDir = "/proc" -type Syscaller interface { - SetPriority(pid int32, priority int) error - GetPriority(pid int32) (int, error) - Kill(pid int32, sig syscall.Signal) error -} - -type UnixSyscaller struct{} - -func (UnixSyscaller) SetPriority(pid int32, nice int) error { - err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice) - if err != nil { - return xerrors.Errorf("set priority: %w", err) - } - return nil -} - -func (UnixSyscaller) GetPriority(pid int32) (int, error) { - nice, err := unix.Getpriority(0, int(pid)) - if err != nil { - return 0, xerrors.Errorf("get priority: %w", err) - } - return nice, nil -} - -func (UnixSyscaller) Kill(pid int, sig syscall.Signal) error { - err := syscall.Kill(pid, sig) - if err != nil { - return xerrors.Errorf("kill: %w", err) - } - - return nil -} - type Process struct { Dir string CmdLine string PID int32 - fs afero.Fs + FS afero.Fs } func (p *Process) SetOOMAdj(score int) error { path := filepath.Join(p.Dir, "oom_score_adj") - err := afero.WriteFile(p.fs, + err := afero.WriteFile(p.FS, path, []byte(strconv.Itoa(score)), 0644, @@ -67,20 +34,20 @@ func (p *Process) SetOOMAdj(score int) error { return nil } -func (p *Process) SetNiceness(sc Syscaller, score int) error { - err := sc.SetPriority(p.PID, score) +func (p *Process) Niceness(sc Syscaller) (int, error) { + nice, err := sc.GetPriority(p.PID) if err != nil { - return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err) + return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err) } - return nil + return nice, nil } -func (p *Process) Nice(sc Syscaller) (int, error) { - nice, err := sc.GetPriority(p.PID) +func (p *Process) SetNiceness(sc Syscaller, score int) error { + err := sc.SetPriority(p.PID, score) if err != nil { - return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err) + return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err) } - return nice, nil + return nil } func (p *Process) Name() string { @@ -108,7 +75,7 @@ func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { } // Check that the process still exists. - exists, err := isProcessExist(syscaller, int32(pid), syscall.Signal(0)) + exists, err := isProcessExist(syscaller, int32(pid)) if err != nil { return nil, xerrors.Errorf("check process exists: %w", err) } @@ -128,15 +95,15 @@ func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { PID: int32(pid), CmdLine: string(cmdline), Dir: filepath.Join(dir, entry), - fs: fs, + FS: fs, }) } return processes, nil } -func isProcessExist(syscaller Syscaller, pid int32, sig syscall.Signal) (bool, error) { - err := syscaller.Kill(pid, sig) +func isProcessExist(syscaller Syscaller, pid int32) (bool, error) { + err := syscaller.Kill(pid, syscall.Signal(0)) if err == nil { return true, nil } @@ -144,10 +111,11 @@ func isProcessExist(syscaller Syscaller, pid int32, sig syscall.Signal) (bool, e return false, nil } - errno, ok := err.(syscall.Errno) - if !ok { + var errno syscall.Errno + if !errors.As(err, &errno) { return false, err } + switch errno { case syscall.ESRCH: return false, nil diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index 42ee8cd7fa140..0534d655f4464 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -1,12 +1,180 @@ package agentproc_test -type mockSyscaller struct { - SetPriorityFn func(int32, int) error +import ( + "fmt" + "strings" + "syscall" + "testing" + + "github.com/golang/mock/gomock" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/agent/agentproc" + "github.com/coder/coder/v2/agent/agentproc/agentproctest" +) + +func TestList(t *testing.T) { + t.Parallel() + + t.Run("OK", func(t *testing.T) { + t.Parallel() + + var ( + fs = afero.NewMemMapFs() + sc = NewMockSyscaller(gomock.NewController(t)) + expectedProcs = make(map[int32]agentproc.Process) + rootDir = agentproc.DefaultProcDir + ) + + for i := 0; i < 4; i++ { + proc := agentproctest.GenerateProcess(t, fs, rootDir) + expectedProcs[proc.PID] = proc + + sc.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(nil) + } + + actualProcs, err := agentproc.List(fs, sc, rootDir) + require.NoError(t, err) + require.Len(t, actualProcs, 4) + for _, proc := range actualProcs { + expected, ok := expectedProcs[proc.PID] + require.True(t, ok) + require.Equal(t, expected.PID, proc.PID) + require.Equal(t, expected.CmdLine, proc.CmdLine) + require.Equal(t, expected.Dir, proc.Dir) + } + }) + + t.Run("FinishedProcess", func(t *testing.T) { + t.Parallel() + + var ( + fs = afero.NewMemMapFs() + sc = NewMockSyscaller(gomock.NewController(t)) + expectedProcs = make(map[int32]agentproc.Process) + rootDir = agentproc.DefaultProcDir + ) + + for i := 0; i < 3; i++ { + proc := agentproctest.GenerateProcess(t, fs, rootDir) + expectedProcs[proc.PID] = proc + + sc.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(nil) + } + + // Create a process that's already finished. We're not adding + // it to the map because it should be skipped over. + proc := agentproctest.GenerateProcess(t, fs, rootDir) + sc.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(xerrors.New("os: process already finished")) + + actualProcs, err := agentproc.List(fs, sc, rootDir) + require.NoError(t, err) + require.Len(t, actualProcs, 3) + for _, proc := range actualProcs { + expected, ok := expectedProcs[proc.PID] + require.True(t, ok) + require.Equal(t, expected.PID, proc.PID) + require.Equal(t, expected.CmdLine, proc.CmdLine) + require.Equal(t, expected.Dir, proc.Dir) + } + }) + + t.Run("NoSuchProcess", func(t *testing.T) { + t.Parallel() + + var ( + fs = afero.NewMemMapFs() + sc = NewMockSyscaller(gomock.NewController(t)) + expectedProcs = make(map[int32]agentproc.Process) + rootDir = agentproc.DefaultProcDir + ) + + for i := 0; i < 3; i++ { + proc := agentproctest.GenerateProcess(t, fs, rootDir) + expectedProcs[proc.PID] = proc + + sc.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(nil) + } + + // Create a process that doesn't exist. We're not adding + // it to the map because it should be skipped over. + proc := agentproctest.GenerateProcess(t, fs, rootDir) + sc.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(syscall.ESRCH) + + actualProcs, err := agentproc.List(fs, sc, rootDir) + require.NoError(t, err) + require.Len(t, actualProcs, 3) + for _, proc := range actualProcs { + expected, ok := expectedProcs[proc.PID] + require.True(t, ok) + require.Equal(t, expected.PID, proc.PID) + require.Equal(t, expected.CmdLine, proc.CmdLine) + require.Equal(t, expected.Dir, proc.Dir) + } + }) } -func (f mockSyscaller) SetPriority(pid int32, nice int) error { - if f.SetPriorityFn == nil { - return nil - } - return f.SetPriorityFn(pid, nice) +// These tests are not very interesting but they provide some modicum of +// confidence. +func TestProcess(t *testing.T) { + t.Parallel() + + t.Run("SetOOMAdj", func(t *testing.T) { + t.Parallel() + + var ( + fs = afero.NewMemMapFs() + dir = agentproc.DefaultProcDir + proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + expectedScore = -1000 + ) + + err := proc.SetOOMAdj(expectedScore) + require.NoError(t, err) + + actualScore, err := afero.ReadFile(fs, fmt.Sprintf("%s/%d/oom_score_adj", dir, proc.PID)) + require.NoError(t, err) + require.Equal(t, fmt.Sprintf("%d", expectedScore), strings.TrimSpace(string(actualScore))) + }) + + t.Run("SetNiceness", func(t *testing.T) { + t.Parallel() + + var ( + sc = NewMockSyscaller(gomock.NewController(t)) + proc = &agentproc.Process{ + PID: 32, + } + score = 20 + ) + + sc.EXPECT().SetPriority(proc.PID, score).Return(nil) + err := proc.SetNiceness(sc, score) + require.NoError(t, err) + }) + + t.Run("Name", func(t *testing.T) { + t.Parallel() + + var ( + proc = &agentproc.Process{ + CmdLine: "helloworld\x00--arg1\x00--arg2", + } + expectedName = "helloworld" + ) + + require.Equal(t, expectedName, proc.Name()) + }) } diff --git a/agent/agentproc/syscaller.go b/agent/agentproc/syscaller.go new file mode 100644 index 0000000000000..d449e159e0b65 --- /dev/null +++ b/agent/agentproc/syscaller.go @@ -0,0 +1,41 @@ +package agentproc + +import ( + "syscall" + + "golang.org/x/sys/unix" + "golang.org/x/xerrors" +) + +type Syscaller interface { + SetPriority(pid int32, priority int) error + GetPriority(pid int32) (int, error) + Kill(pid int32, sig syscall.Signal) error +} + +type UnixSyscaller struct{} + +func (UnixSyscaller) SetPriority(pid int32, nice int) error { + err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice) + if err != nil { + return xerrors.Errorf("set priority: %w", err) + } + return nil +} + +func (UnixSyscaller) GetPriority(pid int32) (int, error) { + nice, err := unix.Getpriority(0, int(pid)) + if err != nil { + return 0, xerrors.Errorf("get priority: %w", err) + } + return nice, nil +} + +func (UnixSyscaller) Kill(pid int, sig syscall.Signal) error { + err := syscall.Kill(pid, sig) + if err != nil { + return xerrors.Errorf("kill: %w", err) + } + + return nil +} diff --git a/agent/agentproc/syscallermock_test.go b/agent/agentproc/syscallermock_test.go new file mode 100644 index 0000000000000..e2a250fbe5736 --- /dev/null +++ b/agent/agentproc/syscallermock_test.go @@ -0,0 +1,78 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coder/coder/v2/agent/agentproc (interfaces: Syscaller) + +// Package agentproc_test is a generated GoMock package. +package agentproc_test + +import ( + reflect "reflect" + syscall "syscall" + + gomock "github.com/golang/mock/gomock" +) + +// MockSyscaller is a mock of Syscaller interface. +type MockSyscaller struct { + ctrl *gomock.Controller + recorder *MockSyscallerMockRecorder +} + +// MockSyscallerMockRecorder is the mock recorder for MockSyscaller. +type MockSyscallerMockRecorder struct { + mock *MockSyscaller +} + +// NewMockSyscaller creates a new mock instance. +func NewMockSyscaller(ctrl *gomock.Controller) *MockSyscaller { + mock := &MockSyscaller{ctrl: ctrl} + mock.recorder = &MockSyscallerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSyscaller) EXPECT() *MockSyscallerMockRecorder { + return m.recorder +} + +// GetPriority mocks base method. +func (m *MockSyscaller) GetPriority(arg0 int32) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPriority", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPriority indicates an expected call of GetPriority. +func (mr *MockSyscallerMockRecorder) GetPriority(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPriority", reflect.TypeOf((*MockSyscaller)(nil).GetPriority), arg0) +} + +// Kill mocks base method. +func (m *MockSyscaller) Kill(arg0 int32, arg1 syscall.Signal) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Kill", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Kill indicates an expected call of Kill. +func (mr *MockSyscallerMockRecorder) Kill(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Kill", reflect.TypeOf((*MockSyscaller)(nil).Kill), arg0, arg1) +} + +// SetPriority mocks base method. +func (m *MockSyscaller) SetPriority(arg0 int32, arg1 int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetPriority", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetPriority indicates an expected call of SetPriority. +func (mr *MockSyscallerMockRecorder) SetPriority(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPriority", reflect.TypeOf((*MockSyscaller)(nil).SetPriority), arg0, arg1) +} From 760cbb38d56bb7f76564cc4e42a6162eb8a14bd3 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 1 Sep 2023 00:11:48 +0000 Subject: [PATCH 04/21] some minor agent tests --- agent/agent.go | 164 +++++++++++++++----------- agent/agent_test.go | 43 +++++++ agent/agentproc/agentproctest/proc.go | 1 - agent/agentproc/syscaller.go | 4 +- 4 files changed, 141 insertions(+), 71 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 1f270c4123f0c..8c7c7d693e5a8 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -73,6 +73,7 @@ type Options struct { ReportMetadataInterval time.Duration ServiceBannerRefreshInterval time.Duration Syscaller agentproc.Syscaller + ProcessManagementInterval time.Duration } type Client interface { @@ -125,6 +126,14 @@ func New(options Options) Agent { prometheusRegistry = prometheus.NewRegistry() } + if options.Syscaller == nil { + options.Syscaller = agentproc.UnixSyscaller{} + } + + if options.ProcessManagementInterval == 0 { + options.ProcessManagementInterval = time.Second + } + ctx, cancelFunc := context.WithCancel(context.Background()) a := &agent{ tailnetListenPort: options.TailnetListenPort, @@ -148,6 +157,8 @@ func New(options Options) Agent { sshMaxTimeout: options.SSHMaxTimeout, subsystems: options.Subsystems, addresses: options.Addresses, + syscaller: options.Syscaller, + processManagementInterval: options.ProcessManagementInterval, prometheusRegistry: prometheusRegistry, metrics: newAgentMetrics(prometheusRegistry), @@ -200,9 +211,10 @@ type agent struct { connCountReconnectingPTY atomic.Int64 - prometheusRegistry *prometheus.Registry - metrics *agentMetrics - syscaller agentproc.Syscaller + prometheusRegistry *prometheus.Registry + metrics *agentMetrics + processManagementInterval time.Duration + syscaller agentproc.Syscaller } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1263,15 +1275,9 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { var prioritizedProcs = []string{"coder"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { - ticker := time.NewTicker(time.Minute) + ticker := time.NewTicker(a.processManagementInterval) defer ticker.Stop() - const ( - procDir = agentproc.DefaultProcDir - niceness = 10 - oomScoreAdj = 100 - ) - if val := a.envVars[EnvProcMemNice]; val == "" || runtime.GOOS != "linux" { a.logger.Info(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", slog.F("env_var", EnvProcMemNice), @@ -1281,81 +1287,103 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { return } + // Do once before falling into loop. + if err := a.manageProcessPriority(ctx); err != nil { + a.logger.Error(ctx, "manage process priority", + slog.F("dir", agentproc.DefaultProcDir), + slog.Error(err), + ) + } + for { select { case <-ticker.C: - procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) - if err != nil { - a.logger.Error(ctx, "failed to list procs", + if err := a.manageProcessPriority(ctx); err != nil { + a.logger.Error(ctx, "manage process priority", slog.F("dir", agentproc.DefaultProcDir), slog.Error(err), ) - continue } - for _, proc := range procs { - // Trim off the path e.g. "./coder" -> "coder" - name := filepath.Base(proc.Name()) - // If the process is prioritized we should adjust - // it's oom_score_adj and avoid lowering its niceness. - if slices.Contains(prioritizedProcs, name) { - err = proc.SetOOMAdj(oomScoreAdj) - if err != nil { - a.logger.Error(ctx, "unable to set proc oom_score_adj", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("oom_score_adj", oomScoreAdj), - slog.Error(err), - ) - continue - } - a.logger.Debug(ctx, "decreased process oom_score", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("oom_score_adj", oomScoreAdj), - ) - continue - } + case <-ctx.Done(): + return + } + } +} - score, err := proc.Niceness(a.syscaller) - if err != nil { - a.logger.Error(ctx, "unable to get proc niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.Error(err), - ) - continue - } - if score != 20 { - a.logger.Error(ctx, "skipping process due to custom niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("niceness", score), - ) - continue - } +func (a *agent) manageProcessPriority(ctx context.Context) error { + const ( + procDir = agentproc.DefaultProcDir + niceness = 10 + oomScoreAdj = 100 + ) - err = proc.SetNiceness(a.syscaller, niceness) - if err != nil { - a.logger.Error(ctx, "unable to set proc niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("niceness", niceness), - slog.Error(err), - ) - continue - } + procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) + if err != nil { + return xerrors.Errorf("list: %w", err) + } - a.logger.Debug(ctx, "deprioritized process", + for _, proc := range procs { + // Trim off the path e.g. "./coder" -> "coder" + name := filepath.Base(proc.Name()) + // If the process is prioritized we should adjust + // it's oom_score_adj and avoid lowering its niceness. + if slices.Contains(prioritizedProcs, name) { + err = proc.SetOOMAdj(oomScoreAdj) + if err != nil { + a.logger.Error(ctx, "unable to set proc oom_score_adj", slog.F("name", proc.Name()), slog.F("pid", proc.PID), - slog.F("niceness", niceness), + slog.F("oom_score_adj", oomScoreAdj), + slog.Error(err), ) + continue } - case <-ctx.Done(): - return + + a.logger.Debug(ctx, "decreased process oom_score", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("oom_score_adj", oomScoreAdj), + ) + continue } + + score, err := proc.Niceness(a.syscaller) + if err != nil { + a.logger.Error(ctx, "unable to get proc niceness", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.Error(err), + ) + continue + } + if score != 20 { + a.logger.Error(ctx, "skipping process due to custom niceness", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", score), + ) + continue + } + + err = proc.SetNiceness(a.syscaller, niceness) + if err != nil { + a.logger.Error(ctx, "unable to set proc niceness", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", niceness), + slog.Error(err), + ) + continue + } + + a.logger.Debug(ctx, "deprioritized process", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", niceness), + ) } + return nil } // isClosed returns whether the API is closed or not. diff --git a/agent/agent_test.go b/agent/agent_test.go index 126e0f4fa4c97..99bad943c6e28 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -41,6 +41,7 @@ import ( "tailscale.com/tailcfg" "cdr.dev/slog" + "cdr.dev/slog/sloggers/sloghuman" "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/agent" "github.com/coder/coder/v2/agent/agentssh" @@ -2395,6 +2396,48 @@ func TestAgent_Metrics_SSH(t *testing.T) { require.NoError(t, err) } +func TestAgent_ManageProcessPriority(t *testing.T) { + t.Parallel() + + t.Run("DisabledByDefault", func(t *testing.T) { + t.Parallel() + + if runtime.GOOS != "linux" { + t.Skip("Skipping non-linux environment") + } + + var buf bytes.Buffer + log := slog.Make(sloghuman.Sink(&buf)) + + _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { + o.Logger = log + }) + + require.Eventually(t, func() bool { + return strings.Contains(buf.String(), "process priority not enabled") + }, testutil.WaitLong, testutil.IntervalFast) + }) + + t.Run("DisabledForNonLinux", func(t *testing.T) { + t.Parallel() + + if runtime.GOOS == "linux" { + t.Skip("Skipping linux environment") + } + + var buf bytes.Buffer + log := slog.Make(sloghuman.Sink(&buf)) + + _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { + o.Logger = log + o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + }) + require.Eventually(t, func() bool { + return strings.Contains(buf.String(), "process priority not enabled") + }, testutil.WaitLong, testutil.IntervalFast) + }) +} + func verifyCollectedMetrics(t *testing.T, expected []agentsdk.AgentMetric, actual []*promgo.MetricFamily) bool { t.Helper() diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go index 1ef599c2bf85b..51ed45112fe86 100644 --- a/agent/agentproc/agentproctest/proc.go +++ b/agent/agentproc/agentproctest/proc.go @@ -40,5 +40,4 @@ func GenerateProcess(t *testing.T, fs afero.Fs, dir string) agentproc.Process { Dir: fmt.Sprintf("%s/%d", dir, pid), FS: fs, } - } diff --git a/agent/agentproc/syscaller.go b/agent/agentproc/syscaller.go index d449e159e0b65..25a067deed67e 100644 --- a/agent/agentproc/syscaller.go +++ b/agent/agentproc/syscaller.go @@ -31,8 +31,8 @@ func (UnixSyscaller) GetPriority(pid int32) (int, error) { return nice, nil } -func (UnixSyscaller) Kill(pid int, sig syscall.Signal) error { - err := syscall.Kill(pid, sig) +func (UnixSyscaller) Kill(pid int32, sig syscall.Signal) error { + err := syscall.Kill(int(pid), sig) if err != nil { return xerrors.Errorf("kill: %w", err) } From f4b864ee86769fcdfc47d670b92539f7b999dd26 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 1 Sep 2023 01:38:38 +0000 Subject: [PATCH 05/21] add a proper test for proc management --- agent/agent.go | 63 +++++++++--------- agent/agent_test.go | 64 +++++++++++++++++++ agent/agentproc/agentproctest/doc.go | 5 ++ agent/agentproc/agentproctest/proc.go | 30 ++++++--- .../syscallermock.go} | 4 +- agent/agentproc/doc.go | 2 - agent/agentproc/proc_test.go | 8 +-- cli/agent.go | 10 ++- 8 files changed, 136 insertions(+), 50 deletions(-) create mode 100644 agent/agentproc/agentproctest/doc.go rename agent/agentproc/{syscallermock_test.go => agentproctest/syscallermock.go} (96%) diff --git a/agent/agent.go b/agent/agent.go index 8c7c7d693e5a8..fb4a9a22aff73 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -73,7 +73,8 @@ type Options struct { ReportMetadataInterval time.Duration ServiceBannerRefreshInterval time.Duration Syscaller agentproc.Syscaller - ProcessManagementInterval time.Duration + ProcessManagementTick <-chan time.Time + ModifiedProcesses chan []*agentproc.Process } type Client interface { @@ -130,10 +131,6 @@ func New(options Options) Agent { options.Syscaller = agentproc.UnixSyscaller{} } - if options.ProcessManagementInterval == 0 { - options.ProcessManagementInterval = time.Second - } - ctx, cancelFunc := context.WithCancel(context.Background()) a := &agent{ tailnetListenPort: options.TailnetListenPort, @@ -158,7 +155,8 @@ func New(options Options) Agent { subsystems: options.Subsystems, addresses: options.Addresses, syscaller: options.Syscaller, - processManagementInterval: options.ProcessManagementInterval, + processManagementTick: options.ProcessManagementTick, + modifiedProcs: options.ModifiedProcesses, prometheusRegistry: prometheusRegistry, metrics: newAgentMetrics(prometheusRegistry), @@ -211,10 +209,11 @@ type agent struct { connCountReconnectingPTY atomic.Int64 - prometheusRegistry *prometheus.Registry - metrics *agentMetrics - processManagementInterval time.Duration - syscaller agentproc.Syscaller + prometheusRegistry *prometheus.Registry + metrics *agentMetrics + processManagementTick <-chan time.Time + modifiedProcs chan []*agentproc.Process + syscaller agentproc.Syscaller } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1275,9 +1274,6 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { var prioritizedProcs = []string{"coder"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { - ticker := time.NewTicker(a.processManagementInterval) - defer ticker.Stop() - if val := a.envVars[EnvProcMemNice]; val == "" || runtime.GOOS != "linux" { a.logger.Info(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", slog.F("env_var", EnvProcMemNice), @@ -1287,42 +1283,45 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { return } - // Do once before falling into loop. - if err := a.manageProcessPriority(ctx); err != nil { - a.logger.Error(ctx, "manage process priority", - slog.F("dir", agentproc.DefaultProcDir), - slog.Error(err), - ) + manage := func() { + // Do once before falling into loop. + procs, err := a.manageProcessPriority(ctx) + if err != nil { + a.logger.Error(ctx, "manage process priority", + slog.F("dir", agentproc.DefaultProcDir), + slog.Error(err), + ) + } + if a.modifiedProcs != nil { + a.modifiedProcs <- procs + } } + manage() + for { select { - case <-ticker.C: - if err := a.manageProcessPriority(ctx); err != nil { - a.logger.Error(ctx, "manage process priority", - slog.F("dir", agentproc.DefaultProcDir), - slog.Error(err), - ) - } - + case <-a.processManagementTick: + manage() case <-ctx.Done(): return } } } -func (a *agent) manageProcessPriority(ctx context.Context) error { +func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process, error) { const ( procDir = agentproc.DefaultProcDir niceness = 10 - oomScoreAdj = 100 + oomScoreAdj = -1000 ) procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) if err != nil { - return xerrors.Errorf("list: %w", err) + return nil, xerrors.Errorf("list: %w", err) } + modProcs := []*agentproc.Process{} for _, proc := range procs { // Trim off the path e.g. "./coder" -> "coder" name := filepath.Base(proc.Name()) @@ -1339,6 +1338,7 @@ func (a *agent) manageProcessPriority(ctx context.Context) error { ) continue } + modProcs = append(modProcs, proc) a.logger.Debug(ctx, "decreased process oom_score", slog.F("name", proc.Name()), @@ -1382,8 +1382,9 @@ func (a *agent) manageProcessPriority(ctx context.Context) error { slog.F("pid", proc.PID), slog.F("niceness", niceness), ) + modProcs = append(modProcs, proc) } - return nil + return modProcs, nil } // isClosed returns whether the API is closed or not. diff --git a/agent/agent_test.go b/agent/agent_test.go index 99bad943c6e28..abc0ce49f64d9 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -21,10 +21,12 @@ import ( "strings" "sync" "sync/atomic" + "syscall" "testing" "time" scp "github.com/bramvdbogaerde/go-scp" + "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/pion/udp" "github.com/pkg/sftp" @@ -44,6 +46,8 @@ import ( "cdr.dev/slog/sloggers/sloghuman" "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/agent" + "github.com/coder/coder/v2/agent/agentproc" + "github.com/coder/coder/v2/agent/agentproc/agentproctest" "github.com/coder/coder/v2/agent/agentssh" "github.com/coder/coder/v2/agent/agenttest" "github.com/coder/coder/v2/coderd/httpapi" @@ -2399,6 +2403,66 @@ func TestAgent_Metrics_SSH(t *testing.T) { func TestAgent_ManageProcessPriority(t *testing.T) { t.Parallel() + t.Run("OK", func(t *testing.T) { + t.Parallel() + + var ( + expectedProcs = map[int32]agentproc.Process{} + fs = afero.NewMemMapFs() + ticker = make(chan time.Time) + syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) + modProcs = make(chan []*agentproc.Process) + logger = slog.Make(sloghuman.Sink(io.Discard)) + ) + + // Create some processes. + for i := 0; i < 4; i++ { + // Create a prioritized process. + var proc agentproc.Process + if i == 0 { + proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir, + func(p *agentproc.Process) { + p.CmdLine = "./coder\x00agent\x00--no-reap" + p.PID = 1 + }, + ) + } else { + proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) + syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) + } + syscaller.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(nil) + + expectedProcs[proc.PID] = proc + } + + _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { + o.ProcessManagementTick = ticker + o.Syscaller = syscaller + o.ModifiedProcesses = modProcs + o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + o.Filesystem = fs + o.Logger = logger + }) + actualProcs := <-modProcs + require.Len(t, actualProcs, 4) + + for _, actual := range actualProcs { + expectedScore := "0" + expected, ok := expectedProcs[actual.PID] + require.True(t, ok) + if expected.PID == 1 { + expectedScore = "-1000" + } + + score, err := afero.ReadFile(fs, filepath.Join(actual.Dir, "oom_score_adj")) + require.NoError(t, err) + require.Equal(t, expectedScore, strings.TrimSpace(string(score))) + } + }) + t.Run("DisabledByDefault", func(t *testing.T) { t.Parallel() diff --git a/agent/agentproc/agentproctest/doc.go b/agent/agentproc/agentproctest/doc.go new file mode 100644 index 0000000000000..5007b36268f76 --- /dev/null +++ b/agent/agentproc/agentproctest/doc.go @@ -0,0 +1,5 @@ +// Package agentproctest contains utility functions +// for testing process management in the agent. +package agentproctest + +//go:generate mockgen -destination ./syscallermock.go -package agentproctest github.com/coder/coder/v2/agent/agentproc Syscaller diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go index 51ed45112fe86..748833addd26f 100644 --- a/agent/agentproc/agentproctest/proc.go +++ b/agent/agentproc/agentproctest/proc.go @@ -11,15 +11,12 @@ import ( "github.com/coder/coder/v2/cryptorand" ) -func GenerateProcess(t *testing.T, fs afero.Fs, dir string) agentproc.Process { +func GenerateProcess(t *testing.T, fs afero.Fs, dir string, muts ...func(*agentproc.Process)) agentproc.Process { t.Helper() pid, err := cryptorand.Intn(1<<31 - 1) require.NoError(t, err) - err = fs.MkdirAll(fmt.Sprintf("/%s/%d", dir, pid), 0555) - require.NoError(t, err) - arg1, err := cryptorand.String(5) require.NoError(t, err) @@ -31,13 +28,26 @@ func GenerateProcess(t *testing.T, fs afero.Fs, dir string) agentproc.Process { cmdline := fmt.Sprintf("%s\x00%s\x00%s", arg1, arg2, arg3) - err = afero.WriteFile(fs, fmt.Sprintf("/%s/%d/cmdline", dir, pid), []byte(cmdline), 0444) - require.NoError(t, err) - - return agentproc.Process{ - PID: int32(pid), + process := agentproc.Process{ CmdLine: cmdline, - Dir: fmt.Sprintf("%s/%d", dir, pid), + PID: int32(pid), FS: fs, } + + for _, mut := range muts { + mut(&process) + } + + process.Dir = fmt.Sprintf("%s/%d", dir, process.PID) + + err = fs.MkdirAll(process.Dir, 0555) + require.NoError(t, err) + + err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0444) + require.NoError(t, err) + + err = afero.WriteFile(fs, fmt.Sprintf("%s/oom_score_adj", process.Dir), []byte("0"), 0444) + require.NoError(t, err) + + return process } diff --git a/agent/agentproc/syscallermock_test.go b/agent/agentproc/agentproctest/syscallermock.go similarity index 96% rename from agent/agentproc/syscallermock_test.go rename to agent/agentproc/agentproctest/syscallermock.go index e2a250fbe5736..8d9697bc559ef 100644 --- a/agent/agentproc/syscallermock_test.go +++ b/agent/agentproc/agentproctest/syscallermock.go @@ -1,8 +1,8 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/coder/coder/v2/agent/agentproc (interfaces: Syscaller) -// Package agentproc_test is a generated GoMock package. -package agentproc_test +// Package agentproctest is a generated GoMock package. +package agentproctest import ( reflect "reflect" diff --git a/agent/agentproc/doc.go b/agent/agentproc/doc.go index 6f65c6ba99d53..8b15c52c5f9fb 100644 --- a/agent/agentproc/doc.go +++ b/agent/agentproc/doc.go @@ -1,5 +1,3 @@ // Package agentproc contains logic for interfacing with local // processes running in the same context as the agent. package agentproc - -//go:generate mockgen -destination ./syscallermock_test.go -package agentproc_test github.com/coder/coder/v2/agent/agentproc Syscaller diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index 0534d655f4464..ea27dd4a7ee49 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -23,7 +23,7 @@ func TestList(t *testing.T) { var ( fs = afero.NewMemMapFs() - sc = NewMockSyscaller(gomock.NewController(t)) + sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) rootDir = agentproc.DefaultProcDir ) @@ -54,7 +54,7 @@ func TestList(t *testing.T) { var ( fs = afero.NewMemMapFs() - sc = NewMockSyscaller(gomock.NewController(t)) + sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) rootDir = agentproc.DefaultProcDir ) @@ -92,7 +92,7 @@ func TestList(t *testing.T) { var ( fs = afero.NewMemMapFs() - sc = NewMockSyscaller(gomock.NewController(t)) + sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) rootDir = agentproc.DefaultProcDir ) @@ -153,7 +153,7 @@ func TestProcess(t *testing.T) { t.Parallel() var ( - sc = NewMockSyscaller(gomock.NewController(t)) + sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) proc = &agentproc.Process{ PID: 32, } diff --git a/cli/agent.go b/cli/agent.go index 8b77c057ef31a..5f1874dc57bd5 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -29,6 +29,7 @@ import ( "cdr.dev/slog/sloggers/slogjson" "cdr.dev/slog/sloggers/slogstackdriver" "github.com/coder/coder/v2/agent" + "github.com/coder/coder/v2/agent/agentproc" "github.com/coder/coder/v2/agent/reaper" "github.com/coder/coder/v2/buildinfo" "github.com/coder/coder/v2/cli/clibase" @@ -267,6 +268,8 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { subsystems = append(subsystems, subsystem) } + procTicker := time.NewTicker(time.Second) + defer procTicker.Stop() agnt := agent.New(agent.Options{ Client: client, Logger: logger, @@ -290,7 +293,12 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { SSHMaxTimeout: sshMaxTimeout, Subsystems: subsystems, - PrometheusRegistry: prometheusRegistry, + PrometheusRegistry: prometheusRegistry, + ProcessManagementTick: procTicker.C, + Syscaller: agentproc.UnixSyscaller{}, + // Intentionally set this to nil. It's mainly used + // for testing. + ModifiedProcesses: nil, }) prometheusSrvClose := ServeHandler(ctx, logger, prometheusMetricsHandler(prometheusRegistry, logger), prometheusAddress, "prometheus") From cbcb854f0cc92d9f2c98ad131610117142e53665 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 1 Sep 2023 01:43:20 +0000 Subject: [PATCH 06/21] custom nice --- agent/agent_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/agent/agent_test.go b/agent/agent_test.go index abc0ce49f64d9..ba73034a973a6 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2463,6 +2463,49 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } }) + t.Run("IgnoreCustomNice", func(t *testing.T) { + t.Parallel() + + var ( + expectedProcs = map[int32]agentproc.Process{} + fs = afero.NewMemMapFs() + ticker = make(chan time.Time) + syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) + modProcs = make(chan []*agentproc.Process) + logger = slog.Make(sloghuman.Sink(io.Discard)) + ) + + // Create some processes. + for i := 0; i < 2; i++ { + // Create a prioritized process. + proc := agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + syscaller.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(nil) + + if i == 0 { + syscaller.EXPECT().GetPriority(proc.PID).Return(25, nil) + } else { + syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) + syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) + } + + expectedProcs[proc.PID] = proc + } + + _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { + o.ProcessManagementTick = ticker + o.Syscaller = syscaller + o.ModifiedProcesses = modProcs + o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + o.Filesystem = fs + o.Logger = logger + }) + actualProcs := <-modProcs + // We should ignore the process with a custom nice score. + require.Len(t, actualProcs, 1) + }) + t.Run("DisabledByDefault", func(t *testing.T) { t.Parallel() From 8230247e32cec554bbcee9de2c09e785ef8adf6d Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 8 Sep 2023 22:21:11 +0000 Subject: [PATCH 07/21] refactor into build files --- agent/agent.go | 27 ++++++++++++------- agent/agent_test.go | 6 +---- agent/agentproc/syscaller.go | 30 --------------------- agent/agentproc/syscaller_other.go | 24 +++++++++++++++++ agent/agentproc/syscaller_unix.go | 42 ++++++++++++++++++++++++++++++ cli/agent.go | 8 +++--- 6 files changed, 88 insertions(+), 49 deletions(-) create mode 100644 agent/agentproc/syscaller_other.go create mode 100644 agent/agentproc/syscaller_unix.go diff --git a/agent/agent.go b/agent/agent.go index fb4a9a22aff73..9d63d4c5a3982 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -53,6 +53,8 @@ const ( ProtocolDial = "dial" ) +// EnvProcMemNice determines whether we attempt to manage +// process CPU and OOM Killer priority. const EnvProcMemNice = "CODER_PROC_MEMNICE_ENABLE" type Options struct { @@ -73,7 +75,6 @@ type Options struct { ReportMetadataInterval time.Duration ServiceBannerRefreshInterval time.Duration Syscaller agentproc.Syscaller - ProcessManagementTick <-chan time.Time ModifiedProcesses chan []*agentproc.Process } @@ -128,7 +129,7 @@ func New(options Options) Agent { } if options.Syscaller == nil { - options.Syscaller = agentproc.UnixSyscaller{} + options.Syscaller = agentproc.NewSyscaller() } ctx, cancelFunc := context.WithCancel(context.Background()) @@ -155,7 +156,6 @@ func New(options Options) Agent { subsystems: options.Subsystems, addresses: options.Addresses, syscaller: options.Syscaller, - processManagementTick: options.ProcessManagementTick, modifiedProcs: options.ModifiedProcesses, prometheusRegistry: prometheusRegistry, @@ -209,11 +209,10 @@ type agent struct { connCountReconnectingPTY atomic.Int64 - prometheusRegistry *prometheus.Registry - metrics *agentMetrics - processManagementTick <-chan time.Time - modifiedProcs chan []*agentproc.Process - syscaller agentproc.Syscaller + prometheusRegistry *prometheus.Registry + metrics *agentMetrics + modifiedProcs chan []*agentproc.Process + syscaller agentproc.Syscaller } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1299,9 +1298,12 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { manage() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { select { - case <-a.processManagementTick: + case <-ticker.C: manage() case <-ctx.Done(): return @@ -1313,7 +1315,7 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process const ( procDir = agentproc.DefaultProcDir niceness = 10 - oomScoreAdj = -1000 + oomScoreAdj = -500 ) procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) @@ -1357,6 +1359,11 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process ) continue } + + // We only want processes that don't have a nice value set + // so we don't override user nice values. + // Getpriority actually returns priority for the nice value + // which is niceness + 20, so here 20 = a niceness of 0 (aka unset). if score != 20 { a.logger.Error(ctx, "skipping process due to custom niceness", slog.F("name", proc.Name()), diff --git a/agent/agent_test.go b/agent/agent_test.go index ba73034a973a6..06c7f54c7ea08 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2409,7 +2409,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) { var ( expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() - ticker = make(chan time.Time) syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) modProcs = make(chan []*agentproc.Process) logger = slog.Make(sloghuman.Sink(io.Discard)) @@ -2439,7 +2438,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { - o.ProcessManagementTick = ticker o.Syscaller = syscaller o.ModifiedProcesses = modProcs o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} @@ -2454,7 +2452,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { expected, ok := expectedProcs[actual.PID] require.True(t, ok) if expected.PID == 1 { - expectedScore = "-1000" + expectedScore = "-500" } score, err := afero.ReadFile(fs, filepath.Join(actual.Dir, "oom_score_adj")) @@ -2469,7 +2467,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) { var ( expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() - ticker = make(chan time.Time) syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) modProcs = make(chan []*agentproc.Process) logger = slog.Make(sloghuman.Sink(io.Discard)) @@ -2494,7 +2491,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { - o.ProcessManagementTick = ticker o.Syscaller = syscaller o.ModifiedProcesses = modProcs o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} diff --git a/agent/agentproc/syscaller.go b/agent/agentproc/syscaller.go index 25a067deed67e..6d346d1fb92ee 100644 --- a/agent/agentproc/syscaller.go +++ b/agent/agentproc/syscaller.go @@ -2,9 +2,6 @@ package agentproc import ( "syscall" - - "golang.org/x/sys/unix" - "golang.org/x/xerrors" ) type Syscaller interface { @@ -12,30 +9,3 @@ type Syscaller interface { GetPriority(pid int32) (int, error) Kill(pid int32, sig syscall.Signal) error } - -type UnixSyscaller struct{} - -func (UnixSyscaller) SetPriority(pid int32, nice int) error { - err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice) - if err != nil { - return xerrors.Errorf("set priority: %w", err) - } - return nil -} - -func (UnixSyscaller) GetPriority(pid int32) (int, error) { - nice, err := unix.Getpriority(0, int(pid)) - if err != nil { - return 0, xerrors.Errorf("get priority: %w", err) - } - return nice, nil -} - -func (UnixSyscaller) Kill(pid int32, sig syscall.Signal) error { - err := syscall.Kill(int(pid), sig) - if err != nil { - return xerrors.Errorf("kill: %w", err) - } - - return nil -} diff --git a/agent/agentproc/syscaller_other.go b/agent/agentproc/syscaller_other.go new file mode 100644 index 0000000000000..ba949d4ac33a5 --- /dev/null +++ b/agent/agentproc/syscaller_other.go @@ -0,0 +1,24 @@ +//go:build !linux +// +build !linux + +package agentproc + +import "syscall" + +func NewSyscaller() Syscaller { + return nopSyscaller{} +} + +type nopSyscaller struct{} + +func (nopSyscaller) SetPriority(pid int32, priority int) error { + return nil +} + +func (nopSyscaller) GetPriority(pid int32) (int, error) { + return 0, nil +} + +func (nopSyscaller) Kill(pid int32, sig syscall.Signal) error { + return nil +} diff --git a/agent/agentproc/syscaller_unix.go b/agent/agentproc/syscaller_unix.go new file mode 100644 index 0000000000000..e63e56b50f724 --- /dev/null +++ b/agent/agentproc/syscaller_unix.go @@ -0,0 +1,42 @@ +//go:build linux +// +build linux + +package agentproc + +import ( + "syscall" + + "golang.org/x/sys/unix" + "golang.org/x/xerrors" +) + +func NewSyscaller() Syscaller { + return UnixSyscaller{} +} + +type UnixSyscaller struct{} + +func (UnixSyscaller) SetPriority(pid int32, nice int) error { + err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice) + if err != nil { + return xerrors.Errorf("set priority: %w", err) + } + return nil +} + +func (UnixSyscaller) GetPriority(pid int32) (int, error) { + nice, err := unix.Getpriority(0, int(pid)) + if err != nil { + return 0, xerrors.Errorf("get priority: %w", err) + } + return nice, nil +} + +func (UnixSyscaller) Kill(pid int32, sig syscall.Signal) error { + err := syscall.Kill(int(pid), sig) + if err != nil { + return xerrors.Errorf("kill: %w", err) + } + + return nil +} diff --git a/cli/agent.go b/cli/agent.go index 5f1874dc57bd5..64693e3803e09 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -287,15 +287,15 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { return resp.SessionToken, nil }, EnvironmentVariables: map[string]string{ - "GIT_ASKPASS": executablePath, + "GIT_ASKPASS": executablePath, + agent.EnvProcMemNice: os.Getenv(agent.EnvProcMemNice), }, IgnorePorts: ignorePorts, SSHMaxTimeout: sshMaxTimeout, Subsystems: subsystems, - PrometheusRegistry: prometheusRegistry, - ProcessManagementTick: procTicker.C, - Syscaller: agentproc.UnixSyscaller{}, + PrometheusRegistry: prometheusRegistry, + Syscaller: agentproc.NewSyscaller(), // Intentionally set this to nil. It's mainly used // for testing. ModifiedProcesses: nil, From 3e1defd13e92a5878bba6bf2277ce5b51fb7441d Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 8 Sep 2023 22:45:26 +0000 Subject: [PATCH 08/21] tick tock --- agent/agent.go | 23 +++++++++++++++++------ agent/agent_test.go | 14 ++++++++++++-- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 9d63d4c5a3982..d1bed60525a4b 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -75,7 +75,10 @@ type Options struct { ReportMetadataInterval time.Duration ServiceBannerRefreshInterval time.Duration Syscaller agentproc.Syscaller - ModifiedProcesses chan []*agentproc.Process + // ModifiedProcesses is used for testing process priority management. + ModifiedProcesses chan []*agentproc.Process + // ProcessManagementTick is used for testing process priority management. + ProcessManagementTick <-chan time.Time } type Client interface { @@ -157,6 +160,7 @@ func New(options Options) Agent { addresses: options.Addresses, syscaller: options.Syscaller, modifiedProcs: options.ModifiedProcesses, + processManagementTick: options.ProcessManagementTick, prometheusRegistry: prometheusRegistry, metrics: newAgentMetrics(prometheusRegistry), @@ -211,8 +215,12 @@ type agent struct { prometheusRegistry *prometheus.Registry metrics *agentMetrics - modifiedProcs chan []*agentproc.Process syscaller agentproc.Syscaller + + // podifiedProcs is used for testing process priority management. + modifiedProcs chan []*agentproc.Process + // processManagementTick is used for testing process priority management. + processManagementTick <-chan time.Time } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1283,7 +1291,6 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { } manage := func() { - // Do once before falling into loop. procs, err := a.manageProcessPriority(ctx) if err != nil { a.logger.Error(ctx, "manage process priority", @@ -1296,14 +1303,18 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { } } + // Do once before falling into loop. manage() - ticker := time.NewTicker(time.Second) - defer ticker.Stop() + if a.processManagementTick == nil { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + a.processManagementTick = ticker.C + } for { select { - case <-ticker.C: + case <-a.processManagementTick: manage() case <-ctx.Done(): return diff --git a/agent/agent_test.go b/agent/agent_test.go index 06c7f54c7ea08..2fcfbbefbab4d 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2410,13 +2410,16 @@ func TestAgent_ManageProcessPriority(t *testing.T) { expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) + ticker = make(chan time.Time) modProcs = make(chan []*agentproc.Process) logger = slog.Make(sloghuman.Sink(io.Discard)) ) // Create some processes. for i := 0; i < 4; i++ { - // Create a prioritized process. + // Create a prioritized process. This process should + // have it's oom_score_adj set to -500 and its nice + // score should be untouched. var proc agentproc.Process if i == 0 { proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir, @@ -2426,6 +2429,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { }, ) } else { + // The rest are peasants. proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) @@ -2443,6 +2447,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} o.Filesystem = fs o.Logger = logger + o.ProcessManagementTick = ticker }) actualProcs := <-modProcs require.Len(t, actualProcs, 4) @@ -2467,6 +2472,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { var ( expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() + ticker = make(chan time.Time) syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) modProcs = make(chan []*agentproc.Process) logger = slog.Make(sloghuman.Sink(io.Discard)) @@ -2474,13 +2480,14 @@ func TestAgent_ManageProcessPriority(t *testing.T) { // Create some processes. for i := 0; i < 2; i++ { - // Create a prioritized process. proc := agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) syscaller.EXPECT(). Kill(proc.PID, syscall.Signal(0)). Return(nil) if i == 0 { + // Set a random nice score. This one should not be adjusted by + // our management loop. syscaller.EXPECT().GetPriority(proc.PID).Return(25, nil) } else { syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) @@ -2496,6 +2503,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} o.Filesystem = fs o.Logger = logger + o.ProcessManagementTick = ticker }) actualProcs := <-modProcs // We should ignore the process with a custom nice score. @@ -2533,6 +2541,8 @@ func TestAgent_ManageProcessPriority(t *testing.T) { _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Logger = log + // Try to enable it so that we can assert that non-linux + // environments are truly disabled. o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} }) require.Eventually(t, func() bool { From 2fe9c70d9c2296b871f879f48e464fd06c88cce5 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 8 Sep 2023 22:48:25 +0000 Subject: [PATCH 09/21] make fmt --- agent/agentproc/agentproctest/proc.go | 6 +++--- agent/agentproc/proc.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go index 748833addd26f..473825cd9d699 100644 --- a/agent/agentproc/agentproctest/proc.go +++ b/agent/agentproc/agentproctest/proc.go @@ -40,13 +40,13 @@ func GenerateProcess(t *testing.T, fs afero.Fs, dir string, muts ...func(*agentp process.Dir = fmt.Sprintf("%s/%d", dir, process.PID) - err = fs.MkdirAll(process.Dir, 0555) + err = fs.MkdirAll(process.Dir, 0o555) require.NoError(t, err) - err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0444) + err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0o444) require.NoError(t, err) - err = afero.WriteFile(fs, fmt.Sprintf("%s/oom_score_adj", process.Dir), []byte("0"), 0444) + err = afero.WriteFile(fs, fmt.Sprintf("%s/oom_score_adj", process.Dir), []byte("0"), 0o444) require.NoError(t, err) return process diff --git a/agent/agentproc/proc.go b/agent/agentproc/proc.go index eaf213eb88faa..29cb9a205bf4f 100644 --- a/agent/agentproc/proc.go +++ b/agent/agentproc/proc.go @@ -25,7 +25,7 @@ func (p *Process) SetOOMAdj(score int) error { err := afero.WriteFile(p.FS, path, []byte(strconv.Itoa(score)), - 0644, + 0o644, ) if err != nil { return xerrors.Errorf("write %q: %w", path, err) From ef41e9a4bcec19817a760bf6cdf48189e4fe73dc Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 12 Sep 2023 22:38:44 +0000 Subject: [PATCH 10/21] pr comments --- agent/agent.go | 74 +++++++----------- agent/agent_test.go | 16 ++-- agent/agentproc/agentproctest/proc.go | 4 +- agent/agentproc/proc_other.go | 28 +++++++ agent/agentproc/proc_test.go | 30 ++++--- agent/agentproc/{proc.go => proc_unix.go} | 95 +++++++++++------------ agent/agentproc/syscaller.go | 11 +++ agent/agentproc/syscaller_other.go | 14 +++- cli/agent.go | 4 +- 9 files changed, 147 insertions(+), 129 deletions(-) create mode 100644 agent/agentproc/proc_other.go rename agent/agentproc/{proc.go => proc_unix.go} (85%) diff --git a/agent/agent.go b/agent/agent.go index d1bed60525a4b..5886f305d2ca9 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -53,9 +53,9 @@ const ( ProtocolDial = "dial" ) -// EnvProcMemNice determines whether we attempt to manage +// EnvProcPrioMgmt determines whether we attempt to manage // process CPU and OOM Killer priority. -const EnvProcMemNice = "CODER_PROC_MEMNICE_ENABLE" +const EnvProcPrioMgmt = "CODER_PROC_PRIO_MGMT" type Options struct { Filesystem afero.Fs @@ -217,7 +217,7 @@ type agent struct { metrics *agentMetrics syscaller agentproc.Syscaller - // podifiedProcs is used for testing process priority management. + // modifiedProcs is used for testing process priority management. modifiedProcs chan []*agentproc.Process // processManagementTick is used for testing process priority management. processManagementTick <-chan time.Time @@ -1281,41 +1281,34 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { var prioritizedProcs = []string{"coder"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { - if val := a.envVars[EnvProcMemNice]; val == "" || runtime.GOOS != "linux" { - a.logger.Info(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", - slog.F("env_var", EnvProcMemNice), + if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { + a.logger.Debug(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", + slog.F("env_var", EnvProcPrioMgmt), slog.F("value", val), slog.F("goos", runtime.GOOS), ) return } - manage := func() { + if a.processManagementTick == nil { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + a.processManagementTick = ticker.C + } + + for { procs, err := a.manageProcessPriority(ctx) if err != nil { a.logger.Error(ctx, "manage process priority", - slog.F("dir", agentproc.DefaultProcDir), slog.Error(err), ) } if a.modifiedProcs != nil { a.modifiedProcs <- procs } - } - - // Do once before falling into loop. - manage() - - if a.processManagementTick == nil { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - a.processManagementTick = ticker.C - } - for { select { case <-a.processManagementTick: - manage() case <-ctx.Done(): return } @@ -1324,18 +1317,26 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process, error) { const ( - procDir = agentproc.DefaultProcDir niceness = 10 oomScoreAdj = -500 ) - procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) + procs, err := agentproc.List(a.filesystem, a.syscaller) if err != nil { return nil, xerrors.Errorf("list: %w", err) } - modProcs := []*agentproc.Process{} + var ( + modProcs = []*agentproc.Process{} + logger slog.Logger + ) + for _, proc := range procs { + logger = a.logger.With( + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + ) + // Trim off the path e.g. "./coder" -> "coder" name := filepath.Base(proc.Name()) // If the process is prioritized we should adjust @@ -1343,29 +1344,19 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process if slices.Contains(prioritizedProcs, name) { err = proc.SetOOMAdj(oomScoreAdj) if err != nil { - a.logger.Error(ctx, "unable to set proc oom_score_adj", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), + logger.Warn(ctx, "unable to set proc oom_score_adj", slog.F("oom_score_adj", oomScoreAdj), slog.Error(err), ) continue } modProcs = append(modProcs, proc) - - a.logger.Debug(ctx, "decreased process oom_score", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("oom_score_adj", oomScoreAdj), - ) continue } score, err := proc.Niceness(a.syscaller) if err != nil { - a.logger.Error(ctx, "unable to get proc niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), + logger.Warn(ctx, "unable to get proc niceness", slog.Error(err), ) continue @@ -1376,9 +1367,7 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process // Getpriority actually returns priority for the nice value // which is niceness + 20, so here 20 = a niceness of 0 (aka unset). if score != 20 { - a.logger.Error(ctx, "skipping process due to custom niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), + logger.Debug(ctx, "skipping process due to custom niceness", slog.F("niceness", score), ) continue @@ -1386,20 +1375,13 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process err = proc.SetNiceness(a.syscaller, niceness) if err != nil { - a.logger.Error(ctx, "unable to set proc niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), + logger.Warn(ctx, "unable to set proc niceness", slog.F("niceness", niceness), slog.Error(err), ) continue } - a.logger.Debug(ctx, "deprioritized process", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("niceness", niceness), - ) modProcs = append(modProcs, proc) } return modProcs, nil diff --git a/agent/agent_test.go b/agent/agent_test.go index 2fcfbbefbab4d..c2d20209f4974 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2422,7 +2422,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { // score should be untouched. var proc agentproc.Process if i == 0 { - proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir, + proc = agentproctest.GenerateProcess(t, fs, func(p *agentproc.Process) { p.CmdLine = "./coder\x00agent\x00--no-reap" p.PID = 1 @@ -2430,7 +2430,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { ) } else { // The rest are peasants. - proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + proc = agentproctest.GenerateProcess(t, fs) syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) } @@ -2444,7 +2444,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Syscaller = syscaller o.ModifiedProcesses = modProcs - o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"} o.Filesystem = fs o.Logger = logger o.ProcessManagementTick = ticker @@ -2480,7 +2480,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { // Create some processes. for i := 0; i < 2; i++ { - proc := agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + proc := agentproctest.GenerateProcess(t, fs) syscaller.EXPECT(). Kill(proc.PID, syscall.Signal(0)). Return(nil) @@ -2500,7 +2500,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Syscaller = syscaller o.ModifiedProcesses = modProcs - o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"} o.Filesystem = fs o.Logger = logger o.ProcessManagementTick = ticker @@ -2518,7 +2518,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } var buf bytes.Buffer - log := slog.Make(sloghuman.Sink(&buf)) + log := slog.Make(sloghuman.Sink(&buf)).Leveled(slog.LevelDebug) _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Logger = log @@ -2537,13 +2537,13 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } var buf bytes.Buffer - log := slog.Make(sloghuman.Sink(&buf)) + log := slog.Make(sloghuman.Sink(&buf)).Leveled(slog.LevelDebug) _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Logger = log // Try to enable it so that we can assert that non-linux // environments are truly disabled. - o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"} }) require.Eventually(t, func() bool { return strings.Contains(buf.String(), "process priority not enabled") diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go index 473825cd9d699..1fe44ad9eb31b 100644 --- a/agent/agentproc/agentproctest/proc.go +++ b/agent/agentproc/agentproctest/proc.go @@ -11,7 +11,7 @@ import ( "github.com/coder/coder/v2/cryptorand" ) -func GenerateProcess(t *testing.T, fs afero.Fs, dir string, muts ...func(*agentproc.Process)) agentproc.Process { +func GenerateProcess(t *testing.T, fs afero.Fs, muts ...func(*agentproc.Process)) agentproc.Process { t.Helper() pid, err := cryptorand.Intn(1<<31 - 1) @@ -38,7 +38,7 @@ func GenerateProcess(t *testing.T, fs afero.Fs, dir string, muts ...func(*agentp mut(&process) } - process.Dir = fmt.Sprintf("%s/%d", dir, process.PID) + process.Dir = fmt.Sprintf("%s/%d", "/proc", process.PID) err = fs.MkdirAll(process.Dir, 0o555) require.NoError(t, err) diff --git a/agent/agentproc/proc_other.go b/agent/agentproc/proc_other.go new file mode 100644 index 0000000000000..b299eac185fd5 --- /dev/null +++ b/agent/agentproc/proc_other.go @@ -0,0 +1,28 @@ +//go:build !linux +// +build !linux + +package agentproc + +import ( + "github.com/spf13/afero" +) + +func (p *Process) SetOOMAdj(score int) error { + return errUnimplimented +} + +func (p *Process) Niceness(sc Syscaller) (int, error) { + return 0, errUnimplimented +} + +func (p *Process) SetNiceness(sc Syscaller, score int) error { + return errUnimplimented +} + +func (p *Process) Name() string { + return "" +} + +func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) { + return nil, errUnimplimented +} diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index ea27dd4a7ee49..6459b05f3f7e8 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -25,11 +25,10 @@ func TestList(t *testing.T) { fs = afero.NewMemMapFs() sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) - rootDir = agentproc.DefaultProcDir ) for i := 0; i < 4; i++ { - proc := agentproctest.GenerateProcess(t, fs, rootDir) + proc := agentproctest.GenerateProcess(t, fs) expectedProcs[proc.PID] = proc sc.EXPECT(). @@ -37,9 +36,9 @@ func TestList(t *testing.T) { Return(nil) } - actualProcs, err := agentproc.List(fs, sc, rootDir) + actualProcs, err := agentproc.List(fs, sc) require.NoError(t, err) - require.Len(t, actualProcs, 4) + require.Len(t, actualProcs, len(expectedProcs)) for _, proc := range actualProcs { expected, ok := expectedProcs[proc.PID] require.True(t, ok) @@ -56,11 +55,10 @@ func TestList(t *testing.T) { fs = afero.NewMemMapFs() sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) - rootDir = agentproc.DefaultProcDir ) for i := 0; i < 3; i++ { - proc := agentproctest.GenerateProcess(t, fs, rootDir) + proc := agentproctest.GenerateProcess(t, fs) expectedProcs[proc.PID] = proc sc.EXPECT(). @@ -70,14 +68,14 @@ func TestList(t *testing.T) { // Create a process that's already finished. We're not adding // it to the map because it should be skipped over. - proc := agentproctest.GenerateProcess(t, fs, rootDir) + proc := agentproctest.GenerateProcess(t, fs) sc.EXPECT(). Kill(proc.PID, syscall.Signal(0)). Return(xerrors.New("os: process already finished")) - actualProcs, err := agentproc.List(fs, sc, rootDir) + actualProcs, err := agentproc.List(fs, sc) require.NoError(t, err) - require.Len(t, actualProcs, 3) + require.Len(t, actualProcs, len(expectedProcs)) for _, proc := range actualProcs { expected, ok := expectedProcs[proc.PID] require.True(t, ok) @@ -94,11 +92,10 @@ func TestList(t *testing.T) { fs = afero.NewMemMapFs() sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) - rootDir = agentproc.DefaultProcDir ) for i := 0; i < 3; i++ { - proc := agentproctest.GenerateProcess(t, fs, rootDir) + proc := agentproctest.GenerateProcess(t, fs) expectedProcs[proc.PID] = proc sc.EXPECT(). @@ -108,14 +105,14 @@ func TestList(t *testing.T) { // Create a process that doesn't exist. We're not adding // it to the map because it should be skipped over. - proc := agentproctest.GenerateProcess(t, fs, rootDir) + proc := agentproctest.GenerateProcess(t, fs) sc.EXPECT(). Kill(proc.PID, syscall.Signal(0)). Return(syscall.ESRCH) - actualProcs, err := agentproc.List(fs, sc, rootDir) + actualProcs, err := agentproc.List(fs, sc) require.NoError(t, err) - require.Len(t, actualProcs, 3) + require.Len(t, actualProcs, len(expectedProcs)) for _, proc := range actualProcs { expected, ok := expectedProcs[proc.PID] require.True(t, ok) @@ -136,15 +133,14 @@ func TestProcess(t *testing.T) { var ( fs = afero.NewMemMapFs() - dir = agentproc.DefaultProcDir - proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + proc = agentproctest.GenerateProcess(t, fs) expectedScore = -1000 ) err := proc.SetOOMAdj(expectedScore) require.NoError(t, err) - actualScore, err := afero.ReadFile(fs, fmt.Sprintf("%s/%d/oom_score_adj", dir, proc.PID)) + actualScore, err := afero.ReadFile(fs, fmt.Sprintf("/proc/%d/oom_score_adj", proc.PID)) require.NoError(t, err) require.Equal(t, fmt.Sprintf("%d", expectedScore), strings.TrimSpace(string(actualScore))) }) diff --git a/agent/agentproc/proc.go b/agent/agentproc/proc_unix.go similarity index 85% rename from agent/agentproc/proc.go rename to agent/agentproc/proc_unix.go index 29cb9a205bf4f..462f7b57c987c 100644 --- a/agent/agentproc/proc.go +++ b/agent/agentproc/proc_unix.go @@ -1,3 +1,6 @@ +//go:build linux +// +build linux + package agentproc import ( @@ -11,56 +14,12 @@ import ( "golang.org/x/xerrors" ) -const DefaultProcDir = "/proc" - -type Process struct { - Dir string - CmdLine string - PID int32 - FS afero.Fs -} - -func (p *Process) SetOOMAdj(score int) error { - path := filepath.Join(p.Dir, "oom_score_adj") - err := afero.WriteFile(p.FS, - path, - []byte(strconv.Itoa(score)), - 0o644, - ) - if err != nil { - return xerrors.Errorf("write %q: %w", path, err) - } - - return nil -} - -func (p *Process) Niceness(sc Syscaller) (int, error) { - nice, err := sc.GetPriority(p.PID) - if err != nil { - return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err) - } - return nice, nil -} - -func (p *Process) SetNiceness(sc Syscaller, score int) error { - err := sc.SetPriority(p.PID, score) - if err != nil { - return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err) - } - return nil -} - -func (p *Process) Name() string { - args := strings.Split(p.CmdLine, "\x00") - // Split will always return at least one element. - return args[0] -} - -func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { - d, err := fs.Open(dir) +func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) { + d, err := fs.Open(defaultProcDir) if err != nil { - return nil, xerrors.Errorf("open dir %q: %w", dir, err) + return nil, xerrors.Errorf("open dir %q: %w", defaultProcDir, err) } + defer d.Close() entries, err := d.Readdirnames(0) if err != nil { @@ -83,7 +42,7 @@ func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { continue } - cmdline, err := afero.ReadFile(fs, filepath.Join(dir, entry, "cmdline")) + cmdline, err := afero.ReadFile(fs, filepath.Join(defaultProcDir, entry, "cmdline")) if err != nil { var errNo syscall.Errno if xerrors.As(err, &errNo) && errNo == syscall.EPERM { @@ -94,7 +53,7 @@ func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { processes = append(processes, &Process{ PID: int32(pid), CmdLine: string(cmdline), - Dir: filepath.Join(dir, entry), + Dir: filepath.Join(defaultProcDir, entry), FS: fs, }) } @@ -125,3 +84,39 @@ func isProcessExist(syscaller Syscaller, pid int32) (bool, error) { return false, xerrors.Errorf("kill: %w", err) } + +func (p *Process) SetOOMAdj(score int) error { + path := filepath.Join(p.Dir, "oom_score_adj") + err := afero.WriteFile(p.FS, + path, + []byte(strconv.Itoa(score)), + 0o644, + ) + if err != nil { + return xerrors.Errorf("write %q: %w", path, err) + } + + return nil +} + +func (p *Process) Niceness(sc Syscaller) (int, error) { + nice, err := sc.GetPriority(p.PID) + if err != nil { + return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err) + } + return nice, nil +} + +func (p *Process) SetNiceness(sc Syscaller, score int) error { + err := sc.SetPriority(p.PID, score) + if err != nil { + return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err) + } + return nil +} + +func (p *Process) Name() string { + args := strings.Split(p.CmdLine, "\x00") + // Split will always return at least one element. + return args[0] +} diff --git a/agent/agentproc/syscaller.go b/agent/agentproc/syscaller.go index 6d346d1fb92ee..cf2c90d7ec77d 100644 --- a/agent/agentproc/syscaller.go +++ b/agent/agentproc/syscaller.go @@ -2,6 +2,8 @@ package agentproc import ( "syscall" + + "github.com/spf13/afero" ) type Syscaller interface { @@ -9,3 +11,12 @@ type Syscaller interface { GetPriority(pid int32) (int, error) Kill(pid int32, sig syscall.Signal) error } + +const defaultProcDir = "/proc" + +type Process struct { + Dir string + CmdLine string + PID int32 + FS afero.Fs +} diff --git a/agent/agentproc/syscaller_other.go b/agent/agentproc/syscaller_other.go index ba949d4ac33a5..5177cac2ddfd1 100644 --- a/agent/agentproc/syscaller_other.go +++ b/agent/agentproc/syscaller_other.go @@ -3,22 +3,28 @@ package agentproc -import "syscall" +import ( + "syscall" + + "golang.org/x/xerrors" +) func NewSyscaller() Syscaller { return nopSyscaller{} } +var errUnimplimented = xerrors.New("unimplemented") + type nopSyscaller struct{} func (nopSyscaller) SetPriority(pid int32, priority int) error { - return nil + return errUnimplimented } func (nopSyscaller) GetPriority(pid int32) (int, error) { - return 0, nil + return 0, errUnimplimented } func (nopSyscaller) Kill(pid int32, sig syscall.Signal) error { - return nil + return errUnimplimented } diff --git a/cli/agent.go b/cli/agent.go index 64693e3803e09..6a06f4d454f91 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -287,8 +287,8 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { return resp.SessionToken, nil }, EnvironmentVariables: map[string]string{ - "GIT_ASKPASS": executablePath, - agent.EnvProcMemNice: os.Getenv(agent.EnvProcMemNice), + "GIT_ASKPASS": executablePath, + agent.EnvProcPrioMgmt: os.Getenv(agent.EnvProcPrioMgmt), }, IgnorePorts: ignorePorts, SSHMaxTimeout: sshMaxTimeout, From 05baba0382a048e7d385e6328f5f10847ceec69e Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 12 Sep 2023 22:44:54 +0000 Subject: [PATCH 11/21] lint --- agent/agentproc/proc_other.go | 8 ++++---- agent/agentproc/syscaller_other.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/agent/agentproc/proc_other.go b/agent/agentproc/proc_other.go index b299eac185fd5..b4d6348003572 100644 --- a/agent/agentproc/proc_other.go +++ b/agent/agentproc/proc_other.go @@ -8,15 +8,15 @@ import ( ) func (p *Process) SetOOMAdj(score int) error { - return errUnimplimented + return errUnimplemented } func (p *Process) Niceness(sc Syscaller) (int, error) { - return 0, errUnimplimented + return 0, errUnimplemented } func (p *Process) SetNiceness(sc Syscaller, score int) error { - return errUnimplimented + return errUnimplemented } func (p *Process) Name() string { @@ -24,5 +24,5 @@ func (p *Process) Name() string { } func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) { - return nil, errUnimplimented + return nil, errUnimplemented } diff --git a/agent/agentproc/syscaller_other.go b/agent/agentproc/syscaller_other.go index 5177cac2ddfd1..2b61910704fd8 100644 --- a/agent/agentproc/syscaller_other.go +++ b/agent/agentproc/syscaller_other.go @@ -13,7 +13,7 @@ func NewSyscaller() Syscaller { return nopSyscaller{} } -var errUnimplimented = xerrors.New("unimplemented") +var errUnimplemented = xerrors.New("unimplemented") type nopSyscaller struct{} From 478d57c51fc5879708c89df5af949a28fb038311 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 12 Sep 2023 23:04:39 +0000 Subject: [PATCH 12/21] whoops --- agent/agentproc/syscaller_other.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agent/agentproc/syscaller_other.go b/agent/agentproc/syscaller_other.go index 2b61910704fd8..114c553e43da2 100644 --- a/agent/agentproc/syscaller_other.go +++ b/agent/agentproc/syscaller_other.go @@ -18,13 +18,13 @@ var errUnimplemented = xerrors.New("unimplemented") type nopSyscaller struct{} func (nopSyscaller) SetPriority(pid int32, priority int) error { - return errUnimplimented + return errUnimplemented } func (nopSyscaller) GetPriority(pid int32) (int, error) { - return 0, errUnimplimented + return 0, errUnimplemented } func (nopSyscaller) Kill(pid int32, sig syscall.Signal) error { - return errUnimplimented + return errUnimplemented } From 0ced5ce78b14da34d90a18e302a3e859618969d9 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 13 Sep 2023 00:12:28 +0000 Subject: [PATCH 13/21] skip non-linux --- agent/agentproc/proc_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index 6459b05f3f7e8..1ecf27a4fe67a 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -2,6 +2,7 @@ package agentproc_test import ( "fmt" + "runtime" "strings" "syscall" "testing" @@ -18,6 +19,10 @@ import ( func TestList(t *testing.T) { t.Parallel() + if runtime.GOOS != "linux" { + t.Skipf("skipping non-linux environment") + } + t.Run("OK", func(t *testing.T) { t.Parallel() @@ -128,6 +133,10 @@ func TestList(t *testing.T) { func TestProcess(t *testing.T) { t.Parallel() + if runtime.GOOS != "linux" { + t.Skipf("skipping non-linux environment") + } + t.Run("SetOOMAdj", func(t *testing.T) { t.Parallel() From 8aaa6d569f9a8a6040c0b74650828618f9918acd Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 13 Sep 2023 01:02:06 +0000 Subject: [PATCH 14/21] skip non-linux --- agent/agent_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/agent/agent_test.go b/agent/agent_test.go index c2d20209f4974..09d1e9a5fd803 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2406,6 +2406,10 @@ func TestAgent_ManageProcessPriority(t *testing.T) { t.Run("OK", func(t *testing.T) { t.Parallel() + if runtime.GOOS != "linux" { + t.Skip("Skipping non-linux environment") + } + var ( expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() @@ -2469,6 +2473,10 @@ func TestAgent_ManageProcessPriority(t *testing.T) { t.Run("IgnoreCustomNice", func(t *testing.T) { t.Parallel() + if runtime.GOOS != "linux" { + t.Skip("Skipping non-linux environment") + } + var ( expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() From cea4851769b71453a8cd471fe5c54ec51030d464 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 13 Sep 2023 01:45:57 +0000 Subject: [PATCH 15/21] prevent race --- agent/agent_test.go | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/agent/agent_test.go b/agent/agent_test.go index 09d1e9a5fd803..6f6f01a7d34c9 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2525,14 +2525,21 @@ func TestAgent_ManageProcessPriority(t *testing.T) { t.Skip("Skipping non-linux environment") } - var buf bytes.Buffer - log := slog.Make(sloghuman.Sink(&buf)).Leveled(slog.LevelDebug) + var ( + buf bytes.Buffer + wr = &syncWriter{ + w: &buf, + } + ) + log := slog.Make(sloghuman.Sink(wr)).Leveled(slog.LevelDebug) _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Logger = log }) require.Eventually(t, func() bool { + wr.mu.Lock() + defer wr.mu.Unlock() return strings.Contains(buf.String(), "process priority not enabled") }, testutil.WaitLong, testutil.IntervalFast) }) @@ -2544,8 +2551,13 @@ func TestAgent_ManageProcessPriority(t *testing.T) { t.Skip("Skipping linux environment") } - var buf bytes.Buffer - log := slog.Make(sloghuman.Sink(&buf)).Leveled(slog.LevelDebug) + var ( + buf bytes.Buffer + wr = &syncWriter{ + w: &buf, + } + ) + log := slog.Make(sloghuman.Sink(wr)).Leveled(slog.LevelDebug) _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Logger = log @@ -2554,6 +2566,9 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"} }) require.Eventually(t, func() bool { + wr.mu.Lock() + defer wr.mu.Unlock() + return strings.Contains(buf.String(), "process priority not enabled") }, testutil.WaitLong, testutil.IntervalFast) }) @@ -2580,3 +2595,14 @@ func verifyCollectedMetrics(t *testing.T, expected []agentsdk.AgentMetric, actua } return true } + +type syncWriter struct { + mu sync.Mutex + w io.Writer +} + +func (s *syncWriter) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.w.Write(p) +} From 5020eb4d11e5453c841b8c63ab552cb599cc55c2 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 14 Sep 2023 20:30:32 +0000 Subject: [PATCH 16/21] only prioritize ourselves --- agent/agent.go | 13 ++++++++----- agent/agent_test.go | 21 +++++++++++++-------- cli/agent.go | 1 + 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 5886f305d2ca9..01fb89f25cfdf 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -79,6 +79,9 @@ type Options struct { ModifiedProcesses chan []*agentproc.Process // ProcessManagementTick is used for testing process priority management. ProcessManagementTick <-chan time.Time + // PrioritizedPIDs are processes that should have preferred CPU allocation + // and be de-prioritized for OOM Killer selection. + PrioritizedPIDs []int } type Client interface { @@ -161,6 +164,7 @@ func New(options Options) Agent { syscaller: options.Syscaller, modifiedProcs: options.ModifiedProcesses, processManagementTick: options.ProcessManagementTick, + prioritizedPIDs: options.PrioritizedPIDs, prometheusRegistry: prometheusRegistry, metrics: newAgentMetrics(prometheusRegistry), @@ -221,6 +225,9 @@ type agent struct { modifiedProcs chan []*agentproc.Process // processManagementTick is used for testing process priority management. processManagementTick <-chan time.Time + // prioritizedPIDs are processes that should have preferred CPU allocation + // and be de-prioritized for OOM Killer selection. + prioritizedPIDs []int } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1278,8 +1285,6 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { } } -var prioritizedProcs = []string{"coder"} - func (a *agent) manageProcessPriorityLoop(ctx context.Context) { if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { a.logger.Debug(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", @@ -1337,11 +1342,9 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process slog.F("pid", proc.PID), ) - // Trim off the path e.g. "./coder" -> "coder" - name := filepath.Base(proc.Name()) // If the process is prioritized we should adjust // it's oom_score_adj and avoid lowering its niceness. - if slices.Contains(prioritizedProcs, name) { + if slices.Contains(a.prioritizedPIDs, int(proc.PID)) { err = proc.SetOOMAdj(oomScoreAdj) if err != nil { logger.Warn(ctx, "unable to set proc oom_score_adj", diff --git a/agent/agent_test.go b/agent/agent_test.go index 6f6f01a7d34c9..5560b5547e8c8 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2411,12 +2411,13 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } var ( - expectedProcs = map[int32]agentproc.Process{} - fs = afero.NewMemMapFs() - syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) - ticker = make(chan time.Time) - modProcs = make(chan []*agentproc.Process) - logger = slog.Make(sloghuman.Sink(io.Discard)) + expectedProcs = map[int32]agentproc.Process{} + fs = afero.NewMemMapFs() + syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) + ticker = make(chan time.Time) + modProcs = make(chan []*agentproc.Process) + logger = slog.Make(sloghuman.Sink(io.Discard)) + prioritizedProc = 123 ) // Create some processes. @@ -2429,12 +2430,15 @@ func TestAgent_ManageProcessPriority(t *testing.T) { proc = agentproctest.GenerateProcess(t, fs, func(p *agentproc.Process) { p.CmdLine = "./coder\x00agent\x00--no-reap" - p.PID = 1 + p.PID = int32(prioritizedProc) }, ) } else { // The rest are peasants. proc = agentproctest.GenerateProcess(t, fs) + if proc.PID == int32(prioritizedProc) { + proc.PID = 1234 + } syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) } @@ -2452,6 +2456,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.Filesystem = fs o.Logger = logger o.ProcessManagementTick = ticker + o.PrioritizedPIDs = []int{prioritizedProc} }) actualProcs := <-modProcs require.Len(t, actualProcs, 4) @@ -2460,7 +2465,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { expectedScore := "0" expected, ok := expectedProcs[actual.PID] require.True(t, ok) - if expected.PID == 1 { + if expected.PID == int32(prioritizedProc) { expectedScore = "-500" } diff --git a/cli/agent.go b/cli/agent.go index 6a06f4d454f91..584e40b5518be 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -299,6 +299,7 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { // Intentionally set this to nil. It's mainly used // for testing. ModifiedProcesses: nil, + PrioritizedPIDs: []int{os.Getpid()}, }) prometheusSrvClose := ServeHandler(ctx, logger, prometheusMetricsHandler(prometheusRegistry, logger), prometheusAddress, "prometheus") From 11ab047a6e8903c2ea72a0ab2694aa9f631c9719 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 14 Sep 2023 20:57:25 +0000 Subject: [PATCH 17/21] Revert "only prioritize ourselves" This reverts commit 5020eb4d11e5453c841b8c63ab552cb599cc55c2. --- agent/agent.go | 13 +++++-------- agent/agent_test.go | 21 ++++++++------------- cli/agent.go | 1 - 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 01fb89f25cfdf..5886f305d2ca9 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -79,9 +79,6 @@ type Options struct { ModifiedProcesses chan []*agentproc.Process // ProcessManagementTick is used for testing process priority management. ProcessManagementTick <-chan time.Time - // PrioritizedPIDs are processes that should have preferred CPU allocation - // and be de-prioritized for OOM Killer selection. - PrioritizedPIDs []int } type Client interface { @@ -164,7 +161,6 @@ func New(options Options) Agent { syscaller: options.Syscaller, modifiedProcs: options.ModifiedProcesses, processManagementTick: options.ProcessManagementTick, - prioritizedPIDs: options.PrioritizedPIDs, prometheusRegistry: prometheusRegistry, metrics: newAgentMetrics(prometheusRegistry), @@ -225,9 +221,6 @@ type agent struct { modifiedProcs chan []*agentproc.Process // processManagementTick is used for testing process priority management. processManagementTick <-chan time.Time - // prioritizedPIDs are processes that should have preferred CPU allocation - // and be de-prioritized for OOM Killer selection. - prioritizedPIDs []int } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1285,6 +1278,8 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { } } +var prioritizedProcs = []string{"coder"} + func (a *agent) manageProcessPriorityLoop(ctx context.Context) { if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { a.logger.Debug(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", @@ -1342,9 +1337,11 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process slog.F("pid", proc.PID), ) + // Trim off the path e.g. "./coder" -> "coder" + name := filepath.Base(proc.Name()) // If the process is prioritized we should adjust // it's oom_score_adj and avoid lowering its niceness. - if slices.Contains(a.prioritizedPIDs, int(proc.PID)) { + if slices.Contains(prioritizedProcs, name) { err = proc.SetOOMAdj(oomScoreAdj) if err != nil { logger.Warn(ctx, "unable to set proc oom_score_adj", diff --git a/agent/agent_test.go b/agent/agent_test.go index 5560b5547e8c8..6f6f01a7d34c9 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2411,13 +2411,12 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } var ( - expectedProcs = map[int32]agentproc.Process{} - fs = afero.NewMemMapFs() - syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) - ticker = make(chan time.Time) - modProcs = make(chan []*agentproc.Process) - logger = slog.Make(sloghuman.Sink(io.Discard)) - prioritizedProc = 123 + expectedProcs = map[int32]agentproc.Process{} + fs = afero.NewMemMapFs() + syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) + ticker = make(chan time.Time) + modProcs = make(chan []*agentproc.Process) + logger = slog.Make(sloghuman.Sink(io.Discard)) ) // Create some processes. @@ -2430,15 +2429,12 @@ func TestAgent_ManageProcessPriority(t *testing.T) { proc = agentproctest.GenerateProcess(t, fs, func(p *agentproc.Process) { p.CmdLine = "./coder\x00agent\x00--no-reap" - p.PID = int32(prioritizedProc) + p.PID = 1 }, ) } else { // The rest are peasants. proc = agentproctest.GenerateProcess(t, fs) - if proc.PID == int32(prioritizedProc) { - proc.PID = 1234 - } syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) } @@ -2456,7 +2452,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.Filesystem = fs o.Logger = logger o.ProcessManagementTick = ticker - o.PrioritizedPIDs = []int{prioritizedProc} }) actualProcs := <-modProcs require.Len(t, actualProcs, 4) @@ -2465,7 +2460,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { expectedScore := "0" expected, ok := expectedProcs[actual.PID] require.True(t, ok) - if expected.PID == int32(prioritizedProc) { + if expected.PID == 1 { expectedScore = "-500" } diff --git a/cli/agent.go b/cli/agent.go index 584e40b5518be..6a06f4d454f91 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -299,7 +299,6 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { // Intentionally set this to nil. It's mainly used // for testing. ModifiedProcesses: nil, - PrioritizedPIDs: []int{os.Getpid()}, }) prometheusSrvClose := ServeHandler(ctx, logger, prometheusMetricsHandler(prometheusRegistry, logger), prometheusAddress, "prometheus") From 46ef05abef24853da4eba8155601e78d269ea47a Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 14 Sep 2023 22:32:19 +0000 Subject: [PATCH 18/21] only prioritize coder agent --- agent/agent.go | 24 +++-- agent/agent_test.go | 14 ++- agent/agentproc/proc_other.go | 2 +- agent/agentproc/proc_test.go | 6 +- agent/agentproc/proc_unix.go | 10 +- agent/out.txt | 167 ++++++++++++++++++++++++++++++++++ 6 files changed, 205 insertions(+), 18 deletions(-) create mode 100644 agent/out.txt diff --git a/agent/agent.go b/agent/agent.go index 5886f305d2ca9..71272aea4121c 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -16,6 +16,7 @@ import ( "os/user" "path/filepath" "runtime" + "runtime/debug" "sort" "strconv" "strings" @@ -1278,7 +1279,7 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { } } -var prioritizedProcs = []string{"coder"} +var prioritizedProcs = []string{"coder agent"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { @@ -1290,6 +1291,15 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { return } + defer func() { + if r := recover(); r != nil { + a.logger.Critical(ctx, "recovered from panic", + slog.F("panic", r), + slog.F("stack", string(debug.Stack())), + ) + } + }() + if a.processManagementTick == nil { ticker := time.NewTicker(time.Second) defer ticker.Stop() @@ -1333,15 +1343,17 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process for _, proc := range procs { logger = a.logger.With( - slog.F("name", proc.Name()), + slog.F("cmd", proc.Cmd()), slog.F("pid", proc.PID), ) - // Trim off the path e.g. "./coder" -> "coder" - name := filepath.Base(proc.Name()) + containsFn := func(e string) bool { + contains := strings.Contains(proc.Cmd(), e) + return contains + } // If the process is prioritized we should adjust // it's oom_score_adj and avoid lowering its niceness. - if slices.Contains(prioritizedProcs, name) { + if slices.ContainsFunc[[]string, string](prioritizedProcs, containsFn) { err = proc.SetOOMAdj(oomScoreAdj) if err != nil { logger.Warn(ctx, "unable to set proc oom_score_adj", @@ -1366,7 +1378,7 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process // so we don't override user nice values. // Getpriority actually returns priority for the nice value // which is niceness + 20, so here 20 = a niceness of 0 (aka unset). - if score != 20 { + if score != 20 && score != niceness { logger.Debug(ctx, "skipping process due to custom niceness", slog.F("niceness", score), ) diff --git a/agent/agent_test.go b/agent/agent_test.go index 6f6f01a7d34c9..55c2e5411cc23 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2429,12 +2429,18 @@ func TestAgent_ManageProcessPriority(t *testing.T) { proc = agentproctest.GenerateProcess(t, fs, func(p *agentproc.Process) { p.CmdLine = "./coder\x00agent\x00--no-reap" - p.PID = 1 + p.PID = int32(i) }, ) } else { - // The rest are peasants. - proc = agentproctest.GenerateProcess(t, fs) + proc = agentproctest.GenerateProcess(t, fs, + func(p *agentproc.Process) { + // Make the cmd something similar to a prioritized + // process but differentiate the arguments. + p.CmdLine = "./coder\x00stat" + }, + ) + syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) } @@ -2460,7 +2466,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { expectedScore := "0" expected, ok := expectedProcs[actual.PID] require.True(t, ok) - if expected.PID == 1 { + if expected.PID == 0 { expectedScore = "-500" } diff --git a/agent/agentproc/proc_other.go b/agent/agentproc/proc_other.go index b4d6348003572..ee1ff194103bb 100644 --- a/agent/agentproc/proc_other.go +++ b/agent/agentproc/proc_other.go @@ -19,7 +19,7 @@ func (p *Process) SetNiceness(sc Syscaller, score int) error { return errUnimplemented } -func (p *Process) Name() string { +func (p *Process) Cmd() string { return "" } diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index 1ecf27a4fe67a..c6081966904c9 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -170,16 +170,16 @@ func TestProcess(t *testing.T) { require.NoError(t, err) }) - t.Run("Name", func(t *testing.T) { + t.Run("Cmd", func(t *testing.T) { t.Parallel() var ( proc = &agentproc.Process{ CmdLine: "helloworld\x00--arg1\x00--arg2", } - expectedName = "helloworld" + expectedName = "helloworld --arg1 --arg2" ) - require.Equal(t, expectedName, proc.Name()) + require.Equal(t, expectedName, proc.Cmd()) }) } diff --git a/agent/agentproc/proc_unix.go b/agent/agentproc/proc_unix.go index 462f7b57c987c..6658c8f43a62e 100644 --- a/agent/agentproc/proc_unix.go +++ b/agent/agentproc/proc_unix.go @@ -115,8 +115,10 @@ func (p *Process) SetNiceness(sc Syscaller, score int) error { return nil } -func (p *Process) Name() string { - args := strings.Split(p.CmdLine, "\x00") - // Split will always return at least one element. - return args[0] +func (p *Process) Cmd() string { + return strings.Join(p.cmdLine(), " ") +} + +func (p *Process) cmdLine() []string { + return strings.Split(p.CmdLine, "\x00") } diff --git a/agent/out.txt b/agent/out.txt new file mode 100644 index 0000000000000..7b435abf43cb0 --- /dev/null +++ b/agent/out.txt @@ -0,0 +1,167 @@ +=== RUN TestAgent_ManageProcessPriority +=== PAUSE TestAgent_ManageProcessPriority +=== CONT TestAgent_ManageProcessPriority +=== RUN TestAgent_ManageProcessPriority/OK +=== PAUSE TestAgent_ManageProcessPriority/OK +=== CONT TestAgent_ManageProcessPriority/OK +YEAH +YUP + t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: get service banner + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) tun device + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) OS network configurator + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) DNS configurator + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: dns: using dns.noopManager + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: link state: interfaces.State{defaultRoute=eth0 ifs={docker0:[172.17.0.1/16] eth0:[172.20.0.7/16]} v4=true v6=false} + t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: post startup req={"version":"v0.0.0-devel","expanded_directory":"","subsystems":null} + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP read buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP write buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP read buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP write buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] couldn't create raw v4 disco listener, using regular listener instead: raw disco listening disabled, SO_MARK unavailable + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] couldn't create raw v6 disco listener, using regular listener instead: raw disco listening disabled, SO_MARK unavailable + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: disco key = d:9bf350ad1946674e + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: Creating WireGuard device... + t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: post lifecycle req={"state":"starting","changed_at":"2023-09-14T22:21:04.162755Z"} + t.go:85: 2023-09-14 22:21:04.163 [debu] agent.client: post lifecycle req={"state":"ready","changed_at":"2023-09-14T22:21:04.162813Z"} + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: Bringing WireGuard device up... + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] UDP bind has been updated + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Interface state was Down, requested Up, now Up + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: Bringing router up... + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - started + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - started + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: [v1] warning: fakeRouter.Up: not implemented. + t.go:85: 2023-09-14 22:21:04.164 [debu] client.net.wgengine: Clearing router settings... + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming receiveDERP - started + t.go:85: 2023-09-14 22:21:04.164 [debu] client.net.wgengine: [v1] warning: fakeRouter.Set: not implemented. + t.go:85: 2023-09-14 22:21:04.165 [debu] client.net.wgengine: Starting network monitor... + t.go:85: 2023-09-14 22:21:04.166 [debu] client.net.wgengine: Engine created. + t.go:85: 2023-09-14 22:21:04.166 [debu] client: magicsock debug logging disabled, use CODER_MAGICSOCK_DEBUG_LOGGING=true to enable + t.go:85: 2023-09-14 22:21:04.166 [debu] client.net.wgengine: magicsock: SetPrivateKey called (init) + t.go:85: 2023-09-14 22:21:04.170 [debu] client: updating network map + t.go:85: 2023-09-14 22:21:04.170 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 0 peers + t.go:85: 2023-09-14 22:21:04.170 [debu] coordinating client client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.170 [debu] client: skipped sending node; no PreferredDERP node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.170813 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:0 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[]}" + t.go:85: 2023-09-14 22:21:04.170 [debu] multiagent node doesn't exist client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.170 [debu] coordinating agent agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.171 [debu] agent node doesn't exist client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.171 [debu] wrote initial client(s) to agent agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes=[] + t.go:85: 2023-09-14 22:21:04.171 [debu] added agent socket agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.171 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer + t.go:85: 2023-09-14 22:21:04.171 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer + t.go:85: 2023-09-14 22:21:04.171 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes=[] + t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (6f93c36c4883ce76647c9e74) in 4.705998ms + t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (3d5303cc01a8f9a035b236a4) in 4.76389ms + t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (a41d11ea5ec51a69012804ae) in 4.792004ms + t.go:85: 2023-09-14 22:21:04.221 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer + t.go:85: 2023-09-14 22:21:04.221 [debu] client.net.wgengine: netcheck: [v1] measuring ICMP latency of test (1): listen ip4:icmp 0.0.0.0: socket: operation not permitted + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: netcheck: [v1] netcheck: measuring HTTP(S) latency of test (1): unexpected status code: 426 (426 Upgrade Required) + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: netcheck: [v1] report: udp=true v6=false v6os=false mapvarydest=false hair= portmap= v4a=127.0.0.1:39073 derp=0 + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: home is now derp-1 (test) + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: adding connection to derp-1 for home-keep-alive + t.go:85: 2023-09-14 22:21:04.228 [debu] client: netinfo callback netinfo="NetInfo{varies=false hairpin= ipv6=false ipv6os=false udp=true icmpv4=false derp=#1 portmap= link=\"\"}" + t.go:85: 2023-09-14 22:21:04.228 [debu] client: sending node node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.228454 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[]}" + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: 1 active derp conns: derp-1=cr0s,wr0s + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: endpoints changed: 127.0.0.1:39073 (stun), 172.17.0.1:39073 (local), 172.20.0.7:39073 (local) + t.go:85: 2023-09-14 22:21:04.228 [debu] got client node update client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 node={"id":1384268018767542531,"as_of":"2023-09-14T22:21:04.228454Z","key":"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24","disco":"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"allowed_ips":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"endpoints":[]} + t.go:85: 2023-09-14 22:21:04.228 [debu] enqueued node to agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_ids=[4db2658f-194a-4b75-9608-1b6b42d6ba8a] + t.go:85: 2023-09-14 22:21:04.228 [debu] client: wireguard status status="&{AsOf:2023-09-14 22:21:04.228713945 +0000 UTC m=+0.212205983 Peers:[] LocalAddrs:[{Addr:127.0.0.1:39073 Type:stun} {Addr:172.17.0.1:39073 Type:local} {Addr:172.20.0.7:39073 Type:local}] DERPs:1}" error= + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: derphttp.Client.Connect: connecting to derp-1 (test) + t.go:85: 2023-09-14 22:21:04.228 [debu] client: wireguard status status="&{AsOf:2023-09-14 22:21:04.228760465 +0000 UTC m=+0.212252503 Peers:[] LocalAddrs:[{Addr:127.0.0.1:39073 Type:stun} {Addr:172.17.0.1:39073 Type:local} {Addr:172.20.0.7:39073 Type:local}] DERPs:1}" error= + t.go:85: 2023-09-14 22:21:04.228 [debu] decoded agent node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a node={"id":7635823020709856630,"as_of":"2023-09-14T22:21:04.228767Z","key":"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d","disco":"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"allowed_ips":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"endpoints":[]} + t.go:85: 2023-09-14 22:21:04.228 [debu] client: sending node node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.228869 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[127.0.0.1:39073 172.17.0.1:39073 172.20.0.7:39073]}" + t.go:85: 2023-09-14 22:21:04.228 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":1384268018767542531,\"as_of\":\"2023-09-14T22:21:04.228454Z\",\"key\":\"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24\",\"disco\":\"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"endpoints\":[]}]" + t.go:85: 2023-09-14 22:21:04.228 [debu] enqueued agent node to client agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 + t.go:85: 2023-09-14 22:21:04.229 [debu] got client node update client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 node={"id":1384268018767542531,"as_of":"2023-09-14T22:21:04.228869Z","key":"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24","disco":"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"allowed_ips":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"endpoints":["127.0.0.1:39073","172.17.0.1:39073","172.20.0.7:39073"]} + t.go:85: 2023-09-14 22:21:04.229 [debu] enqueued node to agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_ids=[4db2658f-194a-4b75-9608-1b6b42d6ba8a] + t.go:85: 2023-09-14 22:21:04.229 [debu] client: adding node node="&{ID:nodeid:69f7e494bd294576 AsOf:2023-09-14 22:21:04.228767 +0000 UTC Key:nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d DiscoKey:discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] AllowedIPs:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] Endpoints:[]}" + t.go:85: 2023-09-14 22:21:04.229 [debu] wrote nodes client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":7635823020709856630,\"as_of\":\"2023-09-14T22:21:04.228767Z\",\"key\":\"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d\",\"disco\":\"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"endpoints\":[]}]" + t.go:85: 2023-09-14 22:21:04.229 [debu] client: updating network map + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 1 peers + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.netstack: [v2] netstack: registered IP fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128 + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wgengine: Reconfig: configuring userspace WireGuard config (with 1/1 peers) + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] UAPI: Updating private key + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Created + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (d1abaaf30fcfbb0a2e30af00) in 68.382µs + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Updating endpoint + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (bbe130a507ad56b3f11e5620) in 104.803µs + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Removing all allowedips + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (f3e4b4d721a0986d001d0c52) in 129.58µs + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Adding allowedip + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Adding allowedip + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Updating persistent keepalive interval + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - Starting + t.go:85: 2023-09-14 22:21:04.229 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":1384268018767542531,\"as_of\":\"2023-09-14T22:21:04.228869Z\",\"key\":\"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24\",\"disco\":\"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"endpoints\":[\"127.0.0.1:39073\",\"172.17.0.1:39073\",\"172.20.0.7:39073\"]}]" + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wgengine: Reconfig: configuring router + t.go:85: 2023-09-14 22:21:04.229 [debu] decoded agent node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a node={"id":7635823020709856630,"as_of":"2023-09-14T22:21:04.229132Z","key":"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d","disco":"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"allowed_ips":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"endpoints":["127.0.0.1:32901","172.17.0.1:32901","172.20.0.7:32901"]} + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] warning: fakeRouter.Set: not implemented. + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: wgengine: Reconfig: configuring DNS + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: Set: {DefaultResolvers:[] Routes:{} SearchDomains:[] Hosts:0} + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: Resolvercfg: {Routes:{} Hosts:0 LocalDomains:[]} + t.go:85: 2023-09-14 22:21:04.230 [debu] enqueued agent node to client agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: OScfg: {Nameservers:[] SearchDomains:[] MatchDomains:[] Hosts:[]} + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] wgengine: Reconfig done + t.go:85: 2023-09-14 22:21:04.230 [debu] wrote nodes client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":7635823020709856630,\"as_of\":\"2023-09-14T22:21:04.229132Z\",\"key\":\"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d\",\"disco\":\"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"endpoints\":[\"127.0.0.1:32901\",\"172.17.0.1:32901\",\"172.20.0.7:32901\"]}]" + t.go:85: 2023-09-14 22:21:04.230 [debu] client: adding node node="&{ID:nodeid:69f7e494bd294576 AsOf:2023-09-14 22:21:04.229132 +0000 UTC Key:nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d DiscoKey:discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] AllowedIPs:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] Endpoints:[127.0.0.1:32901 172.17.0.1:32901 172.20.0.7:32901]}" + t.go:85: 2023-09-14 22:21:04.230 [debu] client: updating network map + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 1 peers + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] wgengine: Reconfig done + t.go:85: 2023-09-14 22:21:04.232 [debu] client.net.wgengine: magicsock: derp-1 connected; connGen=1 + t.go:85: 2023-09-14 22:21:04.280 [debu] client.net.wgengine: netcheck: [v1] measuring ICMP latency of test (1): listen ip4:icmp 0.0.0.0: socket: operation not permitted + t.go:85: 2023-09-14 22:21:04.286 [debu] client.net.wgengine: netcheck: [v1] netcheck: measuring HTTP(S) latency of test (1): unexpected status code: 426 (426 Upgrade Required) + t.go:85: 2023-09-14 22:21:04.286 [debu] client.net.wgengine: netcheck: [v1] report: udp=true v6=false v6os=false mapvarydest=false hair= portmap= v4a=127.0.0.1:39073 derp=0 + t.go:85: 2023-09-14 22:21:04.296 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): sending disco ping to [Vs9bR] ... + t.go:85: 2023-09-14 22:21:04.296 [debu] client.net.wgengine: [v1] magicsock: derp route for [Vs9bR] set to derp-1 (shared home) + t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using 172.17.0.1:32901 + t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using 127.0.0.1:32901 + agent_test.go:2463: + Error Trace: /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2463 + Error: "[]" should have 4 item(s), but has 0 + Test: TestAgent_ManageProcessPriority/OK + t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using DERP only (reset) + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: magicsock: closing connection to derp-1 (conn-close), age 0s + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: magicsock: 0 active derp conns + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: [v1] warning: fakeRouter.Close: not implemented. + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Device closing + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - stopped + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - stopped + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming receiveDERP - stopped + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - Stopping + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Device closed + t.go:85: 2023-09-14 22:21:04.298 [debu] unable to read agent update, connection may be closed agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a ... + error= read json: + github.com/coder/coder/v2/tailnet.(*coordinator).handleNextAgentMessage + /home/coder/go/src/github.com/coder/coder/tailnet/coordinator.go:552 + - io: read/write on closed pipe + t.go:85: 2023-09-14 22:21:04.298 [debu] deleted agent socket and node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.298 [debu] done sending updates agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.298 [debu] unable to read client update, connection may be closed client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a ... + error= read json: + github.com/coder/coder/v2/tailnet.(*coordinator).handleNextClientMessage + /home/coder/go/src/github.com/coder/coder/tailnet/coordinator.go:350 + - EOF + t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client node client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 + t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client connectionSocket from map client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.299 [debu] deleted last client connectionSocket from map client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 + t.go:85: 2023-09-14 22:21:04.299 [debu] done sending updates client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.299 [debu] agent.client: post lifecycle req={"state":"shutting_down","changed_at":"2023-09-14T22:21:04.299336Z"} + t.go:85: 2023-09-14 22:21:04.499 [debu] agent.client: get service banner + t.go:85: 2023-09-14 22:21:04.500 [debu] agent.client: post startup req={"version":"v0.0.0-devel","expanded_directory":"","subsystems":null} + t.go:85: 2023-09-14 22:21:06.300 [debu] agent.client: post lifecycle req={"state":"off","changed_at":"2023-09-14T22:21:04.299369Z"} + stuntest.go:63: STUN server shutdown + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 0 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1965978598 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1248996486 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1788987366 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1965978598 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1248996486 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1788987366 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1965978598 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1248996486 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1788987366 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 + controller.go:137: aborting test due to missing call(s) +--- FAIL: TestAgent_ManageProcessPriority (0.00s) + --- FAIL: TestAgent_ManageProcessPriority/OK (2.26s) +FAIL +exit status 1 +FAIL github.com/coder/coder/v2/agent 2.294s From d132480a017450421d2ea0e6357e6c444b914996 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 14 Sep 2023 23:12:43 +0000 Subject: [PATCH 19/21] remove oom_score_adj --- agent/agent.go | 13 +- agent/agent_test.go | 15 +-- agent/agentproc/agentproctest/proc.go | 4 - agent/agentproc/proc_other.go | 4 - agent/agentproc/proc_test.go | 19 --- agent/agentproc/proc_unix.go | 15 --- agent/agentproc/syscaller.go | 3 - agent/out.txt | 167 -------------------------- 8 files changed, 3 insertions(+), 237 deletions(-) delete mode 100644 agent/out.txt diff --git a/agent/agent.go b/agent/agent.go index 71272aea4121c..9babab81d8475 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1327,8 +1327,7 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process, error) { const ( - niceness = 10 - oomScoreAdj = -500 + niceness = 10 ) procs, err := agentproc.List(a.filesystem, a.syscaller) @@ -1351,18 +1350,10 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process contains := strings.Contains(proc.Cmd(), e) return contains } + // If the process is prioritized we should adjust // it's oom_score_adj and avoid lowering its niceness. if slices.ContainsFunc[[]string, string](prioritizedProcs, containsFn) { - err = proc.SetOOMAdj(oomScoreAdj) - if err != nil { - logger.Warn(ctx, "unable to set proc oom_score_adj", - slog.F("oom_score_adj", oomScoreAdj), - slog.Error(err), - ) - continue - } - modProcs = append(modProcs, proc) continue } diff --git a/agent/agent_test.go b/agent/agent_test.go index 55c2e5411cc23..80fa7435c7437 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2460,20 +2460,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.ProcessManagementTick = ticker }) actualProcs := <-modProcs - require.Len(t, actualProcs, 4) - - for _, actual := range actualProcs { - expectedScore := "0" - expected, ok := expectedProcs[actual.PID] - require.True(t, ok) - if expected.PID == 0 { - expectedScore = "-500" - } - - score, err := afero.ReadFile(fs, filepath.Join(actual.Dir, "oom_score_adj")) - require.NoError(t, err) - require.Equal(t, expectedScore, strings.TrimSpace(string(score))) - } + require.Len(t, actualProcs, len(expectedProcs)-1) }) t.Run("IgnoreCustomNice", func(t *testing.T) { diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go index 1fe44ad9eb31b..c36e04ec1cdc3 100644 --- a/agent/agentproc/agentproctest/proc.go +++ b/agent/agentproc/agentproctest/proc.go @@ -31,7 +31,6 @@ func GenerateProcess(t *testing.T, fs afero.Fs, muts ...func(*agentproc.Process) process := agentproc.Process{ CmdLine: cmdline, PID: int32(pid), - FS: fs, } for _, mut := range muts { @@ -46,8 +45,5 @@ func GenerateProcess(t *testing.T, fs afero.Fs, muts ...func(*agentproc.Process) err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0o444) require.NoError(t, err) - err = afero.WriteFile(fs, fmt.Sprintf("%s/oom_score_adj", process.Dir), []byte("0"), 0o444) - require.NoError(t, err) - return process } diff --git a/agent/agentproc/proc_other.go b/agent/agentproc/proc_other.go index ee1ff194103bb..c0c4e2a25ce32 100644 --- a/agent/agentproc/proc_other.go +++ b/agent/agentproc/proc_other.go @@ -7,10 +7,6 @@ import ( "github.com/spf13/afero" ) -func (p *Process) SetOOMAdj(score int) error { - return errUnimplemented -} - func (p *Process) Niceness(sc Syscaller) (int, error) { return 0, errUnimplemented } diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index c6081966904c9..37991679503c6 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -1,9 +1,7 @@ package agentproc_test import ( - "fmt" "runtime" - "strings" "syscall" "testing" @@ -137,23 +135,6 @@ func TestProcess(t *testing.T) { t.Skipf("skipping non-linux environment") } - t.Run("SetOOMAdj", func(t *testing.T) { - t.Parallel() - - var ( - fs = afero.NewMemMapFs() - proc = agentproctest.GenerateProcess(t, fs) - expectedScore = -1000 - ) - - err := proc.SetOOMAdj(expectedScore) - require.NoError(t, err) - - actualScore, err := afero.ReadFile(fs, fmt.Sprintf("/proc/%d/oom_score_adj", proc.PID)) - require.NoError(t, err) - require.Equal(t, fmt.Sprintf("%d", expectedScore), strings.TrimSpace(string(actualScore))) - }) - t.Run("SetNiceness", func(t *testing.T) { t.Parallel() diff --git a/agent/agentproc/proc_unix.go b/agent/agentproc/proc_unix.go index 6658c8f43a62e..f52caed52ee33 100644 --- a/agent/agentproc/proc_unix.go +++ b/agent/agentproc/proc_unix.go @@ -54,7 +54,6 @@ func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) { PID: int32(pid), CmdLine: string(cmdline), Dir: filepath.Join(defaultProcDir, entry), - FS: fs, }) } @@ -85,20 +84,6 @@ func isProcessExist(syscaller Syscaller, pid int32) (bool, error) { return false, xerrors.Errorf("kill: %w", err) } -func (p *Process) SetOOMAdj(score int) error { - path := filepath.Join(p.Dir, "oom_score_adj") - err := afero.WriteFile(p.FS, - path, - []byte(strconv.Itoa(score)), - 0o644, - ) - if err != nil { - return xerrors.Errorf("write %q: %w", path, err) - } - - return nil -} - func (p *Process) Niceness(sc Syscaller) (int, error) { nice, err := sc.GetPriority(p.PID) if err != nil { diff --git a/agent/agentproc/syscaller.go b/agent/agentproc/syscaller.go index cf2c90d7ec77d..1cd6640e36b43 100644 --- a/agent/agentproc/syscaller.go +++ b/agent/agentproc/syscaller.go @@ -2,8 +2,6 @@ package agentproc import ( "syscall" - - "github.com/spf13/afero" ) type Syscaller interface { @@ -18,5 +16,4 @@ type Process struct { Dir string CmdLine string PID int32 - FS afero.Fs } diff --git a/agent/out.txt b/agent/out.txt deleted file mode 100644 index 7b435abf43cb0..0000000000000 --- a/agent/out.txt +++ /dev/null @@ -1,167 +0,0 @@ -=== RUN TestAgent_ManageProcessPriority -=== PAUSE TestAgent_ManageProcessPriority -=== CONT TestAgent_ManageProcessPriority -=== RUN TestAgent_ManageProcessPriority/OK -=== PAUSE TestAgent_ManageProcessPriority/OK -=== CONT TestAgent_ManageProcessPriority/OK -YEAH -YUP - t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: get service banner - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) tun device - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) OS network configurator - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) DNS configurator - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: dns: using dns.noopManager - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: link state: interfaces.State{defaultRoute=eth0 ifs={docker0:[172.17.0.1/16] eth0:[172.20.0.7/16]} v4=true v6=false} - t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: post startup req={"version":"v0.0.0-devel","expanded_directory":"","subsystems":null} - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP read buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP write buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP read buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP write buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] couldn't create raw v4 disco listener, using regular listener instead: raw disco listening disabled, SO_MARK unavailable - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] couldn't create raw v6 disco listener, using regular listener instead: raw disco listening disabled, SO_MARK unavailable - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: disco key = d:9bf350ad1946674e - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: Creating WireGuard device... - t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: post lifecycle req={"state":"starting","changed_at":"2023-09-14T22:21:04.162755Z"} - t.go:85: 2023-09-14 22:21:04.163 [debu] agent.client: post lifecycle req={"state":"ready","changed_at":"2023-09-14T22:21:04.162813Z"} - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: Bringing WireGuard device up... - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] UDP bind has been updated - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Interface state was Down, requested Up, now Up - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: Bringing router up... - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - started - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - started - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: [v1] warning: fakeRouter.Up: not implemented. - t.go:85: 2023-09-14 22:21:04.164 [debu] client.net.wgengine: Clearing router settings... - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming receiveDERP - started - t.go:85: 2023-09-14 22:21:04.164 [debu] client.net.wgengine: [v1] warning: fakeRouter.Set: not implemented. - t.go:85: 2023-09-14 22:21:04.165 [debu] client.net.wgengine: Starting network monitor... - t.go:85: 2023-09-14 22:21:04.166 [debu] client.net.wgengine: Engine created. - t.go:85: 2023-09-14 22:21:04.166 [debu] client: magicsock debug logging disabled, use CODER_MAGICSOCK_DEBUG_LOGGING=true to enable - t.go:85: 2023-09-14 22:21:04.166 [debu] client.net.wgengine: magicsock: SetPrivateKey called (init) - t.go:85: 2023-09-14 22:21:04.170 [debu] client: updating network map - t.go:85: 2023-09-14 22:21:04.170 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 0 peers - t.go:85: 2023-09-14 22:21:04.170 [debu] coordinating client client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.170 [debu] client: skipped sending node; no PreferredDERP node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.170813 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:0 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[]}" - t.go:85: 2023-09-14 22:21:04.170 [debu] multiagent node doesn't exist client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.170 [debu] coordinating agent agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.171 [debu] agent node doesn't exist client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.171 [debu] wrote initial client(s) to agent agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes=[] - t.go:85: 2023-09-14 22:21:04.171 [debu] added agent socket agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.171 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer - t.go:85: 2023-09-14 22:21:04.171 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer - t.go:85: 2023-09-14 22:21:04.171 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes=[] - t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (6f93c36c4883ce76647c9e74) in 4.705998ms - t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (3d5303cc01a8f9a035b236a4) in 4.76389ms - t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (a41d11ea5ec51a69012804ae) in 4.792004ms - t.go:85: 2023-09-14 22:21:04.221 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer - t.go:85: 2023-09-14 22:21:04.221 [debu] client.net.wgengine: netcheck: [v1] measuring ICMP latency of test (1): listen ip4:icmp 0.0.0.0: socket: operation not permitted - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: netcheck: [v1] netcheck: measuring HTTP(S) latency of test (1): unexpected status code: 426 (426 Upgrade Required) - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: netcheck: [v1] report: udp=true v6=false v6os=false mapvarydest=false hair= portmap= v4a=127.0.0.1:39073 derp=0 - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: home is now derp-1 (test) - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: adding connection to derp-1 for home-keep-alive - t.go:85: 2023-09-14 22:21:04.228 [debu] client: netinfo callback netinfo="NetInfo{varies=false hairpin= ipv6=false ipv6os=false udp=true icmpv4=false derp=#1 portmap= link=\"\"}" - t.go:85: 2023-09-14 22:21:04.228 [debu] client: sending node node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.228454 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[]}" - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: 1 active derp conns: derp-1=cr0s,wr0s - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: endpoints changed: 127.0.0.1:39073 (stun), 172.17.0.1:39073 (local), 172.20.0.7:39073 (local) - t.go:85: 2023-09-14 22:21:04.228 [debu] got client node update client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 node={"id":1384268018767542531,"as_of":"2023-09-14T22:21:04.228454Z","key":"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24","disco":"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"allowed_ips":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"endpoints":[]} - t.go:85: 2023-09-14 22:21:04.228 [debu] enqueued node to agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_ids=[4db2658f-194a-4b75-9608-1b6b42d6ba8a] - t.go:85: 2023-09-14 22:21:04.228 [debu] client: wireguard status status="&{AsOf:2023-09-14 22:21:04.228713945 +0000 UTC m=+0.212205983 Peers:[] LocalAddrs:[{Addr:127.0.0.1:39073 Type:stun} {Addr:172.17.0.1:39073 Type:local} {Addr:172.20.0.7:39073 Type:local}] DERPs:1}" error= - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: derphttp.Client.Connect: connecting to derp-1 (test) - t.go:85: 2023-09-14 22:21:04.228 [debu] client: wireguard status status="&{AsOf:2023-09-14 22:21:04.228760465 +0000 UTC m=+0.212252503 Peers:[] LocalAddrs:[{Addr:127.0.0.1:39073 Type:stun} {Addr:172.17.0.1:39073 Type:local} {Addr:172.20.0.7:39073 Type:local}] DERPs:1}" error= - t.go:85: 2023-09-14 22:21:04.228 [debu] decoded agent node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a node={"id":7635823020709856630,"as_of":"2023-09-14T22:21:04.228767Z","key":"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d","disco":"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"allowed_ips":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"endpoints":[]} - t.go:85: 2023-09-14 22:21:04.228 [debu] client: sending node node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.228869 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[127.0.0.1:39073 172.17.0.1:39073 172.20.0.7:39073]}" - t.go:85: 2023-09-14 22:21:04.228 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":1384268018767542531,\"as_of\":\"2023-09-14T22:21:04.228454Z\",\"key\":\"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24\",\"disco\":\"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"endpoints\":[]}]" - t.go:85: 2023-09-14 22:21:04.228 [debu] enqueued agent node to client agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 - t.go:85: 2023-09-14 22:21:04.229 [debu] got client node update client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 node={"id":1384268018767542531,"as_of":"2023-09-14T22:21:04.228869Z","key":"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24","disco":"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"allowed_ips":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"endpoints":["127.0.0.1:39073","172.17.0.1:39073","172.20.0.7:39073"]} - t.go:85: 2023-09-14 22:21:04.229 [debu] enqueued node to agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_ids=[4db2658f-194a-4b75-9608-1b6b42d6ba8a] - t.go:85: 2023-09-14 22:21:04.229 [debu] client: adding node node="&{ID:nodeid:69f7e494bd294576 AsOf:2023-09-14 22:21:04.228767 +0000 UTC Key:nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d DiscoKey:discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] AllowedIPs:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] Endpoints:[]}" - t.go:85: 2023-09-14 22:21:04.229 [debu] wrote nodes client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":7635823020709856630,\"as_of\":\"2023-09-14T22:21:04.228767Z\",\"key\":\"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d\",\"disco\":\"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"endpoints\":[]}]" - t.go:85: 2023-09-14 22:21:04.229 [debu] client: updating network map - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 1 peers - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.netstack: [v2] netstack: registered IP fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128 - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wgengine: Reconfig: configuring userspace WireGuard config (with 1/1 peers) - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] UAPI: Updating private key - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Created - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (d1abaaf30fcfbb0a2e30af00) in 68.382µs - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Updating endpoint - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (bbe130a507ad56b3f11e5620) in 104.803µs - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Removing all allowedips - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (f3e4b4d721a0986d001d0c52) in 129.58µs - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Adding allowedip - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Adding allowedip - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Updating persistent keepalive interval - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - Starting - t.go:85: 2023-09-14 22:21:04.229 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":1384268018767542531,\"as_of\":\"2023-09-14T22:21:04.228869Z\",\"key\":\"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24\",\"disco\":\"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"endpoints\":[\"127.0.0.1:39073\",\"172.17.0.1:39073\",\"172.20.0.7:39073\"]}]" - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wgengine: Reconfig: configuring router - t.go:85: 2023-09-14 22:21:04.229 [debu] decoded agent node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a node={"id":7635823020709856630,"as_of":"2023-09-14T22:21:04.229132Z","key":"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d","disco":"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"allowed_ips":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"endpoints":["127.0.0.1:32901","172.17.0.1:32901","172.20.0.7:32901"]} - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] warning: fakeRouter.Set: not implemented. - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: wgengine: Reconfig: configuring DNS - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: Set: {DefaultResolvers:[] Routes:{} SearchDomains:[] Hosts:0} - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: Resolvercfg: {Routes:{} Hosts:0 LocalDomains:[]} - t.go:85: 2023-09-14 22:21:04.230 [debu] enqueued agent node to client agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: OScfg: {Nameservers:[] SearchDomains:[] MatchDomains:[] Hosts:[]} - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] wgengine: Reconfig done - t.go:85: 2023-09-14 22:21:04.230 [debu] wrote nodes client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":7635823020709856630,\"as_of\":\"2023-09-14T22:21:04.229132Z\",\"key\":\"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d\",\"disco\":\"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"endpoints\":[\"127.0.0.1:32901\",\"172.17.0.1:32901\",\"172.20.0.7:32901\"]}]" - t.go:85: 2023-09-14 22:21:04.230 [debu] client: adding node node="&{ID:nodeid:69f7e494bd294576 AsOf:2023-09-14 22:21:04.229132 +0000 UTC Key:nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d DiscoKey:discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] AllowedIPs:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] Endpoints:[127.0.0.1:32901 172.17.0.1:32901 172.20.0.7:32901]}" - t.go:85: 2023-09-14 22:21:04.230 [debu] client: updating network map - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 1 peers - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] wgengine: Reconfig done - t.go:85: 2023-09-14 22:21:04.232 [debu] client.net.wgengine: magicsock: derp-1 connected; connGen=1 - t.go:85: 2023-09-14 22:21:04.280 [debu] client.net.wgengine: netcheck: [v1] measuring ICMP latency of test (1): listen ip4:icmp 0.0.0.0: socket: operation not permitted - t.go:85: 2023-09-14 22:21:04.286 [debu] client.net.wgengine: netcheck: [v1] netcheck: measuring HTTP(S) latency of test (1): unexpected status code: 426 (426 Upgrade Required) - t.go:85: 2023-09-14 22:21:04.286 [debu] client.net.wgengine: netcheck: [v1] report: udp=true v6=false v6os=false mapvarydest=false hair= portmap= v4a=127.0.0.1:39073 derp=0 - t.go:85: 2023-09-14 22:21:04.296 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): sending disco ping to [Vs9bR] ... - t.go:85: 2023-09-14 22:21:04.296 [debu] client.net.wgengine: [v1] magicsock: derp route for [Vs9bR] set to derp-1 (shared home) - t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using 172.17.0.1:32901 - t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using 127.0.0.1:32901 - agent_test.go:2463: - Error Trace: /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2463 - Error: "[]" should have 4 item(s), but has 0 - Test: TestAgent_ManageProcessPriority/OK - t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using DERP only (reset) - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: magicsock: closing connection to derp-1 (conn-close), age 0s - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: magicsock: 0 active derp conns - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: [v1] warning: fakeRouter.Close: not implemented. - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Device closing - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - stopped - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - stopped - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming receiveDERP - stopped - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - Stopping - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Device closed - t.go:85: 2023-09-14 22:21:04.298 [debu] unable to read agent update, connection may be closed agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a ... - error= read json: - github.com/coder/coder/v2/tailnet.(*coordinator).handleNextAgentMessage - /home/coder/go/src/github.com/coder/coder/tailnet/coordinator.go:552 - - io: read/write on closed pipe - t.go:85: 2023-09-14 22:21:04.298 [debu] deleted agent socket and node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.298 [debu] done sending updates agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.298 [debu] unable to read client update, connection may be closed client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a ... - error= read json: - github.com/coder/coder/v2/tailnet.(*coordinator).handleNextClientMessage - /home/coder/go/src/github.com/coder/coder/tailnet/coordinator.go:350 - - EOF - t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client node client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 - t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client connectionSocket from map client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.299 [debu] deleted last client connectionSocket from map client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 - t.go:85: 2023-09-14 22:21:04.299 [debu] done sending updates client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.299 [debu] agent.client: post lifecycle req={"state":"shutting_down","changed_at":"2023-09-14T22:21:04.299336Z"} - t.go:85: 2023-09-14 22:21:04.499 [debu] agent.client: get service banner - t.go:85: 2023-09-14 22:21:04.500 [debu] agent.client: post startup req={"version":"v0.0.0-devel","expanded_directory":"","subsystems":null} - t.go:85: 2023-09-14 22:21:06.300 [debu] agent.client: post lifecycle req={"state":"off","changed_at":"2023-09-14T22:21:04.299369Z"} - stuntest.go:63: STUN server shutdown - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 0 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1965978598 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1248996486 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1788987366 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1965978598 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1248996486 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1788987366 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1965978598 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1248996486 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1788987366 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 - controller.go:137: aborting test due to missing call(s) ---- FAIL: TestAgent_ManageProcessPriority (0.00s) - --- FAIL: TestAgent_ManageProcessPriority/OK (2.26s) -FAIL -exit status 1 -FAIL github.com/coder/coder/v2/agent 2.294s From 04ee5cb5b27e4b3cf2e904931fccac8e248f513d Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 15 Sep 2023 00:18:34 +0000 Subject: [PATCH 20/21] defer first --- agent/agent.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 9babab81d8475..d39543324539f 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1282,15 +1282,6 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { var prioritizedProcs = []string{"coder agent"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { - if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { - a.logger.Debug(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", - slog.F("env_var", EnvProcPrioMgmt), - slog.F("value", val), - slog.F("goos", runtime.GOOS), - ) - return - } - defer func() { if r := recover(); r != nil { a.logger.Critical(ctx, "recovered from panic", @@ -1300,6 +1291,15 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { } }() + if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { + a.logger.Debug(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", + slog.F("env_var", EnvProcPrioMgmt), + slog.F("value", val), + slog.F("goos", runtime.GOOS), + ) + return + } + if a.processManagementTick == nil { ticker := time.NewTicker(time.Second) defer ticker.Stop() From ffbeab963fbd392f2e3701eec37382c6e3b8affb Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 15 Sep 2023 00:21:52 +0000 Subject: [PATCH 21/21] avoid resetting niceness for already niced procs --- agent/agent.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index d39543324539f..cdab56d935ce3 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1369,10 +1369,12 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process // so we don't override user nice values. // Getpriority actually returns priority for the nice value // which is niceness + 20, so here 20 = a niceness of 0 (aka unset). - if score != 20 && score != niceness { - logger.Debug(ctx, "skipping process due to custom niceness", - slog.F("niceness", score), - ) + if score != 20 { + if score != niceness { + logger.Debug(ctx, "skipping process due to custom niceness", + slog.F("niceness", score), + ) + } continue } pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy