diff --git a/agent/agent.go b/agent/agent.go
index c2e2670a41257..f755587e793ec 100644
--- a/agent/agent.go
+++ b/agent/agent.go
@@ -64,6 +64,7 @@ type Options struct {
SSHMaxTimeout time.Duration
TailnetListenPort uint16
Subsystem codersdk.AgentSubsystem
+ Addresses []netip.Prefix
PrometheusRegistry *prometheus.Registry
}
@@ -132,6 +133,7 @@ func New(options Options) Agent {
connStatsChan: make(chan *agentsdk.Stats, 1),
sshMaxTimeout: options.SSHMaxTimeout,
subsystem: options.Subsystem,
+ addresses: options.Addresses,
prometheusRegistry: prometheusRegistry,
metrics: newAgentMetrics(prometheusRegistry),
@@ -177,6 +179,7 @@ type agent struct {
lifecycleStates []agentsdk.PostLifecycleRequest
network *tailnet.Conn
+ addresses []netip.Prefix
connStatsChan chan *agentsdk.Stats
latestStat atomic.Pointer[agentsdk.Stats]
@@ -545,6 +548,10 @@ func (a *agent) run(ctx context.Context) error {
}
a.logger.Info(ctx, "fetched manifest", slog.F("manifest", manifest))
+ if manifest.AgentID == uuid.Nil {
+ return xerrors.New("nil agentID returned by manifest")
+ }
+
// Expand the directory and send it back to coderd so external
// applications that rely on the directory can use it.
//
@@ -630,7 +637,7 @@ func (a *agent) run(ctx context.Context) error {
network := a.network
a.closeMutex.Unlock()
if network == nil {
- network, err = a.createTailnet(ctx, manifest.DERPMap, manifest.DisableDirectConnections)
+ network, err = a.createTailnet(ctx, manifest.AgentID, manifest.DERPMap, manifest.DisableDirectConnections)
if err != nil {
return xerrors.Errorf("create tailnet: %w", err)
}
@@ -648,6 +655,11 @@ func (a *agent) run(ctx context.Context) error {
a.startReportingConnectionStats(ctx)
} else {
+ // Update the wireguard IPs if the agent ID changed.
+ err := network.SetAddresses(a.wireguardAddresses(manifest.AgentID))
+ if err != nil {
+ a.logger.Error(ctx, "update tailnet addresses", slog.Error(err))
+ }
// Update the DERP map and allow/disallow direct connections.
network.SetDERPMap(manifest.DERPMap)
network.SetBlockEndpoints(manifest.DisableDirectConnections)
@@ -661,6 +673,20 @@ func (a *agent) run(ctx context.Context) error {
return nil
}
+func (a *agent) wireguardAddresses(agentID uuid.UUID) []netip.Prefix {
+ if len(a.addresses) == 0 {
+ return []netip.Prefix{
+ // This is the IP that should be used primarily.
+ netip.PrefixFrom(tailnet.IPFromUUID(agentID), 128),
+ // We also listen on the legacy codersdk.WorkspaceAgentIP. This
+ // allows for a transition away from wsconncache.
+ netip.PrefixFrom(codersdk.WorkspaceAgentIP, 128),
+ }
+ }
+
+ return a.addresses
+}
+
func (a *agent) trackConnGoroutine(fn func()) error {
a.closeMutex.Lock()
defer a.closeMutex.Unlock()
@@ -675,9 +701,9 @@ func (a *agent) trackConnGoroutine(fn func()) error {
return nil
}
-func (a *agent) createTailnet(ctx context.Context, derpMap *tailcfg.DERPMap, disableDirectConnections bool) (_ *tailnet.Conn, err error) {
+func (a *agent) createTailnet(ctx context.Context, agentID uuid.UUID, derpMap *tailcfg.DERPMap, disableDirectConnections bool) (_ *tailnet.Conn, err error) {
network, err := tailnet.NewConn(&tailnet.Options{
- Addresses: []netip.Prefix{netip.PrefixFrom(codersdk.WorkspaceAgentIP, 128)},
+ Addresses: a.wireguardAddresses(agentID),
DERPMap: derpMap,
Logger: a.logger.Named("tailnet"),
ListenPort: a.tailnetListenPort,
diff --git a/agent/agent_test.go b/agent/agent_test.go
index 8ac7eca050af9..e9b1f485f718a 100644
--- a/agent/agent_test.go
+++ b/agent/agent_test.go
@@ -35,7 +35,6 @@ import (
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"golang.org/x/crypto/ssh"
- "golang.org/x/exp/maps"
"golang.org/x/exp/slices"
"golang.org/x/xerrors"
"tailscale.com/net/speedtest"
@@ -45,6 +44,7 @@ import (
"cdr.dev/slog/sloggers/slogtest"
"github.com/coder/coder/agent"
"github.com/coder/coder/agent/agentssh"
+ "github.com/coder/coder/agent/agenttest"
"github.com/coder/coder/coderd/httpapi"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/codersdk/agentsdk"
@@ -67,7 +67,7 @@ func TestAgent_Stats_SSH(t *testing.T) {
defer cancel()
//nolint:dogsled
- conn, _, stats, _, _ := setupAgent(t, &client{}, 0)
+ conn, _, stats, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
sshClient, err := conn.SSHClient(ctx)
require.NoError(t, err)
@@ -100,7 +100,7 @@ func TestAgent_Stats_ReconnectingPTY(t *testing.T) {
defer cancel()
//nolint:dogsled
- conn, _, stats, _, _ := setupAgent(t, &client{}, 0)
+ conn, _, stats, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
ptyConn, err := conn.ReconnectingPTY(ctx, uuid.New(), 128, 128, "/bin/bash")
require.NoError(t, err)
@@ -130,7 +130,7 @@ func TestAgent_Stats_Magic(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
//nolint:dogsled
- conn, _, _, _, _ := setupAgent(t, &client{}, 0)
+ conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
sshClient, err := conn.SSHClient(ctx)
require.NoError(t, err)
defer sshClient.Close()
@@ -157,7 +157,7 @@ func TestAgent_Stats_Magic(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
//nolint:dogsled
- conn, _, stats, _, _ := setupAgent(t, &client{}, 0)
+ conn, _, stats, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
sshClient, err := conn.SSHClient(ctx)
require.NoError(t, err)
defer sshClient.Close()
@@ -425,20 +425,19 @@ func TestAgent_Session_TTY_MOTD_Update(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
//nolint:dogsled // Allow the blank identifiers.
- conn, client, _, _, _ := setupAgent(t, &client{}, 0)
+ conn, client, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
for _, test := range tests {
test := test
-
+ // Set new banner func and wait for the agent to call it to update the
+ // banner.
ready := make(chan struct{}, 2)
- client.mu.Lock()
- client.getServiceBanner = func() (codersdk.ServiceBannerConfig, error) {
+ client.SetServiceBannerFunc(func() (codersdk.ServiceBannerConfig, error) {
select {
case ready <- struct{}{}:
default:
}
return test.banner, nil
- }
- client.mu.Unlock()
+ })
<-ready
<-ready // Wait for two updates to ensure the value has propagated.
@@ -542,7 +541,7 @@ func TestAgent_Session_TTY_FastCommandHasOutput(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
//nolint:dogsled
- conn, _, _, _, _ := setupAgent(t, &client{}, 0)
+ conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
sshClient, err := conn.SSHClient(ctx)
require.NoError(t, err)
defer sshClient.Close()
@@ -592,7 +591,7 @@ func TestAgent_Session_TTY_HugeOutputIsNotLost(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
//nolint:dogsled
- conn, _, _, _, _ := setupAgent(t, &client{}, 0)
+ conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
sshClient, err := conn.SSHClient(ctx)
require.NoError(t, err)
defer sshClient.Close()
@@ -922,7 +921,7 @@ func TestAgent_SFTP(t *testing.T) {
home = "/" + strings.ReplaceAll(home, "\\", "/")
}
//nolint:dogsled
- conn, _, _, _, _ := setupAgent(t, &client{}, 0)
+ conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
sshClient, err := conn.SSHClient(ctx)
require.NoError(t, err)
defer sshClient.Close()
@@ -954,7 +953,7 @@ func TestAgent_SCP(t *testing.T) {
defer cancel()
//nolint:dogsled
- conn, _, _, _, _ := setupAgent(t, &client{}, 0)
+ conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
sshClient, err := conn.SSHClient(ctx)
require.NoError(t, err)
defer sshClient.Close()
@@ -1062,16 +1061,15 @@ func TestAgent_StartupScript(t *testing.T) {
t.Run("Success", func(t *testing.T) {
t.Parallel()
logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
- client := &client{
- t: t,
- agentID: uuid.New(),
- manifest: agentsdk.Manifest{
+ client := agenttest.NewClient(t,
+ uuid.New(),
+ agentsdk.Manifest{
StartupScript: command,
DERPMap: &tailcfg.DERPMap{},
},
- statsChan: make(chan *agentsdk.Stats),
- coordinator: tailnet.NewCoordinator(logger),
- }
+ make(chan *agentsdk.Stats),
+ tailnet.NewCoordinator(logger),
+ )
closer := agent.New(agent.Options{
Client: client,
Filesystem: afero.NewMemMapFs(),
@@ -1082,36 +1080,35 @@ func TestAgent_StartupScript(t *testing.T) {
_ = closer.Close()
})
assert.Eventually(t, func() bool {
- got := client.getLifecycleStates()
+ got := client.GetLifecycleStates()
return len(got) > 0 && got[len(got)-1] == codersdk.WorkspaceAgentLifecycleReady
}, testutil.WaitShort, testutil.IntervalMedium)
- require.Len(t, client.getStartupLogs(), 1)
- require.Equal(t, output, client.getStartupLogs()[0].Output)
+ require.Len(t, client.GetStartupLogs(), 1)
+ require.Equal(t, output, client.GetStartupLogs()[0].Output)
})
// This ensures that even when coderd sends back that the startup
// script has written too many lines it will still succeed!
t.Run("OverflowsAndSkips", func(t *testing.T) {
t.Parallel()
logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
- client := &client{
- t: t,
- agentID: uuid.New(),
- manifest: agentsdk.Manifest{
+ client := agenttest.NewClient(t,
+ uuid.New(),
+ agentsdk.Manifest{
StartupScript: command,
DERPMap: &tailcfg.DERPMap{},
},
- patchWorkspaceLogs: func() error {
- resp := httptest.NewRecorder()
- httpapi.Write(context.Background(), resp, http.StatusRequestEntityTooLarge, codersdk.Response{
- Message: "Too many lines!",
- })
- res := resp.Result()
- defer res.Body.Close()
- return codersdk.ReadBodyAsError(res)
- },
- statsChan: make(chan *agentsdk.Stats),
- coordinator: tailnet.NewCoordinator(logger),
+ make(chan *agentsdk.Stats, 50),
+ tailnet.NewCoordinator(logger),
+ )
+ client.PatchWorkspaceLogs = func() error {
+ resp := httptest.NewRecorder()
+ httpapi.Write(context.Background(), resp, http.StatusRequestEntityTooLarge, codersdk.Response{
+ Message: "Too many lines!",
+ })
+ res := resp.Result()
+ defer res.Body.Close()
+ return codersdk.ReadBodyAsError(res)
}
closer := agent.New(agent.Options{
Client: client,
@@ -1123,10 +1120,10 @@ func TestAgent_StartupScript(t *testing.T) {
_ = closer.Close()
})
assert.Eventually(t, func() bool {
- got := client.getLifecycleStates()
+ got := client.GetLifecycleStates()
return len(got) > 0 && got[len(got)-1] == codersdk.WorkspaceAgentLifecycleReady
}, testutil.WaitShort, testutil.IntervalMedium)
- require.Len(t, client.getStartupLogs(), 0)
+ require.Len(t, client.GetStartupLogs(), 0)
})
}
@@ -1138,28 +1135,26 @@ func TestAgent_Metadata(t *testing.T) {
t.Run("Once", func(t *testing.T) {
t.Parallel()
//nolint:dogsled
- _, client, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- Metadata: []codersdk.WorkspaceAgentMetadataDescription{
- {
- Key: "greeting",
- Interval: 0,
- Script: echoHello,
- },
+ _, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ Metadata: []codersdk.WorkspaceAgentMetadataDescription{
+ {
+ Key: "greeting",
+ Interval: 0,
+ Script: echoHello,
},
},
}, 0)
var gotMd map[string]agentsdk.PostMetadataRequest
require.Eventually(t, func() bool {
- gotMd = client.getMetadata()
+ gotMd = client.GetMetadata()
return len(gotMd) == 1
}, testutil.WaitShort, testutil.IntervalMedium)
collectedAt := gotMd["greeting"].CollectedAt
require.Never(t, func() bool {
- gotMd = client.getMetadata()
+ gotMd = client.GetMetadata()
if len(gotMd) != 1 {
panic("unexpected number of metadata")
}
@@ -1170,22 +1165,20 @@ func TestAgent_Metadata(t *testing.T) {
t.Run("Many", func(t *testing.T) {
t.Parallel()
//nolint:dogsled
- _, client, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- Metadata: []codersdk.WorkspaceAgentMetadataDescription{
- {
- Key: "greeting",
- Interval: 1,
- Timeout: 100,
- Script: echoHello,
- },
+ _, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ Metadata: []codersdk.WorkspaceAgentMetadataDescription{
+ {
+ Key: "greeting",
+ Interval: 1,
+ Timeout: 100,
+ Script: echoHello,
},
},
}, 0)
var gotMd map[string]agentsdk.PostMetadataRequest
require.Eventually(t, func() bool {
- gotMd = client.getMetadata()
+ gotMd = client.GetMetadata()
return len(gotMd) == 1
}, testutil.WaitShort, testutil.IntervalMedium)
@@ -1195,7 +1188,7 @@ func TestAgent_Metadata(t *testing.T) {
}
if !assert.Eventually(t, func() bool {
- gotMd = client.getMetadata()
+ gotMd = client.GetMetadata()
return gotMd["greeting"].CollectedAt.After(collectedAt1)
}, testutil.WaitShort, testutil.IntervalMedium) {
t.Fatalf("expected metadata to be collected again")
@@ -1221,29 +1214,27 @@ func TestAgentMetadata_Timing(t *testing.T) {
script = "echo hello | tee -a " + greetingPath
)
//nolint:dogsled
- _, client, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- Metadata: []codersdk.WorkspaceAgentMetadataDescription{
- {
- Key: "greeting",
- Interval: reportInterval,
- Script: script,
- },
- {
- Key: "bad",
- Interval: reportInterval,
- Script: "exit 1",
- },
+ _, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ Metadata: []codersdk.WorkspaceAgentMetadataDescription{
+ {
+ Key: "greeting",
+ Interval: reportInterval,
+ Script: script,
+ },
+ {
+ Key: "bad",
+ Interval: reportInterval,
+ Script: "exit 1",
},
},
}, 0)
require.Eventually(t, func() bool {
- return len(client.getMetadata()) == 2
+ return len(client.GetMetadata()) == 2
}, testutil.WaitShort, testutil.IntervalMedium)
for start := time.Now(); time.Since(start) < testutil.WaitMedium; time.Sleep(testutil.IntervalMedium) {
- md := client.getMetadata()
+ md := client.GetMetadata()
require.Len(t, md, 2, "got: %+v", md)
require.Equal(t, "hello\n", md["greeting"].Value)
@@ -1285,11 +1276,9 @@ func TestAgent_Lifecycle(t *testing.T) {
t.Run("StartTimeout", func(t *testing.T) {
t.Parallel()
- _, client, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- StartupScript: "sleep 3",
- StartupScriptTimeout: time.Nanosecond,
- },
+ _, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ StartupScript: "sleep 3",
+ StartupScriptTimeout: time.Nanosecond,
}, 0)
want := []codersdk.WorkspaceAgentLifecycle{
@@ -1299,7 +1288,7 @@ func TestAgent_Lifecycle(t *testing.T) {
var got []codersdk.WorkspaceAgentLifecycle
assert.Eventually(t, func() bool {
- got = client.getLifecycleStates()
+ got = client.GetLifecycleStates()
return slices.Contains(got, want[len(want)-1])
}, testutil.WaitShort, testutil.IntervalMedium)
@@ -1309,11 +1298,9 @@ func TestAgent_Lifecycle(t *testing.T) {
t.Run("StartError", func(t *testing.T) {
t.Parallel()
- _, client, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- StartupScript: "false",
- StartupScriptTimeout: 30 * time.Second,
- },
+ _, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ StartupScript: "false",
+ StartupScriptTimeout: 30 * time.Second,
}, 0)
want := []codersdk.WorkspaceAgentLifecycle{
@@ -1323,7 +1310,7 @@ func TestAgent_Lifecycle(t *testing.T) {
var got []codersdk.WorkspaceAgentLifecycle
assert.Eventually(t, func() bool {
- got = client.getLifecycleStates()
+ got = client.GetLifecycleStates()
return slices.Contains(got, want[len(want)-1])
}, testutil.WaitShort, testutil.IntervalMedium)
@@ -1333,11 +1320,9 @@ func TestAgent_Lifecycle(t *testing.T) {
t.Run("Ready", func(t *testing.T) {
t.Parallel()
- _, client, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- StartupScript: "true",
- StartupScriptTimeout: 30 * time.Second,
- },
+ _, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ StartupScript: "true",
+ StartupScriptTimeout: 30 * time.Second,
}, 0)
want := []codersdk.WorkspaceAgentLifecycle{
@@ -1347,7 +1332,7 @@ func TestAgent_Lifecycle(t *testing.T) {
var got []codersdk.WorkspaceAgentLifecycle
assert.Eventually(t, func() bool {
- got = client.getLifecycleStates()
+ got = client.GetLifecycleStates()
return len(got) > 0 && got[len(got)-1] == want[len(want)-1]
}, testutil.WaitShort, testutil.IntervalMedium)
@@ -1357,15 +1342,13 @@ func TestAgent_Lifecycle(t *testing.T) {
t.Run("ShuttingDown", func(t *testing.T) {
t.Parallel()
- _, client, _, _, closer := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- ShutdownScript: "sleep 3",
- StartupScriptTimeout: 30 * time.Second,
- },
+ _, client, _, _, closer := setupAgent(t, agentsdk.Manifest{
+ ShutdownScript: "sleep 3",
+ StartupScriptTimeout: 30 * time.Second,
}, 0)
assert.Eventually(t, func() bool {
- return slices.Contains(client.getLifecycleStates(), codersdk.WorkspaceAgentLifecycleReady)
+ return slices.Contains(client.GetLifecycleStates(), codersdk.WorkspaceAgentLifecycleReady)
}, testutil.WaitShort, testutil.IntervalMedium)
// Start close asynchronously so that we an inspect the state.
@@ -1387,7 +1370,7 @@ func TestAgent_Lifecycle(t *testing.T) {
var got []codersdk.WorkspaceAgentLifecycle
assert.Eventually(t, func() bool {
- got = client.getLifecycleStates()
+ got = client.GetLifecycleStates()
return slices.Contains(got, want[len(want)-1])
}, testutil.WaitShort, testutil.IntervalMedium)
@@ -1397,15 +1380,13 @@ func TestAgent_Lifecycle(t *testing.T) {
t.Run("ShutdownTimeout", func(t *testing.T) {
t.Parallel()
- _, client, _, _, closer := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- ShutdownScript: "sleep 3",
- ShutdownScriptTimeout: time.Nanosecond,
- },
+ _, client, _, _, closer := setupAgent(t, agentsdk.Manifest{
+ ShutdownScript: "sleep 3",
+ ShutdownScriptTimeout: time.Nanosecond,
}, 0)
assert.Eventually(t, func() bool {
- return slices.Contains(client.getLifecycleStates(), codersdk.WorkspaceAgentLifecycleReady)
+ return slices.Contains(client.GetLifecycleStates(), codersdk.WorkspaceAgentLifecycleReady)
}, testutil.WaitShort, testutil.IntervalMedium)
// Start close asynchronously so that we an inspect the state.
@@ -1428,7 +1409,7 @@ func TestAgent_Lifecycle(t *testing.T) {
var got []codersdk.WorkspaceAgentLifecycle
assert.Eventually(t, func() bool {
- got = client.getLifecycleStates()
+ got = client.GetLifecycleStates()
return slices.Contains(got, want[len(want)-1])
}, testutil.WaitShort, testutil.IntervalMedium)
@@ -1438,15 +1419,13 @@ func TestAgent_Lifecycle(t *testing.T) {
t.Run("ShutdownError", func(t *testing.T) {
t.Parallel()
- _, client, _, _, closer := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- ShutdownScript: "false",
- ShutdownScriptTimeout: 30 * time.Second,
- },
+ _, client, _, _, closer := setupAgent(t, agentsdk.Manifest{
+ ShutdownScript: "false",
+ ShutdownScriptTimeout: 30 * time.Second,
}, 0)
assert.Eventually(t, func() bool {
- return slices.Contains(client.getLifecycleStates(), codersdk.WorkspaceAgentLifecycleReady)
+ return slices.Contains(client.GetLifecycleStates(), codersdk.WorkspaceAgentLifecycleReady)
}, testutil.WaitShort, testutil.IntervalMedium)
// Start close asynchronously so that we an inspect the state.
@@ -1469,7 +1448,7 @@ func TestAgent_Lifecycle(t *testing.T) {
var got []codersdk.WorkspaceAgentLifecycle
assert.Eventually(t, func() bool {
- got = client.getLifecycleStates()
+ got = client.GetLifecycleStates()
return slices.Contains(got, want[len(want)-1])
}, testutil.WaitShort, testutil.IntervalMedium)
@@ -1480,17 +1459,18 @@ func TestAgent_Lifecycle(t *testing.T) {
t.Parallel()
logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
expected := "this-is-shutdown"
- client := &client{
- t: t,
- agentID: uuid.New(),
- manifest: agentsdk.Manifest{
- DERPMap: tailnettest.RunDERPAndSTUN(t),
+ derpMap, _ := tailnettest.RunDERPAndSTUN(t)
+
+ client := agenttest.NewClient(t,
+ uuid.New(),
+ agentsdk.Manifest{
+ DERPMap: derpMap,
StartupScript: "echo 1",
ShutdownScript: "echo " + expected,
},
- statsChan: make(chan *agentsdk.Stats),
- coordinator: tailnet.NewCoordinator(logger),
- }
+ make(chan *agentsdk.Stats, 50),
+ tailnet.NewCoordinator(logger),
+ )
fs := afero.NewMemMapFs()
agent := agent.New(agent.Options{
@@ -1536,71 +1516,63 @@ func TestAgent_Startup(t *testing.T) {
t.Run("EmptyDirectory", func(t *testing.T) {
t.Parallel()
- _, client, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- StartupScript: "true",
- StartupScriptTimeout: 30 * time.Second,
- Directory: "",
- },
+ _, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ StartupScript: "true",
+ StartupScriptTimeout: 30 * time.Second,
+ Directory: "",
}, 0)
assert.Eventually(t, func() bool {
- return client.getStartup().Version != ""
+ return client.GetStartup().Version != ""
}, testutil.WaitShort, testutil.IntervalFast)
- require.Equal(t, "", client.getStartup().ExpandedDirectory)
+ require.Equal(t, "", client.GetStartup().ExpandedDirectory)
})
t.Run("HomeDirectory", func(t *testing.T) {
t.Parallel()
- _, client, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- StartupScript: "true",
- StartupScriptTimeout: 30 * time.Second,
- Directory: "~",
- },
+ _, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ StartupScript: "true",
+ StartupScriptTimeout: 30 * time.Second,
+ Directory: "~",
}, 0)
assert.Eventually(t, func() bool {
- return client.getStartup().Version != ""
+ return client.GetStartup().Version != ""
}, testutil.WaitShort, testutil.IntervalFast)
homeDir, err := os.UserHomeDir()
require.NoError(t, err)
- require.Equal(t, homeDir, client.getStartup().ExpandedDirectory)
+ require.Equal(t, homeDir, client.GetStartup().ExpandedDirectory)
})
t.Run("NotAbsoluteDirectory", func(t *testing.T) {
t.Parallel()
- _, client, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- StartupScript: "true",
- StartupScriptTimeout: 30 * time.Second,
- Directory: "coder/coder",
- },
+ _, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ StartupScript: "true",
+ StartupScriptTimeout: 30 * time.Second,
+ Directory: "coder/coder",
}, 0)
assert.Eventually(t, func() bool {
- return client.getStartup().Version != ""
+ return client.GetStartup().Version != ""
}, testutil.WaitShort, testutil.IntervalFast)
homeDir, err := os.UserHomeDir()
require.NoError(t, err)
- require.Equal(t, filepath.Join(homeDir, "coder/coder"), client.getStartup().ExpandedDirectory)
+ require.Equal(t, filepath.Join(homeDir, "coder/coder"), client.GetStartup().ExpandedDirectory)
})
t.Run("HomeEnvironmentVariable", func(t *testing.T) {
t.Parallel()
- _, client, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- StartupScript: "true",
- StartupScriptTimeout: 30 * time.Second,
- Directory: "$HOME",
- },
+ _, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ StartupScript: "true",
+ StartupScriptTimeout: 30 * time.Second,
+ Directory: "$HOME",
}, 0)
assert.Eventually(t, func() bool {
- return client.getStartup().Version != ""
+ return client.GetStartup().Version != ""
}, testutil.WaitShort, testutil.IntervalFast)
homeDir, err := os.UserHomeDir()
require.NoError(t, err)
- require.Equal(t, homeDir, client.getStartup().ExpandedDirectory)
+ require.Equal(t, homeDir, client.GetStartup().ExpandedDirectory)
})
}
@@ -1617,7 +1589,7 @@ func TestAgent_ReconnectingPTY(t *testing.T) {
defer cancel()
//nolint:dogsled
- conn, _, _, _, _ := setupAgent(t, &client{}, 0)
+ conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
id := uuid.New()
netConn, err := conn.ReconnectingPTY(ctx, id, 100, 100, "/bin/bash")
require.NoError(t, err)
@@ -1719,7 +1691,7 @@ func TestAgent_Dial(t *testing.T) {
}()
//nolint:dogsled
- conn, _, _, _, _ := setupAgent(t, &client{}, 0)
+ conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
require.True(t, conn.AwaitReachable(context.Background()))
conn1, err := conn.DialContext(context.Background(), l.Addr().Network(), l.Addr().String())
require.NoError(t, err)
@@ -1739,12 +1711,10 @@ func TestAgent_Speedtest(t *testing.T) {
t.Skip("This test is relatively flakey because of Tailscale's speedtest code...")
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
- derpMap := tailnettest.RunDERPAndSTUN(t)
+ derpMap, _ := tailnettest.RunDERPAndSTUN(t)
//nolint:dogsled
- conn, _, _, _, _ := setupAgent(t, &client{
- manifest: agentsdk.Manifest{
- DERPMap: derpMap,
- },
+ conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{
+ DERPMap: derpMap,
}, 0)
defer conn.Close()
res, err := conn.Speedtest(ctx, speedtest.Upload, 250*time.Millisecond)
@@ -1761,17 +1731,16 @@ func TestAgent_Reconnect(t *testing.T) {
defer coordinator.Close()
agentID := uuid.New()
- statsCh := make(chan *agentsdk.Stats)
- derpMap := tailnettest.RunDERPAndSTUN(t)
- client := &client{
- t: t,
- agentID: agentID,
- manifest: agentsdk.Manifest{
+ statsCh := make(chan *agentsdk.Stats, 50)
+ derpMap, _ := tailnettest.RunDERPAndSTUN(t)
+ client := agenttest.NewClient(t,
+ agentID,
+ agentsdk.Manifest{
DERPMap: derpMap,
},
- statsChan: statsCh,
- coordinator: coordinator,
- }
+ statsCh,
+ coordinator,
+ )
initialized := atomic.Int32{}
closer := agent.New(agent.Options{
ExchangeToken: func(ctx context.Context) (string, error) {
@@ -1786,7 +1755,7 @@ func TestAgent_Reconnect(t *testing.T) {
require.Eventually(t, func() bool {
return coordinator.Node(agentID) != nil
}, testutil.WaitShort, testutil.IntervalFast)
- client.lastWorkspaceAgent()
+ client.LastWorkspaceAgent()
require.Eventually(t, func() bool {
return initialized.Load() == 2
}, testutil.WaitShort, testutil.IntervalFast)
@@ -1798,16 +1767,15 @@ func TestAgent_WriteVSCodeConfigs(t *testing.T) {
coordinator := tailnet.NewCoordinator(logger)
defer coordinator.Close()
- client := &client{
- t: t,
- agentID: uuid.New(),
- manifest: agentsdk.Manifest{
+ client := agenttest.NewClient(t,
+ uuid.New(),
+ agentsdk.Manifest{
GitAuthConfigs: 1,
DERPMap: &tailcfg.DERPMap{},
},
- statsChan: make(chan *agentsdk.Stats),
- coordinator: coordinator,
- }
+ make(chan *agentsdk.Stats, 50),
+ coordinator,
+ )
filesystem := afero.NewMemMapFs()
closer := agent.New(agent.Options{
ExchangeToken: func(ctx context.Context) (string, error) {
@@ -1830,7 +1798,7 @@ func TestAgent_WriteVSCodeConfigs(t *testing.T) {
func setupSSHCommand(t *testing.T, beforeArgs []string, afterArgs []string) (*ptytest.PTYCmd, pty.Process) {
//nolint:dogsled
- agentConn, _, _, _, _ := setupAgent(t, &client{}, 0)
+ agentConn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0)
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
waitGroup := sync.WaitGroup{}
@@ -1883,12 +1851,11 @@ func setupSSHSession(
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
//nolint:dogsled
- conn, _, _, fs, _ := setupAgent(t, &client{
- manifest: options,
- getServiceBanner: func() (codersdk.ServiceBannerConfig, error) {
+ conn, _, _, fs, _ := setupAgent(t, options, 0, func(c *agenttest.Client, _ *agent.Options) {
+ c.SetServiceBannerFunc(func() (codersdk.ServiceBannerConfig, error) {
return serviceBanner, nil
- },
- }, 0)
+ })
+ })
if prepareFS != nil {
prepareFS(fs)
}
@@ -1905,31 +1872,28 @@ func setupSSHSession(
return session
}
-type closeFunc func() error
-
-func (c closeFunc) Close() error {
- return c()
-}
-
-func setupAgent(t *testing.T, c *client, ptyTimeout time.Duration, opts ...func(agent.Options) agent.Options) (
+func setupAgent(t *testing.T, metadata agentsdk.Manifest, ptyTimeout time.Duration, opts ...func(*agenttest.Client, *agent.Options)) (
*codersdk.WorkspaceAgentConn,
- *client,
+ *agenttest.Client,
<-chan *agentsdk.Stats,
afero.Fs,
io.Closer,
) {
- c.t = t
logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
- if c.manifest.DERPMap == nil {
- c.manifest.DERPMap = tailnettest.RunDERPAndSTUN(t)
+ if metadata.DERPMap == nil {
+ metadata.DERPMap, _ = tailnettest.RunDERPAndSTUN(t)
}
- c.coordinator = tailnet.NewCoordinator(logger)
+ if metadata.AgentID == uuid.Nil {
+ metadata.AgentID = uuid.New()
+ }
+ coordinator := tailnet.NewCoordinator(logger)
t.Cleanup(func() {
- _ = c.coordinator.Close()
+ _ = coordinator.Close()
})
- c.agentID = uuid.New()
- c.statsChan = make(chan *agentsdk.Stats, 50)
+ statsCh := make(chan *agentsdk.Stats, 50)
fs := afero.NewMemMapFs()
+ c := agenttest.NewClient(t, metadata.AgentID, metadata, statsCh, coordinator)
+
options := agent.Options{
Client: c,
Filesystem: fs,
@@ -1938,7 +1902,7 @@ func setupAgent(t *testing.T, c *client, ptyTimeout time.Duration, opts ...func(
}
for _, opt := range opts {
- options = opt(options)
+ opt(c, &options)
}
closer := agent.New(options)
@@ -1947,7 +1911,7 @@ func setupAgent(t *testing.T, c *client, ptyTimeout time.Duration, opts ...func(
})
conn, err := tailnet.NewConn(&tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
- DERPMap: c.manifest.DERPMap,
+ DERPMap: metadata.DERPMap,
Logger: logger.Named("client"),
})
require.NoError(t, err)
@@ -1961,15 +1925,15 @@ func setupAgent(t *testing.T, c *client, ptyTimeout time.Duration, opts ...func(
})
go func() {
defer close(serveClientDone)
- c.coordinator.ServeClient(serverConn, uuid.New(), c.agentID)
+ coordinator.ServeClient(serverConn, uuid.New(), metadata.AgentID)
}()
sendNode, _ := tailnet.ServeCoordinator(clientConn, func(node []*tailnet.Node) error {
return conn.UpdateNodes(node, false)
})
conn.SetNodeCallback(sendNode)
- agentConn := &codersdk.WorkspaceAgentConn{
- Conn: conn,
- }
+ agentConn := codersdk.NewWorkspaceAgentConn(conn, codersdk.WorkspaceAgentConnOptions{
+ AgentID: metadata.AgentID,
+ })
t.Cleanup(func() {
_ = agentConn.Close()
})
@@ -1980,7 +1944,7 @@ func setupAgent(t *testing.T, c *client, ptyTimeout time.Duration, opts ...func(
if !agentConn.AwaitReachable(ctx) {
t.Fatal("agent not reachable")
}
- return agentConn, c, c.statsChan, fs, closer
+ return agentConn, c, statsCh, fs, closer
}
var dialTestPayload = []byte("dean-was-here123")
@@ -2043,146 +2007,6 @@ func testSessionOutput(t *testing.T, session *ssh.Session, expected, unexpected
}
}
-type client struct {
- t *testing.T
- agentID uuid.UUID
- manifest agentsdk.Manifest
- metadata map[string]agentsdk.PostMetadataRequest
- statsChan chan *agentsdk.Stats
- coordinator tailnet.Coordinator
- lastWorkspaceAgent func()
- patchWorkspaceLogs func() error
- getServiceBanner func() (codersdk.ServiceBannerConfig, error)
-
- mu sync.Mutex // Protects following.
- lifecycleStates []codersdk.WorkspaceAgentLifecycle
- startup agentsdk.PostStartupRequest
- logs []agentsdk.StartupLog
-}
-
-func (c *client) Manifest(_ context.Context) (agentsdk.Manifest, error) {
- return c.manifest, nil
-}
-
-func (c *client) Listen(_ context.Context) (net.Conn, error) {
- clientConn, serverConn := net.Pipe()
- closed := make(chan struct{})
- c.lastWorkspaceAgent = func() {
- _ = serverConn.Close()
- _ = clientConn.Close()
- <-closed
- }
- c.t.Cleanup(c.lastWorkspaceAgent)
- go func() {
- _ = c.coordinator.ServeAgent(serverConn, c.agentID, "")
- close(closed)
- }()
- return clientConn, nil
-}
-
-func (c *client) ReportStats(ctx context.Context, _ slog.Logger, statsChan <-chan *agentsdk.Stats, setInterval func(time.Duration)) (io.Closer, error) {
- doneCh := make(chan struct{})
- ctx, cancel := context.WithCancel(ctx)
-
- go func() {
- defer close(doneCh)
-
- setInterval(500 * time.Millisecond)
- for {
- select {
- case <-ctx.Done():
- return
- case stat := <-statsChan:
- select {
- case c.statsChan <- stat:
- case <-ctx.Done():
- return
- default:
- // We don't want to send old stats.
- continue
- }
- }
- }
- }()
- return closeFunc(func() error {
- cancel()
- <-doneCh
- close(c.statsChan)
- return nil
- }), nil
-}
-
-func (c *client) getLifecycleStates() []codersdk.WorkspaceAgentLifecycle {
- c.mu.Lock()
- defer c.mu.Unlock()
- return c.lifecycleStates
-}
-
-func (c *client) PostLifecycle(_ context.Context, req agentsdk.PostLifecycleRequest) error {
- c.mu.Lock()
- defer c.mu.Unlock()
- c.lifecycleStates = append(c.lifecycleStates, req.State)
- return nil
-}
-
-func (*client) PostAppHealth(_ context.Context, _ agentsdk.PostAppHealthsRequest) error {
- return nil
-}
-
-func (c *client) getStartup() agentsdk.PostStartupRequest {
- c.mu.Lock()
- defer c.mu.Unlock()
- return c.startup
-}
-
-func (c *client) getMetadata() map[string]agentsdk.PostMetadataRequest {
- c.mu.Lock()
- defer c.mu.Unlock()
- return maps.Clone(c.metadata)
-}
-
-func (c *client) PostMetadata(_ context.Context, key string, req agentsdk.PostMetadataRequest) error {
- c.mu.Lock()
- defer c.mu.Unlock()
- if c.metadata == nil {
- c.metadata = make(map[string]agentsdk.PostMetadataRequest)
- }
- c.metadata[key] = req
- return nil
-}
-
-func (c *client) PostStartup(_ context.Context, startup agentsdk.PostStartupRequest) error {
- c.mu.Lock()
- defer c.mu.Unlock()
- c.startup = startup
- return nil
-}
-
-func (c *client) getStartupLogs() []agentsdk.StartupLog {
- c.mu.Lock()
- defer c.mu.Unlock()
- return c.logs
-}
-
-func (c *client) PatchStartupLogs(_ context.Context, logs agentsdk.PatchStartupLogs) error {
- c.mu.Lock()
- defer c.mu.Unlock()
- if c.patchWorkspaceLogs != nil {
- return c.patchWorkspaceLogs()
- }
- c.logs = append(c.logs, logs.Logs...)
- return nil
-}
-
-func (c *client) GetServiceBanner(_ context.Context) (codersdk.ServiceBannerConfig, error) {
- c.mu.Lock()
- defer c.mu.Unlock()
- if c.getServiceBanner != nil {
- return c.getServiceBanner()
- }
- return codersdk.ServiceBannerConfig{}, nil
-}
-
// tempDirUnixSocket returns a temporary directory that can safely hold unix
// sockets (probably).
//
@@ -2214,9 +2038,8 @@ func TestAgent_Metrics_SSH(t *testing.T) {
registry := prometheus.NewRegistry()
//nolint:dogsled
- conn, _, _, _, _ := setupAgent(t, &client{}, 0, func(o agent.Options) agent.Options {
+ conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
o.PrometheusRegistry = registry
- return o
})
sshClient, err := conn.SSHClient(ctx)
diff --git a/agent/agenttest/client.go b/agent/agenttest/client.go
new file mode 100644
index 0000000000000..c69ff59eb730b
--- /dev/null
+++ b/agent/agenttest/client.go
@@ -0,0 +1,189 @@
+package agenttest
+
+import (
+ "context"
+ "io"
+ "net"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "golang.org/x/exp/maps"
+
+ "cdr.dev/slog"
+ "github.com/coder/coder/codersdk"
+ "github.com/coder/coder/codersdk/agentsdk"
+ "github.com/coder/coder/tailnet"
+)
+
+func NewClient(t testing.TB,
+ agentID uuid.UUID,
+ manifest agentsdk.Manifest,
+ statsChan chan *agentsdk.Stats,
+ coordinator tailnet.Coordinator,
+) *Client {
+ if manifest.AgentID == uuid.Nil {
+ manifest.AgentID = agentID
+ }
+ return &Client{
+ t: t,
+ agentID: agentID,
+ manifest: manifest,
+ statsChan: statsChan,
+ coordinator: coordinator,
+ }
+}
+
+type Client struct {
+ t testing.TB
+ agentID uuid.UUID
+ manifest agentsdk.Manifest
+ metadata map[string]agentsdk.PostMetadataRequest
+ statsChan chan *agentsdk.Stats
+ coordinator tailnet.Coordinator
+ LastWorkspaceAgent func()
+ PatchWorkspaceLogs func() error
+ GetServiceBannerFunc func() (codersdk.ServiceBannerConfig, error)
+
+ mu sync.Mutex // Protects following.
+ lifecycleStates []codersdk.WorkspaceAgentLifecycle
+ startup agentsdk.PostStartupRequest
+ logs []agentsdk.StartupLog
+}
+
+func (c *Client) Manifest(_ context.Context) (agentsdk.Manifest, error) {
+ return c.manifest, nil
+}
+
+func (c *Client) Listen(_ context.Context) (net.Conn, error) {
+ clientConn, serverConn := net.Pipe()
+ closed := make(chan struct{})
+ c.LastWorkspaceAgent = func() {
+ _ = serverConn.Close()
+ _ = clientConn.Close()
+ <-closed
+ }
+ c.t.Cleanup(c.LastWorkspaceAgent)
+ go func() {
+ _ = c.coordinator.ServeAgent(serverConn, c.agentID, "")
+ close(closed)
+ }()
+ return clientConn, nil
+}
+
+func (c *Client) ReportStats(ctx context.Context, _ slog.Logger, statsChan <-chan *agentsdk.Stats, setInterval func(time.Duration)) (io.Closer, error) {
+ doneCh := make(chan struct{})
+ ctx, cancel := context.WithCancel(ctx)
+
+ go func() {
+ defer close(doneCh)
+
+ setInterval(500 * time.Millisecond)
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case stat := <-statsChan:
+ select {
+ case c.statsChan <- stat:
+ case <-ctx.Done():
+ return
+ default:
+ // We don't want to send old stats.
+ continue
+ }
+ }
+ }
+ }()
+ return closeFunc(func() error {
+ cancel()
+ <-doneCh
+ close(c.statsChan)
+ return nil
+ }), nil
+}
+
+func (c *Client) GetLifecycleStates() []codersdk.WorkspaceAgentLifecycle {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.lifecycleStates
+}
+
+func (c *Client) PostLifecycle(_ context.Context, req agentsdk.PostLifecycleRequest) error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.lifecycleStates = append(c.lifecycleStates, req.State)
+ return nil
+}
+
+func (*Client) PostAppHealth(_ context.Context, _ agentsdk.PostAppHealthsRequest) error {
+ return nil
+}
+
+func (c *Client) GetStartup() agentsdk.PostStartupRequest {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.startup
+}
+
+func (c *Client) GetMetadata() map[string]agentsdk.PostMetadataRequest {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return maps.Clone(c.metadata)
+}
+
+func (c *Client) PostMetadata(_ context.Context, key string, req agentsdk.PostMetadataRequest) error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.metadata == nil {
+ c.metadata = make(map[string]agentsdk.PostMetadataRequest)
+ }
+ c.metadata[key] = req
+ return nil
+}
+
+func (c *Client) PostStartup(_ context.Context, startup agentsdk.PostStartupRequest) error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.startup = startup
+ return nil
+}
+
+func (c *Client) GetStartupLogs() []agentsdk.StartupLog {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.logs
+}
+
+func (c *Client) PatchStartupLogs(_ context.Context, logs agentsdk.PatchStartupLogs) error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.PatchWorkspaceLogs != nil {
+ return c.PatchWorkspaceLogs()
+ }
+ c.logs = append(c.logs, logs.Logs...)
+ return nil
+}
+
+func (c *Client) SetServiceBannerFunc(f func() (codersdk.ServiceBannerConfig, error)) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ c.GetServiceBannerFunc = f
+}
+
+func (c *Client) GetServiceBanner(_ context.Context) (codersdk.ServiceBannerConfig, error) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.GetServiceBannerFunc != nil {
+ return c.GetServiceBannerFunc()
+ }
+ return codersdk.ServiceBannerConfig{}, nil
+}
+
+type closeFunc func() error
+
+func (c closeFunc) Close() error {
+ return c()
+}
diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go
index 328107b39d54b..31970e84477cc 100644
--- a/coderd/apidoc/docs.go
+++ b/coderd/apidoc/docs.go
@@ -5961,6 +5961,9 @@ const docTemplate = `{
"agentsdk.Manifest": {
"type": "object",
"properties": {
+ "agent_id": {
+ "type": "string"
+ },
"apps": {
"type": "array",
"items": {
@@ -7617,6 +7620,7 @@ const docTemplate = `{
"workspace_actions",
"tailnet_ha_coordinator",
"convert-to-oidc",
+ "single_tailnet",
"workspace_build_logs_ui"
],
"x-enum-varnames": [
@@ -7624,6 +7628,7 @@ const docTemplate = `{
"ExperimentWorkspaceActions",
"ExperimentTailnetHACoordinator",
"ExperimentConvertToOIDC",
+ "ExperimentSingleTailnet",
"ExperimentWorkspaceBuildLogsUI"
]
},
diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json
index 7ea1a1de0633c..841f9c50bbe5f 100644
--- a/coderd/apidoc/swagger.json
+++ b/coderd/apidoc/swagger.json
@@ -5251,6 +5251,9 @@
"agentsdk.Manifest": {
"type": "object",
"properties": {
+ "agent_id": {
+ "type": "string"
+ },
"apps": {
"type": "array",
"items": {
@@ -6818,6 +6821,7 @@
"workspace_actions",
"tailnet_ha_coordinator",
"convert-to-oidc",
+ "single_tailnet",
"workspace_build_logs_ui"
],
"x-enum-varnames": [
@@ -6825,6 +6829,7 @@
"ExperimentWorkspaceActions",
"ExperimentTailnetHACoordinator",
"ExperimentConvertToOIDC",
+ "ExperimentSingleTailnet",
"ExperimentWorkspaceBuildLogsUI"
]
},
diff --git a/coderd/coderd.go b/coderd/coderd.go
index dc14727879f08..7104049187235 100644
--- a/coderd/coderd.go
+++ b/coderd/coderd.go
@@ -364,8 +364,23 @@ func New(options *Options) *API {
}
api.Auditor.Store(&options.Auditor)
- api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgentTailnet, 0)
api.TailnetCoordinator.Store(&options.TailnetCoordinator)
+ if api.Experiments.Enabled(codersdk.ExperimentSingleTailnet) {
+ api.agentProvider, err = NewServerTailnet(api.ctx,
+ options.Logger,
+ options.DERPServer,
+ options.DERPMap,
+ &api.TailnetCoordinator,
+ wsconncache.New(api._dialWorkspaceAgentTailnet, 0),
+ )
+ if err != nil {
+ panic("failed to setup server tailnet: " + err.Error())
+ }
+ } else {
+ api.agentProvider = &wsconncache.AgentProvider{
+ Cache: wsconncache.New(api._dialWorkspaceAgentTailnet, 0),
+ }
+ }
api.workspaceAppServer = &workspaceapps.Server{
Logger: options.Logger.Named("workspaceapps"),
@@ -377,7 +392,7 @@ func New(options *Options) *API {
RealIPConfig: options.RealIPConfig,
SignedTokenProvider: api.WorkspaceAppsProvider,
- WorkspaceConnCache: api.workspaceAgentCache,
+ AgentProvider: api.agentProvider,
AppSecurityKey: options.AppSecurityKey,
DisablePathApps: options.DeploymentValues.DisablePathApps.Value(),
@@ -921,10 +936,10 @@ type API struct {
derpCloseFunc func()
metricsCache *metricscache.Cache
- workspaceAgentCache *wsconncache.Cache
updateChecker *updatecheck.Checker
WorkspaceAppsProvider workspaceapps.SignedTokenProvider
workspaceAppServer *workspaceapps.Server
+ agentProvider workspaceapps.AgentProvider
// Experiments contains the list of experiments currently enabled.
// This is used to gate features that are not yet ready for production.
@@ -951,7 +966,8 @@ func (api *API) Close() error {
if coordinator != nil {
_ = (*coordinator).Close()
}
- return api.workspaceAgentCache.Close()
+ _ = api.agentProvider.Close()
+ return nil
}
func compressHandler(h http.Handler) http.Handler {
diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go
index f4bf035311e6a..d073b48824bd4 100644
--- a/coderd/coderdtest/coderdtest.go
+++ b/coderd/coderdtest/coderdtest.go
@@ -109,6 +109,7 @@ type Options struct {
GitAuthConfigs []*gitauth.Config
TrialGenerator func(context.Context, string) error
TemplateScheduleStore schedule.TemplateScheduleStore
+ Coordinator tailnet.Coordinator
HealthcheckFunc func(ctx context.Context, apiKey string) *healthcheck.Report
HealthcheckTimeout time.Duration
diff --git a/coderd/prometheusmetrics/prometheusmetrics_test.go b/coderd/prometheusmetrics/prometheusmetrics_test.go
index 5b53fcaa047e4..2ece768671280 100644
--- a/coderd/prometheusmetrics/prometheusmetrics_test.go
+++ b/coderd/prometheusmetrics/prometheusmetrics_test.go
@@ -302,7 +302,7 @@ func TestAgents(t *testing.T) {
coordinator := tailnet.NewCoordinator(slogtest.Make(t, nil).Leveled(slog.LevelDebug))
coordinatorPtr := atomic.Pointer[tailnet.Coordinator]{}
coordinatorPtr.Store(&coordinator)
- derpMap := tailnettest.RunDERPAndSTUN(t)
+ derpMap, _ := tailnettest.RunDERPAndSTUN(t)
agentInactiveDisconnectTimeout := 1 * time.Hour // don't need to focus on this value in tests
registry := prometheus.NewRegistry()
diff --git a/coderd/tailnet.go b/coderd/tailnet.go
new file mode 100644
index 0000000000000..a1559e4efcd52
--- /dev/null
+++ b/coderd/tailnet.go
@@ -0,0 +1,339 @@
+package coderd
+
+import (
+ "bufio"
+ "context"
+ "net"
+ "net/http"
+ "net/http/httputil"
+ "net/netip"
+ "net/url"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/google/uuid"
+ "golang.org/x/xerrors"
+ "tailscale.com/derp"
+ "tailscale.com/tailcfg"
+
+ "cdr.dev/slog"
+ "github.com/coder/coder/coderd/wsconncache"
+ "github.com/coder/coder/codersdk"
+ "github.com/coder/coder/site"
+ "github.com/coder/coder/tailnet"
+)
+
+var tailnetTransport *http.Transport
+
+func init() {
+ var valid bool
+ tailnetTransport, valid = http.DefaultTransport.(*http.Transport)
+ if !valid {
+ panic("dev error: default transport is the wrong type")
+ }
+}
+
+// NewServerTailnet creates a new tailnet intended for use by coderd. It
+// automatically falls back to wsconncache if a legacy agent is encountered.
+func NewServerTailnet(
+ ctx context.Context,
+ logger slog.Logger,
+ derpServer *derp.Server,
+ derpMap *tailcfg.DERPMap,
+ coord *atomic.Pointer[tailnet.Coordinator],
+ cache *wsconncache.Cache,
+) (*ServerTailnet, error) {
+ logger = logger.Named("servertailnet")
+ conn, err := tailnet.NewConn(&tailnet.Options{
+ Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
+ DERPMap: derpMap,
+ Logger: logger,
+ })
+ if err != nil {
+ return nil, xerrors.Errorf("create tailnet conn: %w", err)
+ }
+
+ serverCtx, cancel := context.WithCancel(ctx)
+ tn := &ServerTailnet{
+ ctx: serverCtx,
+ cancel: cancel,
+ logger: logger,
+ conn: conn,
+ coord: coord,
+ cache: cache,
+ agentNodes: map[uuid.UUID]time.Time{},
+ agentTickets: map[uuid.UUID]map[uuid.UUID]struct{}{},
+ transport: tailnetTransport.Clone(),
+ }
+ tn.transport.DialContext = tn.dialContext
+ tn.transport.MaxIdleConnsPerHost = 10
+ tn.transport.MaxIdleConns = 0
+ agentConn := (*coord.Load()).ServeMultiAgent(uuid.New())
+ tn.agentConn.Store(&agentConn)
+
+ err = tn.getAgentConn().UpdateSelf(conn.Node())
+ if err != nil {
+ tn.logger.Warn(context.Background(), "server tailnet update self", slog.Error(err))
+ }
+ conn.SetNodeCallback(func(node *tailnet.Node) {
+ err := tn.getAgentConn().UpdateSelf(node)
+ if err != nil {
+ tn.logger.Warn(context.Background(), "broadcast server node to agents", slog.Error(err))
+ }
+ })
+
+ // This is set to allow local DERP traffic to be proxied through memory
+ // instead of needing to hit the external access URL. Don't use the ctx
+ // given in this callback, it's only valid while connecting.
+ conn.SetDERPRegionDialer(func(_ context.Context, region *tailcfg.DERPRegion) net.Conn {
+ if !region.EmbeddedRelay {
+ return nil
+ }
+ left, right := net.Pipe()
+ go func() {
+ defer left.Close()
+ defer right.Close()
+ brw := bufio.NewReadWriter(bufio.NewReader(right), bufio.NewWriter(right))
+ derpServer.Accept(ctx, right, brw, "internal")
+ }()
+ return left
+ })
+
+ go tn.watchAgentUpdates()
+ go tn.expireOldAgents()
+ return tn, nil
+}
+
+func (s *ServerTailnet) expireOldAgents() {
+ const (
+ tick = 5 * time.Minute
+ cutoff = 30 * time.Minute
+ )
+
+ ticker := time.NewTicker(tick)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-s.ctx.Done():
+ return
+ case <-ticker.C:
+ }
+
+ s.nodesMu.Lock()
+ agentConn := s.getAgentConn()
+ for agentID, lastConnection := range s.agentNodes {
+ // If no one has connected since the cutoff and there are no active
+ // connections, remove the agent.
+ if time.Since(lastConnection) > cutoff && len(s.agentTickets[agentID]) == 0 {
+ _ = agentConn
+ // err := agentConn.UnsubscribeAgent(agentID)
+ // if err != nil {
+ // s.logger.Error(s.ctx, "unsubscribe expired agent", slog.Error(err), slog.F("agent_id", agentID))
+ // }
+ // delete(s.agentNodes, agentID)
+
+ // TODO(coadler): actually remove from the netmap, then reenable
+ // the above
+ }
+ }
+ s.nodesMu.Unlock()
+ }
+}
+
+func (s *ServerTailnet) watchAgentUpdates() {
+ for {
+ conn := s.getAgentConn()
+ nodes, ok := conn.NextUpdate(s.ctx)
+ if !ok {
+ if conn.IsClosed() && s.ctx.Err() == nil {
+ s.reinitCoordinator()
+ continue
+ }
+ return
+ }
+
+ err := s.conn.UpdateNodes(nodes, false)
+ if err != nil {
+ s.logger.Error(context.Background(), "update node in server tailnet", slog.Error(err))
+ return
+ }
+ }
+}
+
+func (s *ServerTailnet) getAgentConn() tailnet.MultiAgentConn {
+ return *s.agentConn.Load()
+}
+
+func (s *ServerTailnet) reinitCoordinator() {
+ s.nodesMu.Lock()
+ agentConn := (*s.coord.Load()).ServeMultiAgent(uuid.New())
+ s.agentConn.Store(&agentConn)
+
+ // Resubscribe to all of the agents we're tracking.
+ for agentID := range s.agentNodes {
+ err := agentConn.SubscribeAgent(agentID)
+ if err != nil {
+ s.logger.Warn(s.ctx, "resubscribe to agent", slog.Error(err), slog.F("agent_id", agentID))
+ }
+ }
+ s.nodesMu.Unlock()
+}
+
+type ServerTailnet struct {
+ ctx context.Context
+ cancel func()
+
+ logger slog.Logger
+ conn *tailnet.Conn
+ coord *atomic.Pointer[tailnet.Coordinator]
+ agentConn atomic.Pointer[tailnet.MultiAgentConn]
+ cache *wsconncache.Cache
+ nodesMu sync.Mutex
+ // agentNodes is a map of agent tailnetNodes the server wants to keep a
+ // connection to. It contains the last time the agent was connected to.
+ agentNodes map[uuid.UUID]time.Time
+ // agentTockets holds a map of all open connections to an agent.
+ agentTickets map[uuid.UUID]map[uuid.UUID]struct{}
+
+ transport *http.Transport
+}
+
+func (s *ServerTailnet) ReverseProxy(targetURL, dashboardURL *url.URL, agentID uuid.UUID) (_ *httputil.ReverseProxy, release func(), _ error) {
+ proxy := httputil.NewSingleHostReverseProxy(targetURL)
+ proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
+ site.RenderStaticErrorPage(w, r, site.ErrorPageData{
+ Status: http.StatusBadGateway,
+ Title: "Bad Gateway",
+ Description: "Failed to proxy request to application: " + err.Error(),
+ RetryEnabled: true,
+ DashboardURL: dashboardURL.String(),
+ })
+ }
+ proxy.Director = s.director(agentID, proxy.Director)
+ proxy.Transport = s.transport
+
+ return proxy, func() {}, nil
+}
+
+type agentIDKey struct{}
+
+// director makes sure agentIDKey is set on the context in the reverse proxy.
+// This allows the transport to correctly identify which agent to dial to.
+func (*ServerTailnet) director(agentID uuid.UUID, prev func(req *http.Request)) func(req *http.Request) {
+ return func(req *http.Request) {
+ ctx := context.WithValue(req.Context(), agentIDKey{}, agentID)
+ *req = *req.WithContext(ctx)
+ prev(req)
+ }
+}
+
+func (s *ServerTailnet) dialContext(ctx context.Context, network, addr string) (net.Conn, error) {
+ agentID, ok := ctx.Value(agentIDKey{}).(uuid.UUID)
+ if !ok {
+ return nil, xerrors.Errorf("no agent id attached")
+ }
+
+ return s.DialAgentNetConn(ctx, agentID, network, addr)
+}
+
+func (s *ServerTailnet) ensureAgent(agentID uuid.UUID) error {
+ s.nodesMu.Lock()
+ defer s.nodesMu.Unlock()
+
+ _, ok := s.agentNodes[agentID]
+ // If we don't have the node, subscribe.
+ if !ok {
+ s.logger.Debug(s.ctx, "subscribing to agent", slog.F("agent_id", agentID))
+ err := s.getAgentConn().SubscribeAgent(agentID)
+ if err != nil {
+ return xerrors.Errorf("subscribe agent: %w", err)
+ }
+ s.agentTickets[agentID] = map[uuid.UUID]struct{}{}
+ }
+
+ s.agentNodes[agentID] = time.Now()
+ return nil
+}
+
+func (s *ServerTailnet) AgentConn(ctx context.Context, agentID uuid.UUID) (*codersdk.WorkspaceAgentConn, func(), error) {
+ var (
+ conn *codersdk.WorkspaceAgentConn
+ ret = func() {}
+ )
+
+ if s.getAgentConn().AgentIsLegacy(agentID) {
+ s.logger.Debug(s.ctx, "acquiring legacy agent", slog.F("agent_id", agentID))
+ cconn, release, err := s.cache.Acquire(agentID)
+ if err != nil {
+ return nil, nil, xerrors.Errorf("acquire legacy agent conn: %w", err)
+ }
+
+ conn = cconn.WorkspaceAgentConn
+ ret = release
+ } else {
+ err := s.ensureAgent(agentID)
+ if err != nil {
+ return nil, nil, xerrors.Errorf("ensure agent: %w", err)
+ }
+
+ s.logger.Debug(s.ctx, "acquiring agent", slog.F("agent_id", agentID))
+ conn = codersdk.NewWorkspaceAgentConn(s.conn, codersdk.WorkspaceAgentConnOptions{
+ AgentID: agentID,
+ CloseFunc: func() error { return codersdk.ErrSkipClose },
+ })
+ }
+
+ // Since we now have an open conn, be careful to close it if we error
+ // without returning it to the user.
+
+ reachable := conn.AwaitReachable(ctx)
+ if !reachable {
+ ret()
+ conn.Close()
+ return nil, nil, xerrors.New("agent is unreachable")
+ }
+
+ return conn, ret, nil
+}
+
+func (s *ServerTailnet) DialAgentNetConn(ctx context.Context, agentID uuid.UUID, network, addr string) (net.Conn, error) {
+ conn, release, err := s.AgentConn(ctx, agentID)
+ if err != nil {
+ return nil, xerrors.Errorf("acquire agent conn: %w", err)
+ }
+
+ // Since we now have an open conn, be careful to close it if we error
+ // without returning it to the user.
+
+ nc, err := conn.DialContext(ctx, network, addr)
+ if err != nil {
+ release()
+ conn.Close()
+ return nil, xerrors.Errorf("dial context: %w", err)
+ }
+
+ return &netConnCloser{Conn: nc, close: func() {
+ release()
+ conn.Close()
+ }}, err
+}
+
+type netConnCloser struct {
+ net.Conn
+ close func()
+}
+
+func (c *netConnCloser) Close() error {
+ c.close()
+ return c.Conn.Close()
+}
+
+func (s *ServerTailnet) Close() error {
+ s.cancel()
+ _ = s.cache.Close()
+ _ = s.conn.Close()
+ s.transport.CloseIdleConnections()
+ return nil
+}
diff --git a/coderd/tailnet_test.go b/coderd/tailnet_test.go
new file mode 100644
index 0000000000000..16d597607312c
--- /dev/null
+++ b/coderd/tailnet_test.go
@@ -0,0 +1,207 @@
+package coderd_test
+
+import (
+ "context"
+ "fmt"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/netip"
+ "net/url"
+ "sync/atomic"
+ "testing"
+
+ "github.com/google/uuid"
+ "github.com/spf13/afero"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "cdr.dev/slog"
+ "cdr.dev/slog/sloggers/slogtest"
+ "github.com/coder/coder/agent"
+ "github.com/coder/coder/agent/agenttest"
+ "github.com/coder/coder/coderd"
+ "github.com/coder/coder/coderd/wsconncache"
+ "github.com/coder/coder/codersdk"
+ "github.com/coder/coder/codersdk/agentsdk"
+ "github.com/coder/coder/tailnet"
+ "github.com/coder/coder/tailnet/tailnettest"
+ "github.com/coder/coder/testutil"
+)
+
+func TestServerTailnet_AgentConn_OK(t *testing.T) {
+ t.Parallel()
+
+ ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
+ defer cancel()
+
+ // Connect through the ServerTailnet
+ agentID, _, serverTailnet := setupAgent(t, nil)
+
+ conn, release, err := serverTailnet.AgentConn(ctx, agentID)
+ require.NoError(t, err)
+ defer release()
+
+ assert.True(t, conn.AwaitReachable(ctx))
+}
+
+func TestServerTailnet_AgentConn_Legacy(t *testing.T) {
+ t.Parallel()
+
+ ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
+ defer cancel()
+
+ // Force a connection through wsconncache using the legacy hardcoded ip.
+ agentID, _, serverTailnet := setupAgent(t, []netip.Prefix{
+ netip.PrefixFrom(codersdk.WorkspaceAgentIP, 128),
+ })
+
+ conn, release, err := serverTailnet.AgentConn(ctx, agentID)
+ require.NoError(t, err)
+ defer release()
+
+ assert.True(t, conn.AwaitReachable(ctx))
+}
+
+func TestServerTailnet_ReverseProxy_OK(t *testing.T) {
+ t.Parallel()
+
+ ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+ defer cancel()
+
+ // Force a connection through wsconncache using the legacy hardcoded ip.
+ agentID, _, serverTailnet := setupAgent(t, nil)
+
+ u, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", codersdk.WorkspaceAgentHTTPAPIServerPort))
+ require.NoError(t, err)
+
+ rp, release, err := serverTailnet.ReverseProxy(u, u, agentID)
+ require.NoError(t, err)
+ defer release()
+
+ rw := httptest.NewRecorder()
+ req := httptest.NewRequest(
+ http.MethodGet,
+ u.String(),
+ nil,
+ ).WithContext(ctx)
+
+ rp.ServeHTTP(rw, req)
+ res := rw.Result()
+ defer res.Body.Close()
+
+ assert.Equal(t, http.StatusOK, res.StatusCode)
+}
+
+func TestServerTailnet_ReverseProxy_Legacy(t *testing.T) {
+ t.Parallel()
+
+ ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+ defer cancel()
+
+ // Force a connection through wsconncache using the legacy hardcoded ip.
+ agentID, _, serverTailnet := setupAgent(t, []netip.Prefix{
+ netip.PrefixFrom(codersdk.WorkspaceAgentIP, 128),
+ })
+
+ u, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", codersdk.WorkspaceAgentHTTPAPIServerPort))
+ require.NoError(t, err)
+
+ rp, release, err := serverTailnet.ReverseProxy(u, u, agentID)
+ require.NoError(t, err)
+ defer release()
+
+ rw := httptest.NewRecorder()
+ req := httptest.NewRequest(
+ http.MethodGet,
+ u.String(),
+ nil,
+ ).WithContext(ctx)
+
+ rp.ServeHTTP(rw, req)
+ res := rw.Result()
+ defer res.Body.Close()
+
+ assert.Equal(t, http.StatusOK, res.StatusCode)
+}
+
+func setupAgent(t *testing.T, agentAddresses []netip.Prefix) (uuid.UUID, agent.Agent, *coderd.ServerTailnet) {
+ logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+ derpMap, derpServer := tailnettest.RunDERPAndSTUN(t)
+ manifest := agentsdk.Manifest{
+ AgentID: uuid.New(),
+ DERPMap: derpMap,
+ }
+
+ var coordPtr atomic.Pointer[tailnet.Coordinator]
+ coord := tailnet.NewCoordinator(logger)
+ coordPtr.Store(&coord)
+ t.Cleanup(func() {
+ _ = coord.Close()
+ })
+
+ c := agenttest.NewClient(t, manifest.AgentID, manifest, make(chan *agentsdk.Stats, 50), coord)
+
+ options := agent.Options{
+ Client: c,
+ Filesystem: afero.NewMemMapFs(),
+ Logger: logger.Named("agent"),
+ Addresses: agentAddresses,
+ }
+
+ ag := agent.New(options)
+ t.Cleanup(func() {
+ _ = ag.Close()
+ })
+
+ // Wait for the agent to connect.
+ require.Eventually(t, func() bool {
+ return coord.Node(manifest.AgentID) != nil
+ }, testutil.WaitShort, testutil.IntervalFast)
+
+ cache := wsconncache.New(func(id uuid.UUID) (*codersdk.WorkspaceAgentConn, error) {
+ conn, err := tailnet.NewConn(&tailnet.Options{
+ Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
+ DERPMap: manifest.DERPMap,
+ Logger: logger.Named("client"),
+ })
+ require.NoError(t, err)
+ clientConn, serverConn := net.Pipe()
+ serveClientDone := make(chan struct{})
+ t.Cleanup(func() {
+ _ = clientConn.Close()
+ _ = serverConn.Close()
+ _ = conn.Close()
+ <-serveClientDone
+ })
+ go func() {
+ defer close(serveClientDone)
+ coord.ServeClient(serverConn, uuid.New(), manifest.AgentID)
+ }()
+ sendNode, _ := tailnet.ServeCoordinator(clientConn, func(node []*tailnet.Node) error {
+ return conn.UpdateNodes(node, false)
+ })
+ conn.SetNodeCallback(sendNode)
+ return codersdk.NewWorkspaceAgentConn(conn, codersdk.WorkspaceAgentConnOptions{
+ AgentID: manifest.AgentID,
+ AgentIP: codersdk.WorkspaceAgentIP,
+ CloseFunc: func() error { return codersdk.ErrSkipClose },
+ }), nil
+ }, 0)
+
+ serverTailnet, err := coderd.NewServerTailnet(
+ context.Background(),
+ logger,
+ derpServer,
+ manifest.DERPMap,
+ &coordPtr,
+ cache,
+ )
+ require.NoError(t, err)
+
+ t.Cleanup(func() {
+ _ = serverTailnet.Close()
+ })
+
+ return manifest.AgentID, ag, serverTailnet
+}
diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go
index bfe61b4a180df..c1f2e90c02de9 100644
--- a/coderd/workspaceagents.go
+++ b/coderd/workspaceagents.go
@@ -161,6 +161,7 @@ func (api *API) workspaceAgentManifest(rw http.ResponseWriter, r *http.Request)
}
httpapi.Write(ctx, rw, http.StatusOK, agentsdk.Manifest{
+ AgentID: apiAgent.ID,
Apps: convertApps(dbApps),
DERPMap: api.DERPMap,
GitAuthConfigs: len(api.GitAuthConfigs),
@@ -654,7 +655,7 @@ func (api *API) workspaceAgentListeningPorts(rw http.ResponseWriter, r *http.Req
return
}
- agentConn, release, err := api.workspaceAgentCache.Acquire(workspaceAgent.ID)
+ agentConn, release, err := api.agentProvider.AgentConn(ctx, workspaceAgent.ID)
if err != nil {
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error dialing workspace agent.",
@@ -729,7 +730,9 @@ func (api *API) workspaceAgentListeningPorts(rw http.ResponseWriter, r *http.Req
httpapi.Write(ctx, rw, http.StatusOK, portsResponse)
}
-func (api *API) dialWorkspaceAgentTailnet(agentID uuid.UUID) (*codersdk.WorkspaceAgentConn, error) {
+// Deprecated: use api.tailnet.AgentConn instead.
+// See: https://github.com/coder/coder/issues/8218
+func (api *API) _dialWorkspaceAgentTailnet(agentID uuid.UUID) (*codersdk.WorkspaceAgentConn, error) {
clientConn, serverConn := net.Pipe()
conn, err := tailnet.NewConn(&tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
@@ -765,14 +768,16 @@ func (api *API) dialWorkspaceAgentTailnet(agentID uuid.UUID) (*codersdk.Workspac
return nil
})
conn.SetNodeCallback(sendNodes)
- agentConn := &codersdk.WorkspaceAgentConn{
- Conn: conn,
- CloseFunc: func() {
+ agentConn := codersdk.NewWorkspaceAgentConn(conn, codersdk.WorkspaceAgentConnOptions{
+ AgentID: agentID,
+ AgentIP: codersdk.WorkspaceAgentIP,
+ CloseFunc: func() error {
cancel()
_ = clientConn.Close()
_ = serverConn.Close()
+ return nil
},
- }
+ })
go func() {
err := (*api.TailnetCoordinator.Load()).ServeClient(serverConn, uuid.New(), agentID)
if err != nil {
diff --git a/coderd/workspaceapps/apptest/setup.go b/coderd/workspaceapps/apptest/setup.go
index 9432e09c9703d..0f0167f37bf79 100644
--- a/coderd/workspaceapps/apptest/setup.go
+++ b/coderd/workspaceapps/apptest/setup.go
@@ -399,7 +399,8 @@ func doWithRetries(t require.TestingT, client *codersdk.Client, req *http.Reques
return resp, err
}
-func requestWithRetries(ctx context.Context, t require.TestingT, client *codersdk.Client, method, urlOrPath string, body interface{}, opts ...codersdk.RequestOption) (*http.Response, error) {
+func requestWithRetries(ctx context.Context, t testing.TB, client *codersdk.Client, method, urlOrPath string, body interface{}, opts ...codersdk.RequestOption) (*http.Response, error) {
+ t.Helper()
var resp *http.Response
var err error
require.Eventually(t, func() bool {
diff --git a/coderd/workspaceapps/proxy.go b/coderd/workspaceapps/proxy.go
index 1d3e8592d7a1c..9b2d9c4bfa297 100644
--- a/coderd/workspaceapps/proxy.go
+++ b/coderd/workspaceapps/proxy.go
@@ -23,7 +23,6 @@ import (
"github.com/coder/coder/coderd/httpmw"
"github.com/coder/coder/coderd/tracing"
"github.com/coder/coder/coderd/util/slice"
- "github.com/coder/coder/coderd/wsconncache"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/site"
)
@@ -61,6 +60,22 @@ var nonCanonicalHeaders = map[string]string{
"Sec-Websocket-Version": "Sec-WebSocket-Version",
}
+type AgentProvider interface {
+ // ReverseProxy returns an httputil.ReverseProxy for proxying HTTP requests
+ // to the specified agent.
+ //
+ // TODO: after wsconncache is deleted this doesn't need to return an error.
+ ReverseProxy(targetURL, dashboardURL *url.URL, agentID uuid.UUID) (_ *httputil.ReverseProxy, release func(), _ error)
+
+ // AgentConn returns a new connection to the specified agent.
+ //
+ // TODO: after wsconncache is deleted this doesn't need to return a release
+ // func.
+ AgentConn(ctx context.Context, agentID uuid.UUID) (_ *codersdk.WorkspaceAgentConn, release func(), _ error)
+
+ Close() error
+}
+
// Server serves workspace apps endpoints, including:
// - Path-based apps
// - Subdomain app middleware
@@ -83,7 +98,6 @@ type Server struct {
RealIPConfig *httpmw.RealIPConfig
SignedTokenProvider SignedTokenProvider
- WorkspaceConnCache *wsconncache.Cache
AppSecurityKey SecurityKey
// DisablePathApps disables path-based apps. This is a security feature as path
@@ -95,6 +109,8 @@ type Server struct {
DisablePathApps bool
SecureAuthCookie bool
+ AgentProvider AgentProvider
+
websocketWaitMutex sync.Mutex
websocketWaitGroup sync.WaitGroup
}
@@ -106,8 +122,8 @@ func (s *Server) Close() error {
s.websocketWaitGroup.Wait()
s.websocketWaitMutex.Unlock()
- // The caller must close the SignedTokenProvider (if necessary) and the
- // wsconncache.
+ // The caller must close the SignedTokenProvider and the AgentProvider (if
+ // necessary).
return nil
}
@@ -517,18 +533,7 @@ func (s *Server) proxyWorkspaceApp(rw http.ResponseWriter, r *http.Request, appT
r.URL.Path = path
appURL.RawQuery = ""
- proxy := httputil.NewSingleHostReverseProxy(appURL)
- proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
- site.RenderStaticErrorPage(rw, r, site.ErrorPageData{
- Status: http.StatusBadGateway,
- Title: "Bad Gateway",
- Description: "Failed to proxy request to application: " + err.Error(),
- RetryEnabled: true,
- DashboardURL: s.DashboardURL.String(),
- })
- }
-
- conn, release, err := s.WorkspaceConnCache.Acquire(appToken.AgentID)
+ proxy, release, err := s.AgentProvider.ReverseProxy(appURL, s.DashboardURL, appToken.AgentID)
if err != nil {
site.RenderStaticErrorPage(rw, r, site.ErrorPageData{
Status: http.StatusBadGateway,
@@ -540,7 +545,6 @@ func (s *Server) proxyWorkspaceApp(rw http.ResponseWriter, r *http.Request, appT
return
}
defer release()
- proxy.Transport = conn.HTTPTransport()
proxy.ModifyResponse = func(r *http.Response) error {
r.Header.Del(httpmw.AccessControlAllowOriginHeader)
@@ -658,13 +662,14 @@ func (s *Server) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) {
go httpapi.Heartbeat(ctx, conn)
- agentConn, release, err := s.WorkspaceConnCache.Acquire(appToken.AgentID)
+ agentConn, release, err := s.AgentProvider.AgentConn(ctx, appToken.AgentID)
if err != nil {
log.Debug(ctx, "dial workspace agent", slog.Error(err))
_ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("dial workspace agent: %s", err))
return
}
defer release()
+ defer agentConn.Close()
log.Debug(ctx, "dialed workspace agent")
ptNetConn, err := agentConn.ReconnectingPTY(ctx, reconnect, uint16(height), uint16(width), r.URL.Query().Get("command"))
if err != nil {
diff --git a/coderd/wsconncache/wsconncache.go b/coderd/wsconncache/wsconncache.go
index 19c7f65f9fb74..13d1588384954 100644
--- a/coderd/wsconncache/wsconncache.go
+++ b/coderd/wsconncache/wsconncache.go
@@ -1,9 +1,12 @@
// Package wsconncache caches workspace agent connections by UUID.
+// Deprecated: Use ServerTailnet instead.
package wsconncache
import (
"context"
"net/http"
+ "net/http/httputil"
+ "net/url"
"sync"
"time"
@@ -13,13 +16,57 @@ import (
"golang.org/x/xerrors"
"github.com/coder/coder/codersdk"
+ "github.com/coder/coder/site"
)
-// New creates a new workspace connection cache that closes
-// connections after the inactive timeout provided.
+type AgentProvider struct {
+ Cache *Cache
+}
+
+func (a *AgentProvider) AgentConn(_ context.Context, agentID uuid.UUID) (*codersdk.WorkspaceAgentConn, func(), error) {
+ conn, rel, err := a.Cache.Acquire(agentID)
+ if err != nil {
+ return nil, nil, xerrors.Errorf("acquire agent connection: %w", err)
+ }
+
+ return conn.WorkspaceAgentConn, rel, nil
+}
+
+func (a *AgentProvider) ReverseProxy(targetURL *url.URL, dashboardURL *url.URL, agentID uuid.UUID) (*httputil.ReverseProxy, func(), error) {
+ proxy := httputil.NewSingleHostReverseProxy(targetURL)
+ proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
+ site.RenderStaticErrorPage(w, r, site.ErrorPageData{
+ Status: http.StatusBadGateway,
+ Title: "Bad Gateway",
+ Description: "Failed to proxy request to application: " + err.Error(),
+ RetryEnabled: true,
+ DashboardURL: dashboardURL.String(),
+ })
+ }
+
+ conn, release, err := a.Cache.Acquire(agentID)
+ if err != nil {
+ return nil, nil, xerrors.Errorf("acquire agent connection: %w", err)
+ }
+
+ proxy.Transport = conn.HTTPTransport()
+
+ return proxy, release, nil
+}
+
+func (a *AgentProvider) Close() error {
+ return a.Cache.Close()
+}
+
+// New creates a new workspace connection cache that closes connections after
+// the inactive timeout provided.
+//
+// Agent connections are cached due to Wireguard negotiation taking a few
+// hundred milliseconds, depending on latency.
//
-// Agent connections are cached due to WebRTC negotiation
-// taking a few hundred milliseconds.
+// Deprecated: Use coderd.NewServerTailnet instead. wsconncache is being phased
+// out because it creates a unique Tailnet for each agent.
+// See: https://github.com/coder/coder/issues/8218
func New(dialer Dialer, inactiveTimeout time.Duration) *Cache {
if inactiveTimeout == 0 {
inactiveTimeout = 5 * time.Minute
diff --git a/coderd/wsconncache/wsconncache_test.go b/coderd/wsconncache/wsconncache_test.go
index 34b92267080e5..276e528313751 100644
--- a/coderd/wsconncache/wsconncache_test.go
+++ b/coderd/wsconncache/wsconncache_test.go
@@ -157,22 +157,23 @@ func TestCache(t *testing.T) {
func setupAgent(t *testing.T, manifest agentsdk.Manifest, ptyTimeout time.Duration) *codersdk.WorkspaceAgentConn {
t.Helper()
logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
- manifest.DERPMap = tailnettest.RunDERPAndSTUN(t)
+ manifest.DERPMap, _ = tailnettest.RunDERPAndSTUN(t)
coordinator := tailnet.NewCoordinator(logger)
t.Cleanup(func() {
_ = coordinator.Close()
})
- agentID := uuid.New()
+ manifest.AgentID = uuid.New()
closer := agent.New(agent.Options{
Client: &client{
t: t,
- agentID: agentID,
+ agentID: manifest.AgentID,
manifest: manifest,
coordinator: coordinator,
},
Logger: logger.Named("agent"),
ReconnectingPTYTimeout: ptyTimeout,
+ Addresses: []netip.Prefix{netip.PrefixFrom(codersdk.WorkspaceAgentIP, 128)},
})
t.Cleanup(func() {
_ = closer.Close()
@@ -189,14 +190,15 @@ func setupAgent(t *testing.T, manifest agentsdk.Manifest, ptyTimeout time.Durati
_ = serverConn.Close()
_ = conn.Close()
})
- go coordinator.ServeClient(serverConn, uuid.New(), agentID)
+ go coordinator.ServeClient(serverConn, uuid.New(), manifest.AgentID)
sendNode, _ := tailnet.ServeCoordinator(clientConn, func(node []*tailnet.Node) error {
return conn.UpdateNodes(node, false)
})
conn.SetNodeCallback(sendNode)
- agentConn := &codersdk.WorkspaceAgentConn{
- Conn: conn,
- }
+ agentConn := codersdk.NewWorkspaceAgentConn(conn, codersdk.WorkspaceAgentConnOptions{
+ AgentID: manifest.AgentID,
+ AgentIP: codersdk.WorkspaceAgentIP,
+ })
t.Cleanup(func() {
_ = agentConn.Close()
})
diff --git a/codersdk/agentsdk/agentsdk.go b/codersdk/agentsdk/agentsdk.go
index 1e281ef494099..bf150cd84940f 100644
--- a/codersdk/agentsdk/agentsdk.go
+++ b/codersdk/agentsdk/agentsdk.go
@@ -84,6 +84,7 @@ func (c *Client) PostMetadata(ctx context.Context, key string, req PostMetadataR
}
type Manifest struct {
+ AgentID uuid.UUID `json:"agent_id"`
// GitAuthConfigs stores the number of Git configurations
// the Coder deployment has. If this number is >0, we
// set up special configuration in the workspace.
diff --git a/codersdk/deployment.go b/codersdk/deployment.go
index 3921963e86f4b..79266441b6dc6 100644
--- a/codersdk/deployment.go
+++ b/codersdk/deployment.go
@@ -1764,6 +1764,12 @@ const (
// oidc.
ExperimentConvertToOIDC Experiment = "convert-to-oidc"
+ // ExperimentSingleTailnet replaces workspace connections inside coderd to
+ // all use a single tailnet, instead of the previous behavior of creating a
+ // single tailnet for each agent.
+ // WARNING: This cannot be enabled when using HA.
+ ExperimentSingleTailnet Experiment = "single_tailnet"
+
ExperimentWorkspaceBuildLogsUI Experiment = "workspace_build_logs_ui"
// Add new experiments here!
// ExperimentExample Experiment = "example"
diff --git a/codersdk/workspaceagentconn.go b/codersdk/workspaceagentconn.go
index 64bd4fe2f8bfa..6b9b6f0d33f44 100644
--- a/codersdk/workspaceagentconn.go
+++ b/codersdk/workspaceagentconn.go
@@ -15,6 +15,7 @@ import (
"time"
"github.com/google/uuid"
+ "github.com/hashicorp/go-multierror"
"golang.org/x/crypto/ssh"
"golang.org/x/xerrors"
"tailscale.com/ipn/ipnstate"
@@ -27,8 +28,14 @@ import (
// WorkspaceAgentIP is a static IPv6 address with the Tailscale prefix that is used to route
// connections from clients to this node. A dynamic address is not required because a Tailnet
// client only dials a single agent at a time.
+//
+// Deprecated: use tailnet.IP() instead. This is kept for backwards
+// compatibility with wsconncache.
+// See: https://github.com/coder/coder/issues/8218
var WorkspaceAgentIP = netip.MustParseAddr("fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4")
+var ErrSkipClose = xerrors.New("skip tailnet close")
+
const (
WorkspaceAgentSSHPort = tailnet.WorkspaceAgentSSHPort
WorkspaceAgentReconnectingPTYPort = tailnet.WorkspaceAgentReconnectingPTYPort
@@ -120,11 +127,38 @@ func init() {
}
}
+// NewWorkspaceAgentConn creates a new WorkspaceAgentConn. `conn` may be unique
+// to the WorkspaceAgentConn, or it may be shared in the case of coderd. If the
+// conn is shared and closing it is undesirable, you may return ErrNoClose from
+// opts.CloseFunc. This will ensure the underlying conn is not closed.
+func NewWorkspaceAgentConn(conn *tailnet.Conn, opts WorkspaceAgentConnOptions) *WorkspaceAgentConn {
+ return &WorkspaceAgentConn{
+ Conn: conn,
+ opts: opts,
+ }
+}
+
// WorkspaceAgentConn represents a connection to a workspace agent.
// @typescript-ignore WorkspaceAgentConn
type WorkspaceAgentConn struct {
*tailnet.Conn
- CloseFunc func()
+ opts WorkspaceAgentConnOptions
+}
+
+// @typescript-ignore WorkspaceAgentConnOptions
+type WorkspaceAgentConnOptions struct {
+ AgentID uuid.UUID
+ AgentIP netip.Addr
+ CloseFunc func() error
+}
+
+func (c *WorkspaceAgentConn) agentAddress() netip.Addr {
+ var emptyIP netip.Addr
+ if cmp := c.opts.AgentIP.Compare(emptyIP); cmp != 0 {
+ return c.opts.AgentIP
+ }
+
+ return tailnet.IPFromUUID(c.opts.AgentID)
}
// AwaitReachable waits for the agent to be reachable.
@@ -132,7 +166,7 @@ func (c *WorkspaceAgentConn) AwaitReachable(ctx context.Context) bool {
ctx, span := tracing.StartSpan(ctx)
defer span.End()
- return c.Conn.AwaitReachable(ctx, WorkspaceAgentIP)
+ return c.Conn.AwaitReachable(ctx, c.agentAddress())
}
// Ping pings the agent and returns the round-trip time.
@@ -141,13 +175,20 @@ func (c *WorkspaceAgentConn) Ping(ctx context.Context) (time.Duration, bool, *ip
ctx, span := tracing.StartSpan(ctx)
defer span.End()
- return c.Conn.Ping(ctx, WorkspaceAgentIP)
+ return c.Conn.Ping(ctx, c.agentAddress())
}
// Close ends the connection to the workspace agent.
func (c *WorkspaceAgentConn) Close() error {
- if c.CloseFunc != nil {
- c.CloseFunc()
+ var cerr error
+ if c.opts.CloseFunc != nil {
+ cerr = c.opts.CloseFunc()
+ if xerrors.Is(cerr, ErrSkipClose) {
+ return nil
+ }
+ }
+ if cerr != nil {
+ return multierror.Append(cerr, c.Conn.Close())
}
return c.Conn.Close()
}
@@ -176,10 +217,12 @@ type ReconnectingPTYRequest struct {
func (c *WorkspaceAgentConn) ReconnectingPTY(ctx context.Context, id uuid.UUID, height, width uint16, command string) (net.Conn, error) {
ctx, span := tracing.StartSpan(ctx)
defer span.End()
+
if !c.AwaitReachable(ctx) {
return nil, xerrors.Errorf("workspace agent not reachable in time: %v", ctx.Err())
}
- conn, err := c.DialContextTCP(ctx, netip.AddrPortFrom(WorkspaceAgentIP, WorkspaceAgentReconnectingPTYPort))
+
+ conn, err := c.Conn.DialContextTCP(ctx, netip.AddrPortFrom(c.agentAddress(), WorkspaceAgentReconnectingPTYPort))
if err != nil {
return nil, err
}
@@ -209,10 +252,12 @@ func (c *WorkspaceAgentConn) ReconnectingPTY(ctx context.Context, id uuid.UUID,
func (c *WorkspaceAgentConn) SSH(ctx context.Context) (net.Conn, error) {
ctx, span := tracing.StartSpan(ctx)
defer span.End()
+
if !c.AwaitReachable(ctx) {
return nil, xerrors.Errorf("workspace agent not reachable in time: %v", ctx.Err())
}
- return c.DialContextTCP(ctx, netip.AddrPortFrom(WorkspaceAgentIP, WorkspaceAgentSSHPort))
+
+ return c.Conn.DialContextTCP(ctx, netip.AddrPortFrom(c.agentAddress(), WorkspaceAgentSSHPort))
}
// SSHClient calls SSH to create a client that uses a weak cipher
@@ -220,10 +265,12 @@ func (c *WorkspaceAgentConn) SSH(ctx context.Context) (net.Conn, error) {
func (c *WorkspaceAgentConn) SSHClient(ctx context.Context) (*ssh.Client, error) {
ctx, span := tracing.StartSpan(ctx)
defer span.End()
+
netConn, err := c.SSH(ctx)
if err != nil {
return nil, xerrors.Errorf("ssh: %w", err)
}
+
sshConn, channels, requests, err := ssh.NewClientConn(netConn, "localhost:22", &ssh.ClientConfig{
// SSH host validation isn't helpful, because obtaining a peer
// connection already signifies user-intent to dial a workspace.
@@ -233,6 +280,7 @@ func (c *WorkspaceAgentConn) SSHClient(ctx context.Context) (*ssh.Client, error)
if err != nil {
return nil, xerrors.Errorf("ssh conn: %w", err)
}
+
return ssh.NewClient(sshConn, channels, requests), nil
}
@@ -240,17 +288,21 @@ func (c *WorkspaceAgentConn) SSHClient(ctx context.Context) (*ssh.Client, error)
func (c *WorkspaceAgentConn) Speedtest(ctx context.Context, direction speedtest.Direction, duration time.Duration) ([]speedtest.Result, error) {
ctx, span := tracing.StartSpan(ctx)
defer span.End()
+
if !c.AwaitReachable(ctx) {
return nil, xerrors.Errorf("workspace agent not reachable in time: %v", ctx.Err())
}
- speedConn, err := c.DialContextTCP(ctx, netip.AddrPortFrom(WorkspaceAgentIP, WorkspaceAgentSpeedtestPort))
+
+ speedConn, err := c.Conn.DialContextTCP(ctx, netip.AddrPortFrom(c.agentAddress(), WorkspaceAgentSpeedtestPort))
if err != nil {
return nil, xerrors.Errorf("dial speedtest: %w", err)
}
+
results, err := speedtest.RunClientWithConn(direction, duration, speedConn)
if err != nil {
return nil, xerrors.Errorf("run speedtest: %w", err)
}
+
return results, err
}
@@ -259,19 +311,23 @@ func (c *WorkspaceAgentConn) Speedtest(ctx context.Context, direction speedtest.
func (c *WorkspaceAgentConn) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) {
ctx, span := tracing.StartSpan(ctx)
defer span.End()
- if network == "unix" {
- return nil, xerrors.New("network must be tcp or udp")
- }
- _, rawPort, _ := net.SplitHostPort(addr)
- port, _ := strconv.ParseUint(rawPort, 10, 16)
- ipp := netip.AddrPortFrom(WorkspaceAgentIP, uint16(port))
+
if !c.AwaitReachable(ctx) {
return nil, xerrors.Errorf("workspace agent not reachable in time: %v", ctx.Err())
}
- if network == "udp" {
+
+ _, rawPort, _ := net.SplitHostPort(addr)
+ port, _ := strconv.ParseUint(rawPort, 10, 16)
+ ipp := netip.AddrPortFrom(c.agentAddress(), uint16(port))
+
+ switch network {
+ case "tcp":
+ return c.Conn.DialContextTCP(ctx, ipp)
+ case "udp":
return c.Conn.DialContextUDP(ctx, ipp)
+ default:
+ return nil, xerrors.Errorf("unknown network %q", network)
}
- return c.Conn.DialContextTCP(ctx, ipp)
}
type WorkspaceAgentListeningPortsResponse struct {
@@ -309,7 +365,8 @@ func (c *WorkspaceAgentConn) ListeningPorts(ctx context.Context) (WorkspaceAgent
func (c *WorkspaceAgentConn) apiRequest(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
ctx, span := tracing.StartSpan(ctx)
defer span.End()
- host := net.JoinHostPort(WorkspaceAgentIP.String(), strconv.Itoa(WorkspaceAgentHTTPAPIServerPort))
+
+ host := net.JoinHostPort(c.agentAddress().String(), strconv.Itoa(WorkspaceAgentHTTPAPIServerPort))
url := fmt.Sprintf("http://%s%s", host, path)
req, err := http.NewRequestWithContext(ctx, method, url, body)
@@ -332,13 +389,14 @@ func (c *WorkspaceAgentConn) apiClient() *http.Client {
if network != "tcp" {
return nil, xerrors.Errorf("network must be tcp")
}
+
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, xerrors.Errorf("split host port %q: %w", addr, err)
}
- // Verify that host is TailnetIP and port is
- // TailnetStatisticsPort.
- if host != WorkspaceAgentIP.String() || port != strconv.Itoa(WorkspaceAgentHTTPAPIServerPort) {
+
+ // Verify that the port is TailnetStatisticsPort.
+ if port != strconv.Itoa(WorkspaceAgentHTTPAPIServerPort) {
return nil, xerrors.Errorf("request %q does not appear to be for http api", addr)
}
@@ -346,7 +404,12 @@ func (c *WorkspaceAgentConn) apiClient() *http.Client {
return nil, xerrors.Errorf("workspace agent not reachable in time: %v", ctx.Err())
}
- conn, err := c.DialContextTCP(ctx, netip.AddrPortFrom(WorkspaceAgentIP, WorkspaceAgentHTTPAPIServerPort))
+ ipAddr, err := netip.ParseAddr(host)
+ if err != nil {
+ return nil, xerrors.Errorf("parse host addr: %w", err)
+ }
+
+ conn, err := c.Conn.DialContextTCP(ctx, netip.AddrPortFrom(ipAddr, WorkspaceAgentHTTPAPIServerPort))
if err != nil {
return nil, xerrors.Errorf("dial http api: %w", err)
}
diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go
index 208c4511d261a..b76ebba9344f5 100644
--- a/codersdk/workspaceagents.go
+++ b/codersdk/workspaceagents.go
@@ -307,8 +307,8 @@ func (c *Client) DialWorkspaceAgent(ctx context.Context, agentID uuid.UUID, opti
options.Logger.Debug(ctx, "failed to dial", slog.Error(err))
continue
}
- sendNode, errChan := tailnet.ServeCoordinator(websocket.NetConn(ctx, ws, websocket.MessageBinary), func(node []*tailnet.Node) error {
- return conn.UpdateNodes(node, false)
+ sendNode, errChan := tailnet.ServeCoordinator(websocket.NetConn(ctx, ws, websocket.MessageBinary), func(nodes []*tailnet.Node) error {
+ return conn.UpdateNodes(nodes, false)
})
conn.SetNodeCallback(sendNode)
options.Logger.Debug(ctx, "serving coordinator")
@@ -330,13 +330,15 @@ func (c *Client) DialWorkspaceAgent(ctx context.Context, agentID uuid.UUID, opti
return nil, err
}
- agentConn = &WorkspaceAgentConn{
- Conn: conn,
- CloseFunc: func() {
+ agentConn = NewWorkspaceAgentConn(conn, WorkspaceAgentConnOptions{
+ AgentID: agentID,
+ CloseFunc: func() error {
cancel()
<-closed
+ return conn.Close()
},
- }
+ })
+
if !agentConn.AwaitReachable(ctx) {
_ = agentConn.Close()
return nil, xerrors.Errorf("timed out waiting for agent to become reachable: %w", ctx.Err())
diff --git a/docs/api/agents.md b/docs/api/agents.md
index 69ff2fbe72318..7dcce5d52e847 100644
--- a/docs/api/agents.md
+++ b/docs/api/agents.md
@@ -292,6 +292,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaceagents/me/manifest \
```json
{
+ "agent_id": "string",
"apps": [
{
"command": "string",
diff --git a/docs/api/schemas.md b/docs/api/schemas.md
index 2a0861d413573..a042b6d1f6e04 100644
--- a/docs/api/schemas.md
+++ b/docs/api/schemas.md
@@ -161,6 +161,7 @@
```json
{
+ "agent_id": "string",
"apps": [
{
"command": "string",
@@ -260,6 +261,7 @@
| Name | Type | Required | Restrictions | Description |
| ---------------------------- | ------------------------------------------------------------------------------------------------- | -------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `agent_id` | string | false | | |
| `apps` | array of [codersdk.WorkspaceApp](#codersdkworkspaceapp) | false | | |
| `derpmap` | [tailcfg.DERPMap](#tailcfgderpmap) | false | | |
| `directory` | string | false | | |
@@ -2543,6 +2545,7 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in
| `workspace_actions` |
| `tailnet_ha_coordinator` |
| `convert-to-oidc` |
+| `single_tailnet` |
| `workspace_build_logs_ui` |
## codersdk.Feature
diff --git a/enterprise/coderd/appearance_test.go b/enterprise/coderd/appearance_test.go
index dc6ce99052b60..6f564eaa3a680 100644
--- a/enterprise/coderd/appearance_test.go
+++ b/enterprise/coderd/appearance_test.go
@@ -6,9 +6,8 @@ import (
"net/http"
"testing"
- "github.com/stretchr/testify/require"
-
"github.com/google/uuid"
+ "github.com/stretchr/testify/require"
"github.com/coder/coder/cli/clibase"
"github.com/coder/coder/coderd/coderdtest"
diff --git a/enterprise/tailnet/coordinator.go b/enterprise/tailnet/coordinator.go
index b0d9cfa64032f..889df136710c5 100644
--- a/enterprise/tailnet/coordinator.go
+++ b/enterprise/tailnet/coordinator.go
@@ -17,6 +17,7 @@ import (
"cdr.dev/slog"
"github.com/coder/coder/coderd/database/pubsub"
+ "github.com/coder/coder/codersdk"
agpl "github.com/coder/coder/tailnet"
)
@@ -37,9 +38,12 @@ func NewCoordinator(logger slog.Logger, ps pubsub.Pubsub) (agpl.Coordinator, err
closeFunc: cancelFunc,
close: make(chan struct{}),
nodes: map[uuid.UUID]*agpl.Node{},
- agentSockets: map[uuid.UUID]*agpl.TrackedConn{},
- agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]*agpl.TrackedConn{},
+ agentSockets: map[uuid.UUID]agpl.Queue{},
+ agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]agpl.Queue{},
agentNameCache: nameCache,
+ clients: map[uuid.UUID]agpl.Queue{},
+ clientsToAgents: map[uuid.UUID]map[uuid.UUID]agpl.Queue{},
+ legacyAgents: map[uuid.UUID]struct{}{},
}
if err := coord.runPubsub(ctx); err != nil {
@@ -49,6 +53,56 @@ func NewCoordinator(logger slog.Logger, ps pubsub.Pubsub) (agpl.Coordinator, err
return coord, nil
}
+func (c *haCoordinator) ServeMultiAgent(id uuid.UUID) agpl.MultiAgentConn {
+ m := (&agpl.MultiAgent{
+ ID: id,
+ Logger: c.log,
+ AgentIsLegacyFunc: c.agentIsLegacy,
+ OnSubscribe: c.clientSubscribeToAgent,
+ OnNodeUpdate: c.clientNodeUpdate,
+ OnRemove: c.clientDisconnected,
+ }).Init()
+ c.addClient(id, m)
+ return m
+}
+
+func (c *haCoordinator) addClient(id uuid.UUID, q agpl.Queue) {
+ c.mutex.Lock()
+ c.clients[id] = q
+ c.clientsToAgents[id] = map[uuid.UUID]agpl.Queue{}
+ c.mutex.Unlock()
+}
+
+func (c *haCoordinator) clientSubscribeToAgent(enq agpl.Queue, agentID uuid.UUID) (*agpl.Node, error) {
+ c.mutex.Lock()
+ defer c.mutex.Unlock()
+
+ c.initOrSetAgentConnectionSocketLocked(agentID, enq)
+
+ node := c.nodes[enq.UniqueID()]
+ if node != nil {
+ err := c.sendNodeToAgentLocked(agentID, node)
+ if err != nil {
+ return nil, xerrors.Errorf("handle client update: %w", err)
+ }
+ }
+
+ agentNode, ok := c.nodes[agentID]
+ // If we have the node locally, give it back to the multiagent.
+ if ok {
+ return agentNode, nil
+ }
+
+ // If we don't have the node locally, notify other coordinators.
+ err := c.publishClientHello(agentID)
+ if err != nil {
+ return nil, xerrors.Errorf("publish client hello: %w", err)
+ }
+
+ // nolint:nilnil
+ return nil, nil
+}
+
type haCoordinator struct {
id uuid.UUID
log slog.Logger
@@ -60,14 +114,26 @@ type haCoordinator struct {
// nodes maps agent and connection IDs their respective node.
nodes map[uuid.UUID]*agpl.Node
// agentSockets maps agent IDs to their open websocket.
- agentSockets map[uuid.UUID]*agpl.TrackedConn
+ agentSockets map[uuid.UUID]agpl.Queue
// agentToConnectionSockets maps agent IDs to connection IDs of conns that
// are subscribed to updates for that agent.
- agentToConnectionSockets map[uuid.UUID]map[uuid.UUID]*agpl.TrackedConn
+ agentToConnectionSockets map[uuid.UUID]map[uuid.UUID]agpl.Queue
+
+ // clients holds a map of all clients connected to the coordinator. This is
+ // necessary because a client may not be subscribed into any agents.
+ clients map[uuid.UUID]agpl.Queue
+ // clientsToAgents is an index of clients to all of their subscribed agents.
+ clientsToAgents map[uuid.UUID]map[uuid.UUID]agpl.Queue
// agentNameCache holds a cache of agent names. If one of them disappears,
// it's helpful to have a name cached for debugging.
agentNameCache *lru.Cache[uuid.UUID, string]
+
+ // legacyAgents holda a mapping of all agents detected as legacy, meaning
+ // they only listen on codersdk.WorkspaceAgentIP. They aren't compatible
+ // with the new ServerTailnet, so they must be connected through
+ // wsconncache.
+ legacyAgents map[uuid.UUID]struct{}
}
// Node returns an in-memory node by ID.
@@ -88,47 +154,62 @@ func (c *haCoordinator) agentLogger(agent uuid.UUID) slog.Logger {
// ServeClient accepts a WebSocket connection that wants to connect to an agent
// with the specified ID.
-func (c *haCoordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error {
+func (c *haCoordinator) ServeClient(conn net.Conn, id, agentID uuid.UUID) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- logger := c.clientLogger(id, agent)
-
- c.mutex.Lock()
- connectionSockets, ok := c.agentToConnectionSockets[agent]
- if !ok {
- connectionSockets = map[uuid.UUID]*agpl.TrackedConn{}
- c.agentToConnectionSockets[agent] = connectionSockets
- }
+ logger := c.clientLogger(id, agentID)
tc := agpl.NewTrackedConn(ctx, cancel, conn, id, logger, 0)
- // Insert this connection into a map so the agent
- // can publish node updates.
- connectionSockets[id] = tc
+ defer tc.Close()
- // When a new connection is requested, we update it with the latest
- // node of the agent. This allows the connection to establish.
- node, ok := c.nodes[agent]
- if ok {
- err := tc.Enqueue([]*agpl.Node{node})
- c.mutex.Unlock()
+ c.addClient(id, tc)
+ defer c.clientDisconnected(id)
+
+ agentNode, err := c.clientSubscribeToAgent(tc, agentID)
+ if err != nil {
+ return xerrors.Errorf("subscribe agent: %w", err)
+ }
+
+ if agentNode != nil {
+ err := tc.Enqueue([]*agpl.Node{agentNode})
if err != nil {
- return xerrors.Errorf("enqueue node: %w", err)
+ logger.Debug(ctx, "enqueue initial node", slog.Error(err))
}
- } else {
- c.mutex.Unlock()
- err := c.publishClientHello(agent)
+ }
+
+ go tc.SendUpdates()
+
+ decoder := json.NewDecoder(conn)
+ // Indefinitely handle messages from the client websocket.
+ for {
+ err := c.handleNextClientMessage(id, decoder)
if err != nil {
- return xerrors.Errorf("publish client hello: %w", err)
+ if errors.Is(err, io.EOF) || errors.Is(err, io.ErrClosedPipe) {
+ return nil
+ }
+ return xerrors.Errorf("handle next client message: %w", err)
}
}
- go tc.SendUpdates()
+}
- defer func() {
- c.mutex.Lock()
- defer c.mutex.Unlock()
+func (c *haCoordinator) initOrSetAgentConnectionSocketLocked(agentID uuid.UUID, enq agpl.Queue) {
+ connectionSockets, ok := c.agentToConnectionSockets[agentID]
+ if !ok {
+ connectionSockets = map[uuid.UUID]agpl.Queue{}
+ c.agentToConnectionSockets[agentID] = connectionSockets
+ }
+ connectionSockets[enq.UniqueID()] = enq
+ c.clientsToAgents[enq.UniqueID()][agentID] = c.agentSockets[agentID]
+}
+
+func (c *haCoordinator) clientDisconnected(id uuid.UUID) {
+ c.mutex.Lock()
+ defer c.mutex.Unlock()
+
+ for agentID := range c.clientsToAgents[id] {
// Clean all traces of this connection from the map.
delete(c.nodes, id)
- connectionSockets, ok := c.agentToConnectionSockets[agent]
+ connectionSockets, ok := c.agentToConnectionSockets[agentID]
if !ok {
return
}
@@ -136,51 +217,65 @@ func (c *haCoordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID
if len(connectionSockets) != 0 {
return
}
- delete(c.agentToConnectionSockets, agent)
- }()
-
- decoder := json.NewDecoder(conn)
- // Indefinitely handle messages from the client websocket.
- for {
- err := c.handleNextClientMessage(id, agent, decoder)
- if err != nil {
- if errors.Is(err, io.EOF) || errors.Is(err, io.ErrClosedPipe) {
- return nil
- }
- return xerrors.Errorf("handle next client message: %w", err)
- }
+ delete(c.agentToConnectionSockets, agentID)
}
+
+ delete(c.clients, id)
+ delete(c.clientsToAgents, id)
}
-func (c *haCoordinator) handleNextClientMessage(id, agent uuid.UUID, decoder *json.Decoder) error {
+func (c *haCoordinator) handleNextClientMessage(id uuid.UUID, decoder *json.Decoder) error {
var node agpl.Node
err := decoder.Decode(&node)
if err != nil {
return xerrors.Errorf("read json: %w", err)
}
+ return c.clientNodeUpdate(id, &node)
+}
+
+func (c *haCoordinator) clientNodeUpdate(id uuid.UUID, node *agpl.Node) error {
c.mutex.Lock()
+ defer c.mutex.Unlock()
// Update the node of this client in our in-memory map. If an agent entirely
// shuts down and reconnects, it needs to be aware of all clients attempting
// to establish connections.
- c.nodes[id] = &node
- // Write the new node from this client to the actively connected agent.
- agentSocket, ok := c.agentSockets[agent]
+ c.nodes[id] = node
+ for agentID, agentSocket := range c.clientsToAgents[id] {
+ if agentSocket == nil {
+ // If we don't own the agent locally, send it over pubsub to a node that
+ // owns the agent.
+ err := c.publishNodesToAgent(agentID, []*agpl.Node{node})
+ if err != nil {
+ c.log.Error(context.Background(), "publish node to agent", slog.Error(err), slog.F("agent_id", agentID))
+ }
+ } else {
+ // Write the new node from this client to the actively connected agent.
+ err := agentSocket.Enqueue([]*agpl.Node{node})
+ if err != nil {
+ c.log.Error(context.Background(), "enqueue node to agent", slog.Error(err), slog.F("agent_id", agentID))
+ }
+ }
+ }
+
+ return nil
+}
+
+func (c *haCoordinator) sendNodeToAgentLocked(agentID uuid.UUID, node *agpl.Node) error {
+ agentSocket, ok := c.agentSockets[agentID]
if !ok {
- c.mutex.Unlock()
// If we don't own the agent locally, send it over pubsub to a node that
// owns the agent.
- err := c.publishNodesToAgent(agent, []*agpl.Node{&node})
+ err := c.publishNodesToAgent(agentID, []*agpl.Node{node})
if err != nil {
return xerrors.Errorf("publish node to agent")
}
return nil
}
- err = agentSocket.Enqueue([]*agpl.Node{&node})
- c.mutex.Unlock()
+ err := agentSocket.Enqueue([]*agpl.Node{node})
if err != nil {
- return xerrors.Errorf("enqueu nodes: %w", err)
+ return xerrors.Errorf("enqueue node: %w", err)
}
return nil
}
@@ -202,7 +297,7 @@ func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID, name string) err
// dead.
oldAgentSocket, ok := c.agentSockets[id]
if ok {
- overwrites = oldAgentSocket.Overwrites + 1
+ overwrites = oldAgentSocket.Overwrites() + 1
_ = oldAgentSocket.Close()
}
// This uniquely identifies a connection that belongs to this goroutine.
@@ -219,6 +314,9 @@ func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID, name string) err
}
}
c.agentSockets[id] = tc
+ for clientID := range c.agentToConnectionSockets[id] {
+ c.clientsToAgents[clientID][id] = tc
+ }
c.mutex.Unlock()
go tc.SendUpdates()
@@ -234,10 +332,13 @@ func (c *haCoordinator) ServeAgent(conn net.Conn, id uuid.UUID, name string) err
// Only delete the connection if it's ours. It could have been
// overwritten.
- if idConn, ok := c.agentSockets[id]; ok && idConn.ID == unique {
+ if idConn, ok := c.agentSockets[id]; ok && idConn.UniqueID() == unique {
delete(c.agentSockets, id)
delete(c.nodes, id)
}
+ for clientID := range c.agentToConnectionSockets[id] {
+ c.clientsToAgents[clientID][id] = nil
+ }
}()
decoder := json.NewDecoder(conn)
@@ -285,6 +386,13 @@ func (c *haCoordinator) handleClientHello(id uuid.UUID) error {
return c.publishAgentToNodes(id, node)
}
+func (c *haCoordinator) agentIsLegacy(agentID uuid.UUID) bool {
+ c.mutex.RLock()
+ _, ok := c.legacyAgents[agentID]
+ c.mutex.RUnlock()
+ return ok
+}
+
func (c *haCoordinator) handleAgentUpdate(id uuid.UUID, decoder *json.Decoder) (*agpl.Node, error) {
var node agpl.Node
err := decoder.Decode(&node)
@@ -293,6 +401,11 @@ func (c *haCoordinator) handleAgentUpdate(id uuid.UUID, decoder *json.Decoder) (
}
c.mutex.Lock()
+ // Keep a cache of all legacy agents.
+ if len(node.Addresses) > 0 && node.Addresses[0].Addr() == codersdk.WorkspaceAgentIP {
+ c.legacyAgents[id] = struct{}{}
+ }
+
oldNode := c.nodes[id]
if oldNode != nil {
if oldNode.AsOf.After(node.AsOf) {
@@ -311,7 +424,9 @@ func (c *haCoordinator) handleAgentUpdate(id uuid.UUID, decoder *json.Decoder) (
for _, connectionSocket := range connectionSockets {
_ = connectionSocket.Enqueue([]*agpl.Node{&node})
}
+
c.mutex.Unlock()
+
return &node, nil
}
@@ -334,20 +449,18 @@ func (c *haCoordinator) Close() error {
for _, socket := range c.agentSockets {
socket := socket
go func() {
- _ = socket.Close()
+ _ = socket.CoordinatorClose()
wg.Done()
}()
}
- for _, connMap := range c.agentToConnectionSockets {
- wg.Add(len(connMap))
- for _, socket := range connMap {
- socket := socket
- go func() {
- _ = socket.Close()
- wg.Done()
- }()
- }
+ wg.Add(len(c.clients))
+ for _, client := range c.clients {
+ client := client
+ go func() {
+ _ = client.CoordinatorClose()
+ wg.Done()
+ }()
}
wg.Wait()
@@ -422,13 +535,12 @@ func (c *haCoordinator) runPubsub(ctx context.Context) error {
}
go func() {
for {
- var message []byte
select {
case <-ctx.Done():
return
- case message = <-messageQueue:
+ case message := <-messageQueue:
+ c.handlePubsubMessage(ctx, message)
}
- c.handlePubsubMessage(ctx, message)
}
}()
diff --git a/enterprise/tailnet/pgcoord.go b/enterprise/tailnet/pgcoord.go
index 6c3cc73ac9e9d..37c516f5aa65e 100644
--- a/enterprise/tailnet/pgcoord.go
+++ b/enterprise/tailnet/pgcoord.go
@@ -125,6 +125,11 @@ func NewPGCoord(ctx context.Context, logger slog.Logger, ps pubsub.Pubsub, store
return c, nil
}
+func (c *pgCoord) ServeMultiAgent(id uuid.UUID) agpl.MultiAgentConn {
+ _, _ = c, id
+ panic("not implemented") // TODO: Implement
+}
+
func (*pgCoord) ServeHTTPDebug(w http.ResponseWriter, _ *http.Request) {
// TODO(spikecurtis) I'd like to hold off implementing this until after the rest of this is code reviewed.
w.WriteHeader(http.StatusOK)
diff --git a/enterprise/wsproxy/wsproxy.go b/enterprise/wsproxy/wsproxy.go
index fce5e0cc7a3b1..ae5da832054e2 100644
--- a/enterprise/wsproxy/wsproxy.go
+++ b/enterprise/wsproxy/wsproxy.go
@@ -183,9 +183,12 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
SecurityKey: secKey,
Logger: s.Logger.Named("proxy_token_provider"),
},
- WorkspaceConnCache: wsconncache.New(s.DialWorkspaceAgent, 0),
- AppSecurityKey: secKey,
+ AppSecurityKey: secKey,
+ // TODO: Convert wsproxy to use coderd.ServerTailnet.
+ AgentProvider: &wsconncache.AgentProvider{
+ Cache: wsconncache.New(s.DialWorkspaceAgent, 0),
+ },
DisablePathApps: opts.DisablePathApps,
SecureAuthCookie: opts.SecureAuthCookie,
}
@@ -273,6 +276,7 @@ func (s *Server) Close() error {
tmp, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
_ = s.SDKClient.WorkspaceProxyGoingAway(tmp)
+ _ = s.AppServer.AgentProvider.Close()
return s.AppServer.Close()
}
diff --git a/scaletest/agentconn/run.go b/scaletest/agentconn/run.go
index cefbb660485e5..6593e7c79b52c 100644
--- a/scaletest/agentconn/run.go
+++ b/scaletest/agentconn/run.go
@@ -6,7 +6,6 @@ import (
"io"
"net"
"net/http"
- "net/netip"
"net/url"
"strconv"
"time"
@@ -377,7 +376,10 @@ func agentHTTPClient(conn *codersdk.WorkspaceAgentConn) *http.Client {
if err != nil {
return nil, xerrors.Errorf("parse port %q: %w", port, err)
}
- return conn.DialContextTCP(ctx, netip.AddrPortFrom(codersdk.WorkspaceAgentIP, uint16(portUint)))
+
+ // Addr doesn't matter here, besides the port. DialContext will
+ // automatically choose the right IP to dial.
+ return conn.DialContext(ctx, "tcp", fmt.Sprintf("127.0.0.1:%d", portUint))
},
},
}
diff --git a/scaletest/reconnectingpty/run_test.go b/scaletest/reconnectingpty/run_test.go
index f6f70bbf574bf..382a3718436f9 100644
--- a/scaletest/reconnectingpty/run_test.go
+++ b/scaletest/reconnectingpty/run_test.go
@@ -9,6 +9,7 @@ import (
"github.com/google/uuid"
"github.com/stretchr/testify/require"
+ "cdr.dev/slog"
"cdr.dev/slog/sloggers/slogtest"
"github.com/coder/coder/agent"
"github.com/coder/coder/coderd/coderdtest"
@@ -243,7 +244,7 @@ func Test_Runner(t *testing.T) {
func setupRunnerTest(t *testing.T) (client *codersdk.Client, agentID uuid.UUID) {
t.Helper()
- client = coderdtest.New(t, &coderdtest.Options{
+ client, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{
IncludeProvisionerDaemon: true,
})
user := coderdtest.CreateFirstUser(t, client)
@@ -282,12 +283,16 @@ func setupRunnerTest(t *testing.T) (client *codersdk.Client, agentID uuid.UUID)
agentClient.SetSessionToken(authToken)
agentCloser := agent.New(agent.Options{
Client: agentClient,
- Logger: slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Named("agent"),
+ Logger: slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Named("agent").Leveled(slog.LevelDebug),
})
t.Cleanup(func() {
_ = agentCloser.Close()
})
resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
+ require.Eventually(t, func() bool {
+ t.Log("agent id", resources[0].Agents[0].ID)
+ return (*api.TailnetCoordinator.Load()).Node(resources[0].Agents[0].ID) != nil
+ }, testutil.WaitLong, testutil.IntervalMedium, "agent never connected")
return client, resources[0].Agents[0].ID
}
diff --git a/scaletest/workspacetraffic/run_test.go b/scaletest/workspacetraffic/run_test.go
index e53d408bcd428..c070a906be228 100644
--- a/scaletest/workspacetraffic/run_test.go
+++ b/scaletest/workspacetraffic/run_test.go
@@ -68,6 +68,7 @@ func TestRun(t *testing.T) {
agentCloser := agent.New(agent.Options{
Client: agentClient,
})
+
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
t.Cleanup(func() {
diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts
index ee1bcda3736e9..5ba2158c93b36 100644
--- a/site/src/api/typesGenerated.ts
+++ b/site/src/api/typesGenerated.ts
@@ -1431,12 +1431,14 @@ export const Entitlements: Entitlement[] = [
export type Experiment =
| "convert-to-oidc"
| "moons"
+ | "single_tailnet"
| "tailnet_ha_coordinator"
| "workspace_actions"
| "workspace_build_logs_ui"
export const Experiments: Experiment[] = [
"convert-to-oidc",
"moons",
+ "single_tailnet",
"tailnet_ha_coordinator",
"workspace_actions",
"workspace_build_logs_ui",
diff --git a/tailnet/conn.go b/tailnet/conn.go
index 363ccb80ff48c..0534cf961771a 100644
--- a/tailnet/conn.go
+++ b/tailnet/conn.go
@@ -139,6 +139,7 @@ func NewConn(options *Options) (conn *Conn, err error) {
}
}()
+ IP()
dialer := &tsdial.Dialer{
Logf: Logger(options.Logger.Named("tsdial")),
}
@@ -182,10 +183,17 @@ func NewConn(options *Options) (conn *Conn, err error) {
netMap.SelfNode.DiscoKey = magicConn.DiscoPublicKey()
netStack, err := netstack.Create(
- Logger(options.Logger.Named("netstack")), tunDevice, wireguardEngine, magicConn, dialer, dnsManager)
+ Logger(options.Logger.Named("netstack")),
+ tunDevice,
+ wireguardEngine,
+ magicConn,
+ dialer,
+ dnsManager,
+ )
if err != nil {
return nil, xerrors.Errorf("create netstack: %w", err)
}
+
dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
return netStack.DialContextTCP(ctx, dst)
}
@@ -203,7 +211,14 @@ func NewConn(options *Options) (conn *Conn, err error) {
localIPs, _ := localIPSet.IPSet()
logIPSet := netipx.IPSetBuilder{}
logIPs, _ := logIPSet.IPSet()
- wireguardEngine.SetFilter(filter.New(netMap.PacketFilter, localIPs, logIPs, nil, Logger(options.Logger.Named("packet-filter"))))
+ wireguardEngine.SetFilter(filter.New(
+ netMap.PacketFilter,
+ localIPs,
+ logIPs,
+ nil,
+ Logger(options.Logger.Named("packet-filter")),
+ ))
+
dialContext, dialCancel := context.WithCancel(context.Background())
server := &Conn{
blockEndpoints: options.BlockEndpoints,
@@ -230,6 +245,7 @@ func NewConn(options *Options) (conn *Conn, err error) {
_ = server.Close()
}
}()
+
wireguardEngine.SetStatusCallback(func(s *wgengine.Status, err error) {
server.logger.Debug(context.Background(), "wireguard status", slog.F("status", s), slog.Error(err))
if err != nil {
@@ -251,6 +267,7 @@ func NewConn(options *Options) (conn *Conn, err error) {
server.lastMutex.Unlock()
server.sendNode()
})
+
wireguardEngine.SetNetInfoCallback(func(ni *tailcfg.NetInfo) {
server.logger.Debug(context.Background(), "netinfo callback", slog.F("netinfo", ni))
server.lastMutex.Lock()
@@ -262,6 +279,7 @@ func NewConn(options *Options) (conn *Conn, err error) {
server.lastMutex.Unlock()
server.sendNode()
})
+
magicConn.SetDERPForcedWebsocketCallback(func(region int, reason string) {
server.logger.Debug(context.Background(), "derp forced websocket", slog.F("region", region), slog.F("reason", reason))
server.lastMutex.Lock()
@@ -273,6 +291,7 @@ func NewConn(options *Options) (conn *Conn, err error) {
server.lastMutex.Unlock()
server.sendNode()
})
+
netStack.ForwardTCPIn = server.forwardTCP
netStack.ForwardTCPSockOpts = server.forwardTCPSockOpts
@@ -284,22 +303,30 @@ func NewConn(options *Options) (conn *Conn, err error) {
return server, nil
}
-// IP generates a new IP with a static service prefix.
-func IP() netip.Addr {
- // This is Tailscale's ephemeral service prefix.
- // This can be changed easily later-on, because
- // all of our nodes are ephemeral.
+func maskUUID(uid uuid.UUID) uuid.UUID {
+ // This is Tailscale's ephemeral service prefix. This can be changed easily
+ // later-on, because all of our nodes are ephemeral.
// fd7a:115c:a1e0
- uid := uuid.New()
uid[0] = 0xfd
uid[1] = 0x7a
uid[2] = 0x11
uid[3] = 0x5c
uid[4] = 0xa1
uid[5] = 0xe0
+ return uid
+}
+
+// IP generates a random IP with a static service prefix.
+func IP() netip.Addr {
+ uid := maskUUID(uuid.New())
return netip.AddrFrom16(uid)
}
+// IP generates a new IP from a UUID.
+func IPFromUUID(uid uuid.UUID) netip.Addr {
+ return netip.AddrFrom16(maskUUID(uid))
+}
+
// Conn is an actively listening Wireguard connection.
type Conn struct {
dialContext context.Context
@@ -334,6 +361,29 @@ type Conn struct {
trafficStats *connstats.Statistics
}
+func (c *Conn) SetAddresses(ips []netip.Prefix) error {
+ c.mutex.Lock()
+ defer c.mutex.Unlock()
+
+ c.netMap.Addresses = ips
+
+ netMapCopy := *c.netMap
+ c.logger.Debug(context.Background(), "updating network map")
+ c.wireguardEngine.SetNetworkMap(&netMapCopy)
+ err := c.reconfig()
+ if err != nil {
+ return xerrors.Errorf("reconfig: %w", err)
+ }
+
+ return nil
+}
+
+func (c *Conn) Addresses() []netip.Prefix {
+ c.mutex.Lock()
+ defer c.mutex.Unlock()
+ return c.netMap.Addresses
+}
+
func (c *Conn) SetNodeCallback(callback func(node *Node)) {
c.lastMutex.Lock()
c.nodeCallback = callback
@@ -366,32 +416,6 @@ func (c *Conn) SetDERPRegionDialer(dialer func(ctx context.Context, region *tail
c.magicConn.SetDERPRegionDialer(dialer)
}
-func (c *Conn) RemoveAllPeers() error {
- c.mutex.Lock()
- defer c.mutex.Unlock()
-
- c.netMap.Peers = []*tailcfg.Node{}
- c.peerMap = map[tailcfg.NodeID]*tailcfg.Node{}
- netMapCopy := *c.netMap
- c.logger.Debug(context.Background(), "updating network map")
- c.wireguardEngine.SetNetworkMap(&netMapCopy)
- cfg, err := nmcfg.WGCfg(c.netMap, Logger(c.logger.Named("wgconfig")), netmap.AllowSingleHosts, "")
- if err != nil {
- return xerrors.Errorf("update wireguard config: %w", err)
- }
- err = c.wireguardEngine.Reconfig(cfg, c.wireguardRouter, &dns.Config{}, &tailcfg.Debug{})
- if err != nil {
- if c.isClosed() {
- return nil
- }
- if errors.Is(err, wgengine.ErrNoChanges) {
- return nil
- }
- return xerrors.Errorf("reconfig: %w", err)
- }
- return nil
-}
-
// UpdateNodes connects with a set of peers. This can be constantly updated,
// and peers will continually be reconnected as necessary. If replacePeers is
// true, all peers will be removed before adding the new ones.
@@ -423,6 +447,7 @@ func (c *Conn) UpdateNodes(nodes []*Node, replacePeers bool) error {
}
delete(c.peerMap, peer.ID)
}
+
for _, node := range nodes {
// If no preferred DERP is provided, we can't reach the node.
if node.PreferredDERP == 0 {
@@ -452,17 +477,29 @@ func (c *Conn) UpdateNodes(nodes []*Node, replacePeers bool) error {
}
c.peerMap[node.ID] = peerNode
}
+
c.netMap.Peers = make([]*tailcfg.Node, 0, len(c.peerMap))
for _, peer := range c.peerMap {
c.netMap.Peers = append(c.netMap.Peers, peer.Clone())
}
+
netMapCopy := *c.netMap
c.logger.Debug(context.Background(), "updating network map")
c.wireguardEngine.SetNetworkMap(&netMapCopy)
+ err := c.reconfig()
+ if err != nil {
+ return xerrors.Errorf("reconfig: %w", err)
+ }
+
+ return nil
+}
+
+func (c *Conn) reconfig() error {
cfg, err := nmcfg.WGCfg(c.netMap, Logger(c.logger.Named("wgconfig")), netmap.AllowSingleHosts, "")
if err != nil {
return xerrors.Errorf("update wireguard config: %w", err)
}
+
err = c.wireguardEngine.Reconfig(cfg, c.wireguardRouter, &dns.Config{}, &tailcfg.Debug{})
if err != nil {
if c.isClosed() {
@@ -473,6 +510,7 @@ func (c *Conn) UpdateNodes(nodes []*Node, replacePeers bool) error {
}
return xerrors.Errorf("reconfig: %w", err)
}
+
return nil
}
diff --git a/tailnet/conn_test.go b/tailnet/conn_test.go
index 2e19379e6df03..0dd0812b94777 100644
--- a/tailnet/conn_test.go
+++ b/tailnet/conn_test.go
@@ -23,7 +23,7 @@ func TestMain(m *testing.M) {
func TestTailnet(t *testing.T) {
t.Parallel()
logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
- derpMap := tailnettest.RunDERPAndSTUN(t)
+ derpMap, _ := tailnettest.RunDERPAndSTUN(t)
t.Run("InstantClose", func(t *testing.T) {
t.Parallel()
conn, err := tailnet.NewConn(&tailnet.Options{
@@ -172,7 +172,7 @@ func TestConn_PreferredDERP(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
defer cancel()
logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
- derpMap := tailnettest.RunDERPAndSTUN(t)
+ derpMap, _ := tailnettest.RunDERPAndSTUN(t)
conn, err := tailnet.NewConn(&tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
Logger: logger.Named("w1"),
diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go
index 7ff279a508b46..93cf8c67af56b 100644
--- a/tailnet/coordinator.go
+++ b/tailnet/coordinator.go
@@ -1,7 +1,6 @@
package tailnet
import (
- "bytes"
"context"
"encoding/json"
"errors"
@@ -11,17 +10,16 @@ import (
"net/http"
"net/netip"
"sync"
- "sync/atomic"
"time"
- "cdr.dev/slog"
-
"github.com/google/uuid"
lru "github.com/hashicorp/golang-lru/v2"
"golang.org/x/exp/slices"
"golang.org/x/xerrors"
"tailscale.com/tailcfg"
"tailscale.com/types/key"
+
+ "cdr.dev/slog"
)
// Coordinator exchanges nodes with agents to establish connections.
@@ -44,6 +42,8 @@ type Coordinator interface {
ServeAgent(conn net.Conn, id uuid.UUID, name string) error
// Close closes the coordinator.
Close() error
+
+ ServeMultiAgent(id uuid.UUID) MultiAgentConn
}
// Node represents a node in the network.
@@ -54,10 +54,11 @@ type Node struct {
AsOf time.Time `json:"as_of"`
// Key is the Wireguard public key of the node.
Key key.NodePublic `json:"key"`
- // DiscoKey is used for discovery messages over DERP to establish peer-to-peer connections.
+ // DiscoKey is used for discovery messages over DERP to establish
+ // peer-to-peer connections.
DiscoKey key.DiscoPublic `json:"disco"`
- // PreferredDERP is the DERP server that peered connections
- // should meet at to establish.
+ // PreferredDERP is the DERP server that peered connections should meet at
+ // to establish.
PreferredDERP int `json:"preferred_derp"`
// DERPLatency is the latency in seconds to each DERP server.
DERPLatency map[string]float64 `json:"derp_latency"`
@@ -68,8 +69,8 @@ type Node struct {
DERPForcedWebsocket map[int]string `json:"derp_forced_websockets"`
// Addresses are the IP address ranges this connection exposes.
Addresses []netip.Prefix `json:"addresses"`
- // AllowedIPs specify what addresses can dial the connection.
- // We allow all by default.
+ // AllowedIPs specify what addresses can dial the connection. We allow all
+ // by default.
AllowedIPs []netip.Prefix `json:"allowed_ips"`
// Endpoints are ip:port combinations that can be used to establish
// peer-to-peer connections.
@@ -130,12 +131,33 @@ func NewCoordinator(logger slog.Logger) Coordinator {
// ┌──────────────────┐ ┌────────────────────┐ ┌───────────────────┐ ┌──────────────────┐
// │tailnet.Coordinate├──►│tailnet.AcceptClient│◄─►│tailnet.AcceptAgent│◄──┤tailnet.Coordinate│
// └──────────────────┘ └────────────────────┘ └───────────────────┘ └──────────────────┘
-// This coordinator is incompatible with multiple Coder
-// replicas as all node data is in-memory.
+// This coordinator is incompatible with multiple Coder replicas as all node
+// data is in-memory.
type coordinator struct {
core *core
}
+func (c *coordinator) ServeMultiAgent(id uuid.UUID) MultiAgentConn {
+ m := (&MultiAgent{
+ ID: id,
+ Logger: c.core.logger,
+ AgentIsLegacyFunc: c.core.agentIsLegacy,
+ OnSubscribe: c.core.clientSubscribeToAgent,
+ OnUnsubscribe: c.core.clientUnsubscribeFromAgent,
+ OnNodeUpdate: c.core.clientNodeUpdate,
+ OnRemove: c.core.clientDisconnected,
+ }).Init()
+ c.core.addClient(id, m)
+ return m
+}
+
+func (c *core) addClient(id uuid.UUID, ma Queue) {
+ c.mutex.Lock()
+ c.clients[id] = ma
+ c.clientsToAgents[id] = map[uuid.UUID]Queue{}
+ c.mutex.Unlock()
+}
+
// core is an in-memory structure of Node and TrackedConn mappings. Its methods may be called from multiple goroutines;
// it is protected by a mutex to ensure data stay consistent.
type core struct {
@@ -146,14 +168,38 @@ type core struct {
// nodes maps agent and connection IDs their respective node.
nodes map[uuid.UUID]*Node
// agentSockets maps agent IDs to their open websocket.
- agentSockets map[uuid.UUID]*TrackedConn
+ agentSockets map[uuid.UUID]Queue
// agentToConnectionSockets maps agent IDs to connection IDs of conns that
// are subscribed to updates for that agent.
- agentToConnectionSockets map[uuid.UUID]map[uuid.UUID]*TrackedConn
+ agentToConnectionSockets map[uuid.UUID]map[uuid.UUID]Queue
+
+ // clients holds a map of all clients connected to the coordinator. This is
+ // necessary because a client may not be subscribed into any agents.
+ clients map[uuid.UUID]Queue
+ // clientsToAgents is an index of clients to all of their subscribed agents.
+ clientsToAgents map[uuid.UUID]map[uuid.UUID]Queue
// agentNameCache holds a cache of agent names. If one of them disappears,
// it's helpful to have a name cached for debugging.
agentNameCache *lru.Cache[uuid.UUID, string]
+
+ // legacyAgents holda a mapping of all agents detected as legacy, meaning
+ // they only listen on codersdk.WorkspaceAgentIP. They aren't compatible
+ // with the new ServerTailnet, so they must be connected through
+ // wsconncache.
+ legacyAgents map[uuid.UUID]struct{}
+}
+
+type Queue interface {
+ UniqueID() uuid.UUID
+ Enqueue(n []*Node) error
+ Name() string
+ Stats() (start, lastWrite int64)
+ Overwrites() int64
+ // CoordinatorClose is used by the coordinator when closing a Queue. It
+ // should skip removing itself from the coordinator.
+ CoordinatorClose() error
+ Close() error
}
func newCore(logger slog.Logger) *core {
@@ -165,128 +211,18 @@ func newCore(logger slog.Logger) *core {
return &core{
logger: logger,
closed: false,
- nodes: make(map[uuid.UUID]*Node),
- agentSockets: map[uuid.UUID]*TrackedConn{},
- agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]*TrackedConn{},
+ nodes: map[uuid.UUID]*Node{},
+ agentSockets: map[uuid.UUID]Queue{},
+ agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]Queue{},
agentNameCache: nameCache,
+ legacyAgents: map[uuid.UUID]struct{}{},
+ clients: map[uuid.UUID]Queue{},
+ clientsToAgents: map[uuid.UUID]map[uuid.UUID]Queue{},
}
}
var ErrWouldBlock = xerrors.New("would block")
-type TrackedConn struct {
- ctx context.Context
- cancel func()
- conn net.Conn
- updates chan []*Node
- logger slog.Logger
- lastData []byte
-
- // ID is an ephemeral UUID used to uniquely identify the owner of the
- // connection.
- ID uuid.UUID
-
- Name string
- Start int64
- LastWrite int64
- Overwrites int64
-}
-
-func (t *TrackedConn) Enqueue(n []*Node) (err error) {
- atomic.StoreInt64(&t.LastWrite, time.Now().Unix())
- select {
- case t.updates <- n:
- return nil
- default:
- return ErrWouldBlock
- }
-}
-
-// Close the connection and cancel the context for reading node updates from the queue
-func (t *TrackedConn) Close() error {
- t.cancel()
- return t.conn.Close()
-}
-
-// WriteTimeout is the amount of time we wait to write a node update to a connection before we declare it hung.
-// It is exported so that tests can use it.
-const WriteTimeout = time.Second * 5
-
-// SendUpdates reads node updates and writes them to the connection. Ends when writes hit an error or context is
-// canceled.
-func (t *TrackedConn) SendUpdates() {
- for {
- select {
- case <-t.ctx.Done():
- t.logger.Debug(t.ctx, "done sending updates")
- return
- case nodes := <-t.updates:
- data, err := json.Marshal(nodes)
- if err != nil {
- t.logger.Error(t.ctx, "unable to marshal nodes update", slog.Error(err), slog.F("nodes", nodes))
- return
- }
- if bytes.Equal(t.lastData, data) {
- t.logger.Debug(t.ctx, "skipping duplicate update", slog.F("nodes", string(data)))
- continue
- }
-
- // Set a deadline so that hung connections don't put back pressure on the system.
- // Node updates are tiny, so even the dinkiest connection can handle them if it's not hung.
- err = t.conn.SetWriteDeadline(time.Now().Add(WriteTimeout))
- if err != nil {
- // often, this is just because the connection is closed/broken, so only log at debug.
- t.logger.Debug(t.ctx, "unable to set write deadline", slog.Error(err))
- _ = t.Close()
- return
- }
- _, err = t.conn.Write(data)
- if err != nil {
- // often, this is just because the connection is closed/broken, so only log at debug.
- t.logger.Debug(t.ctx, "could not write nodes to connection",
- slog.Error(err), slog.F("nodes", string(data)))
- _ = t.Close()
- return
- }
- t.logger.Debug(t.ctx, "wrote nodes", slog.F("nodes", string(data)))
-
- // nhooyr.io/websocket has a bugged implementation of deadlines on a websocket net.Conn. What they are
- // *supposed* to do is set a deadline for any subsequent writes to complete, otherwise the call to Write()
- // fails. What nhooyr.io/websocket does is set a timer, after which it expires the websocket write context.
- // If this timer fires, then the next write will fail *even if we set a new write deadline*. So, after
- // our successful write, it is important that we reset the deadline before it fires.
- err = t.conn.SetWriteDeadline(time.Time{})
- if err != nil {
- // often, this is just because the connection is closed/broken, so only log at debug.
- t.logger.Debug(t.ctx, "unable to extend write deadline", slog.Error(err))
- _ = t.Close()
- return
- }
- t.lastData = data
- }
- }
-}
-
-func NewTrackedConn(ctx context.Context, cancel func(), conn net.Conn, id uuid.UUID, logger slog.Logger, overwrites int64) *TrackedConn {
- // buffer updates so they don't block, since we hold the
- // coordinator mutex while queuing. Node updates don't
- // come quickly, so 512 should be plenty for all but
- // the most pathological cases.
- updates := make(chan []*Node, 512)
- now := time.Now().Unix()
- return &TrackedConn{
- ctx: ctx,
- conn: conn,
- cancel: cancel,
- updates: updates,
- logger: logger,
- ID: id,
- Start: now,
- LastWrite: now,
- Overwrites: overwrites,
- }
-}
-
// Node returns an in-memory node by ID.
// If the node does not exist, nil is returned.
func (c *coordinator) Node(id uuid.UUID) *Node {
@@ -321,16 +257,29 @@ func (c *core) agentCount() int {
// ServeClient accepts a WebSocket connection that wants to connect to an agent
// with the specified ID.
-func (c *coordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error {
+func (c *coordinator) ServeClient(conn net.Conn, id, agentID uuid.UUID) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- logger := c.core.clientLogger(id, agent)
+ logger := c.core.clientLogger(id, agentID)
logger.Debug(ctx, "coordinating client")
- tc, err := c.core.initAndTrackClient(ctx, cancel, conn, id, agent)
+
+ tc := NewTrackedConn(ctx, cancel, conn, id, logger, 0)
+ defer tc.Close()
+
+ c.core.addClient(id, tc)
+ defer c.core.clientDisconnected(id)
+
+ agentNode, err := c.core.clientSubscribeToAgent(tc, agentID)
if err != nil {
- return err
+ return xerrors.Errorf("subscribe agent: %w", err)
+ }
+
+ if agentNode != nil {
+ err := tc.Enqueue([]*Node{agentNode})
+ if err != nil {
+ logger.Debug(ctx, "enqueue initial node", slog.Error(err))
+ }
}
- defer c.core.clientDisconnected(id, agent)
// On this goroutine, we read updates from the client and publish them. We start a second goroutine
// to write updates back to the client.
@@ -338,7 +287,7 @@ func (c *coordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID)
decoder := json.NewDecoder(conn)
for {
- err := c.handleNextClientMessage(id, agent, decoder)
+ err := c.handleNextClientMessage(id, decoder)
if err != nil {
logger.Debug(ctx, "unable to read client update, connection may be closed", slog.Error(err))
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrClosedPipe) || errors.Is(err, context.Canceled) {
@@ -353,99 +302,133 @@ func (c *core) clientLogger(id, agent uuid.UUID) slog.Logger {
return c.logger.With(slog.F("client_id", id), slog.F("agent_id", agent))
}
-// initAndTrackClient creates a TrackedConn for the client, and sends any initial Node updates if we have any. It is
-// one function that does two things because it is critical that we hold the mutex for both things, lest we miss some
-// updates.
-func (c *core) initAndTrackClient(
- ctx context.Context, cancel func(), conn net.Conn, id, agent uuid.UUID,
-) (
- *TrackedConn, error,
-) {
- logger := c.clientLogger(id, agent)
- c.mutex.Lock()
- defer c.mutex.Unlock()
- if c.closed {
- return nil, xerrors.New("coordinator is closed")
- }
- tc := NewTrackedConn(ctx, cancel, conn, id, logger, 0)
-
- // When a new connection is requested, we update it with the latest
- // node of the agent. This allows the connection to establish.
- node, ok := c.nodes[agent]
- if ok {
- err := tc.Enqueue([]*Node{node})
- // this should never error since we're still the only goroutine that
- // knows about the TrackedConn. If we hit an error something really
- // wrong is happening
- if err != nil {
- logger.Critical(ctx, "unable to queue initial node", slog.Error(err))
- return nil, err
- }
- }
-
- // Insert this connection into a map so the agent
- // can publish node updates.
- connectionSockets, ok := c.agentToConnectionSockets[agent]
+func (c *core) initOrSetAgentConnectionSocketLocked(agentID uuid.UUID, enq Queue) {
+ connectionSockets, ok := c.agentToConnectionSockets[agentID]
if !ok {
- connectionSockets = map[uuid.UUID]*TrackedConn{}
- c.agentToConnectionSockets[agent] = connectionSockets
+ connectionSockets = map[uuid.UUID]Queue{}
+ c.agentToConnectionSockets[agentID] = connectionSockets
}
- connectionSockets[id] = tc
- logger.Debug(ctx, "added tracked connection")
- return tc, nil
+ connectionSockets[enq.UniqueID()] = enq
+
+ c.clientsToAgents[enq.UniqueID()][agentID] = c.agentSockets[agentID]
}
-func (c *core) clientDisconnected(id, agent uuid.UUID) {
- logger := c.clientLogger(id, agent)
+func (c *core) clientDisconnected(id uuid.UUID) {
+ logger := c.clientLogger(id, uuid.Nil)
c.mutex.Lock()
defer c.mutex.Unlock()
// Clean all traces of this connection from the map.
delete(c.nodes, id)
logger.Debug(context.Background(), "deleted client node")
- connectionSockets, ok := c.agentToConnectionSockets[agent]
- if !ok {
- return
- }
- delete(connectionSockets, id)
- logger.Debug(context.Background(), "deleted client connectionSocket from map")
- if len(connectionSockets) != 0 {
- return
+
+ for agentID := range c.clientsToAgents[id] {
+ connectionSockets, ok := c.agentToConnectionSockets[agentID]
+ if !ok {
+ continue
+ }
+ delete(connectionSockets, id)
+ logger.Debug(context.Background(), "deleted client connectionSocket from map", slog.F("agent_id", agentID))
+
+ if len(connectionSockets) == 0 {
+ delete(c.agentToConnectionSockets, agentID)
+ logger.Debug(context.Background(), "deleted last client connectionSocket from map", slog.F("agent_id", agentID))
+ }
}
- delete(c.agentToConnectionSockets, agent)
- logger.Debug(context.Background(), "deleted last client connectionSocket from map")
+
+ delete(c.clients, id)
+ delete(c.clientsToAgents, id)
+ logger.Debug(context.Background(), "deleted client agents")
}
-func (c *coordinator) handleNextClientMessage(id, agent uuid.UUID, decoder *json.Decoder) error {
- logger := c.core.clientLogger(id, agent)
+func (c *coordinator) handleNextClientMessage(id uuid.UUID, decoder *json.Decoder) error {
+ logger := c.core.clientLogger(id, uuid.Nil)
+
var node Node
err := decoder.Decode(&node)
if err != nil {
return xerrors.Errorf("read json: %w", err)
}
+
logger.Debug(context.Background(), "got client node update", slog.F("node", node))
- return c.core.clientNodeUpdate(id, agent, &node)
+ return c.core.clientNodeUpdate(id, &node)
}
-func (c *core) clientNodeUpdate(id, agent uuid.UUID, node *Node) error {
- logger := c.clientLogger(id, agent)
+func (c *core) clientNodeUpdate(id uuid.UUID, node *Node) error {
c.mutex.Lock()
defer c.mutex.Unlock()
+
// Update the node of this client in our in-memory map. If an agent entirely
// shuts down and reconnects, it needs to be aware of all clients attempting
// to establish connections.
c.nodes[id] = node
- agentSocket, ok := c.agentSockets[agent]
- if !ok {
- logger.Debug(context.Background(), "no agent socket, unable to send node")
- return nil
+ return c.clientNodeUpdateLocked(id, node)
+}
+
+func (c *core) clientNodeUpdateLocked(id uuid.UUID, node *Node) error {
+ logger := c.clientLogger(id, uuid.Nil)
+
+ agents := []uuid.UUID{}
+ for agentID, agentSocket := range c.clientsToAgents[id] {
+ if agentSocket == nil {
+ logger.Debug(context.Background(), "enqueue node to agent; socket is nil", slog.F("agent_id", agentID))
+ continue
+ }
+
+ err := agentSocket.Enqueue([]*Node{node})
+ if err != nil {
+ logger.Debug(context.Background(), "unable to Enqueue node to agent", slog.Error(err), slog.F("agent_id", agentID))
+ continue
+ }
+ agents = append(agents, agentID)
}
- err := agentSocket.Enqueue([]*Node{node})
- if err != nil {
- return xerrors.Errorf("Enqueue node: %w", err)
+ logger.Debug(context.Background(), "enqueued node to agents", slog.F("agent_ids", agents))
+ return nil
+}
+
+func (c *core) clientSubscribeToAgent(enq Queue, agentID uuid.UUID) (*Node, error) {
+ c.mutex.Lock()
+ defer c.mutex.Unlock()
+
+ logger := c.clientLogger(enq.UniqueID(), agentID)
+
+ c.initOrSetAgentConnectionSocketLocked(agentID, enq)
+
+ node, ok := c.nodes[enq.UniqueID()]
+ if ok {
+ // If we have the client node, send it to the agent. If not, it will be
+ // sent async.
+ agentSocket, ok := c.agentSockets[agentID]
+ if !ok {
+ logger.Debug(context.Background(), "subscribe to agent; socket is nil")
+ } else {
+ err := agentSocket.Enqueue([]*Node{node})
+ if err != nil {
+ return nil, xerrors.Errorf("enqueue client to agent: %w", err)
+ }
+ }
+ } else {
+ logger.Debug(context.Background(), "multiagent node doesn't exist")
}
- logger.Debug(context.Background(), "enqueued node to agent")
+
+ agentNode, ok := c.nodes[agentID]
+ if !ok {
+ // This is ok, once the agent connects the node will be sent over.
+ logger.Debug(context.Background(), "agent node doesn't exist", slog.F("agent_id", agentID))
+ }
+
+ // Send the subscribed agent back to the multi agent.
+ return agentNode, nil
+}
+
+func (c *core) clientUnsubscribeFromAgent(enq Queue, agentID uuid.UUID) error {
+ c.mutex.Lock()
+ defer c.mutex.Unlock()
+
+ delete(c.clientsToAgents[enq.UniqueID()], agentID)
+ delete(c.agentToConnectionSockets[agentID], enq.UniqueID())
+
return nil
}
@@ -493,11 +476,14 @@ func (c *core) agentDisconnected(id, unique uuid.UUID) {
// Only delete the connection if it's ours. It could have been
// overwritten.
- if idConn, ok := c.agentSockets[id]; ok && idConn.ID == unique {
+ if idConn, ok := c.agentSockets[id]; ok && idConn.UniqueID() == unique {
delete(c.agentSockets, id)
delete(c.nodes, id)
logger.Debug(context.Background(), "deleted agent socket and node")
}
+ for clientID := range c.agentToConnectionSockets[id] {
+ c.clientsToAgents[clientID][id] = nil
+ }
}
// initAndTrackAgent creates a TrackedConn for the agent, and sends any initial nodes updates if we have any. It is
@@ -519,7 +505,7 @@ func (c *core) initAndTrackAgent(ctx context.Context, cancel func(), conn net.Co
// dead.
oldAgentSocket, ok := c.agentSockets[id]
if ok {
- overwrites = oldAgentSocket.Overwrites + 1
+ overwrites = oldAgentSocket.Overwrites() + 1
_ = oldAgentSocket.Close()
}
tc := NewTrackedConn(ctx, cancel, conn, unique, logger, overwrites)
@@ -549,6 +535,10 @@ func (c *core) initAndTrackAgent(ctx context.Context, cancel func(), conn net.Co
}
c.agentSockets[id] = tc
+ for clientID := range c.agentToConnectionSockets[id] {
+ c.clientsToAgents[clientID][id] = tc
+ }
+
logger.Debug(ctx, "added agent socket")
return tc, nil
}
@@ -564,11 +554,31 @@ func (c *coordinator) handleNextAgentMessage(id uuid.UUID, decoder *json.Decoder
return c.core.agentNodeUpdate(id, &node)
}
+// This is copied from codersdk because importing it here would cause an import
+// cycle. This is just temporary until wsconncache is phased out.
+var legacyAgentIP = netip.MustParseAddr("fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4")
+
+// This is temporary until we no longer need to detect for agent backwards
+// compatibility.
+// See: https://github.com/coder/coder/issues/8218
+func (c *core) agentIsLegacy(agentID uuid.UUID) bool {
+ c.mutex.RLock()
+ _, ok := c.legacyAgents[agentID]
+ c.mutex.RUnlock()
+ return ok
+}
+
func (c *core) agentNodeUpdate(id uuid.UUID, node *Node) error {
logger := c.agentLogger(id)
c.mutex.Lock()
defer c.mutex.Unlock()
c.nodes[id] = node
+
+ // Keep a cache of all legacy agents.
+ if len(node.Addresses) > 0 && node.Addresses[0].Addr() == legacyAgentIP {
+ c.legacyAgents[id] = struct{}{}
+ }
+
connectionSockets, ok := c.agentToConnectionSockets[id]
if !ok {
logger.Debug(context.Background(), "no client sockets; unable to send node")
@@ -588,6 +598,7 @@ func (c *core) agentNodeUpdate(id uuid.UUID, node *Node) error {
slog.F("client_id", clientID), slog.Error(err))
}
}
+
return nil
}
@@ -611,20 +622,18 @@ func (c *core) close() error {
for _, socket := range c.agentSockets {
socket := socket
go func() {
- _ = socket.Close()
+ _ = socket.CoordinatorClose()
wg.Done()
}()
}
- for _, connMap := range c.agentToConnectionSockets {
- wg.Add(len(connMap))
- for _, socket := range connMap {
- socket := socket
- go func() {
- _ = socket.Close()
- wg.Done()
- }()
- }
+ wg.Add(len(c.clients))
+ for _, client := range c.clients {
+ client := client
+ go func() {
+ _ = client.CoordinatorClose()
+ wg.Done()
+ }()
}
c.mutex.Unlock()
@@ -649,8 +658,8 @@ func (c *core) serveHTTPDebug(w http.ResponseWriter, r *http.Request) {
}
func CoordinatorHTTPDebug(
- agentSocketsMap map[uuid.UUID]*TrackedConn,
- agentToConnectionSocketsMap map[uuid.UUID]map[uuid.UUID]*TrackedConn,
+ agentSocketsMap map[uuid.UUID]Queue,
+ agentToConnectionSocketsMap map[uuid.UUID]map[uuid.UUID]Queue,
agentNameCache *lru.Cache[uuid.UUID, string],
) func(w http.ResponseWriter, _ *http.Request) {
return func(w http.ResponseWriter, _ *http.Request) {
@@ -658,7 +667,7 @@ func CoordinatorHTTPDebug(
type idConn struct {
id uuid.UUID
- conn *TrackedConn
+ conn Queue
}
{
@@ -671,16 +680,17 @@ func CoordinatorHTTPDebug(
}
slices.SortFunc(agentSockets, func(a, b idConn) bool {
- return a.conn.Name < b.conn.Name
+ return a.conn.Name() < b.conn.Name()
})
for _, agent := range agentSockets {
+ start, lastWrite := agent.conn.Stats()
_, _ = fmt.Fprintf(w, "%s (%s): created %v ago, write %v ago, overwrites %d \n",
- agent.conn.Name,
+ agent.conn.Name(),
agent.id.String(),
- now.Sub(time.Unix(agent.conn.Start, 0)).Round(time.Second),
- now.Sub(time.Unix(agent.conn.LastWrite, 0)).Round(time.Second),
- agent.conn.Overwrites,
+ now.Sub(time.Unix(start, 0)).Round(time.Second),
+ now.Sub(time.Unix(lastWrite, 0)).Round(time.Second),
+ agent.conn.Overwrites(),
)
if conns := agentToConnectionSocketsMap[agent.id]; len(conns) > 0 {
@@ -696,11 +706,12 @@ func CoordinatorHTTPDebug(
_, _ = fmt.Fprintln(w, "")
for _, connSocket := range connSockets {
+ start, lastWrite := connSocket.conn.Stats()
_, _ = fmt.Fprintf(w, "- %s (
%s): created %v ago, write %v ago \n",
- connSocket.conn.Name,
+ connSocket.conn.Name(),
connSocket.id.String(),
- now.Sub(time.Unix(connSocket.conn.Start, 0)).Round(time.Second),
- now.Sub(time.Unix(connSocket.conn.LastWrite, 0)).Round(time.Second),
+ now.Sub(time.Unix(start, 0)).Round(time.Second),
+ now.Sub(time.Unix(lastWrite, 0)).Round(time.Second),
)
}
_, _ = fmt.Fprintln(w, "
")
@@ -755,11 +766,12 @@ func CoordinatorHTTPDebug(
_, _ = fmt.Fprintf(w, "connections: total %d
\n", len(agentConns.conns))
_, _ = fmt.Fprintln(w, "")
for _, agentConn := range agentConns.conns {
+ start, lastWrite := agentConn.conn.Stats()
_, _ = fmt.Fprintf(w, "- %s (
%s): created %v ago, write %v ago \n",
- agentConn.conn.Name,
+ agentConn.conn.Name(),
agentConn.id.String(),
- now.Sub(time.Unix(agentConn.conn.Start, 0)).Round(time.Second),
- now.Sub(time.Unix(agentConn.conn.LastWrite, 0)).Round(time.Second),
+ now.Sub(time.Unix(start, 0)).Round(time.Second),
+ now.Sub(time.Unix(lastWrite, 0)).Round(time.Second),
)
}
_, _ = fmt.Fprintln(w, "
")
diff --git a/tailnet/multiagent.go b/tailnet/multiagent.go
new file mode 100644
index 0000000000000..13300fdce677a
--- /dev/null
+++ b/tailnet/multiagent.go
@@ -0,0 +1,167 @@
+package tailnet
+
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/google/uuid"
+ "golang.org/x/xerrors"
+
+ "cdr.dev/slog"
+)
+
+type MultiAgentConn interface {
+ UpdateSelf(node *Node) error
+ SubscribeAgent(agentID uuid.UUID) error
+ UnsubscribeAgent(agentID uuid.UUID) error
+ NextUpdate(ctx context.Context) ([]*Node, bool)
+ AgentIsLegacy(agentID uuid.UUID) bool
+ Close() error
+ IsClosed() bool
+}
+
+type MultiAgent struct {
+ mu sync.RWMutex
+
+ closed bool
+
+ ID uuid.UUID
+ Logger slog.Logger
+
+ AgentIsLegacyFunc func(agentID uuid.UUID) bool
+ OnSubscribe func(enq Queue, agent uuid.UUID) (*Node, error)
+ OnUnsubscribe func(enq Queue, agent uuid.UUID) error
+ OnNodeUpdate func(id uuid.UUID, node *Node) error
+ OnRemove func(id uuid.UUID)
+
+ updates chan []*Node
+ closeOnce sync.Once
+ start int64
+ lastWrite int64
+ // Client nodes normally generate a unique id for each connection so
+ // overwrites are really not an issue, but is provided for compatibility.
+ overwrites int64
+}
+
+func (m *MultiAgent) Init() *MultiAgent {
+ m.updates = make(chan []*Node, 128)
+ m.start = time.Now().Unix()
+ return m
+}
+
+func (m *MultiAgent) UniqueID() uuid.UUID {
+ return m.ID
+}
+
+func (m *MultiAgent) AgentIsLegacy(agentID uuid.UUID) bool {
+ return m.AgentIsLegacyFunc(agentID)
+}
+
+var ErrMultiAgentClosed = xerrors.New("multiagent is closed")
+
+func (m *MultiAgent) UpdateSelf(node *Node) error {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ if m.closed {
+ return ErrMultiAgentClosed
+ }
+
+ return m.OnNodeUpdate(m.ID, node)
+}
+
+func (m *MultiAgent) SubscribeAgent(agentID uuid.UUID) error {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ if m.closed {
+ return ErrMultiAgentClosed
+ }
+
+ node, err := m.OnSubscribe(m, agentID)
+ if err != nil {
+ return err
+ }
+
+ if node != nil {
+ return m.enqueueLocked([]*Node{node})
+ }
+
+ return nil
+}
+
+func (m *MultiAgent) UnsubscribeAgent(agentID uuid.UUID) error {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ if m.closed {
+ return ErrMultiAgentClosed
+ }
+
+ return m.OnUnsubscribe(m, agentID)
+}
+
+func (m *MultiAgent) NextUpdate(ctx context.Context) ([]*Node, bool) {
+ select {
+ case <-ctx.Done():
+ return nil, false
+
+ case nodes, ok := <-m.updates:
+ return nodes, ok
+ }
+}
+
+func (m *MultiAgent) Enqueue(nodes []*Node) error {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+
+ if m.closed {
+ return nil
+ }
+
+ return m.enqueueLocked(nodes)
+}
+
+func (m *MultiAgent) enqueueLocked(nodes []*Node) error {
+ atomic.StoreInt64(&m.lastWrite, time.Now().Unix())
+
+ select {
+ case m.updates <- nodes:
+ return nil
+ default:
+ return ErrWouldBlock
+ }
+}
+
+func (m *MultiAgent) Name() string {
+ return m.ID.String()
+}
+
+func (m *MultiAgent) Stats() (start int64, lastWrite int64) {
+ return m.start, atomic.LoadInt64(&m.lastWrite)
+}
+
+func (m *MultiAgent) Overwrites() int64 {
+ return m.overwrites
+}
+
+func (m *MultiAgent) IsClosed() bool {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ return m.closed
+}
+
+func (m *MultiAgent) CoordinatorClose() error {
+ m.mu.Lock()
+ if !m.closed {
+ m.closed = true
+ close(m.updates)
+ }
+ m.mu.Unlock()
+ return nil
+}
+
+func (m *MultiAgent) Close() error {
+ _ = m.CoordinatorClose()
+ m.closeOnce.Do(func() { m.OnRemove(m.ID) })
+ return nil
+}
diff --git a/tailnet/tailnettest/tailnettest.go b/tailnet/tailnettest/tailnettest.go
index 482c1232e258a..0cb7dbd330ed3 100644
--- a/tailnet/tailnettest/tailnettest.go
+++ b/tailnet/tailnettest/tailnettest.go
@@ -22,7 +22,7 @@ import (
)
// RunDERPAndSTUN creates a DERP mapping for tests.
-func RunDERPAndSTUN(t *testing.T) *tailcfg.DERPMap {
+func RunDERPAndSTUN(t *testing.T) (*tailcfg.DERPMap, *derp.Server) {
logf := tailnet.Logger(slogtest.Make(t, nil))
d := derp.NewServer(key.NewNode(), logf)
server := httptest.NewUnstartedServer(derphttp.Handler(d))
@@ -61,7 +61,7 @@ func RunDERPAndSTUN(t *testing.T) *tailcfg.DERPMap {
},
},
},
- }
+ }, d
}
// RunDERPOnlyWebSockets creates a DERP mapping for tests that
diff --git a/tailnet/tailnettest/tailnettest_test.go b/tailnet/tailnettest/tailnettest_test.go
index 6424bd94db0c0..fda818a1cebca 100644
--- a/tailnet/tailnettest/tailnettest_test.go
+++ b/tailnet/tailnettest/tailnettest_test.go
@@ -14,7 +14,7 @@ func TestMain(m *testing.M) {
func TestRunDERPAndSTUN(t *testing.T) {
t.Parallel()
- _ = tailnettest.RunDERPAndSTUN(t)
+ _, _ = tailnettest.RunDERPAndSTUN(t)
}
func TestRunDERPOnlyWebSockets(t *testing.T) {
diff --git a/tailnet/trackedconn.go b/tailnet/trackedconn.go
new file mode 100644
index 0000000000000..cedd6e37dbc8d
--- /dev/null
+++ b/tailnet/trackedconn.go
@@ -0,0 +1,147 @@
+package tailnet
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "net"
+ "sync/atomic"
+ "time"
+
+ "github.com/google/uuid"
+
+ "cdr.dev/slog"
+)
+
+// WriteTimeout is the amount of time we wait to write a node update to a connection before we declare it hung.
+// It is exported so that tests can use it.
+const WriteTimeout = time.Second * 5
+
+type TrackedConn struct {
+ ctx context.Context
+ cancel func()
+ conn net.Conn
+ updates chan []*Node
+ logger slog.Logger
+ lastData []byte
+
+ // ID is an ephemeral UUID used to uniquely identify the owner of the
+ // connection.
+ id uuid.UUID
+
+ name string
+ start int64
+ lastWrite int64
+ overwrites int64
+}
+
+func NewTrackedConn(ctx context.Context, cancel func(), conn net.Conn, id uuid.UUID, logger slog.Logger, overwrites int64) *TrackedConn {
+ // buffer updates so they don't block, since we hold the
+ // coordinator mutex while queuing. Node updates don't
+ // come quickly, so 512 should be plenty for all but
+ // the most pathological cases.
+ updates := make(chan []*Node, 512)
+ now := time.Now().Unix()
+ return &TrackedConn{
+ ctx: ctx,
+ conn: conn,
+ cancel: cancel,
+ updates: updates,
+ logger: logger,
+ id: id,
+ start: now,
+ lastWrite: now,
+ overwrites: overwrites,
+ }
+}
+
+func (t *TrackedConn) Enqueue(n []*Node) (err error) {
+ atomic.StoreInt64(&t.lastWrite, time.Now().Unix())
+ select {
+ case t.updates <- n:
+ return nil
+ default:
+ return ErrWouldBlock
+ }
+}
+
+func (t *TrackedConn) UniqueID() uuid.UUID {
+ return t.id
+}
+
+func (t *TrackedConn) Name() string {
+ return t.name
+}
+
+func (t *TrackedConn) Stats() (start, lastWrite int64) {
+ return t.start, atomic.LoadInt64(&t.lastWrite)
+}
+
+func (t *TrackedConn) Overwrites() int64 {
+ return t.overwrites
+}
+
+func (t *TrackedConn) CoordinatorClose() error {
+ return t.Close()
+}
+
+// Close the connection and cancel the context for reading node updates from the queue
+func (t *TrackedConn) Close() error {
+ t.cancel()
+ return t.conn.Close()
+}
+
+// SendUpdates reads node updates and writes them to the connection. Ends when writes hit an error or context is
+// canceled.
+func (t *TrackedConn) SendUpdates() {
+ for {
+ select {
+ case <-t.ctx.Done():
+ t.logger.Debug(t.ctx, "done sending updates")
+ return
+ case nodes := <-t.updates:
+ data, err := json.Marshal(nodes)
+ if err != nil {
+ t.logger.Error(t.ctx, "unable to marshal nodes update", slog.Error(err), slog.F("nodes", nodes))
+ return
+ }
+ if bytes.Equal(t.lastData, data) {
+ t.logger.Debug(t.ctx, "skipping duplicate update", slog.F("nodes", string(data)))
+ continue
+ }
+
+ // Set a deadline so that hung connections don't put back pressure on the system.
+ // Node updates are tiny, so even the dinkiest connection can handle them if it's not hung.
+ err = t.conn.SetWriteDeadline(time.Now().Add(WriteTimeout))
+ if err != nil {
+ // often, this is just because the connection is closed/broken, so only log at debug.
+ t.logger.Debug(t.ctx, "unable to set write deadline", slog.Error(err))
+ _ = t.Close()
+ return
+ }
+ _, err = t.conn.Write(data)
+ if err != nil {
+ // often, this is just because the connection is closed/broken, so only log at debug.
+ t.logger.Debug(t.ctx, "could not write nodes to connection",
+ slog.Error(err), slog.F("nodes", string(data)))
+ _ = t.Close()
+ return
+ }
+ t.logger.Debug(t.ctx, "wrote nodes", slog.F("nodes", string(data)))
+
+ // nhooyr.io/websocket has a bugged implementation of deadlines on a websocket net.Conn. What they are
+ // *supposed* to do is set a deadline for any subsequent writes to complete, otherwise the call to Write()
+ // fails. What nhooyr.io/websocket does is set a timer, after which it expires the websocket write context.
+ // If this timer fires, then the next write will fail *even if we set a new write deadline*. So, after
+ // our successful write, it is important that we reset the deadline before it fires.
+ err = t.conn.SetWriteDeadline(time.Time{})
+ if err != nil {
+ // often, this is just because the connection is closed/broken, so only log at debug.
+ t.logger.Debug(t.ctx, "unable to extend write deadline", slog.Error(err))
+ _ = t.Close()
+ return
+ }
+ t.lastData = data
+ }
+ }
+}
pFad - Phonifier reborn
Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies:
Alternative Proxy
pFad Proxy
pFad v3 Proxy
pFad v4 Proxy