diff --git a/coderd/rbac/roles_test.go b/coderd/rbac/roles_test.go index d90f045284c5b..e6680d4d628cc 100644 --- a/coderd/rbac/roles_test.go +++ b/coderd/rbac/roles_test.go @@ -77,7 +77,7 @@ func TestOwnerExec(t *testing.T) { }) } -// nolint:tparallel,paralleltest -- subtests share a map, just run sequentially. +// nolint:tparallel,paralleltest // subtests share a map, just run sequentially. func TestRolePermissions(t *testing.T) { t.Parallel() @@ -557,7 +557,7 @@ func TestRolePermissions(t *testing.T) { // nolint:tparallel,paralleltest for _, c := range testCases { c := c - // nolint:tparallel,paralleltest -- These share the same remainingPermissions map + // nolint:tparallel,paralleltest // These share the same remainingPermissions map t.Run(c.Name, func(t *testing.T) { remainingSubjs := make(map[string]struct{}) for _, subj := range requiredSubjects { @@ -600,7 +600,7 @@ func TestRolePermissions(t *testing.T) { // Only run these if the tests on top passed. Otherwise, the error output is too noisy. if passed { for rtype, v := range remainingPermissions { - // nolint:tparallel,paralleltest -- Making a subtest for easier diagnosing failures. + // nolint:tparallel,paralleltest // Making a subtest for easier diagnosing failures. t.Run(fmt.Sprintf("%s-AllActions", rtype), func(t *testing.T) { if len(v) > 0 { assert.Equal(t, map[policy.Action]bool{}, v, "remaining permissions should be empty for type %q", rtype) diff --git a/tailnet/test/integration/integration.go b/tailnet/test/integration/integration.go index b26365ea3ee8b..3877542c8eafc 100644 --- a/tailnet/test/integration/integration.go +++ b/tailnet/test/integration/integration.go @@ -41,12 +41,34 @@ import ( "github.com/coder/coder/v2/testutil" ) -// IDs used in tests. -var ( - Client1ID = uuid.MustParse("00000000-0000-0000-0000-000000000001") - Client2ID = uuid.MustParse("00000000-0000-0000-0000-000000000002") +type ClientNumber int + +const ( + ClientNumber1 ClientNumber = 1 + ClientNumber2 ClientNumber = 2 ) +type Client struct { + Number ClientNumber + ID uuid.UUID + ListenPort uint16 + ShouldRunTests bool +} + +var Client1 = Client{ + Number: ClientNumber1, + ID: uuid.MustParse("00000000-0000-0000-0000-000000000001"), + ListenPort: client1Port, + ShouldRunTests: true, +} + +var Client2 = Client{ + Number: ClientNumber2, + ID: uuid.MustParse("00000000-0000-0000-0000-000000000002"), + ListenPort: client2Port, + ShouldRunTests: false, +} + type TestTopology struct { Name string // SetupNetworking creates interfaces and network namespaces for the test. @@ -59,12 +81,12 @@ type TestTopology struct { Server ServerStarter // StartClient gets called in each client subprocess. It's expected to // create the tailnet.Conn and ensure connectivity to it's peer. - StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn + StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me Client, peer Client) *tailnet.Conn // RunTests is the main test function. It's called in each of the client // subprocesses. If tests can only run once, they should check the client ID // and return early if it's not the expected one. - RunTests func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID, conn *tailnet.Conn) + RunTests func(t *testing.T, logger slog.Logger, serverURL *url.URL, conn *tailnet.Conn, me Client, peer Client) } type ServerStarter interface { @@ -264,13 +286,14 @@ http { // StartClientDERP creates a client connection to the server for coordination // and creates a tailnet.Conn which will only use DERP to connect to the peer. -func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn { - return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{ - Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)}, - DERPMap: basicDERPMap(t, serverURL), +func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn { + return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)}, + DERPMap: derpMap, BlockEndpoints: true, Logger: logger, DERPForceWebSockets: false, + ListenPort: me.ListenPort, // These tests don't have internet connection, so we need to force // magicsock to do anything. ForceNetworkUp: true, @@ -279,13 +302,14 @@ func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, // StartClientDERPWebSockets does the same thing as StartClientDERP but will // only use DERP WebSocket fallback. -func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn { - return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{ - Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)}, - DERPMap: basicDERPMap(t, serverURL), +func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn { + return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)}, + DERPMap: derpMap, BlockEndpoints: true, Logger: logger, DERPForceWebSockets: true, + ListenPort: me.ListenPort, // These tests don't have internet connection, so we need to force // magicsock to do anything. ForceNetworkUp: true, @@ -295,20 +319,21 @@ func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url. // StartClientDirect does the same thing as StartClientDERP but disables // BlockEndpoints (which enables Direct connections), and waits for a direct // connection to be established between the two peers. -func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn { - conn := startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{ - Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)}, - DERPMap: basicDERPMap(t, serverURL), +func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn { + conn := startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(me.ID), 128)}, + DERPMap: derpMap, BlockEndpoints: false, Logger: logger, DERPForceWebSockets: true, + ListenPort: me.ListenPort, // These tests don't have internet connection, so we need to force // magicsock to do anything. ForceNetworkUp: true, }) // Wait for direct connection to be established. - peerIP := tailnet.IPFromUUID(peerID) + peerIP := tailnet.IPFromUUID(peer.ID) require.Eventually(t, func() bool { t.Log("attempting ping to peer to judge direct connection") ctx := testutil.Context(t, testutil.WaitShort) @@ -332,8 +357,8 @@ type ClientStarter struct { Options *tailnet.Options } -func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID, options *tailnet.Options) *tailnet.Conn { - u, err := serverURL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", myID.String())) +func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, me, peer Client, options *tailnet.Options) *tailnet.Conn { + u, err := serverURL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", me.ID.String())) require.NoError(t, err) //nolint:bodyclose ws, _, err := websocket.Dial(context.Background(), u.String(), nil) @@ -357,7 +382,7 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, my _ = conn.Close() }) - coordination := tailnet.NewRemoteCoordination(logger, coord, conn, peerID) + coordination := tailnet.NewRemoteCoordination(logger, coord, conn, peer.ID) t.Cleanup(func() { _ = coordination.Close() }) @@ -365,10 +390,17 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, my return conn } -func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap { +func basicDERPMap(serverURLStr string) (*tailcfg.DERPMap, error) { + serverURL, err := url.Parse(serverURLStr) + if err != nil { + return nil, xerrors.Errorf("parse server URL %q: %w", serverURLStr, err) + } + portStr := serverURL.Port() port, err := strconv.Atoi(portStr) - require.NoError(t, err, "parse server port") + if err != nil { + return nil, xerrors.Errorf("parse port %q: %w", portStr, err) + } hostname := serverURL.Hostname() ipv4 := "" @@ -399,7 +431,7 @@ func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap { }, }, }, - } + }, nil } // ExecBackground starts a subprocess with the given flags and returns a diff --git a/tailnet/test/integration/integration_test.go b/tailnet/test/integration/integration_test.go index 45d88145216c1..e23b716096048 100644 --- a/tailnet/test/integration/integration_test.go +++ b/tailnet/test/integration/integration_test.go @@ -4,19 +4,27 @@ package integration_test import ( + "context" + "encoding/json" "flag" "fmt" + "net" "net/http" "net/url" "os" "os/signal" + "path/filepath" "runtime" + "strconv" "syscall" "testing" "time" - "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "tailscale.com/net/stun/stuntest" + "tailscale.com/tailcfg" + "tailscale.com/types/nettype" "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" @@ -30,17 +38,19 @@ const runTestEnv = "CODER_TAILNET_TESTS" var ( isSubprocess = flag.Bool("subprocess", false, "Signifies that this is a test subprocess") testID = flag.String("test-name", "", "Which test is being run") - role = flag.String("role", "", "The role of the test subprocess: server, client") + role = flag.String("role", "", "The role of the test subprocess: server, stun, client") // Role: server serverListenAddr = flag.String("server-listen-addr", "", "The address to listen on for the server") + // Role: stun + stunListenAddr = flag.String("stun-listen-addr", "", "The address to listen on for the STUN server") + // Role: client - clientName = flag.String("client-name", "", "The name of the client for logs") - clientServerURL = flag.String("client-server-url", "", "The url to connect to the server") - clientMyID = flag.String("client-id", "", "The id of the client") - clientPeerID = flag.String("client-peer-id", "", "The id of the other client") - clientRunTests = flag.Bool("client-run-tests", false, "Run the tests in the client subprocess") + clientName = flag.String("client-name", "", "The name of the client for logs") + clientNumber = flag.Int("client-number", 0, "The number of the client") + clientServerURL = flag.String("client-server-url", "", "The url to connect to the server") + clientDERPMapPath = flag.String("client-derp-map-path", "", "The path to the DERP map file to use on this client") ) func TestMain(m *testing.M) { @@ -87,7 +97,7 @@ var topologies = []integration.TestTopology{ // endpoints to connect as routing is enabled between client 1 and // client 2. Name: "EasyNATDirect", - SetupNetworking: integration.SetupNetworkingEasyNAT, + SetupNetworking: integration.SetupNetworkingEasyNATWithSTUN, Server: integration.SimpleServerOptions{}, StartClient: integration.StartClientDirect, RunTests: integration.TestSuite, @@ -143,17 +153,41 @@ func TestIntegration(t *testing.T) { log := slogtest.Make(t, nil).Leveled(slog.LevelDebug) networking := topo.SetupNetworking(t, log) - // Fork the three child processes. + // Useful for debugging network namespaces by avoiding cleanup. + // t.Cleanup(func() { + // time.Sleep(time.Minute * 15) + // }) + closeServer := startServerSubprocess(t, topo.Name, networking) + + closeSTUN := func() error { return nil } + if networking.STUN.ListenAddr != "" { + closeSTUN = startSTUNSubprocess(t, topo.Name, networking) + } + + // Write the DERP maps to a file. + tempDir := t.TempDir() + client1DERPMapPath := filepath.Join(tempDir, "client1-derp-map.json") + client1DERPMap, err := networking.Client1.ResolveDERPMap() + require.NoError(t, err, "resolve client 1 DERP map") + err = writeDERPMapToFile(client1DERPMapPath, client1DERPMap) + require.NoError(t, err, "write client 1 DERP map") + client2DERPMapPath := filepath.Join(tempDir, "client2-derp-map.json") + client2DERPMap, err := networking.Client2.ResolveDERPMap() + require.NoError(t, err, "resolve client 2 DERP map") + err = writeDERPMapToFile(client2DERPMapPath, client2DERPMap) + require.NoError(t, err, "write client 2 DERP map") + // client1 runs the tests. - client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, 1) - _, closeClient2 := startClientSubprocess(t, topo.Name, networking, 2) + client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, integration.Client1, client1DERPMapPath) + _, closeClient2 := startClientSubprocess(t, topo.Name, networking, integration.Client2, client2DERPMapPath) // Wait for client1 to exit. require.NoError(t, <-client1ErrCh, "client 1 exited") // Close client2 and the server. require.NoError(t, closeClient2(), "client 2 exited") + require.NoError(t, closeSTUN(), "stun exited") require.NoError(t, closeServer(), "server exited") }) } @@ -169,10 +203,11 @@ func handleTestSubprocess(t *testing.T) { } } require.NotEmptyf(t, topo.Name, "unknown test topology %q", *testID) + require.Contains(t, []string{"server", "stun", "client"}, *role, "unknown role %q", *role) testName := topo.Name + "/" - if *role == "server" { - testName += "server" + if *role == "server" || *role == "stun" { + testName += *role } else { testName += *clientName } @@ -185,27 +220,44 @@ func handleTestSubprocess(t *testing.T) { topo.Server.StartServer(t, logger, *serverListenAddr) // no exit + case "stun": + launchSTUNServer(t, *stunListenAddr) + // no exit + case "client": logger = logger.Named(*clientName) + if *clientNumber != int(integration.ClientNumber1) && *clientNumber != int(integration.ClientNumber2) { + t.Fatalf("invalid client number %d", clientNumber) + } + me, peer := integration.Client1, integration.Client2 + if *clientNumber == int(integration.ClientNumber2) { + me, peer = peer, me + } + serverURL, err := url.Parse(*clientServerURL) require.NoErrorf(t, err, "parse server url %q", *clientServerURL) - myID, err := uuid.Parse(*clientMyID) - require.NoErrorf(t, err, "parse client id %q", *clientMyID) - peerID, err := uuid.Parse(*clientPeerID) - require.NoErrorf(t, err, "parse peer id %q", *clientPeerID) + + // Load the DERP map. + var derpMap tailcfg.DERPMap + derpMapPath := *clientDERPMapPath + f, err := os.Open(derpMapPath) + require.NoErrorf(t, err, "open DERP map %q", derpMapPath) + err = json.NewDecoder(f).Decode(&derpMap) + _ = f.Close() + require.NoErrorf(t, err, "decode DERP map %q", derpMapPath) waitForServerAvailable(t, serverURL) - conn := topo.StartClient(t, logger, serverURL, myID, peerID) + conn := topo.StartClient(t, logger, serverURL, &derpMap, me, peer) - if *clientRunTests { + if me.ShouldRunTests { // Wait for connectivity. - peerIP := tailnet.IPFromUUID(peerID) + peerIP := tailnet.IPFromUUID(peer.ID) if !conn.AwaitReachable(testutil.Context(t, testutil.WaitLong), peerIP) { t.Fatalf("peer %v did not become reachable", peerIP) } - topo.RunTests(t, logger, serverURL, myID, peerID, conn) + topo.RunTests(t, logger, serverURL, conn, me, peer) // then exit return } @@ -218,6 +270,23 @@ func handleTestSubprocess(t *testing.T) { }) } +type forcedAddrPacketListener struct { + addr string +} + +var _ nettype.PacketListener = forcedAddrPacketListener{} + +func (ln forcedAddrPacketListener) ListenPacket(ctx context.Context, network, _ string) (net.PacketConn, error) { + return nettype.Std{}.ListenPacket(ctx, network, ln.addr) +} + +func launchSTUNServer(t *testing.T, listenAddr string) { + ln := forcedAddrPacketListener{addr: listenAddr} + addr, cleanup := stuntest.ServeWithPacketListener(t, ln) + t.Cleanup(cleanup) + assert.Equal(t, listenAddr, addr.String(), "listen address should match forced addr") +} + func waitForServerAvailable(t *testing.T, serverURL *url.URL) { const delay = 100 * time.Millisecond const reqTimeout = 2 * time.Second @@ -247,29 +316,32 @@ func waitForServerAvailable(t *testing.T, serverURL *url.URL) { } func startServerSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error { - _, closeFn := startSubprocess(t, "server", networking.ProcessServer.NetNS, []string{ + _, closeFn := startSubprocess(t, "server", networking.Server.Process.NetNS, []string{ "--subprocess", "--test-name=" + topologyName, "--role=server", - "--server-listen-addr=" + networking.ServerListenAddr, + "--server-listen-addr=" + networking.Server.ListenAddr, }) return closeFn } -func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, clientNumber int) (<-chan error, func() error) { - require.True(t, clientNumber == 1 || clientNumber == 2) +func startSTUNSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error { + _, closeFn := startSubprocess(t, "stun", networking.STUN.Process.NetNS, []string{ + "--subprocess", + "--test-name=" + topologyName, + "--role=stun", + "--stun-listen-addr=" + networking.STUN.ListenAddr, + }) + return closeFn +} +func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, me integration.Client, derpMapPath string) (<-chan error, func() error) { var ( - clientName = fmt.Sprintf("client%d", clientNumber) - myID = integration.Client1ID - peerID = integration.Client2ID - accessURL = networking.ServerAccessURLClient1 - netNS = networking.ProcessClient1.NetNS + clientName = fmt.Sprintf("client%d", me.Number) + clientProcessConfig = networking.Client1 ) - if clientNumber == 2 { - myID, peerID = peerID, myID - accessURL = networking.ServerAccessURLClient2 - netNS = networking.ProcessClient2.NetNS + if me.Number == integration.ClientNumber2 { + clientProcessConfig = networking.Client2 } flags := []string{ @@ -277,15 +349,12 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra "--test-name=" + topologyName, "--role=client", "--client-name=" + clientName, - "--client-server-url=" + accessURL, - "--client-id=" + myID.String(), - "--client-peer-id=" + peerID.String(), - } - if clientNumber == 1 { - flags = append(flags, "--client-run-tests") + "--client-number=" + strconv.Itoa(int(me.Number)), + "--client-server-url=" + clientProcessConfig.ServerAccessURL, + "--client-derp-map-path=" + derpMapPath, } - return startSubprocess(t, clientName, netNS, flags) + return startSubprocess(t, clientName, clientProcessConfig.Process.NetNS, flags) } // startSubprocess launches the test binary with the same flags as the test, but @@ -295,6 +364,22 @@ func startClientSubprocess(t *testing.T, topologyName string, networking integra func startSubprocess(t *testing.T, processName string, netNS *os.File, flags []string) (<-chan error, func() error) { name := os.Args[0] // Always use verbose mode since it gets piped to the parent test anyways. - args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...) + args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...) //nolint:gocritic return integration.ExecBackground(t, processName, netNS, name, args) } + +func writeDERPMapToFile(path string, derpMap *tailcfg.DERPMap) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + err = enc.Encode(derpMap) + if err != nil { + return err + } + return nil +} diff --git a/tailnet/test/integration/network.go b/tailnet/test/integration/network.go index 80eeb6048bd66..e0d8f7109c167 100644 --- a/tailnet/test/integration/network.go +++ b/tailnet/test/integration/network.go @@ -13,27 +13,55 @@ import ( "github.com/stretchr/testify/require" "github.com/tailscale/netlink" "golang.org/x/xerrors" + "tailscale.com/tailcfg" "cdr.dev/slog" "github.com/coder/coder/v2/cryptorand" ) +const ( + client1Port = 48001 + client1RouterPort = 48011 + client2Port = 48002 + client2RouterPort = 48012 +) + type TestNetworking struct { - // ServerListenAddr is the IP address and port that the server listens on, - // passed to StartServer. - ServerListenAddr string - // ServerAccessURLClient1 is the hostname and port that the first client - // uses to access the server. - ServerAccessURLClient1 string - // ServerAccessURLClient2 is the hostname and port that the second client - // uses to access the server. - ServerAccessURLClient2 string - - // Networking settings for each subprocess. - ProcessServer TestNetworkingProcess - ProcessClient1 TestNetworkingProcess - ProcessClient2 TestNetworkingProcess + Server TestNetworkingServer + STUN TestNetworkingSTUN + Client1 TestNetworkingClient + Client2 TestNetworkingClient +} + +type TestNetworkingServer struct { + Process TestNetworkingProcess + ListenAddr string +} + +type TestNetworkingSTUN struct { + Process TestNetworkingProcess + // If empty, no STUN subprocess is launched. + ListenAddr string +} + +type TestNetworkingClient struct { + Process TestNetworkingProcess + // ServerAccessURL is the hostname and port that the client uses to access + // the server over HTTP for coordination. + ServerAccessURL string + // DERPMap is the DERP map that the client uses. If nil, a basic DERP map + // containing only a single DERP with `ServerAccessURL` is used with no + // STUN servers. + DERPMap *tailcfg.DERPMap +} + +func (c TestNetworkingClient) ResolveDERPMap() (*tailcfg.DERPMap, error) { + if c.DERPMap != nil { + return c.DERPMap, nil + } + + return basicDERPMap(c.ServerAccessURL) } type TestNetworkingProcess struct { @@ -46,14 +74,9 @@ type TestNetworkingProcess struct { // namespace only exists for isolation on the host and doesn't serve any routing // purpose. func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking { - netNSName := "codertest_netns_" - randStr, err := cryptorand.String(4) - require.NoError(t, err, "generate random string for netns name") - netNSName += randStr - // Create a single network namespace for all tests so we can have an // isolated loopback interface. - netNSFile := createNetNS(t, netNSName) + netNSFile := createNetNS(t, uniqNetName(t)) var ( listenAddr = "127.0.0.1:8080" @@ -62,176 +85,323 @@ func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking { } ) return TestNetworking{ - ServerListenAddr: listenAddr, - ServerAccessURLClient1: "http://" + listenAddr, - ServerAccessURLClient2: "http://" + listenAddr, - ProcessServer: process, - ProcessClient1: process, - ProcessClient2: process, + Server: TestNetworkingServer{ + Process: process, + ListenAddr: listenAddr, + }, + Client1: TestNetworkingClient{ + Process: process, + ServerAccessURL: "http://" + listenAddr, + }, + Client2: TestNetworkingClient{ + Process: process, + ServerAccessURL: "http://" + listenAddr, + }, } } -// SetupNetworkingEasyNAT creates a network namespace with a router that NATs -// packets between two clients and a server. -// See createFakeRouter for the full topology. +func easyNAT(t *testing.T) fakeInternet { + internet := createFakeInternet(t) + + _, err := commandInNetNS(internet.BridgeNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() + require.NoError(t, wrapExitErr(err), "enable IP forwarding in bridge NetNS") + + // Set up iptables masquerade rules to allow each router to NAT packets. + leaves := []struct { + fakeRouterLeaf + clientPort int + natPort int + }{ + {internet.Client1, client1Port, client1RouterPort}, + {internet.Client2, client2Port, client2RouterPort}, + } + for _, leaf := range leaves { + _, err := commandInNetNS(leaf.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() + require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS") + + // All non-UDP traffic should use regular masquerade e.g. for HTTP. + _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ + "-t", "nat", + "-A", "POSTROUTING", + // Every interface except loopback. + "!", "-o", "lo", + // Every protocol except UDP. + "!", "-p", "udp", + "-j", "MASQUERADE", + }).Output() + require.NoError(t, wrapExitErr(err), "add iptables non-UDP masquerade rule") + + // Outgoing traffic should get NATed to the router's IP. + _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ + "-t", "nat", + "-A", "POSTROUTING", + "-p", "udp", + "--sport", fmt.Sprint(leaf.clientPort), + "-j", "SNAT", + "--to-source", fmt.Sprintf("%s:%d", leaf.RouterIP, leaf.natPort), + }).Output() + require.NoError(t, wrapExitErr(err), "add iptables SNAT rule") + + // Incoming traffic should be forwarded to the client's IP. + _, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{ + "-t", "nat", + "-A", "PREROUTING", + "-p", "udp", + "--dport", fmt.Sprint(leaf.natPort), + "-j", "DNAT", + "--to-destination", fmt.Sprintf("%s:%d", leaf.ClientIP, leaf.clientPort), + }).Output() + require.NoError(t, wrapExitErr(err), "add iptables DNAT rule") + } + + return internet +} + +// SetupNetworkingEasyNAT creates a fake internet and sets up "easy NAT" +// forwarding rules. +// See createFakeInternet. // NAT is achieved through a single iptables masquerade rule. func SetupNetworkingEasyNAT(t *testing.T, _ slog.Logger) TestNetworking { - router := createFakeRouter(t) - - // Set up iptables masquerade rules to allow the router to NAT packets - // between the Three Kingdoms. - _, err := commandInNetNS(router.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output() - require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS") - _, err = commandInNetNS(router.RouterNetNS, "iptables", []string{ - "-t", "nat", - "-A", "POSTROUTING", - // Every interface except loopback. - "!", "-o", "lo", - "-j", "MASQUERADE", - }).Output() - require.NoError(t, wrapExitErr(err), "add iptables masquerade rule") - - return router.Net + return easyNAT(t).Net } -type fakeRouter struct { - Net TestNetworking +// SetupNetworkingEasyNATWithSTUN does the same as SetupNetworkingEasyNAT, but +// also creates a namespace and bridge address for a STUN server. +func SetupNetworkingEasyNATWithSTUN(t *testing.T, _ slog.Logger) TestNetworking { + internet := easyNAT(t) - RouterNetNS *os.File - RouterVeths struct { - Server string - Client1 string - Client2 string + // Create another network namespace for the STUN server. + stunNetNS := createNetNS(t, internet.NamePrefix+"stun") + internet.Net.STUN.Process = TestNetworkingProcess{ + NetNS: stunNetNS, + } + + const ip = "10.0.0.64" + err := joinBridge(joinBridgeOpts{ + bridgeNetNS: internet.BridgeNetNS, + netNS: stunNetNS, + bridgeName: internet.BridgeName, + vethPair: vethPair{ + Outer: internet.NamePrefix + "b-stun", + Inner: internet.NamePrefix + "stun-b", + }, + ip: ip, + }) + require.NoError(t, err, "join bridge with STUN server") + internet.Net.STUN.ListenAddr = ip + ":3478" + + // Define custom DERP map. + stunRegion := &tailcfg.DERPRegion{ + RegionID: 10000, + RegionCode: "stun0", + RegionName: "STUN0", + Nodes: []*tailcfg.DERPNode{ + { + Name: "stun0a", + RegionID: 1, + IPv4: ip, + IPv6: "none", + STUNPort: 3478, + STUNOnly: true, + }, + }, } - ServerNetNS *os.File - ServerVeth string - Client1NetNS *os.File - Client1Veth string - Client2NetNS *os.File - Client2Veth string + client1DERP, err := internet.Net.Client1.ResolveDERPMap() + require.NoError(t, err, "resolve DERP map for client 1") + client1DERP.Regions[stunRegion.RegionID] = stunRegion + internet.Net.Client1.DERPMap = client1DERP + client2DERP, err := internet.Net.Client2.ResolveDERPMap() + require.NoError(t, err, "resolve DERP map for client 2") + client2DERP.Regions[stunRegion.RegionID] = stunRegion + internet.Net.Client2.DERPMap = client2DERP + + return internet.Net } -// fakeRouter creates multiple namespaces with veth pairs between them with -// the following topology: -// -// namespaces: -// - router -// - server -// - client1 -// - client2 +type vethPair struct { + Outer string + Inner string +} + +type fakeRouterLeaf struct { + // RouterIP is the IP address of the router on the bridge. + RouterIP string + // ClientIP is the IP address of the client on the router. + ClientIP string + // RouterNetNS is the router for this specific leaf. + RouterNetNS *os.File + // ClientNetNS is where the "user" is. + ClientNetNS *os.File + // Veth pair between the router and the bridge. + OuterVethPair vethPair + // Veth pair between the user and the router. + InnerVethPair vethPair +} + +type fakeInternet struct { + Net TestNetworking + + NamePrefix string + BridgeNetNS *os.File + BridgeName string + ServerNetNS *os.File + ServerVethPair vethPair // between bridge and server NS + Client1 fakeRouterLeaf + Client2 fakeRouterLeaf +} + +// createFakeInternet creates multiple namespaces with veth pairs between them +// with the following topology: // -// veth pairs: -// - router-server (10.0.1.1) <-> server-router (10.0.1.2) -// - router-client1 (10.0.2.1) <-> client1-router (10.0.2.2) -// - router-client2 (10.0.3.1) <-> client2-router (10.0.3.2) +// . veth ┌────────┐ veth +// . ┌─────────────────┤ Bridge ├───────────────────┐ +// . │ └───┬────┘ │ +// . │ │ │ +// . │10.0.0.1 veth│10.0.0.2 │10.0.0.3 +// . ┌───────┴───────┐ ┌───────┴─────────┐ ┌────────┴────────┐ +// . │ Server │ │ Client 1 router │ │ Client 2 router │ +// . └───────────────┘ └───────┬─────────┘ └────────┬────────┘ +// . │10.0.2.1 │10.0.3.1 +// . veth│ veth│ +// . │10.0.2.2 │10.0.3.2 +// . ┌───────┴─────────┐ ┌────────┴────────┐ +// . │ Client 1 │ │ Client 2 │ +// . └─────────────────┘ └─────────────────┘ // // No iptables rules are created, so packets will not be forwarded out of the -// box. Routes are created between all namespaces based on the veth pairs, -// however. -func createFakeRouter(t *testing.T) fakeRouter { +// box. Default routes are created from the edge namespaces (client1, client2) +// to their respective routers, but no NAT rules are created. +func createFakeInternet(t *testing.T) fakeInternet { t.Helper() const ( - routerServerPrefix = "10.0.1." - routerServerIP = routerServerPrefix + "1" - serverIP = routerServerPrefix + "2" - routerClient1Prefix = "10.0.2." - routerClient1IP = routerClient1Prefix + "1" - client1IP = routerClient1Prefix + "2" - routerClient2Prefix = "10.0.3." - routerClient2IP = routerClient2Prefix + "1" - client2IP = routerClient2Prefix + "2" + bridgePrefix = "10.0.0." + serverIP = bridgePrefix + "1" + client1Prefix = "10.0.2." + client2Prefix = "10.0.3." ) + var ( + namePrefix = uniqNetName(t) + "_" + router = fakeInternet{ + NamePrefix: namePrefix, + BridgeName: namePrefix + "b", + } + ) + + // Create bridge namespace and bridge interface. + router.BridgeNetNS = createNetNS(t, router.BridgeName) + err := createBridge(router.BridgeNetNS, router.BridgeName) + require.NoError(t, err, "create bridge in netns") - prefix := uniqNetName(t) + "_" - router := fakeRouter{} - router.RouterVeths.Server = prefix + "r-s" - router.RouterVeths.Client1 = prefix + "r-c1" - router.RouterVeths.Client2 = prefix + "r-c2" - router.ServerVeth = prefix + "s-r" - router.Client1Veth = prefix + "c1-r" - router.Client2Veth = prefix + "c2-r" - - // Create namespaces. - router.RouterNetNS = createNetNS(t, prefix+"r") - serverNS := createNetNS(t, prefix+"s") - client1NS := createNetNS(t, prefix+"c1") - client2NS := createNetNS(t, prefix+"c2") - - vethPairs := []struct { - parentName string - peerName string - parentNS *os.File - peerNS *os.File - parentIP string - peerIP string + // Create server namespace and veth pair between bridge and server. + router.ServerNetNS = createNetNS(t, namePrefix+"s") + router.ServerVethPair = vethPair{ + Outer: namePrefix + "b-s", + Inner: namePrefix + "s-b", + } + err = joinBridge(joinBridgeOpts{ + bridgeNetNS: router.BridgeNetNS, + netNS: router.ServerNetNS, + bridgeName: router.BridgeName, + vethPair: router.ServerVethPair, + ip: serverIP, + }) + require.NoError(t, err, "join bridge with server") + + leaves := []struct { + leaf *fakeRouterLeaf + routerName string + clientName string + routerBridgeIP string + routerClientIP string + clientIP string }{ { - parentName: router.RouterVeths.Server, - peerName: router.ServerVeth, - parentNS: router.RouterNetNS, - peerNS: serverNS, - parentIP: routerServerIP, - peerIP: serverIP, - }, - { - parentName: router.RouterVeths.Client1, - peerName: router.Client1Veth, - parentNS: router.RouterNetNS, - peerNS: client1NS, - parentIP: routerClient1IP, - peerIP: client1IP, + leaf: &router.Client1, + routerName: "c1r", + clientName: "c1", + routerBridgeIP: bridgePrefix + "2", + routerClientIP: client1Prefix + "1", + clientIP: client1Prefix + "2", }, { - parentName: router.RouterVeths.Client2, - peerName: router.Client2Veth, - parentNS: router.RouterNetNS, - peerNS: client2NS, - parentIP: routerClient2IP, - peerIP: client2IP, + leaf: &router.Client2, + routerName: "c2r", + clientName: "c2", + routerBridgeIP: bridgePrefix + "3", + routerClientIP: client2Prefix + "1", + clientIP: client2Prefix + "2", }, } - for _, vethPair := range vethPairs { - err := createVethPair(vethPair.parentName, vethPair.peerName) - require.NoErrorf(t, err, "create veth pair %q <-> %q", vethPair.parentName, vethPair.peerName) - - // Move the veth interfaces to the respective network namespaces. - err = setVethNetNS(vethPair.parentName, int(vethPair.parentNS.Fd())) - require.NoErrorf(t, err, "set veth %q to NetNS", vethPair.parentName) - err = setVethNetNS(vethPair.peerName, int(vethPair.peerNS.Fd())) - require.NoErrorf(t, err, "set veth %q to NetNS", vethPair.peerName) + for _, leaf := range leaves { + leaf.leaf.RouterIP = leaf.routerBridgeIP + leaf.leaf.ClientIP = leaf.clientIP - // Set IP addresses on the interfaces. - err = setInterfaceIP(vethPair.parentNS, vethPair.parentName, vethPair.parentIP) - require.NoErrorf(t, err, "set IP %q on interface %q", vethPair.parentIP, vethPair.parentName) - err = setInterfaceIP(vethPair.peerNS, vethPair.peerName, vethPair.peerIP) - require.NoErrorf(t, err, "set IP %q on interface %q", vethPair.peerIP, vethPair.peerName) + // Create two network namespaces for each leaf: one for the router and + // one for the "client". + leaf.leaf.RouterNetNS = createNetNS(t, namePrefix+leaf.routerName) + leaf.leaf.ClientNetNS = createNetNS(t, namePrefix+leaf.clientName) - // Bring up both interfaces. - err = setInterfaceUp(vethPair.parentNS, vethPair.parentName) - require.NoErrorf(t, err, "bring up interface %q", vethPair.parentName) - err = setInterfaceUp(vethPair.peerNS, vethPair.peerName) - require.NoErrorf(t, err, "bring up interface %q", vethPair.parentName) + // Join the bridge. + leaf.leaf.OuterVethPair = vethPair{ + Outer: namePrefix + "b-" + leaf.routerName, + Inner: namePrefix + leaf.routerName + "-b", + } + err = joinBridge(joinBridgeOpts{ + bridgeNetNS: router.BridgeNetNS, + netNS: leaf.leaf.RouterNetNS, + bridgeName: router.BridgeName, + vethPair: leaf.leaf.OuterVethPair, + ip: leaf.routerBridgeIP, + }) + require.NoError(t, err, "join bridge with router") + + // Create inner veth pair between the router and the client. + leaf.leaf.InnerVethPair = vethPair{ + Outer: namePrefix + leaf.routerName + "-" + leaf.clientName, + Inner: namePrefix + leaf.clientName + "-" + leaf.routerName, + } + err = createVethPair(leaf.leaf.InnerVethPair.Outer, leaf.leaf.InnerVethPair.Inner) + require.NoErrorf(t, err, "create veth pair %q <-> %q", leaf.leaf.InnerVethPair.Outer, leaf.leaf.InnerVethPair.Inner) + + // Move the network interfaces to the respective network namespaces. + err = setVethNetNS(leaf.leaf.InnerVethPair.Outer, int(leaf.leaf.RouterNetNS.Fd())) + require.NoErrorf(t, err, "set veth %q to NetNS", leaf.leaf.InnerVethPair.Outer) + err = setVethNetNS(leaf.leaf.InnerVethPair.Inner, int(leaf.leaf.ClientNetNS.Fd())) + require.NoErrorf(t, err, "set veth %q to NetNS", leaf.leaf.InnerVethPair.Inner) + + // Set router's "local" IP on the veth. + err = setInterfaceIP(leaf.leaf.RouterNetNS, leaf.leaf.InnerVethPair.Outer, leaf.routerClientIP) + require.NoErrorf(t, err, "set IP %q on interface %q", leaf.routerClientIP, leaf.leaf.InnerVethPair.Outer) + // Set client's IP on the veth. + err = setInterfaceIP(leaf.leaf.ClientNetNS, leaf.leaf.InnerVethPair.Inner, leaf.clientIP) + require.NoErrorf(t, err, "set IP %q on interface %q", leaf.clientIP, leaf.leaf.InnerVethPair.Inner) + + // Bring up the interfaces. + err = setInterfaceUp(leaf.leaf.RouterNetNS, leaf.leaf.InnerVethPair.Outer) + require.NoErrorf(t, err, "bring up interface %q", leaf.leaf.OuterVethPair.Outer) + err = setInterfaceUp(leaf.leaf.ClientNetNS, leaf.leaf.InnerVethPair.Inner) + require.NoErrorf(t, err, "bring up interface %q", leaf.leaf.InnerVethPair.Inner) // We don't need to add a route from parent to peer since the kernel // already adds a default route for the /24. We DO need to add a default // route from peer to parent, however. - err = addRouteInNetNS(vethPair.peerNS, []string{"default", "via", vethPair.parentIP, "dev", vethPair.peerName}) - require.NoErrorf(t, err, "add peer default route to %q", vethPair.peerName) + err = addRouteInNetNS(leaf.leaf.ClientNetNS, []string{"default", "via", leaf.routerClientIP, "dev", leaf.leaf.InnerVethPair.Inner}) + require.NoErrorf(t, err, "add peer default route to %q", leaf.leaf.InnerVethPair.Inner) } router.Net = TestNetworking{ - ServerListenAddr: serverIP + ":8080", - ServerAccessURLClient1: "http://" + serverIP + ":8080", - ServerAccessURLClient2: "http://" + serverIP + ":8080", - ProcessServer: TestNetworkingProcess{ - NetNS: serverNS, + Server: TestNetworkingServer{ + Process: TestNetworkingProcess{NetNS: router.ServerNetNS}, + ListenAddr: serverIP + ":8080", }, - ProcessClient1: TestNetworkingProcess{ - NetNS: client1NS, + Client1: TestNetworkingClient{ + Process: TestNetworkingProcess{NetNS: router.Client1.ClientNetNS}, + ServerAccessURL: "http://" + serverIP + ":8080", }, - ProcessClient2: TestNetworkingProcess{ - NetNS: client2NS, + Client2: TestNetworkingClient{ + Process: TestNetworkingProcess{NetNS: router.Client2.ClientNetNS}, + ServerAccessURL: "http://" + serverIP + ":8080", }, } return router @@ -246,6 +416,60 @@ func uniqNetName(t *testing.T) string { return netNSName } +type joinBridgeOpts struct { + bridgeNetNS *os.File + netNS *os.File + bridgeName string + // This vethPair will be created and should not already exist. + vethPair vethPair + ip string +} + +// joinBridge joins the given network namespace to the bridge. It creates a veth +// pair between the specified NetNS and the bridge NetNS, sets the IP address on +// the "child" veth, and brings up the interfaces. +func joinBridge(opts joinBridgeOpts) error { + // Create outer veth pair between the router and the bridge. + err := createVethPair(opts.vethPair.Outer, opts.vethPair.Inner) + if err != nil { + return xerrors.Errorf("create veth pair %q <-> %q: %w", opts.vethPair.Outer, opts.vethPair.Inner, err) + } + + // Move the network interfaces to the respective network namespaces. + err = setVethNetNS(opts.vethPair.Outer, int(opts.bridgeNetNS.Fd())) + if err != nil { + return xerrors.Errorf("set veth %q to NetNS: %w", opts.vethPair.Outer, err) + } + err = setVethNetNS(opts.vethPair.Inner, int(opts.netNS.Fd())) + if err != nil { + return xerrors.Errorf("set veth %q to NetNS: %w", opts.vethPair.Inner, err) + } + + // Connect the outer veth to the bridge. + err = setInterfaceBridge(opts.bridgeNetNS, opts.vethPair.Outer, opts.bridgeName) + if err != nil { + return xerrors.Errorf("set interface %q master to %q: %w", opts.vethPair.Outer, opts.bridgeName, err) + } + + // Set the bridge IP on the inner veth. + err = setInterfaceIP(opts.netNS, opts.vethPair.Inner, opts.ip) + if err != nil { + return xerrors.Errorf("set IP %q on interface %q: %w", opts.ip, opts.vethPair.Inner, err) + } + + // Bring up the interfaces. + err = setInterfaceUp(opts.bridgeNetNS, opts.vethPair.Outer) + if err != nil { + return xerrors.Errorf("bring up interface %q: %w", opts.vethPair.Outer, err) + } + err = setInterfaceUp(opts.netNS, opts.vethPair.Inner) + if err != nil { + return xerrors.Errorf("bring up interface %q: %w", opts.vethPair.Inner, err) + } + + return nil +} + // createNetNS creates a new network namespace with the given name. The returned // file is a file descriptor to the network namespace. // Note: all cleanup is handled for you, you do not need to call Close on the @@ -283,18 +507,48 @@ func createNetNS(t *testing.T, name string) *os.File { return file } +// createBridge creates a bridge in the given network namespace. The bridge is +// automatically brought up. +func createBridge(netNS *os.File, name string) error { + // While it might be possible to create a bridge directly in a NetNS or move + // an existing bridge to a NetNS, I couldn't figure out a way to do it. + // Creating it directly within the NetNS is the simplest way. + _, err := commandInNetNS(netNS, "ip", []string{"link", "add", name, "type", "bridge"}).Output() + if err != nil { + return xerrors.Errorf("create bridge %q in netns: %w", name, wrapExitErr(err)) + } + + _, err = commandInNetNS(netNS, "ip", []string{"link", "set", name, "up"}).Output() + if err != nil { + return xerrors.Errorf("set bridge %q up in netns: %w", name, wrapExitErr(err)) + } + + return nil +} + +// setInterfaceBridge sets the master of the given interface to the specified +// bridge. +func setInterfaceBridge(netNS *os.File, ifaceName, bridgeName string) error { + _, err := commandInNetNS(netNS, "ip", []string{"link", "set", ifaceName, "master", bridgeName}).Output() + if err != nil { + return xerrors.Errorf("set interface %q master to %q in netns: %w", ifaceName, bridgeName, wrapExitErr(err)) + } + + return nil +} + // createVethPair creates a veth pair with the given names. func createVethPair(parentVethName, peerVethName string) error { - vethLinkAttrs := netlink.NewLinkAttrs() - vethLinkAttrs.Name = parentVethName + linkAttrs := netlink.NewLinkAttrs() + linkAttrs.Name = parentVethName veth := &netlink.Veth{ - LinkAttrs: vethLinkAttrs, + LinkAttrs: linkAttrs, PeerName: peerVethName, } err := netlink.LinkAdd(veth) if err != nil { - return xerrors.Errorf("LinkAdd(name: %q, peerName: %q): %w", parentVethName, peerVethName, err) + return xerrors.Errorf("LinkAdd(type: veth, name: %q, peerName: %q): %w", parentVethName, peerVethName, err) } return nil diff --git a/tailnet/test/integration/remove_test_ns.sh b/tailnet/test/integration/remove_test_ns.sh new file mode 100755 index 0000000000000..464aac6c8eff0 --- /dev/null +++ b/tailnet/test/integration/remove_test_ns.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -euo pipefail + +if [[ $(id -u) -ne 0 ]]; then + echo "Please run with sudo" + exit 1 +fi + +to_delete=$(ip netns list | grep -o 'cdr_.*_.*' | cut -d' ' -f1) +echo "Will delete:" +for ns in $to_delete; do + echo "- $ns" +done + +read -p "Continue? [y/N] " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 +fi + +for ns in $to_delete; do + ip netns delete "$ns" +done diff --git a/tailnet/test/integration/suite.go b/tailnet/test/integration/suite.go index 54fb0856a21af..32d9adb2e4a14 100644 --- a/tailnet/test/integration/suite.go +++ b/tailnet/test/integration/suite.go @@ -7,7 +7,6 @@ import ( "net/url" "testing" - "github.com/google/uuid" "github.com/stretchr/testify/require" "cdr.dev/slog" @@ -17,12 +16,12 @@ import ( // TODO: instead of reusing one conn for each suite, maybe we should make a new // one for each subtest? -func TestSuite(t *testing.T, _ slog.Logger, _ *url.URL, _, peerID uuid.UUID, conn *tailnet.Conn) { +func TestSuite(t *testing.T, _ slog.Logger, _ *url.URL, conn *tailnet.Conn, _, peer Client) { t.Parallel() t.Run("Connectivity", func(t *testing.T) { t.Parallel() - peerIP := tailnet.IPFromUUID(peerID) + peerIP := tailnet.IPFromUUID(peer.ID) _, _, _, err := conn.Ping(testutil.Context(t, testutil.WaitLong), peerIP) require.NoError(t, err, "ping peer") })
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies: