Skip to content

Commit af4a668

Browse files
spikecurtisspikecurtis
authored
fix: use tailscale that avoids small MTU paths (#18323)
Fixes #15523 Uses latest https://github.com/coder/tailscale which includes coder/tailscale#85 to stop selecting paths with small MTU for direct connections. Also updates the tailnet integration test to reproduce the issue. The previous version had the 2 peers connected by a single veth, but this allows the OS to fragment the packet. In the new version, the 2 peers (and server) are all connected by a central router. The link between peer 1 and the router has an adjustable MTU. IPv6 does not allow packets to be fragmented by intermediate routers, so sending a too-large packet in this scenario forces the router to drop packets and reproduce the issue (without the tailscale changes).
1 parent 2377d76 commit af4a668

File tree

5 files changed

+109
-85
lines changed

5 files changed

+109
-85
lines changed

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ replace github.com/tcnksm/go-httpstat => github.com/coder/go-httpstat v0.0.0-202
3636

3737
// There are a few minor changes we make to Tailscale that we're slowly upstreaming. Compare here:
3838
// https://github.com/tailscale/tailscale/compare/main...coder:tailscale:main
39-
replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20250422090654-5090e715905e
39+
replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20250611020837-f14d20d23d8c
4040

4141
// This is replaced to include
4242
// 1. a fix for a data race: c.f. https://github.com/tailscale/wireguard-go/pull/25

‎go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -920,8 +920,8 @@ github.com/coder/serpent v0.10.0 h1:ofVk9FJXSek+SmL3yVE3GoArP83M+1tX+H7S4t8BSuM=
920920
github.com/coder/serpent v0.10.0/go.mod h1:cZFW6/fP+kE9nd/oRkEHJpG6sXCtQ+AX7WMMEHv0Y3Q=
921921
github.com/coder/ssh v0.0.0-20231128192721-70855dedb788 h1:YoUSJ19E8AtuUFVYBpXuOD6a/zVP3rcxezNsoDseTUw=
922922
github.com/coder/ssh v0.0.0-20231128192721-70855dedb788/go.mod h1:aGQbuCLyhRLMzZF067xc84Lh7JDs1FKwCmF1Crl9dxQ=
923-
github.com/coder/tailscale v1.1.1-0.20250422090654-5090e715905e h1:nope/SZfoLB9MCOB9wdCE6gW5+8l3PhFrDC5IWPL8bk=
924-
github.com/coder/tailscale v1.1.1-0.20250422090654-5090e715905e/go.mod h1:1ggFFdHTRjPRu9Yc1yA7nVHBYB50w9Ce7VIXNqcW6Ko=
923+
github.com/coder/tailscale v1.1.1-0.20250611020837-f14d20d23d8c h1:d/qBIi3Ez7KkopRgNtfdvTMqvqBg47d36qVfkd3C5EQ=
924+
github.com/coder/tailscale v1.1.1-0.20250611020837-f14d20d23d8c/go.mod h1:l7ml5uu7lFh5hY28lGYM4b/oFSmuPHYX6uk4RAu23Lc=
925925
github.com/coder/terraform-config-inspect v0.0.0-20250107175719-6d06d90c630e h1:JNLPDi2P73laR1oAclY6jWzAbucf70ASAvf5mh2cME0=
926926
github.com/coder/terraform-config-inspect v0.0.0-20250107175719-6d06d90c630e/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI=
927927
github.com/coder/terraform-provider-coder/v2 v2.5.3 h1:EwqIIQKe/j8bsR4WyDJ3bD0dVdkfVqJ43TwClyGneUU=

‎tailnet/test/integration/integration.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/go-chi/chi/v5"
2626
"github.com/google/uuid"
2727
"github.com/stretchr/testify/require"
28+
"golang.org/x/sys/unix"
2829
"golang.org/x/xerrors"
2930
"tailscale.com/derp"
3031
"tailscale.com/derp/derphttp"
@@ -458,6 +459,16 @@ func (UDPEchoService) StartService(t *testing.T, logger slog.Logger, _ *tailnet.
458459
Port: EchoPort,
459460
})
460461
require.NoError(t, err)
462+
463+
// set path MTU discovery so that we don't fragment the responses.
464+
c, err := l.SyscallConn()
465+
require.NoError(t, err)
466+
var sockErr error
467+
err = c.Control(func(fd uintptr) {
468+
sockErr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_MTU_DISCOVER, unix.IP_PMTUDISC_DO)
469+
})
470+
require.NoError(t, err)
471+
require.NoError(t, sockErr)
461472
logger.Info(context.Background(), "started UDPEcho server")
462473
t.Cleanup(func() {
463474
lCloseErr := l.Close()

‎tailnet/test/integration/integration_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ var topologies = []integration.TestTopology{
112112
{
113113
// Test that direct over normal MTU works.
114114
Name: "DirectMTU1500",
115-
NetworkingProvider: integration.TriangleNetwork{InterClientMTU: 1500},
115+
NetworkingProvider: integration.TriangleNetwork{Client1MTU: 1500},
116116
Server: integration.SimpleServerOptions{},
117117
ClientStarter: integration.BasicClientStarter{
118118
WaitForDirect: true,
@@ -124,7 +124,7 @@ var topologies = []integration.TestTopology{
124124
{
125125
// Test that small MTU works.
126126
Name: "MTU1280",
127-
NetworkingProvider: integration.TriangleNetwork{InterClientMTU: 1280},
127+
NetworkingProvider: integration.TriangleNetwork{Client1MTU: 1280},
128128
Server: integration.SimpleServerOptions{},
129129
ClientStarter: integration.BasicClientStarter{Service: integration.UDPEchoService{}, LogPackets: true},
130130
RunTests: integration.TestBigUDP,

‎tailnet/test/integration/network.go

Lines changed: 93 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -390,33 +390,38 @@ func createFakeInternet(t *testing.T) fakeInternet {
390390
}
391391

392392
type TriangleNetwork struct {
393-
InterClientMTU int
393+
Client1MTU int
394394
}
395395

396396
type fakeTriangleNetwork struct {
397-
NamePrefix string
398-
ServerNetNS *os.File
399-
Client1NetNS *os.File
400-
Client2NetNS *os.File
401-
ServerClient1VethPair vethPair
402-
ServerClient2VethPair vethPair
403-
Client1Client2VethPair vethPair
397+
NamePrefix string
398+
ServerNetNS *os.File
399+
Client1NetNS *os.File
400+
Client2NetNS *os.File
401+
RouterNetNS *os.File
402+
ServerVethPair vethPair
403+
Client1VethPair vethPair
404+
Client2VethPair vethPair
404405
}
405406

406-
// SetupNetworking creates multiple namespaces with veth pairs between them
407-
// with the following topology:
407+
// SetupNetworking creates multiple namespaces with a central router in the following topology
408408
// .
409-
// . ┌────────────────────────────────────────────┐
410-
// . │ Server │
411-
// . └─────┬───────────────────────────────────┬──┘
412-
// . │fdac:38fa:ffff:2::3 │fdac:38fa:ffff:3::3
413-
// . veth│ veth│
414-
// . │fdac:38fa:ffff:2::1 │fdac:38fa:ffff:3::2
415-
// . ┌───────┴──────┐ ┌─────┴───────┐
416-
// . │ │ fdac:38fa:ffff:1::2│ │
417-
// . │ Client 1 ├──────────────────────┤ Client 2 │
418-
// . │ │fdac:38fa:ffff:1::1 │ │
419-
// . └──────────────┘ └─────────────┘
409+
// . ┌──────────────┐
410+
// . │ │
411+
// . │ Server ├─────────────────────────────────────┐
412+
// . │ │fdac:38fa:ffff:3::2 │
413+
// . └──────────────┘ │ fdac:38fa:ffff:3::1
414+
// . ┌──────────────┐ ┌─────┴───────┐
415+
// . │ │ fdac:38fa:ffff:1::1│ │
416+
// . │ Client 1 ├───────────────────────────────┤ Router │
417+
// . │ │fdac:38fa:ffff:1::2 │ │
418+
// . └──────────────┘ └─────┬───────┘
419+
// . ┌──────────────┐ │ fdac:38fa:ffff:2::1
420+
// . │ │ │
421+
// . │ Client 2 ├─────────────────────────────────────┘
422+
// . │ │fdac:38fa:ffff:2::2
423+
// . └──────────────┘
424+
// The veth link between Client 1 and the router has a configurable MTU via Client1MTU.
420425
func (n TriangleNetwork) SetupNetworking(t *testing.T, l slog.Logger) TestNetworking {
421426
logger := l.Named("setup-networking").Leveled(slog.LevelDebug)
422427
t.Helper()
@@ -433,101 +438,109 @@ func (n TriangleNetwork) SetupNetworking(t *testing.T, l slog.Logger) TestNetwor
433438
network.ServerNetNS = createNetNS(t, namePrefix+"server")
434439
network.Client1NetNS = createNetNS(t, namePrefix+"client1")
435440
network.Client2NetNS = createNetNS(t, namePrefix+"client2")
441+
network.RouterNetNS = createNetNS(t, namePrefix+"router")
436442

437-
// Create veth pair between server and client1
438-
network.ServerClient1VethPair = vethPair{
439-
Outer: namePrefix + "s-1",
440-
Inner: namePrefix + "1-s",
443+
// Create veth pair between server and router
444+
network.ServerVethPair = vethPair{
445+
Outer: namePrefix + "s-r",
446+
Inner: namePrefix + "r-s",
441447
}
442-
err := createVethPair(network.ServerClient1VethPair.Outer, network.ServerClient1VethPair.Inner)
448+
err := createVethPair(network.ServerVethPair.Outer, network.ServerVethPair.Inner)
443449
require.NoErrorf(t, err, "create veth pair %q <-> %q",
444-
network.ServerClient1VethPair.Outer, network.ServerClient1VethPair.Inner)
450+
network.ServerVethPair.Outer, network.ServerVethPair.Inner)
445451

446-
// Move server-client1 veth ends to their respective namespaces
447-
err = setVethNetNS(network.ServerClient1VethPair.Outer, int(network.ServerNetNS.Fd()))
448-
require.NoErrorf(t, err, "set veth %q to server NetNS", network.ServerClient1VethPair.Outer)
449-
err = setVethNetNS(network.ServerClient1VethPair.Inner, int(network.Client1NetNS.Fd()))
450-
require.NoErrorf(t, err, "set veth %q to client1 NetNS", network.ServerClient1VethPair.Inner)
452+
// Move server-router veth ends to their respective namespaces
453+
err = setVethNetNS(network.ServerVethPair.Outer, int(network.ServerNetNS.Fd()))
454+
require.NoErrorf(t, err, "set veth %q to server NetNS", network.ServerVethPair.Outer)
455+
err = setVethNetNS(network.ServerVethPair.Inner, int(network.RouterNetNS.Fd()))
456+
require.NoErrorf(t, err, "set veth %q to router NetNS", network.ServerVethPair.Inner)
451457

452-
// Create veth pair between server and client2
453-
network.ServerClient2VethPair = vethPair{
454-
Outer: namePrefix + "s-2",
455-
Inner: namePrefix + "2-s",
458+
// Create veth pair between client1 and router
459+
network.Client1VethPair = vethPair{
460+
Outer: namePrefix + "1-r",
461+
Inner: namePrefix + "r-1",
456462
}
457-
err = createVethPair(network.ServerClient2VethPair.Outer, network.ServerClient2VethPair.Inner)
463+
logger.Debug(context.Background(), "creating client1 link", slog.F("mtu", n.Client1MTU))
464+
err = createVethPair(network.Client1VethPair.Outer, network.Client1VethPair.Inner, withMTU(n.Client1MTU))
458465
require.NoErrorf(t, err, "create veth pair %q <-> %q",
459-
network.ServerClient2VethPair.Outer, network.ServerClient2VethPair.Inner)
466+
network.Client1VethPair.Outer, network.Client1VethPair.Inner)
460467

461-
// Move server-client2 veth ends to their respective namespaces
462-
err = setVethNetNS(network.ServerClient2VethPair.Outer, int(network.ServerNetNS.Fd()))
463-
require.NoErrorf(t, err, "set veth %q to server NetNS", network.ServerClient2VethPair.Outer)
464-
err = setVethNetNS(network.ServerClient2VethPair.Inner, int(network.Client2NetNS.Fd()))
465-
require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.ServerClient2VethPair.Inner)
468+
// Move client1-router veth ends to their respective namespaces
469+
err = setVethNetNS(network.Client1VethPair.Outer, int(network.Client1NetNS.Fd()))
470+
require.NoErrorf(t, err, "set veth %q to server NetNS", network.Client1VethPair.Outer)
471+
err = setVethNetNS(network.Client1VethPair.Inner, int(network.RouterNetNS.Fd()))
472+
require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.Client1VethPair.Inner)
466473

467474
// Create veth pair between client1 and client2
468-
network.Client1Client2VethPair = vethPair{
469-
Outer: namePrefix + "1-2",
470-
Inner: namePrefix + "2-1",
475+
network.Client2VethPair = vethPair{
476+
Outer: namePrefix + "2-r",
477+
Inner: namePrefix + "r-2",
471478
}
472-
logger.Debug(context.Background(), "creating inter-client link", slog.F("mtu", n.InterClientMTU))
473-
err = createVethPair(network.Client1Client2VethPair.Outer, network.Client1Client2VethPair.Inner,
474-
withMTU(n.InterClientMTU))
479+
480+
err = createVethPair(network.Client2VethPair.Outer, network.Client2VethPair.Inner)
475481
require.NoErrorf(t, err, "create veth pair %q <-> %q",
476-
network.Client1Client2VethPair.Outer, network.Client1Client2VethPair.Inner)
482+
network.Client2VethPair.Outer, network.Client2VethPair.Inner)
477483

478484
// Move client1-client2 veth ends to their respective namespaces
479-
err = setVethNetNS(network.Client1Client2VethPair.Outer, int(network.Client1NetNS.Fd()))
480-
require.NoErrorf(t, err, "set veth %q to client1 NetNS", network.Client1Client2VethPair.Outer)
481-
err = setVethNetNS(network.Client1Client2VethPair.Inner, int(network.Client2NetNS.Fd()))
482-
require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.Client1Client2VethPair.Inner)
485+
err = setVethNetNS(network.Client2VethPair.Outer, int(network.Client2NetNS.Fd()))
486+
require.NoErrorf(t, err, "set veth %q to client1 NetNS", network.Client2VethPair.Outer)
487+
err = setVethNetNS(network.Client2VethPair.Inner, int(network.RouterNetNS.Fd()))
488+
require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.Client2VethPair.Inner)
483489

484490
// Set IP addresses according to the diagram:
485-
err = setInterfaceIP6(network.ServerNetNS, network.ServerClient1VethPair.Outer, ula+"2::3")
486-
require.NoErrorf(t, err, "set IP on server-client1 interface")
487-
err = setInterfaceIP6(network.ServerNetNS, network.ServerClient2VethPair.Outer, ula+"3::3")
488-
require.NoErrorf(t, err, "set IP on server-client2 interface")
489-
490-
err = setInterfaceIP6(network.Client1NetNS, network.ServerClient1VethPair.Inner, ula+"2::1")
491-
require.NoErrorf(t, err, "set IP on client1-server interface")
492-
err = setInterfaceIP6(network.Client1NetNS, network.Client1Client2VethPair.Outer, ula+"1::1")
493-
require.NoErrorf(t, err, "set IP on client1-client2 interface")
494-
495-
err = setInterfaceIP6(network.Client2NetNS, network.ServerClient2VethPair.Inner, ula+"3::2")
496-
require.NoErrorf(t, err, "set IP on client2-server interface")
497-
err = setInterfaceIP6(network.Client2NetNS, network.Client1Client2VethPair.Inner, ula+"1::2")
498-
require.NoErrorf(t, err, "set IP on client2-client1 interface")
491+
err = setInterfaceIP6(network.ServerNetNS, network.ServerVethPair.Outer, ula+"3::2")
492+
require.NoErrorf(t, err, "set IP on server interface")
493+
err = setInterfaceIP6(network.Client1NetNS, network.Client1VethPair.Outer, ula+"1::2")
494+
require.NoErrorf(t, err, "set IP on client1 interface")
495+
err = setInterfaceIP6(network.Client2NetNS, network.Client2VethPair.Outer, ula+"2::2")
496+
require.NoErrorf(t, err, "set IP on client2 interface")
497+
498+
err = setInterfaceIP6(network.RouterNetNS, network.ServerVethPair.Inner, ula+"3::1")
499+
require.NoErrorf(t, err, "set IP on router-server interface")
500+
err = setInterfaceIP6(network.RouterNetNS, network.Client1VethPair.Inner, ula+"1::1")
501+
require.NoErrorf(t, err, "set IP on router-client1 interface")
502+
err = setInterfaceIP6(network.RouterNetNS, network.Client2VethPair.Inner, ula+"2::1")
503+
require.NoErrorf(t, err, "set IP on router-client2 interface")
499504

500505
// Bring up all interfaces
501506
interfaces := []struct {
502-
netNS *os.File
503-
ifaceName string
507+
netNS *os.File
508+
ifaceName string
509+
defaultRoute string
504510
}{
505-
{network.ServerNetNS, network.ServerClient1VethPair.Outer},
506-
{network.ServerNetNS, network.ServerClient2VethPair.Outer},
507-
{network.Client1NetNS, network.ServerClient1VethPair.Inner},
508-
{network.Client1NetNS, network.Client1Client2VethPair.Outer},
509-
{network.Client2NetNS, network.ServerClient2VethPair.Inner},
510-
{network.Client2NetNS, network.Client1Client2VethPair.Inner},
511+
{network.ServerNetNS, network.ServerVethPair.Outer, ula + "3::1"},
512+
{network.Client1NetNS, network.Client1VethPair.Outer, ula + "1::1"},
513+
{network.Client2NetNS, network.Client2VethPair.Outer, ula + "2::1"},
514+
{network.RouterNetNS, network.ServerVethPair.Inner, ""},
515+
{network.RouterNetNS, network.Client1VethPair.Inner, ""},
516+
{network.RouterNetNS, network.Client2VethPair.Inner, ""},
511517
}
512518
for _, iface := range interfaces {
513519
err = setInterfaceUp(iface.netNS, iface.ifaceName)
514520
require.NoErrorf(t, err, "bring up interface %q", iface.ifaceName)
515-
// Note: routes are not needed as we are fully connected, so nothing needs to forward IP to a further
516-
// destination.
521+
522+
if iface.defaultRoute != "" {
523+
err = addRouteInNetNS(iface.netNS, []string{"default", "via", iface.defaultRoute, "dev", iface.ifaceName})
524+
require.NoErrorf(t, err, "add peer default route to %s", iface.defaultRoute)
525+
}
517526
}
518527

528+
// enable IP forwarding in the router
529+
_, err = commandInNetNS(network.RouterNetNS, "sysctl", []string{"-w", "net.ipv6.conf.all.forwarding=1"}).Output()
530+
require.NoError(t, wrapExitErr(err), "enable IPv6 forwarding in router NetNS")
531+
519532
return TestNetworking{
520533
Server: TestNetworkingServer{
521534
Process: TestNetworkingProcess{NetNS: network.ServerNetNS},
522535
ListenAddr: "[::]:8080", // Server listens on all IPs
523536
},
524537
Client1: TestNetworkingClient{
525538
Process: TestNetworkingProcess{NetNS: network.Client1NetNS},
526-
ServerAccessURL: "http://[" + ula + "2::3]:8080", // Client1 accesses server directly
539+
ServerAccessURL: "http://[" + ula + "3::2]:8080",
527540
},
528541
Client2: TestNetworkingClient{
529542
Process: TestNetworkingProcess{NetNS: network.Client2NetNS},
530-
ServerAccessURL: "http://[" + ula + "3::3]:8080", // Client2 accesses server directly
543+
ServerAccessURL: "http://[" + ula + "3::2]:8080",
531544
},
532545
}
533546
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy