// Tests for the localhost proxy registry. The C-exported functions are thin // wrappers around the Go-typed functions tested here (cgo is not usable // from test files); the wrappers only convert types and fetch the tsnet // dialer. Tests inject a plain net.Dialer instead of a live Tailnet. // // Run with: go test -race ./... package main import ( "context" "errors" "fmt" "io" "net" "net/netip" "os" "strconv" "strings" "sync" "sync/atomic" "testing" "time" ) // netDial is the test stand-in for tsnet.Server.Dial. func netDial(ctx context.Context, network, addr string) (net.Conn, error) { var d net.Dialer return d.DialContext(ctx, network, addr) } // startEchoServer starts a TCP echo server and returns its IP and port. func startEchoServer(t *testing.T) (string, int) { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("echo listen: %v", err) } t.Cleanup(func() { ln.Close() }) go func() { for { conn, err := ln.Accept() if err != nil { return } go func(c net.Conn) { defer c.Close() io.Copy(c, c) }(conn) } }() host, portStr, err := net.SplitHostPort(ln.Addr().String()) if err != nil { t.Fatalf("split echo addr: %v", err) } port, err := strconv.Atoi(portStr) if err != nil { t.Fatalf("parse echo port: %v", err) } return host, port } // echoRoundTrip sends payload through the proxy at localPort and verifies // the echo server sends it back unchanged. func echoRoundTrip(t *testing.T, localPort int, payload string) { t.Helper() conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 5*time.Second) if err != nil { t.Fatalf("dial proxy 127.0.0.1:%d: %v", localPort, err) } defer conn.Close() conn.SetDeadline(time.Now().Add(5 * time.Second)) if _, err := conn.Write([]byte(payload)); err != nil { t.Fatalf("write through proxy: %v", err) } buf := make([]byte, len(payload)) if _, err := io.ReadFull(conn, buf); err != nil { t.Fatalf("read echo through proxy: %v", err) } if string(buf) != payload { t.Fatalf("echo mismatch: sent %q, got %q", payload, string(buf)) } } // expectClosed verifies that no proxy accepts connections on localPort // anymore. The listener is closed synchronously by stopProxy, so a fresh // dial must be refused. func expectClosed(t *testing.T, localPort int) { t.Helper() conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 1*time.Second) if err == nil { conn.Close() t.Fatalf("port %d still accepts connections after stop", localPort) } } // resetProxies cleans global proxy state after each test. func resetProxies(t *testing.T) { t.Helper() t.Cleanup(stopAllProxies) } func proxyCount() int { proxyMu.Lock() defer proxyMu.Unlock() return len(proxies) } func currentLegacyID() int64 { proxyMu.Lock() defer proxyMu.Unlock() return legacyProxyID } func TestConcurrentProxyHandles(t *testing.T) { resetProxies(t) ip, port := startEchoServer(t) const n = 5 ids := make(map[int64]bool, n) ports := make(map[int]bool, n) localPorts := make([]int, 0, n) for i := 0; i < n; i++ { id, localPort, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("startProxy %d: %v", i, err) } if id <= 0 { t.Fatalf("startProxy %d: non-positive handle %d", i, id) } if ids[id] { t.Fatalf("duplicate handle %d", id) } if ports[localPort] { t.Fatalf("duplicate local port %d", localPort) } ids[id] = true ports[localPort] = true localPorts = append(localPorts, localPort) } // All proxies forward concurrently. var wg sync.WaitGroup for i, lp := range localPorts { wg.Add(1) go func(i, lp int) { defer wg.Done() echoRoundTrip(t, lp, fmt.Sprintf("payload-%d", i)) }(i, lp) } wg.Wait() if got := proxyCount(); got != n { t.Fatalf("proxy count = %d, want %d", got, n) } } func TestIndependentStop(t *testing.T) { resetProxies(t) ip, port := startEchoServer(t) idA, portA, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("start A: %v", err) } _, portB, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("start B: %v", err) } _, portC, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("start C: %v", err) } stopProxy(idA) expectClosed(t, portA) echoRoundTrip(t, portB, "b-still-alive") echoRoundTrip(t, portC, "c-still-alive") if got := proxyCount(); got != 2 { t.Fatalf("proxy count after one stop = %d, want 2", got) } } func TestStopIsIdempotent(t *testing.T) { resetProxies(t) ip, port := startEchoServer(t) id, localPort, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("start: %v", err) } stopProxy(id) stopProxy(id) // double stop: no-op stopProxy(id + 1000) // unknown handle: no-op stopProxy(0) // zero handle: no-op stopProxy(-1) // negative handle: no-op expectClosed(t, localPort) if got := proxyCount(); got != 0 { t.Fatalf("proxy count = %d, want 0", got) } } func TestRestartGetsFreshHandleAndPort(t *testing.T) { resetProxies(t) ip, port := startEchoServer(t) id1, port1, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("first start: %v", err) } echoRoundTrip(t, port1, "first-life") stopProxy(id1) expectClosed(t, port1) id2, port2, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("restart: %v", err) } if id2 <= id1 { t.Fatalf("handle not monotonic: first %d, restart %d", id1, id2) } echoRoundTrip(t, port2, "second-life") } func TestSameRemoteGetsDistinctLocalPorts(t *testing.T) { // Local ports cannot collide: every proxy listens on 127.0.0.1:0 and // the kernel assigns the port. Two proxies to the same remote are // allowed and independent. resetProxies(t) ip, port := startEchoServer(t) id1, port1, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("start 1: %v", err) } id2, port2, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("start 2: %v", err) } if port1 == port2 { t.Fatalf("same local port %d for two proxies", port1) } echoRoundTrip(t, port1, "one") echoRoundTrip(t, port2, "two") stopProxy(id1) expectClosed(t, port1) echoRoundTrip(t, port2, "two-survives") stopProxy(id2) } func TestLegacyProxyOneAtATime(t *testing.T) { resetProxies(t) ip, port := startEchoServer(t) // A handle-based proxy that must survive every legacy operation. _, handlePort, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("start handle proxy: %v", err) } legacyPort1, err := startLegacyProxy(netDial, ip, port) if err != nil { t.Fatalf("legacy start 1: %v", err) } echoRoundTrip(t, legacyPort1, "legacy-1") // Second legacy start replaces the first. legacyPort2, err := startLegacyProxy(netDial, ip, port) if err != nil { t.Fatalf("legacy start 2: %v", err) } expectClosed(t, legacyPort1) echoRoundTrip(t, legacyPort2, "legacy-2") echoRoundTrip(t, handlePort, "handle-survives-legacy-restart") // Legacy stop touches only the legacy proxy. stopLegacyProxy() expectClosed(t, legacyPort2) echoRoundTrip(t, handlePort, "handle-survives-legacy-stop") if got := currentLegacyID(); got != 0 { t.Fatalf("legacyProxyID = %d after stop, want 0", got) } stopLegacyProxy() // idempotent with none running if got := proxyCount(); got != 1 { t.Fatalf("proxy count = %d, want 1 (the handle proxy)", got) } } func TestStopAllProxies(t *testing.T) { resetProxies(t) ip, port := startEchoServer(t) _, p1, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("start 1: %v", err) } _, p2, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("start 2: %v", err) } lp, err := startLegacyProxy(netDial, ip, port) if err != nil { t.Fatalf("legacy start: %v", err) } stopAllProxies() expectClosed(t, p1) expectClosed(t, p2) expectClosed(t, lp) if got := proxyCount(); got != 0 { t.Fatalf("proxy count = %d, want 0", got) } if got := currentLegacyID(); got != 0 { t.Fatalf("legacyProxyID = %d, want 0", got) } } func TestDeadRemoteClosesLocalConn(t *testing.T) { resetProxies(t) // Reserve a port and close it so nothing listens there. ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("reserve: %v", err) } deadPort := ln.Addr().(*net.TCPAddr).Port ln.Close() _, localPort, err := startProxy(netDial, "127.0.0.1", deadPort) if err != nil { t.Fatalf("start: %v", err) } conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 5*time.Second) if err != nil { t.Fatalf("dial proxy: %v", err) } defer conn.Close() conn.SetReadDeadline(time.Now().Add(5 * time.Second)) buf := make([]byte, 1) if _, err := conn.Read(buf); err == nil { t.Fatal("expected the proxy to close the connection when the remote dial fails") } } // guardConn fails the test if the proxy touches the upstream connection // after stopProxy has returned: the drain guarantee says a returned stop // left no goroutine behind. type guardConn struct { net.Conn afterStop *atomic.Bool t *testing.T } func (g *guardConn) Read(p []byte) (int, error) { if g.afterStop.Load() { g.t.Error("Read on upstream conn after stopProxy returned") } return g.Conn.Read(p) } func (g *guardConn) Write(p []byte) (int, error) { if g.afterStop.Load() { g.t.Error("Write on upstream conn after stopProxy returned") } return g.Conn.Write(p) } // TestConcurrentDoubleStop covers the app calling stopProxyHandle twice // for one handle from two teardown paths, at the same time. Both calls // must be safe and both must return only after the proxy is drained. func TestConcurrentDoubleStop(t *testing.T) { resetProxies(t) ip, port := startEchoServer(t) id, localPort, err := startProxy(netDial, ip, port) if err != nil { t.Fatalf("start: %v", err) } echoRoundTrip(t, localPort, "alive-before-stops") const stoppers = 4 var wg sync.WaitGroup for i := 0; i < stoppers; i++ { wg.Add(1) go func() { defer wg.Done() stopProxy(id) }() } wg.Wait() stopProxy(id) // and once more after the dust settles expectClosed(t, localPort) if got := proxyCount(); got != 0 { t.Fatalf("proxy count = %d, want 0", got) } } // TestStopWaitsForHungDial is register item 87 trigger (b): a TCP client // has connected to the local listener, the upstream dial to an // unreachable peer is still pending, and the handle is stopped. The stop // must abort the dial through the proxy context and must not return // before the dial has actually unblocked. func TestStopWaitsForHungDial(t *testing.T) { resetProxies(t) dialStarted := make(chan struct{}, 1) var dialReturned atomic.Bool hung := func(ctx context.Context, network, addr string) (net.Conn, error) { dialStarted <- struct{}{} <-ctx.Done() // hangs until the proxy context is canceled dialReturned.Store(true) return nil, ctx.Err() } id, localPort, err := startProxy(hung, "100.64.0.99", 5050) if err != nil { t.Fatalf("start: %v", err) } conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 5*time.Second) if err != nil { t.Fatalf("dial proxy: %v", err) } defer conn.Close() select { case <-dialStarted: case <-time.After(5 * time.Second): t.Fatal("upstream dial never started") } stopped := make(chan struct{}) go func() { stopProxy(id) close(stopped) }() select { case <-stopped: case <-time.After(5 * time.Second): t.Fatal("stopProxy hung on an in-flight dial (context not honored)") } if !dialReturned.Load() { t.Fatal("stopProxy returned before the in-flight dial was aborted") } expectClosed(t, localPort) if got := proxyCount(); got != 0 { t.Fatalf("proxy count = %d, want 0", got) } } // TestStopDrainsActiveConnection is register item 87 trigger (a): the // handle is stopped while a live connection is mid-copy. After stopProxy // returns, no copy loop may touch the upstream connection anymore. func TestStopDrainsActiveConnection(t *testing.T) { resetProxies(t) ip, port := startEchoServer(t) var afterStop atomic.Bool guarded := func(ctx context.Context, network, addr string) (net.Conn, error) { c, err := netDial(ctx, network, addr) if err != nil { return nil, err } return &guardConn{Conn: c, afterStop: &afterStop, t: t}, nil } id, localPort, err := startProxy(guarded, ip, port) if err != nil { t.Fatalf("start: %v", err) } conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 5*time.Second) if err != nil { t.Fatalf("dial proxy: %v", err) } defer conn.Close() // Mid-conversation: both copy loops are live. conn.SetDeadline(time.Now().Add(5 * time.Second)) if _, err := conn.Write([]byte("mid-copy")); err != nil { t.Fatalf("write: %v", err) } buf := make([]byte, len("mid-copy")) if _, err := io.ReadFull(conn, buf); err != nil { t.Fatalf("read echo: %v", err) } stopProxy(id) afterStop.Store(true) // The drain guarantee: any straggler copy loop would now trip the // guard. Give one a moment to do so. time.Sleep(50 * time.Millisecond) expectClosed(t, localPort) if got := proxyCount(); got != 0 { t.Fatalf("proxy count = %d, want 0", got) } } // TestStopAllProxiesDrainsInFlightDials is the TailscaleStop fence: // stopNode may only close the tsnet server after stopAllProxies returns, // so stopAllProxies must not return while any proxy still has a dial in // flight. func TestStopAllProxiesDrainsInFlightDials(t *testing.T) { resetProxies(t) const n = 3 var dialsLive, dialsAborted atomic.Int32 started := make(chan struct{}, n) hung := func(ctx context.Context, network, addr string) (net.Conn, error) { dialsLive.Add(1) started <- struct{}{} <-ctx.Done() dialsAborted.Add(1) dialsLive.Add(-1) return nil, ctx.Err() } for i := 0; i < n; i++ { _, localPort, err := startProxy(hung, "100.64.0.99", 5050) if err != nil { t.Fatalf("start %d: %v", i, err) } conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 5*time.Second) if err != nil { t.Fatalf("dial proxy %d: %v", i, err) } defer conn.Close() } for i := 0; i < n; i++ { select { case <-started: case <-time.After(5 * time.Second): t.Fatalf("upstream dial %d never started", i) } } stopAllProxies() if live := dialsLive.Load(); live != 0 { t.Fatalf("stopAllProxies returned with %d dials still in flight", live) } if aborted := dialsAborted.Load(); aborted != n { t.Fatalf("aborted dials = %d, want %d", aborted, n) } if got := proxyCount(); got != 0 { t.Fatalf("proxy count = %d, want 0", got) } } // TestConcurrentStopDialStorm hammers stop against in-flight dials and // double stops from many goroutines. Run with -race. func TestConcurrentStopDialStorm(t *testing.T) { resetProxies(t) ip, port := startEchoServer(t) // A dial that is deliberately still in flight when the stop lands. slow := func(ctx context.Context, network, addr string) (net.Conn, error) { select { case <-time.After(2 * time.Millisecond): case <-ctx.Done(): return nil, ctx.Err() } return netDial(ctx, network, addr) } const workers = 12 var wg sync.WaitGroup for w := 0; w < workers; w++ { wg.Add(1) go func(w int) { defer wg.Done() for i := 0; i < 20; i++ { id, localPort, err := startProxy(slow, ip, port) if err != nil { t.Errorf("worker %d start %d: %v", w, i, err) return } // Race a local connection (and its upstream dial) // against two concurrent stops of the same handle. conn, _ := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), time.Second) var stops sync.WaitGroup for s := 0; s < 2; s++ { stops.Add(1) go func() { defer stops.Done() stopProxy(id) }() } stops.Wait() if conn != nil { conn.Close() } } }(w) } wg.Wait() if got := proxyCount(); got != 0 { t.Fatalf("proxy count after storm = %d, want 0", got) } } // ============================================================ // Peer disco ping (register item 90) // ============================================================ // TestPingPeerNotUp: pinging while the node is stopped must return the // clean not-started error, never crash. The global server is nil in // tests, which is exactly the stopped state. func TestPingPeerNotUp(t *testing.T) { if got := pingPeerOnNode("100.64.0.7"); got != errNotStarted { t.Fatalf("pingPeerOnNode with no node = %q, want %q", got, errNotStarted) } } // TestPingPeerDispatch: the parsed peer address reaches the injected // ping function, the context carries a deadline within the budget, and // success maps to the empty error string. func TestPingPeerDispatch(t *testing.T) { var got netip.Addr var hadDeadline bool fake := func(ctx context.Context, addr netip.Addr) error { got = addr _, hadDeadline = ctx.Deadline() return nil } if msg := pingPeerAddr(fake, "100.64.0.7", time.Second); msg != "" { t.Fatalf("pingPeerAddr success = %q, want empty", msg) } if want := netip.MustParseAddr("100.64.0.7"); got != want { t.Fatalf("pinged addr = %v, want %v", got, want) } if !hadDeadline { t.Fatal("ping context has no deadline; the ping must be bounded") } } // TestPingPeerBadIP: an unparseable IP fails fast without dispatching a // ping at all. func TestPingPeerBadIP(t *testing.T) { fake := func(ctx context.Context, addr netip.Addr) error { t.Fatal("ping dispatched for an unparseable IP") return nil } msg := pingPeerAddr(fake, "not-an-ip", time.Second) if msg == "" { t.Fatal("pingPeerAddr accepted an unparseable IP") } } // TestPingPeerFailurePropagates: a failing ping surfaces its reason in // the bridge error string. func TestPingPeerFailurePropagates(t *testing.T) { fake := func(ctx context.Context, addr netip.Addr) error { return errors.New("no matching peer") } msg := pingPeerAddr(fake, "100.64.0.7", time.Second) if !strings.Contains(msg, "no matching peer") { t.Fatalf("pingPeerAddr failure = %q, want it to contain the ping error", msg) } } // TestPingPeerTimeout: a ping that never answers is cut off by the // bounded context, so the bridge call always returns. func TestPingPeerTimeout(t *testing.T) { fake := func(ctx context.Context, addr netip.Addr) error { <-ctx.Done() // never answers; unblocks only via the deadline return ctx.Err() } done := make(chan string, 1) go func() { done <- pingPeerAddr(fake, "100.64.0.7", 50*time.Millisecond) }() select { case msg := <-done: if msg == "" { t.Fatal("timed-out ping reported success") } case <-time.After(5 * time.Second): t.Fatal("pingPeerAddr did not return after its timeout") } } // ============================================================ // Path recovery and netmon injection (register item 90) // ============================================================ // recoverRecorder records the recovery calls in order. type recoverRecorder struct { mu sync.Mutex calls []string } func (r *recoverRecorder) record(s string) { r.mu.Lock() defer r.mu.Unlock() r.calls = append(r.calls, s) } func (r *recoverRecorder) SetNetworkUp(up bool) { r.record(fmt.Sprintf("SetNetworkUp(%v)", up)) } func (r *recoverRecorder) Rebind() { r.record("Rebind") } func (r *recoverRecorder) ReSTUN(why string) { r.record("ReSTUN(" + why + ")") } // TestRecoverPathsSequence: the recovery tier runs exactly // SetNetworkUp(true), Rebind, ReSTUN("app-recover"), in that order — // network-up first so a wedged-down node reconnects its home DERP, // Rebind before ReSTUN so the fresh endpoints are advertised from the // re-bound sockets. func TestRecoverPathsSequence(t *testing.T) { rec := &recoverRecorder{} recoverPaths(rec) want := []string{"SetNetworkUp(true)", "Rebind", "ReSTUN(app-recover)"} if len(rec.calls) != len(want) { t.Fatalf("recoverPaths calls = %v, want %v", rec.calls, want) } for i := range want { if rec.calls[i] != want[i] { t.Fatalf("recoverPaths call %d = %q, want %q (all: %v)", i, rec.calls[i], want[i], rec.calls) } } } // TestRecoverPathsNotUp: recovery on a never-started node must return // the clean not-started error, never touch magicsock, never crash. The // global server is nil in tests, which is exactly the stopped state. func TestRecoverPathsNotUp(t *testing.T) { if got := recoverPathsOnNode(); got != errNotStarted { t.Fatalf("recoverPathsOnNode with no node = %q, want %q", got, errNotStarted) } } // TestInjectNetmonEventNotUp: the netmon inject on a stopped node is a // clean error, never a crash. func TestInjectNetmonEventNotUp(t *testing.T) { if got := injectNetmonEventOnNode(); got != errNotStarted { t.Fatalf("injectNetmonEventOnNode with no node = %q, want %q", got, errNotStarted) } } // TestUpdateDefaultRouteInterfaceStoppedNode: the default-route push is // package-level netmon state and must be safe with no node at all, // for any input. func TestUpdateDefaultRouteInterfaceStoppedNode(t *testing.T) { updateDefaultRouteInterface("") // empty name: upstream no-op updateDefaultRouteInterface("en0") // plausible interface updateDefaultRouteInterface("no-such-if9") // unknown interface: logs, no panic } // TestRecoverPathsConcurrentWithStop: recovery, netmon injection, and // TailscaleStop racing from many goroutines must not race or panic, // and every recovery on the (always-stopped) node reports the clean // not-started error. Exercises the node-lock fencing shared with // stopNode. Run with -race. func TestRecoverPathsConcurrentWithStop(t *testing.T) { resetProxies(t) const workers = 8 var wg sync.WaitGroup for w := 0; w < workers; w++ { wg.Add(1) go func() { defer wg.Done() for i := 0; i < 50; i++ { if got := recoverPathsOnNode(); got != errNotStarted { t.Errorf("recoverPathsOnNode during stop storm = %q, want %q", got, errNotStarted) return } if got := injectNetmonEventOnNode(); got != errNotStarted { t.Errorf("injectNetmonEventOnNode during stop storm = %q, want %q", got, errNotStarted) return } updateDefaultRouteInterface("en0") } }() } for i := 0; i < 25; i++ { if err := stopNode(); err != nil { t.Errorf("stopNode %d: %v", i, err) } } wg.Wait() } // TestSetDebugNetLogging: the toggle sets and clears both log-only // knobs in the process environment (envknob.Setenv keeps the cached // knob values in sync with the env, which os.Getenv observes). func TestSetDebugNetLogging(t *testing.T) { t.Cleanup(func() { setDebugNetLogging(false) }) setDebugNetLogging(true) for _, k := range []string{"TS_DEBUG_DERP", "TS_DEBUG_DISCO"} { if got := os.Getenv(k); got != "true" { t.Fatalf("%s after enable = %q, want %q", k, got, "true") } } setDebugNetLogging(false) for _, k := range []string{"TS_DEBUG_DERP", "TS_DEBUG_DISCO"} { if got := os.Getenv(k); got != "false" { t.Fatalf("%s after disable = %q, want %q", k, got, "false") } } } func TestConcurrentStartStopStress(t *testing.T) { resetProxies(t) ip, port := startEchoServer(t) const workers = 16 var wg sync.WaitGroup for w := 0; w < workers; w++ { wg.Add(1) go func(w int) { defer wg.Done() for i := 0; i < 8; i++ { id, localPort, err := startProxy(netDial, ip, port) if err != nil { t.Errorf("worker %d start %d: %v", w, i, err) return } echoRoundTrip(t, localPort, fmt.Sprintf("w%d-i%d", w, i)) stopProxy(id) } }(w) } wg.Wait() if got := proxyCount(); got != 0 { t.Fatalf("proxy count after stress = %d, want 0", got) } }