ca3a8b6298
The deep dive (constellation-docs research/tsnet-deafness-deep-dive.md) established the embedded node goes deaf because a zombie home-DERP connection is only detected by the 120 s read deadline and the Rebind/ ReSTUN repair machinery runs solely on netmon events an embedding never receives. TailscaleRecoverPaths() runs SetNetworkUp + Rebind + ReSTUN on the running node (3 s DERP probe bound; holds the node lock because Conn.Rebind has no closed-conn fence upstream, unlike the LocalClient paths), and the Swift layer now feeds real NWPathMonitor transitions into netmon exactly as the official Apple client does, so the repair machinery finally runs on network changes. The TailscalePingPeer docs no longer claim a disco ping re-registers DERP: it is a detector, not a cure. Opt-in debugNetLogging start flag (envknob; compiled out of GOOS=ios by upstream, effective on desktop/Android bench builds). Go suite 29/29 under -race, Dart 13/13, both xcframework slices export the new symbols. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
852 lines
23 KiB
Go
852 lines
23 KiB
Go
// Tests for the localhost proxy registry. The C-exported functions are thin
|
|
// wrappers around the Go-typed functions tested here (cgo is not usable
|
|
// from test files); the wrappers only convert types and fetch the tsnet
|
|
// dialer. Tests inject a plain net.Dialer instead of a live Tailnet.
|
|
//
|
|
// Run with: go test -race ./...
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/netip"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// netDial is the test stand-in for tsnet.Server.Dial.
|
|
func netDial(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
var d net.Dialer
|
|
return d.DialContext(ctx, network, addr)
|
|
}
|
|
|
|
// startEchoServer starts a TCP echo server and returns its IP and port.
|
|
func startEchoServer(t *testing.T) (string, int) {
|
|
t.Helper()
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("echo listen: %v", err)
|
|
}
|
|
t.Cleanup(func() { ln.Close() })
|
|
|
|
go func() {
|
|
for {
|
|
conn, err := ln.Accept()
|
|
if err != nil {
|
|
return
|
|
}
|
|
go func(c net.Conn) {
|
|
defer c.Close()
|
|
io.Copy(c, c)
|
|
}(conn)
|
|
}
|
|
}()
|
|
|
|
host, portStr, err := net.SplitHostPort(ln.Addr().String())
|
|
if err != nil {
|
|
t.Fatalf("split echo addr: %v", err)
|
|
}
|
|
port, err := strconv.Atoi(portStr)
|
|
if err != nil {
|
|
t.Fatalf("parse echo port: %v", err)
|
|
}
|
|
return host, port
|
|
}
|
|
|
|
// echoRoundTrip sends payload through the proxy at localPort and verifies
|
|
// the echo server sends it back unchanged.
|
|
func echoRoundTrip(t *testing.T, localPort int, payload string) {
|
|
t.Helper()
|
|
conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 5*time.Second)
|
|
if err != nil {
|
|
t.Fatalf("dial proxy 127.0.0.1:%d: %v", localPort, err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
conn.SetDeadline(time.Now().Add(5 * time.Second))
|
|
if _, err := conn.Write([]byte(payload)); err != nil {
|
|
t.Fatalf("write through proxy: %v", err)
|
|
}
|
|
buf := make([]byte, len(payload))
|
|
if _, err := io.ReadFull(conn, buf); err != nil {
|
|
t.Fatalf("read echo through proxy: %v", err)
|
|
}
|
|
if string(buf) != payload {
|
|
t.Fatalf("echo mismatch: sent %q, got %q", payload, string(buf))
|
|
}
|
|
}
|
|
|
|
// expectClosed verifies that no proxy accepts connections on localPort
|
|
// anymore. The listener is closed synchronously by stopProxy, so a fresh
|
|
// dial must be refused.
|
|
func expectClosed(t *testing.T, localPort int) {
|
|
t.Helper()
|
|
conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 1*time.Second)
|
|
if err == nil {
|
|
conn.Close()
|
|
t.Fatalf("port %d still accepts connections after stop", localPort)
|
|
}
|
|
}
|
|
|
|
// resetProxies cleans global proxy state after each test.
|
|
func resetProxies(t *testing.T) {
|
|
t.Helper()
|
|
t.Cleanup(stopAllProxies)
|
|
}
|
|
|
|
func proxyCount() int {
|
|
proxyMu.Lock()
|
|
defer proxyMu.Unlock()
|
|
return len(proxies)
|
|
}
|
|
|
|
func currentLegacyID() int64 {
|
|
proxyMu.Lock()
|
|
defer proxyMu.Unlock()
|
|
return legacyProxyID
|
|
}
|
|
|
|
func TestConcurrentProxyHandles(t *testing.T) {
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
const n = 5
|
|
ids := make(map[int64]bool, n)
|
|
ports := make(map[int]bool, n)
|
|
localPorts := make([]int, 0, n)
|
|
|
|
for i := 0; i < n; i++ {
|
|
id, localPort, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("startProxy %d: %v", i, err)
|
|
}
|
|
if id <= 0 {
|
|
t.Fatalf("startProxy %d: non-positive handle %d", i, id)
|
|
}
|
|
if ids[id] {
|
|
t.Fatalf("duplicate handle %d", id)
|
|
}
|
|
if ports[localPort] {
|
|
t.Fatalf("duplicate local port %d", localPort)
|
|
}
|
|
ids[id] = true
|
|
ports[localPort] = true
|
|
localPorts = append(localPorts, localPort)
|
|
}
|
|
|
|
// All proxies forward concurrently.
|
|
var wg sync.WaitGroup
|
|
for i, lp := range localPorts {
|
|
wg.Add(1)
|
|
go func(i, lp int) {
|
|
defer wg.Done()
|
|
echoRoundTrip(t, lp, fmt.Sprintf("payload-%d", i))
|
|
}(i, lp)
|
|
}
|
|
wg.Wait()
|
|
|
|
if got := proxyCount(); got != n {
|
|
t.Fatalf("proxy count = %d, want %d", got, n)
|
|
}
|
|
}
|
|
|
|
func TestIndependentStop(t *testing.T) {
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
idA, portA, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start A: %v", err)
|
|
}
|
|
_, portB, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start B: %v", err)
|
|
}
|
|
_, portC, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start C: %v", err)
|
|
}
|
|
|
|
stopProxy(idA)
|
|
|
|
expectClosed(t, portA)
|
|
echoRoundTrip(t, portB, "b-still-alive")
|
|
echoRoundTrip(t, portC, "c-still-alive")
|
|
|
|
if got := proxyCount(); got != 2 {
|
|
t.Fatalf("proxy count after one stop = %d, want 2", got)
|
|
}
|
|
}
|
|
|
|
func TestStopIsIdempotent(t *testing.T) {
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
id, localPort, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start: %v", err)
|
|
}
|
|
|
|
stopProxy(id)
|
|
stopProxy(id) // double stop: no-op
|
|
stopProxy(id + 1000) // unknown handle: no-op
|
|
stopProxy(0) // zero handle: no-op
|
|
stopProxy(-1) // negative handle: no-op
|
|
|
|
expectClosed(t, localPort)
|
|
if got := proxyCount(); got != 0 {
|
|
t.Fatalf("proxy count = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestRestartGetsFreshHandleAndPort(t *testing.T) {
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
id1, port1, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("first start: %v", err)
|
|
}
|
|
echoRoundTrip(t, port1, "first-life")
|
|
stopProxy(id1)
|
|
expectClosed(t, port1)
|
|
|
|
id2, port2, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("restart: %v", err)
|
|
}
|
|
if id2 <= id1 {
|
|
t.Fatalf("handle not monotonic: first %d, restart %d", id1, id2)
|
|
}
|
|
echoRoundTrip(t, port2, "second-life")
|
|
}
|
|
|
|
func TestSameRemoteGetsDistinctLocalPorts(t *testing.T) {
|
|
// Local ports cannot collide: every proxy listens on 127.0.0.1:0 and
|
|
// the kernel assigns the port. Two proxies to the same remote are
|
|
// allowed and independent.
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
id1, port1, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start 1: %v", err)
|
|
}
|
|
id2, port2, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start 2: %v", err)
|
|
}
|
|
if port1 == port2 {
|
|
t.Fatalf("same local port %d for two proxies", port1)
|
|
}
|
|
|
|
echoRoundTrip(t, port1, "one")
|
|
echoRoundTrip(t, port2, "two")
|
|
|
|
stopProxy(id1)
|
|
expectClosed(t, port1)
|
|
echoRoundTrip(t, port2, "two-survives")
|
|
stopProxy(id2)
|
|
}
|
|
|
|
func TestLegacyProxyOneAtATime(t *testing.T) {
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
// A handle-based proxy that must survive every legacy operation.
|
|
_, handlePort, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start handle proxy: %v", err)
|
|
}
|
|
|
|
legacyPort1, err := startLegacyProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("legacy start 1: %v", err)
|
|
}
|
|
echoRoundTrip(t, legacyPort1, "legacy-1")
|
|
|
|
// Second legacy start replaces the first.
|
|
legacyPort2, err := startLegacyProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("legacy start 2: %v", err)
|
|
}
|
|
expectClosed(t, legacyPort1)
|
|
echoRoundTrip(t, legacyPort2, "legacy-2")
|
|
echoRoundTrip(t, handlePort, "handle-survives-legacy-restart")
|
|
|
|
// Legacy stop touches only the legacy proxy.
|
|
stopLegacyProxy()
|
|
expectClosed(t, legacyPort2)
|
|
echoRoundTrip(t, handlePort, "handle-survives-legacy-stop")
|
|
|
|
if got := currentLegacyID(); got != 0 {
|
|
t.Fatalf("legacyProxyID = %d after stop, want 0", got)
|
|
}
|
|
stopLegacyProxy() // idempotent with none running
|
|
if got := proxyCount(); got != 1 {
|
|
t.Fatalf("proxy count = %d, want 1 (the handle proxy)", got)
|
|
}
|
|
}
|
|
|
|
func TestStopAllProxies(t *testing.T) {
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
_, p1, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start 1: %v", err)
|
|
}
|
|
_, p2, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start 2: %v", err)
|
|
}
|
|
lp, err := startLegacyProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("legacy start: %v", err)
|
|
}
|
|
|
|
stopAllProxies()
|
|
|
|
expectClosed(t, p1)
|
|
expectClosed(t, p2)
|
|
expectClosed(t, lp)
|
|
if got := proxyCount(); got != 0 {
|
|
t.Fatalf("proxy count = %d, want 0", got)
|
|
}
|
|
if got := currentLegacyID(); got != 0 {
|
|
t.Fatalf("legacyProxyID = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
func TestDeadRemoteClosesLocalConn(t *testing.T) {
|
|
resetProxies(t)
|
|
|
|
// Reserve a port and close it so nothing listens there.
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("reserve: %v", err)
|
|
}
|
|
deadPort := ln.Addr().(*net.TCPAddr).Port
|
|
ln.Close()
|
|
|
|
_, localPort, err := startProxy(netDial, "127.0.0.1", deadPort)
|
|
if err != nil {
|
|
t.Fatalf("start: %v", err)
|
|
}
|
|
|
|
conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 5*time.Second)
|
|
if err != nil {
|
|
t.Fatalf("dial proxy: %v", err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
conn.SetReadDeadline(time.Now().Add(5 * time.Second))
|
|
buf := make([]byte, 1)
|
|
if _, err := conn.Read(buf); err == nil {
|
|
t.Fatal("expected the proxy to close the connection when the remote dial fails")
|
|
}
|
|
}
|
|
|
|
// guardConn fails the test if the proxy touches the upstream connection
|
|
// after stopProxy has returned: the drain guarantee says a returned stop
|
|
// left no goroutine behind.
|
|
type guardConn struct {
|
|
net.Conn
|
|
afterStop *atomic.Bool
|
|
t *testing.T
|
|
}
|
|
|
|
func (g *guardConn) Read(p []byte) (int, error) {
|
|
if g.afterStop.Load() {
|
|
g.t.Error("Read on upstream conn after stopProxy returned")
|
|
}
|
|
return g.Conn.Read(p)
|
|
}
|
|
|
|
func (g *guardConn) Write(p []byte) (int, error) {
|
|
if g.afterStop.Load() {
|
|
g.t.Error("Write on upstream conn after stopProxy returned")
|
|
}
|
|
return g.Conn.Write(p)
|
|
}
|
|
|
|
// TestConcurrentDoubleStop covers the app calling stopProxyHandle twice
|
|
// for one handle from two teardown paths, at the same time. Both calls
|
|
// must be safe and both must return only after the proxy is drained.
|
|
func TestConcurrentDoubleStop(t *testing.T) {
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
id, localPort, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start: %v", err)
|
|
}
|
|
echoRoundTrip(t, localPort, "alive-before-stops")
|
|
|
|
const stoppers = 4
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < stoppers; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
stopProxy(id)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
|
|
stopProxy(id) // and once more after the dust settles
|
|
expectClosed(t, localPort)
|
|
if got := proxyCount(); got != 0 {
|
|
t.Fatalf("proxy count = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
// TestStopWaitsForHungDial is register item 87 trigger (b): a TCP client
|
|
// has connected to the local listener, the upstream dial to an
|
|
// unreachable peer is still pending, and the handle is stopped. The stop
|
|
// must abort the dial through the proxy context and must not return
|
|
// before the dial has actually unblocked.
|
|
func TestStopWaitsForHungDial(t *testing.T) {
|
|
resetProxies(t)
|
|
|
|
dialStarted := make(chan struct{}, 1)
|
|
var dialReturned atomic.Bool
|
|
hung := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
dialStarted <- struct{}{}
|
|
<-ctx.Done() // hangs until the proxy context is canceled
|
|
dialReturned.Store(true)
|
|
return nil, ctx.Err()
|
|
}
|
|
|
|
id, localPort, err := startProxy(hung, "100.64.0.99", 5050)
|
|
if err != nil {
|
|
t.Fatalf("start: %v", err)
|
|
}
|
|
|
|
conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 5*time.Second)
|
|
if err != nil {
|
|
t.Fatalf("dial proxy: %v", err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
select {
|
|
case <-dialStarted:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("upstream dial never started")
|
|
}
|
|
|
|
stopped := make(chan struct{})
|
|
go func() {
|
|
stopProxy(id)
|
|
close(stopped)
|
|
}()
|
|
|
|
select {
|
|
case <-stopped:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("stopProxy hung on an in-flight dial (context not honored)")
|
|
}
|
|
if !dialReturned.Load() {
|
|
t.Fatal("stopProxy returned before the in-flight dial was aborted")
|
|
}
|
|
expectClosed(t, localPort)
|
|
if got := proxyCount(); got != 0 {
|
|
t.Fatalf("proxy count = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
// TestStopDrainsActiveConnection is register item 87 trigger (a): the
|
|
// handle is stopped while a live connection is mid-copy. After stopProxy
|
|
// returns, no copy loop may touch the upstream connection anymore.
|
|
func TestStopDrainsActiveConnection(t *testing.T) {
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
var afterStop atomic.Bool
|
|
guarded := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
c, err := netDial(ctx, network, addr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &guardConn{Conn: c, afterStop: &afterStop, t: t}, nil
|
|
}
|
|
|
|
id, localPort, err := startProxy(guarded, ip, port)
|
|
if err != nil {
|
|
t.Fatalf("start: %v", err)
|
|
}
|
|
|
|
conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 5*time.Second)
|
|
if err != nil {
|
|
t.Fatalf("dial proxy: %v", err)
|
|
}
|
|
defer conn.Close()
|
|
|
|
// Mid-conversation: both copy loops are live.
|
|
conn.SetDeadline(time.Now().Add(5 * time.Second))
|
|
if _, err := conn.Write([]byte("mid-copy")); err != nil {
|
|
t.Fatalf("write: %v", err)
|
|
}
|
|
buf := make([]byte, len("mid-copy"))
|
|
if _, err := io.ReadFull(conn, buf); err != nil {
|
|
t.Fatalf("read echo: %v", err)
|
|
}
|
|
|
|
stopProxy(id)
|
|
afterStop.Store(true)
|
|
|
|
// The drain guarantee: any straggler copy loop would now trip the
|
|
// guard. Give one a moment to do so.
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
expectClosed(t, localPort)
|
|
if got := proxyCount(); got != 0 {
|
|
t.Fatalf("proxy count = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
// TestStopAllProxiesDrainsInFlightDials is the TailscaleStop fence:
|
|
// stopNode may only close the tsnet server after stopAllProxies returns,
|
|
// so stopAllProxies must not return while any proxy still has a dial in
|
|
// flight.
|
|
func TestStopAllProxiesDrainsInFlightDials(t *testing.T) {
|
|
resetProxies(t)
|
|
|
|
const n = 3
|
|
var dialsLive, dialsAborted atomic.Int32
|
|
started := make(chan struct{}, n)
|
|
hung := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
dialsLive.Add(1)
|
|
started <- struct{}{}
|
|
<-ctx.Done()
|
|
dialsAborted.Add(1)
|
|
dialsLive.Add(-1)
|
|
return nil, ctx.Err()
|
|
}
|
|
|
|
for i := 0; i < n; i++ {
|
|
_, localPort, err := startProxy(hung, "100.64.0.99", 5050)
|
|
if err != nil {
|
|
t.Fatalf("start %d: %v", i, err)
|
|
}
|
|
conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), 5*time.Second)
|
|
if err != nil {
|
|
t.Fatalf("dial proxy %d: %v", i, err)
|
|
}
|
|
defer conn.Close()
|
|
}
|
|
for i := 0; i < n; i++ {
|
|
select {
|
|
case <-started:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatalf("upstream dial %d never started", i)
|
|
}
|
|
}
|
|
|
|
stopAllProxies()
|
|
|
|
if live := dialsLive.Load(); live != 0 {
|
|
t.Fatalf("stopAllProxies returned with %d dials still in flight", live)
|
|
}
|
|
if aborted := dialsAborted.Load(); aborted != n {
|
|
t.Fatalf("aborted dials = %d, want %d", aborted, n)
|
|
}
|
|
if got := proxyCount(); got != 0 {
|
|
t.Fatalf("proxy count = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
// TestConcurrentStopDialStorm hammers stop against in-flight dials and
|
|
// double stops from many goroutines. Run with -race.
|
|
func TestConcurrentStopDialStorm(t *testing.T) {
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
// A dial that is deliberately still in flight when the stop lands.
|
|
slow := func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
select {
|
|
case <-time.After(2 * time.Millisecond):
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
}
|
|
return netDial(ctx, network, addr)
|
|
}
|
|
|
|
const workers = 12
|
|
var wg sync.WaitGroup
|
|
for w := 0; w < workers; w++ {
|
|
wg.Add(1)
|
|
go func(w int) {
|
|
defer wg.Done()
|
|
for i := 0; i < 20; i++ {
|
|
id, localPort, err := startProxy(slow, ip, port)
|
|
if err != nil {
|
|
t.Errorf("worker %d start %d: %v", w, i, err)
|
|
return
|
|
}
|
|
// Race a local connection (and its upstream dial)
|
|
// against two concurrent stops of the same handle.
|
|
conn, _ := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", localPort), time.Second)
|
|
var stops sync.WaitGroup
|
|
for s := 0; s < 2; s++ {
|
|
stops.Add(1)
|
|
go func() {
|
|
defer stops.Done()
|
|
stopProxy(id)
|
|
}()
|
|
}
|
|
stops.Wait()
|
|
if conn != nil {
|
|
conn.Close()
|
|
}
|
|
}
|
|
}(w)
|
|
}
|
|
wg.Wait()
|
|
|
|
if got := proxyCount(); got != 0 {
|
|
t.Fatalf("proxy count after storm = %d, want 0", got)
|
|
}
|
|
}
|
|
|
|
// ============================================================
|
|
// Peer disco ping (register item 90)
|
|
// ============================================================
|
|
|
|
// TestPingPeerNotUp: pinging while the node is stopped must return the
|
|
// clean not-started error, never crash. The global server is nil in
|
|
// tests, which is exactly the stopped state.
|
|
func TestPingPeerNotUp(t *testing.T) {
|
|
if got := pingPeerOnNode("100.64.0.7"); got != errNotStarted {
|
|
t.Fatalf("pingPeerOnNode with no node = %q, want %q", got, errNotStarted)
|
|
}
|
|
}
|
|
|
|
// TestPingPeerDispatch: the parsed peer address reaches the injected
|
|
// ping function, the context carries a deadline within the budget, and
|
|
// success maps to the empty error string.
|
|
func TestPingPeerDispatch(t *testing.T) {
|
|
var got netip.Addr
|
|
var hadDeadline bool
|
|
fake := func(ctx context.Context, addr netip.Addr) error {
|
|
got = addr
|
|
_, hadDeadline = ctx.Deadline()
|
|
return nil
|
|
}
|
|
|
|
if msg := pingPeerAddr(fake, "100.64.0.7", time.Second); msg != "" {
|
|
t.Fatalf("pingPeerAddr success = %q, want empty", msg)
|
|
}
|
|
if want := netip.MustParseAddr("100.64.0.7"); got != want {
|
|
t.Fatalf("pinged addr = %v, want %v", got, want)
|
|
}
|
|
if !hadDeadline {
|
|
t.Fatal("ping context has no deadline; the ping must be bounded")
|
|
}
|
|
}
|
|
|
|
// TestPingPeerBadIP: an unparseable IP fails fast without dispatching a
|
|
// ping at all.
|
|
func TestPingPeerBadIP(t *testing.T) {
|
|
fake := func(ctx context.Context, addr netip.Addr) error {
|
|
t.Fatal("ping dispatched for an unparseable IP")
|
|
return nil
|
|
}
|
|
msg := pingPeerAddr(fake, "not-an-ip", time.Second)
|
|
if msg == "" {
|
|
t.Fatal("pingPeerAddr accepted an unparseable IP")
|
|
}
|
|
}
|
|
|
|
// TestPingPeerFailurePropagates: a failing ping surfaces its reason in
|
|
// the bridge error string.
|
|
func TestPingPeerFailurePropagates(t *testing.T) {
|
|
fake := func(ctx context.Context, addr netip.Addr) error {
|
|
return errors.New("no matching peer")
|
|
}
|
|
msg := pingPeerAddr(fake, "100.64.0.7", time.Second)
|
|
if !strings.Contains(msg, "no matching peer") {
|
|
t.Fatalf("pingPeerAddr failure = %q, want it to contain the ping error", msg)
|
|
}
|
|
}
|
|
|
|
// TestPingPeerTimeout: a ping that never answers is cut off by the
|
|
// bounded context, so the bridge call always returns.
|
|
func TestPingPeerTimeout(t *testing.T) {
|
|
fake := func(ctx context.Context, addr netip.Addr) error {
|
|
<-ctx.Done() // never answers; unblocks only via the deadline
|
|
return ctx.Err()
|
|
}
|
|
|
|
done := make(chan string, 1)
|
|
go func() { done <- pingPeerAddr(fake, "100.64.0.7", 50*time.Millisecond) }()
|
|
|
|
select {
|
|
case msg := <-done:
|
|
if msg == "" {
|
|
t.Fatal("timed-out ping reported success")
|
|
}
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("pingPeerAddr did not return after its timeout")
|
|
}
|
|
}
|
|
|
|
// ============================================================
|
|
// Path recovery and netmon injection (register item 90)
|
|
// ============================================================
|
|
|
|
// recoverRecorder records the recovery calls in order.
|
|
type recoverRecorder struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
}
|
|
|
|
func (r *recoverRecorder) record(s string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.calls = append(r.calls, s)
|
|
}
|
|
|
|
func (r *recoverRecorder) SetNetworkUp(up bool) { r.record(fmt.Sprintf("SetNetworkUp(%v)", up)) }
|
|
func (r *recoverRecorder) Rebind() { r.record("Rebind") }
|
|
func (r *recoverRecorder) ReSTUN(why string) { r.record("ReSTUN(" + why + ")") }
|
|
|
|
// TestRecoverPathsSequence: the recovery tier runs exactly
|
|
// SetNetworkUp(true), Rebind, ReSTUN("app-recover"), in that order —
|
|
// network-up first so a wedged-down node reconnects its home DERP,
|
|
// Rebind before ReSTUN so the fresh endpoints are advertised from the
|
|
// re-bound sockets.
|
|
func TestRecoverPathsSequence(t *testing.T) {
|
|
rec := &recoverRecorder{}
|
|
recoverPaths(rec)
|
|
|
|
want := []string{"SetNetworkUp(true)", "Rebind", "ReSTUN(app-recover)"}
|
|
if len(rec.calls) != len(want) {
|
|
t.Fatalf("recoverPaths calls = %v, want %v", rec.calls, want)
|
|
}
|
|
for i := range want {
|
|
if rec.calls[i] != want[i] {
|
|
t.Fatalf("recoverPaths call %d = %q, want %q (all: %v)", i, rec.calls[i], want[i], rec.calls)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestRecoverPathsNotUp: recovery on a never-started node must return
|
|
// the clean not-started error, never touch magicsock, never crash. The
|
|
// global server is nil in tests, which is exactly the stopped state.
|
|
func TestRecoverPathsNotUp(t *testing.T) {
|
|
if got := recoverPathsOnNode(); got != errNotStarted {
|
|
t.Fatalf("recoverPathsOnNode with no node = %q, want %q", got, errNotStarted)
|
|
}
|
|
}
|
|
|
|
// TestInjectNetmonEventNotUp: the netmon inject on a stopped node is a
|
|
// clean error, never a crash.
|
|
func TestInjectNetmonEventNotUp(t *testing.T) {
|
|
if got := injectNetmonEventOnNode(); got != errNotStarted {
|
|
t.Fatalf("injectNetmonEventOnNode with no node = %q, want %q", got, errNotStarted)
|
|
}
|
|
}
|
|
|
|
// TestUpdateDefaultRouteInterfaceStoppedNode: the default-route push is
|
|
// package-level netmon state and must be safe with no node at all,
|
|
// for any input.
|
|
func TestUpdateDefaultRouteInterfaceStoppedNode(t *testing.T) {
|
|
updateDefaultRouteInterface("") // empty name: upstream no-op
|
|
updateDefaultRouteInterface("en0") // plausible interface
|
|
updateDefaultRouteInterface("no-such-if9") // unknown interface: logs, no panic
|
|
}
|
|
|
|
// TestRecoverPathsConcurrentWithStop: recovery, netmon injection, and
|
|
// TailscaleStop racing from many goroutines must not race or panic,
|
|
// and every recovery on the (always-stopped) node reports the clean
|
|
// not-started error. Exercises the node-lock fencing shared with
|
|
// stopNode. Run with -race.
|
|
func TestRecoverPathsConcurrentWithStop(t *testing.T) {
|
|
resetProxies(t)
|
|
|
|
const workers = 8
|
|
var wg sync.WaitGroup
|
|
for w := 0; w < workers; w++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for i := 0; i < 50; i++ {
|
|
if got := recoverPathsOnNode(); got != errNotStarted {
|
|
t.Errorf("recoverPathsOnNode during stop storm = %q, want %q", got, errNotStarted)
|
|
return
|
|
}
|
|
if got := injectNetmonEventOnNode(); got != errNotStarted {
|
|
t.Errorf("injectNetmonEventOnNode during stop storm = %q, want %q", got, errNotStarted)
|
|
return
|
|
}
|
|
updateDefaultRouteInterface("en0")
|
|
}
|
|
}()
|
|
}
|
|
|
|
for i := 0; i < 25; i++ {
|
|
if err := stopNode(); err != nil {
|
|
t.Errorf("stopNode %d: %v", i, err)
|
|
}
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
// TestSetDebugNetLogging: the toggle sets and clears both log-only
|
|
// knobs in the process environment (envknob.Setenv keeps the cached
|
|
// knob values in sync with the env, which os.Getenv observes).
|
|
func TestSetDebugNetLogging(t *testing.T) {
|
|
t.Cleanup(func() { setDebugNetLogging(false) })
|
|
|
|
setDebugNetLogging(true)
|
|
for _, k := range []string{"TS_DEBUG_DERP", "TS_DEBUG_DISCO"} {
|
|
if got := os.Getenv(k); got != "true" {
|
|
t.Fatalf("%s after enable = %q, want %q", k, got, "true")
|
|
}
|
|
}
|
|
|
|
setDebugNetLogging(false)
|
|
for _, k := range []string{"TS_DEBUG_DERP", "TS_DEBUG_DISCO"} {
|
|
if got := os.Getenv(k); got != "false" {
|
|
t.Fatalf("%s after disable = %q, want %q", k, got, "false")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestConcurrentStartStopStress(t *testing.T) {
|
|
resetProxies(t)
|
|
ip, port := startEchoServer(t)
|
|
|
|
const workers = 16
|
|
var wg sync.WaitGroup
|
|
for w := 0; w < workers; w++ {
|
|
wg.Add(1)
|
|
go func(w int) {
|
|
defer wg.Done()
|
|
for i := 0; i < 8; i++ {
|
|
id, localPort, err := startProxy(netDial, ip, port)
|
|
if err != nil {
|
|
t.Errorf("worker %d start %d: %v", w, i, err)
|
|
return
|
|
}
|
|
echoRoundTrip(t, localPort, fmt.Sprintf("w%d-i%d", w, i))
|
|
stopProxy(id)
|
|
}
|
|
}(w)
|
|
}
|
|
wg.Wait()
|
|
|
|
if got := proxyCount(); got != 0 {
|
|
t.Fatalf("proxy count after stress = %d, want 0", got)
|
|
}
|
|
}
|