android: coalesce peer routes to skip rebuild storm on large tailnets

libtailscale/backend.go

@@ -101,7 +101,12 @@ type backend struct {
 	settings   settingsFunc
 	lastCfg    *router.Config
 	lastDNSCfg *dns.OSConfig
-	netMon     *netmon.Monitor
+	// lastEffective is the snapshot of the VpnService.Builder-relevant
+	// subset of the last-applied config. updateTUN uses it to skip a
+	// rebuild when only fields outside that subset (e.g. peer /32 churn
+	// that we coalesce) have changed. See tailscale/tailscale#19591.
+	lastEffective vpnEffectiveCfg
+	netMon        *netmon.Monitor
 
 	logIDPublic logid.PublicID
 	logger      *logtail.Logger

libtailscale/net.go

@@ -9,11 +9,13 @@ import (
 	"log"
 	"net/netip"
 	"runtime/debug"
+	"slices"
 	"strings"
 	"syscall"
 
 	"github.com/tailscale/tailscale-android/libtailscale/ifaceparse"
 	rangescalc "github.com/tailscale/tailscale-android/libtailscale/ranges_calc"
+	routesutil "github.com/tailscale/tailscale-android/libtailscale/routesutil"
 	"github.com/tailscale/wireguard-go/tun"
 	"tailscale.com/net/dns"
 	"tailscale.com/net/netmon"
@@ -77,6 +79,20 @@ var googleDNSServers = []netip.Addr{
 }
 
 func (b *backend) updateTUN(rcfg *router.Config, dcfg *dns.OSConfig) (err error) {
+	// On large tailnets, wgengine fires a router.Config update on every peer
+	// up/down — but the change is just a per-peer /32 inside the tailnet's
+	// CGNAT range, which the Android VPN layer doesn't need (peer-level
+	// routing is handled inside wgengine). Coalescing those /32s into a
+	// single parent prefix lets us no-op when only peer membership changed
+	// and skip the VpnService.Builder.establish() rebuild that was tearing
+	// down every TCP socket on tun0. See tailscale/tailscale#19591.
+	eff := computeEffective(rcfg, dcfg)
+	if rcfg != nil && len(rcfg.LocalAddrs) > 0 && b.lastEffective.Equal(eff) {
+		b.logger.Logf("updateTUN: VPN-effective cfg unchanged; skipping rebuild (routes %d, coalesced %d)",
+			len(rcfg.Routes), len(eff.routes))
+		return nil
+	}
+
 	b.logger.Logf("updateTUN: changed")
 	defer b.logger.Logf("updateTUN: finished")
 
@@ -144,7 +160,8 @@ func (b *backend) updateTUN(rcfg *router.Config, dcfg *dns.OSConfig) (err error)
 
 	if useExclude {
 		// For API 33+, use ExcludeRoute for LocalRoutes and AddRoute for Routes.
-		for _, route := range rcfg.Routes {
+		// eff.routes is the coalesced set — see computeEffective.
+		for _, route := range eff.routes {
 			// Normalize route address; Builder.addRoute does not accept non-zero masked bits.
 			route = route.Masked()
 			if err := builder.AddRoute(route.Addr().String(), int32(route.Bits())); err != nil {
@@ -163,10 +180,10 @@ func (b *backend) updateTUN(rcfg *router.Config, dcfg *dns.OSConfig) (err error)
 			}
 		}
 
-		b.logger.Logf("updateTUN: added %d routes (exclude-mode), localRoutes=%d", len(rcfg.Routes), len(rcfg.LocalRoutes))
+		b.logger.Logf("updateTUN: added %d routes (exclude-mode), localRoutes=%d", len(eff.routes), len(rcfg.LocalRoutes))
 	} else {
 		// Older APIs: compute allowed-minus-disallowed prefixes and AddRoute them.
-		prefixesV4, prefixesV6, err := rangescalc.Calculate(rcfg.Routes, rcfg.LocalRoutes)
+		prefixesV4, prefixesV6, err := rangescalc.Calculate(eff.routes, rcfg.LocalRoutes)
 		if err != nil {
 			b.logger.Logf("updateTUN: route calculation error: %v", err)
 			return err
@@ -253,6 +270,7 @@ func (b *backend) updateTUN(rcfg *router.Config, dcfg *dns.OSConfig) (err error)
 
 	b.lastCfg = rcfg
 	b.lastDNSCfg = dcfg
+	b.lastEffective = eff
 	return nil
 }
 
@@ -269,6 +287,7 @@ func closeFileDescriptor() error {
 // CloseVPN closes any active TUN devices.
 func (b *backend) CloseTUNs() {
 	b.lastCfg = nil
+	b.lastEffective = vpnEffectiveCfg{}
 	b.devices.Shutdown()
 }
 
@@ -339,3 +358,47 @@ func (b *backend) getPlatformDNSConfig() string {
 func (b *backend) setCfg(rcfg *router.Config, dcfg *dns.OSConfig) error {
 	return b.settings(rcfg, dcfg)
 }
+
+// vpnEffectiveCfg captures the subset of router.Config + dns.OSConfig
+// that VpnService.Builder actually consumes. updateTUN compares this
+// against the last-applied snapshot to decide whether a tun rebuild is
+// needed.
+type vpnEffectiveCfg struct {
+	localAddrs    []netip.Prefix
+	routes        []netip.Prefix // peer routes coalesced via routesutil.CoalescePeerRoutes
+	localRoutes   []netip.Prefix
+	mtu           int
+	nameservers   []netip.Addr
+	searchDomains []dnsname.FQDN
+}
+
+func computeEffective(rcfg *router.Config, dcfg *dns.OSConfig) vpnEffectiveCfg {
+	if rcfg == nil {
+		return vpnEffectiveCfg{}
+	}
+	eff := vpnEffectiveCfg{
+		localAddrs:  slices.Clone(rcfg.LocalAddrs),
+		routes:      routesutil.CoalescePeerRoutes(rcfg.Routes),
+		localRoutes: slices.Clone(rcfg.LocalRoutes),
+		mtu:         rcfg.NewMTU,
+	}
+	if dcfg != nil {
+		eff.nameservers = slices.Clone(dcfg.Nameservers)
+		eff.searchDomains = slices.Clone(dcfg.SearchDomains)
+	}
+	return eff
+}
+
+// Equal reports whether two effective configs are identical. The routes
+// slice is sorted by routesutil.CoalescePeerRoutes; the other slices
+// preserve wgengine's emission order, which is deterministic for a given
+// netmap. A spurious mismatch would only cause an extra rebuild, never a
+// missed one.
+func (e vpnEffectiveCfg) Equal(o vpnEffectiveCfg) bool {
+	return e.mtu == o.mtu &&
+		slices.Equal(e.localAddrs, o.localAddrs) &&
+		slices.Equal(e.routes, o.routes) &&
+		slices.Equal(e.localRoutes, o.localRoutes) &&
+		slices.Equal(e.nameservers, o.nameservers) &&
+		slices.Equal(e.searchDomains, o.searchDomains)
+}

libtailscale/routesutil/routesutil.go

@@ -0,0 +1,82 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package routesutil provides helpers for manipulating the route slices
+// produced by wgengine before they are pushed into Android's
+// VpnService.Builder. It lives in its own package (rather than inside
+// libtailscale) so the pure-Go helpers can be unit-tested without an
+// Android NDK toolchain — Makefile's go-test target excludes only the
+// libtailscale package itself.
+package routesutil
+
+import (
+	"cmp"
+	"net/netip"
+	"slices"
+
+	"tailscale.com/net/tsaddr"
+)
+
+// CoalescePeerRoutes folds per-peer /32 (IPv4) and /128 (IPv6) routes
+// inside Tailscale's well-known address ranges (the CGNAT range
+// 100.64.0.0/10 and the Tailscale ULA fd7a:115c:a1e0::/48) into a single
+// parent prefix per family. Other prefixes — subnet routers, exit nodes,
+// custom advertised routes — pass through unchanged.
+//
+// Peer-level routing inside the tailnet is handled by wgengine's
+// userspace WireGuard, so the Android VPN layer only needs the parent
+// prefix to deliver tailnet-bound traffic into tun0. On a tailnet with
+// N peers and subnets-off, the input shrinks from ~N entries to ~2 (one
+// /10, one /48) plus any non-tailnet routes.
+//
+// The output is sorted (IPv4 before IPv6, then ascending prefix length,
+// then ascending address) so callers can compare two coalesced sets
+// with slices.Equal.
+//
+// See tailscale/tailscale#19591 for context.
+func CoalescePeerRoutes(in []netip.Prefix) []netip.Prefix {
+	if len(in) == 0 {
+		return nil
+	}
+	v4 := tsaddr.CGNATRange()
+	v6 := tsaddr.TailscaleULARange()
+
+	var hasV4Peer, hasV6Peer bool
+	out := make([]netip.Prefix, 0, len(in)+2)
+	for _, r := range in {
+		switch {
+		case r.Bits() == 32 && v4.Contains(r.Addr()):
+			hasV4Peer = true
+		case r.Bits() == 128 && v6.Contains(r.Addr()):
+			hasV6Peer = true
+		default:
+			out = append(out, r)
+		}
+	}
+	// Avoid emitting a duplicate parent prefix if wgengine already
+	// included it among non-peer routes (theoretically possible if the
+	// tailnet admin advertises the parent as a subnet route).
+	if hasV4Peer && !slices.Contains(out, v4) {
+		out = append(out, v4)
+	}
+	if hasV6Peer && !slices.Contains(out, v6) {
+		out = append(out, v6)
+	}
+	slices.SortFunc(out, prefixLess)
+	return out
+}
+
+// prefixLess sorts prefixes for stable comparison: IPv4 before IPv6,
+// then ascending prefix length, then ascending address.
+func prefixLess(a, b netip.Prefix) int {
+	if a.Addr().Is4() != b.Addr().Is4() {
+		if a.Addr().Is4() {
+			return -1
+		}
+		return 1
+	}
+	if c := cmp.Compare(a.Bits(), b.Bits()); c != 0 {
+		return c
+	}
+	return a.Addr().Compare(b.Addr())
+}