Router that virtually distributes inference across connected devices in the home.
Go
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package discovery
import (
"context"
"errors"
"fmt"
"net"
"sync/atomic"
"testing"
"time"
)
func node(id string, txt ...string) Node {
return Node{ID: id, Host: id + ".local.", Port: 14318, Addresses: []string{"192.168.1.10"}, TXT: txt}
}
func seenSet(ns ...Node) map[string]Node {
m := make(map[string]Node, len(ns))
for _, n := range ns {
m[n.ID] = n
}
return m
}
// eventsByType tallies events for concise assertions.
func eventsByType(evs []Event) map[string]int {
m := map[string]int{}
for _, e := range evs {
m[e.Type]++
}
return m
}
func TestReconcileDiscoveredUpdatedRemoved(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithMissThreshold(3))
if got := eventsByType(b.reconcile(seenSet(node("a")))); got[Discovered] != 1 {
t.Fatalf("first scan: got %v, want 1 discovered", got)
}
// Same node, no change → no event.
if evs := b.reconcile(seenSet(node("a"))); len(evs) != 0 {
t.Fatalf("unchanged scan emitted %v", evs)
}
// Changed TXT → updated.
if got := eventsByType(b.reconcile(seenSet(node("a", "v=2")))); got[Updated] != 1 {
t.Fatalf("changed scan: got %v, want 1 updated", got)
}
// Absent for < threshold: no removal yet.
for i := 0; i < 2; i++ {
if evs := b.reconcile(seenSet()); len(evs) != 0 {
t.Fatalf("miss %d emitted %v, want none before threshold", i+1, evs)
}
}
// Third consecutive miss reaches the threshold → removed.
if got := eventsByType(b.reconcile(seenSet())); got[Removed] != 1 {
t.Fatalf("threshold miss: got %v, want 1 removed", got)
}
if len(b.Nodes()) != 0 {
t.Fatalf("node not evicted: %v", b.Nodes())
}
}
// TestDefaultMissThresholdOutlastsASaturatedNode pins the eviction window a
// browser gets when it does not choose one. A node saturated by its own
// inference load stops answering mDNS for as long as the load lasts, and the
// window that shipped (three scans, 15s) was short enough to read that as a
// departure — so a working node was evicted mid-request. Nothing else in the
// package asserts the default, and the whole point of the constant is the
// elapsed time it buys.
func TestDefaultMissThresholdOutlastsASaturatedNode(t *testing.T) {
b := New("_nvpair-test._tcp", "local")
if window := time.Duration(b.opt.missThreshold) * b.opt.interval; window < time.Minute {
t.Fatalf("default eviction window is %s; too short to outlast a load burst", window)
}
b.reconcile(seenSet(node("a")))
for i := 1; i < missThresholdDefault; i++ {
if evs := b.reconcile(seenSet()); len(evs) != 0 {
t.Fatalf("miss %d of %d emitted %v; the node must be held until the threshold", i, missThresholdDefault, evs)
}
}
if got := eventsByType(b.reconcile(seenSet())); got[Removed] != 1 {
t.Fatalf("miss %d: got %v, want 1 removed", missThresholdDefault, got)
}
}
// A longer window must not mean a blind one. Probing and giving up are separate
// decisions: the probe starts a quarter of the way in and runs every scan, so the
// minute is spent gathering evidence rather than waiting, and eviction rests on a
// run of failed probes instead of the one that lands on the deadline.
func TestLivenessProbeRunsThroughoutTheWindowNotJustAtTheEnd(t *testing.T) {
var probes atomic.Int32
b := New("_nvpair-test._tcp", "local", WithLivenessProbe(func(Node) bool {
probes.Add(1)
return false
}))
b.reconcile(seenSet(node("a")))
for i := 1; i < probeAfterMisses; i++ {
b.reconcile(seenSet())
}
if got := probes.Load(); got != 0 {
t.Fatalf("probe ran %d times before probeAfterMisses; want 0", got)
}
// From probeAfterMisses onward every scan asks again, and no failed answer may
// evict before the threshold.
for i := probeAfterMisses; i < missThresholdDefault; i++ {
if evs := b.reconcile(seenSet()); len(evs) != 0 {
t.Fatalf("miss %d emitted %v; a failed probe must not evict before the threshold", i, evs)
}
}
asked := probes.Load()
if want := int32(missThresholdDefault - probeAfterMisses); asked != want {
t.Fatalf("probe ran %d times across the window; want %d (once per scan from probeAfterMisses)", asked, want)
}
if got := eventsByType(b.reconcile(seenSet()))[Removed]; got != 1 {
t.Fatalf("node not evicted at the threshold after %d failed probes (removed=%d)", asked, got)
}
}
// The reason for probing early is rescue: one answer clears the miss counter, so
// a peer whose multicast is being dropped while it remains perfectly reachable
// never reaches the threshold at all.
func TestAProbeThatAnswersMidWindowFullyRecoversTheNode(t *testing.T) {
alive := false
b := New("_nvpair-test._tcp", "local", WithLivenessProbe(func(Node) bool { return alive }))
b.reconcile(seenSet(node("a")))
for i := 1; i < missThresholdDefault; i++ {
b.reconcile(seenSet())
}
// One answer, one scan short of eviction.
alive = true
if evs := b.reconcile(seenSet()); len(evs) != 0 {
t.Fatalf("a node that answered its probe emitted %v", evs)
}
if b.misses["a"] != 0 {
t.Fatalf("miss counter = %d after a successful probe; want 0", b.misses["a"])
}
// And the full window is available again from scratch.
alive = false
for i := 1; i < missThresholdDefault; i++ {
if evs := b.reconcile(seenSet()); len(evs) != 0 {
t.Fatalf("miss %d after recovery emitted %v; the window must restart", i, evs)
}
}
}
// A browser that picks a threshold shorter than probeAfterMisses must still get
// its probe: one that first ran after the point of no return could never rescue
// anything, and the node would be evicted without ever being asked.
func TestAShortThresholdStillProbesBeforeEvicting(t *testing.T) {
var probes atomic.Int32
b := New("_nvpair-test._tcp", "local", WithMissThreshold(1), WithLivenessProbe(func(Node) bool {
probes.Add(1)
return true
}))
b.reconcile(seenSet(node("a")))
if evs := b.reconcile(seenSet()); len(evs) != 0 {
t.Fatalf("a reachable node was evicted at a threshold of 1: %v", evs)
}
if got := probes.Load(); got != 1 {
t.Fatalf("probe ran %d times; want 1", got)
}
}
// TestEmptyScanDoesNotEvictTheWholeFleet is the regression that motivated the
// guard. A machine saturated by its own inference load could not drain its
// multicast socket inside scanTimeout, so its scans came back completely empty
// and every known node took a miss on the same pass — including idle peers doing
// no work at all. Three such passes evicted the entire cluster in one reconcile.
func TestEmptyScanDoesNotEvictTheWholeFleet(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithLivenessProbe(func(Node) bool { return false }))
b.reconcile(seenSet(node("a"), node("b"), node("c")))
for i := 1; i <= emptyScanGrace; i++ {
if evs := b.reconcile(seenSet()); len(evs) != 0 {
t.Fatalf("empty scan %d emitted %v; an empty scan is evidence about this browser, not the fleet", i, evs)
}
}
if got := len(b.Nodes()); got != 3 {
t.Fatalf("%d of 3 nodes survived %d empty scans; want all 3", got, emptyScanGrace)
}
for _, k := range []string{"a", "b", "c"} {
if b.misses[k] != 0 {
t.Fatalf("node %q took %d misses from empty scans; want 0", k, b.misses[k])
}
}
}
// The guard is a grace, not a veto: a whole fleet can genuinely go — a switch
// losing power, the last peers shutting down together — and those records still
// have to age out.
func TestSustainedEmptyScansStillEvictEventually(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithLivenessProbe(func(Node) bool { return false }))
b.reconcile(seenSet(node("a"), node("b")))
removed := 0
// Past the grace, empty scans count normally, so the nodes need the full miss
// threshold on top before they go.
for i := 0; i < emptyScanGrace+missThresholdDefault+1 && removed < 2; i++ {
removed += eventsByType(b.reconcile(seenSet()))[Removed]
}
if removed != 2 {
t.Fatalf("%d of 2 nodes aged out after %d empty scans; want both",
removed, emptyScanGrace+missThresholdDefault+1)
}
}
// The suppression is justified by several independent machines going quiet at
// once, so it does not apply when there is only one node to lose. Losing a lone
// peer looks identical whether it left or we stopped listening, and delaying that
// eviction would buy nothing — the miss threshold and the liveness probe already
// cover it.
func TestALoneNodeIsNotShieldedByTheEmptyScanGuard(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithMissThreshold(2), WithLivenessProbe(func(Node) bool { return false }))
b.reconcile(seenSet(node("a")))
b.reconcile(seenSet())
if got := eventsByType(b.reconcile(seenSet()))[Removed]; got != 1 {
t.Fatalf("a single known node was not evicted at its threshold (removed=%d)", got)
}
}
// One node answering proves the receive path works, so the run of excuses ends
// there — otherwise a browser that alternated one good scan with six empty ones
// would never evict anything.
func TestAnySeenNodeResetsTheEmptyRun(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithLivenessProbe(func(Node) bool { return false }))
b.reconcile(seenSet(node("a"), node("b")))
for i := 0; i < emptyScanGrace; i++ {
b.reconcile(seenSet())
}
if b.emptyScans != emptyScanGrace {
t.Fatalf("empty run = %d, want %d", b.emptyScans, emptyScanGrace)
}
// "b" alone answers: partial, but proof the socket is being drained.
b.reconcile(seenSet(node("b")))
if b.emptyScans != 0 {
t.Fatalf("empty run = %d after a node answered; want 0", b.emptyScans)
}
}
// The guard must not soften the ordinary case. A scan that returns some nodes and
// not others says something real about the ones that are missing.
func TestPartialScanStillPenalizesTheMissingNode(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithMissThreshold(2), WithLivenessProbe(func(Node) bool { return false }))
b.reconcile(seenSet(node("a"), node("gone")))
b.reconcile(seenSet(node("a")))
if got := eventsByType(b.reconcile(seenSet(node("a"))))[Removed]; got != 1 {
t.Fatalf("a node missing from scans that returned other nodes was not evicted (removed=%d)", got)
}
}
// A browser that knows about nobody is not failing when it hears nobody, so it
// must not bank excuses it would spend later against nodes it has only just
// discovered.
func TestEmptyScanWithNoKnownNodesBanksNothing(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithLivenessProbe(func(Node) bool { return false }))
for i := 0; i < emptyScanGrace*2; i++ {
b.reconcile(seenSet())
}
if b.emptyScans != 0 {
t.Fatalf("empty run = %d with no nodes known; want 0", b.emptyScans)
}
// And the excuses it did not bank are not spent on the nodes that arrive next.
b.reconcile(seenSet(node("a"), node("b")))
if evs := b.reconcile(seenSet()); len(evs) != 0 {
t.Fatalf("first empty scan after discovery emitted %v; the grace should start fresh", evs)
}
if b.emptyScans != 1 {
t.Fatalf("empty run = %d after one empty scan; want 1", b.emptyScans)
}
}
func TestReconcileOrderInsensitive(t *testing.T) {
b := New("_nvpair-test._tcp", "local")
a1 := Node{ID: "a", Host: "a.local.", Port: 1, Addresses: []string{"10.0.0.1", "10.0.0.2"}, TXT: []string{"x=1", "y=2"}}
a2 := Node{ID: "a", Host: "a.local.", Port: 1, Addresses: []string{"10.0.0.2", "10.0.0.1"}, TXT: []string{"y=2", "x=1"}}
b.reconcile(seenSet(a1))
if evs := b.reconcile(seenSet(a2)); len(evs) != 0 {
t.Fatalf("reordered addresses/TXT emitted a spurious event %v", evs)
}
}
func TestNoEviction(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithNoEviction(), WithMissThreshold(1))
b.reconcile(seenSet(node("a")))
for i := 0; i < 5; i++ {
if evs := b.reconcile(seenSet()); len(evs) != 0 {
t.Fatalf("no-evict browser emitted %v on miss %d", evs, i+1)
}
}
if len(b.Nodes()) != 1 {
t.Fatalf("no-evict browser dropped the node: %v", b.Nodes())
}
}
func TestLivenessProbeRetainsReachable(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithMissThreshold(2), WithLivenessProbe(func(Node) bool { return true }))
b.reconcile(seenSet(node("a")))
// Two misses cross the threshold, but the probe says the node is still up.
b.reconcile(seenSet())
if evs := b.reconcile(seenSet()); len(evs) != 0 {
t.Fatalf("reachable threshold-missed node was evicted: %v", evs)
}
if len(b.Nodes()) != 1 {
t.Fatalf("reachable node dropped: %v", b.Nodes())
}
}
func TestLivenessProbeEvictsUnreachable(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithMissThreshold(2), WithLivenessProbe(func(Node) bool { return false }))
b.reconcile(seenSet(node("a")))
b.reconcile(seenSet())
if got := eventsByType(b.reconcile(seenSet())); got[Removed] != 1 {
t.Fatalf("unreachable threshold-missed node not evicted: %v", got)
}
if len(b.Nodes()) != 0 {
t.Fatalf("unreachable node retained: %v", b.Nodes())
}
}
// TestLivenessProbesOverlap pins the concurrency bound: one probe stuck on an
// unreachable address must not hold up the rest. Each probe blocks until
// probeConcurrency of them are in flight, which can only complete if they run
// together.
func TestLivenessProbesOverlap(t *testing.T) {
const nodes = probeConcurrency
release := make(chan struct{})
var started atomic.Int32
b := New("_nvpair-test._tcp", "local", WithMissThreshold(1), WithLivenessProbe(func(Node) bool {
if started.Add(1) == nodes {
close(release)
}
select {
case <-release:
case <-time.After(2 * time.Second):
// Serial probes never overlap, so release is never closed and each
// one waits out this deadline instead of hanging the test.
}
return true
}))
seed := make([]Node, 0, nodes)
for i := range nodes {
seed = append(seed, node(fmt.Sprintf("n%d", i)))
}
b.reconcile(seenSet(seed...))
start := time.Now()
b.reconcile(seenSet())
if elapsed := time.Since(start); elapsed > time.Second {
t.Fatalf("probes did not overlap: %d nodes took %v", nodes, elapsed)
}
if len(b.Nodes()) != nodes {
t.Fatalf("reachable nodes evicted: %d of %d retained", len(b.Nodes()), nodes)
}
}
func TestKeyFunc(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithKeyFunc(func(n Node) string { return UUIDFromTXT(n.TXT) }))
if got := b.key(node("host", "uuid=abc")); got != "abc" {
t.Errorf("key with uuid = %q, want abc", got)
}
// Falls back to instance name when the key func returns "".
if got := b.key(node("host")); got != "host" {
t.Errorf("key without uuid = %q, want host (fallback)", got)
}
}
func TestUUIDFromTXT(t *testing.T) {
if got := UUIDFromTXT([]string{"v=1", "uuid=xyz", "ip=1.2.3.4"}); got != "xyz" {
t.Errorf("UUIDFromTXT = %q, want xyz", got)
}
if got := UUIDFromTXT([]string{"v=1"}); got != "" {
t.Errorf("UUIDFromTXT with no uuid = %q, want empty", got)
}
}
func TestSeed(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithKeyFunc(func(n Node) string { return UUIDFromTXT(n.TXT) }))
b.Seed(node("host-a", "uuid=a"), node("host-b", "uuid=b"))
if len(b.Nodes()) != 2 {
t.Fatalf("Seed: got %d nodes, want 2", len(b.Nodes()))
}
// A subsequent scan that omits a seeded node must not immediately drop it
// (miss counting starts fresh); and re-seeding replaces in place.
b.Seed(node("host-a", "uuid=a", "ip=1.2.3.4"))
if len(b.Nodes()) != 2 {
t.Fatalf("re-seed changed node count: %d", len(b.Nodes()))
}
}
func TestPollReturnsSnapshot(t *testing.T) {
b := New("_nvpair-test._tcp", "local")
b.browseFunc = func(context.Context) map[string]Node { return seenSet(node("a"), node("b")) }
got := b.Poll(context.Background())
if len(got) != 2 {
t.Fatalf("Poll returned %d nodes, want 2", len(got))
}
}
func TestRunEmitsAndCloses(t *testing.T) {
b := New("_nvpair-test._tcp", "local", WithInterval(5*time.Millisecond), WithMissThreshold(1))
// First scan sees "a"; every subsequent scan is empty so it's removed.
var calls int
b.browseFunc = func(context.Context) map[string]Node {
calls++
if calls == 1 {
return seenSet(node("a"))
}
return seenSet()
}
ctx, cancel := context.WithCancel(context.Background())
events := make(chan Event, 16)
done := make(chan struct{})
go func() { b.Run(ctx, events); close(done) }()
got := map[string]int{}
timeout := time.After(2 * time.Second)
for {
select {
case e, ok := <-events:
if !ok {
t.Fatal("events channel closed before observing discovered+removed")
}
got[e.Type]++
if got[Discovered] >= 1 && got[Removed] >= 1 {
cancel()
<-done // Run must close the channel on return
return
}
case <-timeout:
cancel()
t.Fatalf("timed out; events seen: %v", got)
}
}
}
func TestSendMulticastQueryOnInterfaceUsesFirstIPv4AndMDNSTarget(t *testing.T) {
ifi := &net.Interface{Index: 7, Name: "eth0"}
wantSource := net.IPv4(192, 0, 2, 10)
addrs := []net.Addr{
&net.IPNet{IP: net.ParseIP("2001:db8::10")},
&net.IPNet{IP: wantSource},
&net.IPNet{IP: net.IPv4(198, 51, 100, 20)},
}
payload := []byte("PTR query")
var gotPayload []byte
var gotInterface *net.Interface
var gotSource net.IP
var gotTarget *net.UDPAddr
source, err := sendMulticastQueryOnInterface(
payload,
ifi,
addrs,
func(buf []byte, sentIfi *net.Interface, src net.IP, target *net.UDPAddr) error {
gotPayload = append([]byte(nil), buf...)
gotInterface = sentIfi
gotSource = append(net.IP(nil), src...)
gotTarget = target
return nil
},
)
if err != nil {
t.Fatalf("sendMulticastQueryOnInterface: %v", err)
}
if !source.Equal(wantSource) || !gotSource.Equal(wantSource) {
t.Fatalf("source = %s / sent %s, want %s", source, gotSource, wantSource)
}
if gotInterface != ifi {
t.Errorf("interface = %v, want %v", gotInterface, ifi)
}
if string(gotPayload) != string(payload) {
t.Errorf("payload = %q, want %q", gotPayload, payload)
}
if gotTarget == nil || !gotTarget.IP.Equal(net.IPv4(224, 0, 0, 251)) || gotTarget.Port != 5353 {
t.Errorf("target = %v, want 224.0.0.251:5353", gotTarget)
}
}
func TestSendMulticastQueryOnInterfaceReturnsSenderFailure(t *testing.T) {
wantErr := errors.New("send refused")
wantSource := net.IPv4(192, 0, 2, 10)
source, err := sendMulticastQueryOnInterface(
[]byte("PTR query"),
&net.Interface{Index: 7, Name: "eth0"},
[]net.Addr{&net.IPNet{IP: wantSource}},
func([]byte, *net.Interface, net.IP, *net.UDPAddr) error {
return wantErr
},
)
if !source.Equal(wantSource) {
t.Fatalf("source = %s, want %s", source, wantSource)
}
if !errors.Is(err, wantErr) {
t.Fatalf("error = %v, want %v", err, wantErr)
}
}
func TestSendMulticastQueryOnInterfaceSkipsInterfacesWithoutIPv4(t *testing.T) {
called := false
source, err := sendMulticastQueryOnInterface(
[]byte("PTR query"),
&net.Interface{Index: 7, Name: "eth0"},
[]net.Addr{&net.IPNet{IP: net.ParseIP("2001:db8::10")}},
func([]byte, *net.Interface, net.IP, *net.UDPAddr) error {
called = true
return nil
},
)
if err != nil {
t.Fatalf("sendMulticastQueryOnInterface: %v", err)
}
if source != nil {
t.Fatalf("source = %s, want nil", source)
}
if called {
t.Fatal("sender called without an IPv4 address")
}
}
// TestSendFailuresNeedARunAndClearOnRecovery: this feeds address selection, so a
// single blip must not move a host's canonical address, and one success must undo
// the suppression immediately.
func TestSendFailuresNeedARunAndClearOnRecovery(t *testing.T) {
b := New("_x._tcp", "local")
for range sendFailureThreshold - 1 {
b.recordSendOutcomes(map[string]bool{"eth0": false})
}
if got := b.SendFailures(); got["eth0"] {
t.Fatalf("eth0 reported failed after %d misses, want a full run of %d first",
sendFailureThreshold-1, sendFailureThreshold)
}
b.recordSendOutcomes(map[string]bool{"eth0": false})
if got := b.SendFailures(); !got["eth0"] {
t.Fatalf("eth0 not reported failed after %d consecutive misses", sendFailureThreshold)
}
b.recordSendOutcomes(map[string]bool{"eth0": true})
if got := b.SendFailures(); got["eth0"] {
t.Fatal("one successful send must clear the suppression outright")
}
}
// TestSendFailuresForgetAnInterfaceThatIsGone: a VPN adapter that comes and goes
// would otherwise leave a run of failures behind that suppresses an address on an
// interface no longer present, and grow the counter map for the process's life.
func TestSendFailuresForgetAnInterfaceThatIsGone(t *testing.T) {
b := New("_x._tcp", "local")
for range sendFailureThreshold {
b.recordSendOutcomes(map[string]bool{"eth0": false, "tun0": false})
}
if got := b.SendFailures(); !got["tun0"] {
t.Fatalf("tun0 not reported failed after a full run: %v", got)
}
// tun0 is gone: the next scan attempts eth0 only.
b.recordSendOutcomes(map[string]bool{"eth0": false})
got := b.SendFailures()
if got["tun0"] {
t.Error("an interface that is no longer attempted must not stay asserted as failed")
}
if !got["eth0"] {
t.Error("eth0 is still being attempted and still failing; its run must survive")
}
b.mu.RLock()
_, stale := b.sendMisses["tun0"]
b.mu.RUnlock()
if stale {
t.Error("the counter for a departed interface was retained")
}
}