Router that virtually distributes inference across connected devices in the home.
Go
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package main
import (
"context"
"crypto/tls"
"encoding/json"
"io"
"log/slog"
"maps"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"nvpair-shared/clustertrust"
"nvpair-shared/mdns"
"nvpair-shared/netmon"
"nvpair-shared/netpick"
"nvpair-shared/nodeid"
"nvpair-shared/noderec"
"nvpair-shared/reach"
)
// daemon is the promoted node-scanner: it advertises this
// node as ONE _nvpair-node record built from a registry of relayed service
// registrations, browses _nvpair-node to build a directory of every LAN node, and
// serves the discovery:* RPC (register/unregister/update-txt/get-nodes/
// reload-identity) while emitting discovery:node-* events.
//
// Post-cutover this is the sole discovery path: the legacy _nvpair-node-info
// browse and node/* stream are gone, and every consumer registers/subscribes
// through the broker relay, which the broker builds from this daemon's
// discovery:node-* stream and fans out to consumers as discovery:nodes snapshots.
type daemon struct {
codec *Codec
mesh *clustertrust.Mesh
reg *registry
dir *directory
responder *mdns.Responder
browser *Discovery
// instance is this node's mDNS instance name (the hostname). It names the
// registry-driven self directory entry (publishSelf), which is anchored to
// our own identity rather than to hearing our own multicast.
instance string
http *http.Client
// modelsHTTP fetches a peer's /v1/models. It gets a longer timeout than http
// (node-info): engine-manager's models sweep is bounded by its own, larger
// deadline, so a peer with several engines can legitimately take longer than
// a node-info probe. A shorter client would give up mid-sweep and silently
// drop that peer's models.
modelsHTTP *http.Client
// modelsClients is the long-lived per-peer mTLS pool for LAN /v1/models
// fetches. A throwaway Transport every 15s paid a full handshake each time.
modelsClients *clustertrust.PeerClientPool
// tlsHTTP, when non-nil (the operator passed --client-cert/--client-key/
// --ca-bundle), fetches node-info over HTTPS on the ni port instead of plain
// HTTP. Dormant by default (nil): node-info is plain per the transport policy,
// so this is opt-in scaffolding for a future TLS node-info rollout, gated only
// on the flags — never inferred from a node's cluster-uuid or a TXT hint.
tlsHTTP *http.Client
// nodeInfoOrigins queues same-origin requests before http.Client starts its
// timeout, while nodeInfoTransport enforces the matching connection bound.
nodeInfoOrigins nodeInfoOriginGate
// baseDir is the config base nodeid.Resolve reads identity from. Derived
// from --cluster-dir (its parent) so the scanner's uuid= tracks the same
// identity.json cluster-manager writes, even under a non-default cluster dir;
// empty selects nodeid's shared default (production's default layout).
baseDir string
// clusterDir is the cluster trust subtree (--cluster-dir) holding this node's
// keypair, pins, and admission record. It gates whether we advertise a
// cluster-uuid=: we advertise only while actually a member
// (clustertrust.Clustered), never merely because a keypair from a past
// membership is still on disk — a left/removed node keeps its keypair, and
// advertising off keypair presence makes it falsely appear clustered so peers
// can never invite it back.
clusterDir string
// selfMu serializes every mutation of this node's own directory entry
// (publishSelf / reloadIdentity / dropSelf) so a burst of service
// registrations and a concurrent network/identity change can't interleave
// into a torn self entry. The self key is written nowhere else — onBrowse
// ignores our own uuid — so this is the sole self-key writer lock.
selfMu sync.Mutex
// lastInfo / lastModels cache each node's last successful enrichment (keyed by
// hostUuid) — GPU/CPU/memory from node-info and the model list from
// engine-manager — so a transient fetch failure reuses the prior values
// instead of blanking the node card. Same resilience the legacy scanner path
// had via its own lastInfo cache.
infoMu sync.Mutex
lastInfo map[string]NodeInfoResponse
lastInfoAt map[string]time.Time
lastModels map[string][]string
// lastModelsByEngine caches the per-engine attribution (engine name ->
// models) alongside the flat lastModels union, so a transient /v1/models
// miss reuses both rather than blanking the per-engine card. Keyed by
// hostUuid like the other enrichment caches.
lastModelsByEngine map[string]map[string][]string
// lastLoadedByEngine caches the per-engine loaded (in-memory) set from the
// same /v1/models fetch, so a transient miss reuses it rather than blanking
// a remote node's loaded state. Keyed by hostUuid.
lastLoadedByEngine map[string]map[string][]string
// nodeInfoDown holds the nodes whose last node-info enrichment failed, so
// the outage is reported when it starts and when it ends rather than once
// per sweep. Guarded by infoMu with the caches it explains: the reason a
// card keeps showing an old inventory is exactly this entry.
nodeInfoDown map[string]bool
// activityMu guards lastActivityAt, when each peer was last seen returning
// inference response bytes to one of this node's proxies (keyed by hostUuid,
// relayed by the broker from the proxies).
//
// It is separate from the enrichment caches above because it is evidence of a
// different kind. Those record what we managed to ask a peer; this records
// what a peer sent us unprompted, on the data plane, as a by-product of doing
// real work. That makes it the one liveness signal that gets STRONGER as a
// node gets busier, which is precisely when the control-plane probes fail.
activityMu sync.Mutex
lastActivityAt map[string]time.Time
// enrichHosts remembers which of a node's published addresses answered, per
// service, so a sweep asks that one instead of walking the list from the top
// every peerRefreshInterval. It carries its own lock (see hostMemory).
enrichHosts hostMemory
// telemetryRetries owns one active telemetry attempt per node and suppresses
// repeated remote attempts after failures at the same node-info target.
// Local results always finish successfully so loopback stays on the healthy
// cadence.
telemetryRetries telemetryRetryGate
// observedMu guards observed, this host's own addresses that remote peers
// have actually connected to. nvpair-node-info sees those connections and the
// broker relays the set here, because this daemon decides what the node
// publishes and a peer-completed connection is the only direct proof that an
// address works from somewhere other than this machine.
observedMu sync.Mutex
observed map[string]bool
}
// nodeInfoFetchTimeout bounds each node-info enrichment HTTP(S) GET, shared by
// the plain and (opt-in) TLS clients.
const nodeInfoFetchTimeout = 3 * time.Second
const nodeInfoResponseDrainLimit = 64 << 10
// nodeInfoDialTimeout bounds only the TCP connect of that GET.
//
// The two are separated because the failures they cover are not the same. A peer
// that answers slowly is doing work — reading GPU inventory — and deserves the
// whole request budget. A peer whose address silently drops packets, which is
// what a host firewall or an address only its own side can reach looks like from
// here, never gets as far as a request, and waiting the full budget for it costs
// that much on every sweep and never learns anything. Connecting is a round trip
// on any working link, so a connect that has not completed in this long is not
// going to.
//
// It is reach.DefaultTimeout because an address a reachability probe would have
// already written off is not worth waiting longer for here.
const nodeInfoDialTimeout = reach.DefaultTimeout
// nodeInfoTransport is the transport both node-info clients dial through: the
// plain one built here and, when the operator opts enrichment onto HTTPS, the
// TLS one built in tls.go. The TLS handshake gets the same budget as the
// connect, for the same reason.
func nodeInfoTransport(tlsConfig *tls.Config) *http.Transport {
return &http.Transport{
DialContext: (&net.Dialer{Timeout: nodeInfoDialTimeout}).DialContext,
TLSClientConfig: tlsConfig,
TLSHandshakeTimeout: nodeInfoDialTimeout,
MaxConnsPerHost: 1,
MaxIdleConnsPerHost: 1,
IdleConnTimeout: 90 * time.Second,
}
}
// modelsFetchTimeout bounds a peer /v1/models GET. It exceeds engine-manager's
// own models-sweep cap (modelsTimeout, 5s) so the fetch waits for a bounded
// sweep to finish instead of timing out partway through it.
const modelsFetchTimeout = 6 * time.Second
// peerRefreshInterval is the cadence of the mDNS-independent convergence loop
// (refreshPeersLoop). The event-driven enrichment still handles fast initial
// discovery; this is the background sweep that catches changes that never move
// the mDNS record (a model pull/delete, a late engine start) and announcements
// that did move it but were not received (a peer joining or leaving a cluster).
// It comfortably exceeds modelsFetchTimeout so a worst-case sweep finishes well
// before the next tick.
const peerRefreshInterval = 15 * time.Second
// telemetryRefreshInterval keeps utilization observations fresh enough for the
// scheduler while leaving node-info's one-second collector as the faster
// sampling layer. Slow requests remain in flight across ticks without delaying
// dispatch to healthy nodes.
const telemetryRefreshInterval = 2 * time.Second
const telemetryRefreshJitter = 250 * time.Millisecond
const telemetryRetryMax = 30 * time.Second
// modelsRefreshConcurrency caps how many peer /v1/models fetches a single sweep
// runs at once. It sits well above realistic LAN node counts, so it only bounds
// a pathological fan-out (an unexpectedly large network) rather than throttling
// normal operation — the per-node goroutines still run concurrently so one slow
// peer never consumes another's timeout budget.
const modelsRefreshConcurrency = 16
// telemetryRefreshConcurrency limits simultaneous node-info GETs across every
// telemetry tick.
const telemetryRefreshConcurrency = 16
type telemetryRetryState struct {
consecutiveFailures int
retryAt time.Time
retryTargetKey string
token uint64
inFlight bool
activeTargetKey string
discardActive bool
}
// telemetryRetryGate is zero-value ready. Active-attempt ownership and retry
// timing are deliberately separate: a changed endpoint can discard another
// target's backoff, but it cannot release a request that has not finished.
// Removal clears retry timing and marks any active result stale while preserving
// the claim until its matching finish.
type telemetryRetryGate struct {
mu sync.Mutex
nextToken uint64
peers map[string]telemetryRetryState
}
func (g *telemetryRetryGate) claim(hostUUID, targetKey string, now time.Time) (uint64, bool) {
g.mu.Lock()
defer g.mu.Unlock()
state, exists := g.peers[hostUUID]
if exists && state.inFlight {
return 0, false
}
if exists && state.retryTargetKey == targetKey && now.Before(state.retryAt) {
return 0, false
}
if state.retryTargetKey != targetKey {
state.consecutiveFailures = 0
state.retryAt = time.Time{}
state.retryTargetKey = ""
}
g.nextToken++
state.token = g.nextToken
state.inFlight = true
state.activeTargetKey = targetKey
state.discardActive = false
if g.peers == nil {
g.peers = make(map[string]telemetryRetryState)
}
g.peers[hostUUID] = state
return state.token, true
}
func (g *telemetryRetryGate) finish(hostUUID string, token uint64, ok bool, now time.Time) {
g.mu.Lock()
defer g.mu.Unlock()
state, exists := g.peers[hostUUID]
if !exists || !state.inFlight || state.token != token {
return
}
state.inFlight = false
if state.discardActive {
delete(g.peers, hostUUID)
return
}
if ok {
delete(g.peers, hostUUID)
return
}
if state.retryTargetKey != state.activeTargetKey {
state.consecutiveFailures = 0
}
state.consecutiveFailures++
state.retryTargetKey = state.activeTargetKey
state.retryAt = now.Add(telemetryRetryDelay(state.consecutiveFailures))
state.activeTargetKey = ""
g.peers[hostUUID] = state
}
func (g *telemetryRetryGate) forget(hostUUID string) {
g.mu.Lock()
defer g.mu.Unlock()
state, exists := g.peers[hostUUID]
if !exists {
return
}
if !state.inFlight {
delete(g.peers, hostUUID)
return
}
state.consecutiveFailures = 0
state.retryAt = time.Time{}
state.retryTargetKey = ""
state.discardActive = true
g.peers[hostUUID] = state
}
func telemetryTargetKey(hosts []string, port int) string {
return strconv.Itoa(port) + "\x00" + strings.Join(hosts, "\x00")
}
func telemetryRetryDelay(consecutiveFailures int) time.Duration {
delay := telemetryRefreshInterval
for failure := 0; failure < consecutiveFailures; failure++ {
if delay >= telemetryRetryMax/2 {
return telemetryRetryMax
}
delay *= 2
}
return delay
}
// newDaemon builds the daemon. tlsHTTP is nil unless the operator opted node-info
// enrichment onto HTTPS via the --client-cert/--client-key/--ca-bundle flags; nil
// keeps the default plain-HTTP fetch.
func newDaemon(codec *Codec, mesh *clustertrust.Mesh, clusterDir string, tlsHTTP *http.Client) *daemon {
// nodeid reads <base>/cluster/identity.json; --cluster-dir is that cluster
// subdir, so its parent is the base. Empty -> nodeid's shared default.
baseDir := ""
if clusterDir != "" {
baseDir = filepath.Dir(clusterDir)
}
// The daemon is the single stamper of node identity: hostUuid from
// cluster-manager's persisted identity when present (nodeid.Resolve prefers
// it), clusterUuid from the cluster trust dir, addresses from the local
// ranker.
hostUUID := nodeid.Resolve(baseDir)
var cluUUID string
if mesh.Clustered() {
cluUUID = mesh.NodeUUID()
}
// Cold start has no peer or send evidence yet, so this is the route- and
// interface-only answer. refreshAdvertisedAddressesOnce converges it as
// evidence arrives.
reg := newRegistry(hostUUID, cluUUID, netpick.LocalCandidates(netpick.Evidence{}))
instance, _ := os.Hostname()
d := &daemon{
codec: codec,
mesh: mesh,
reg: reg,
instance: instance,
baseDir: baseDir,
clusterDir: clusterDir,
dir: newDirectory(),
http: &http.Client{Timeout: nodeInfoFetchTimeout, Transport: nodeInfoTransport(nil)},
modelsHTTP: &http.Client{Timeout: modelsFetchTimeout},
modelsClients: clustertrust.NewPeerClientPool(mesh, modelsFetchTimeout),
tlsHTTP: tlsHTTP,
lastInfo: make(map[string]NodeInfoResponse),
lastInfoAt: make(map[string]time.Time),
lastModels: make(map[string][]string),
lastModelsByEngine: make(map[string]map[string][]string),
lastLoadedByEngine: make(map[string]map[string][]string),
nodeInfoDown: make(map[string]bool),
lastActivityAt: make(map[string]time.Time),
}
// Anti-flap: before evicting a node that's missed the mDNS threshold,
// confirm it still answers on node-info (or engine-manager as fallback).
// Inference ports (ol/lm) are never probed — they are the promoted proxies
// and SYN spam there contributed to accept-queue exhaustion on macOS.
//
// Key by uuid= (not the mDNS instance name): two hosts sharing a hostname but
// holding distinct UUIDs must not collapse into one directory entry (matching
// cluster-manager's browser). Self re-stamps are handled separately in
// reloadIdentity, which moves this node's own directory entry to the new uuid.
d.browser = NewDiscovery(noderec.ServiceType, noderec.Domain,
WithLivenessProbe(func(n RawNode) bool { return d.reachable(n) }),
WithKeyFunc(func(n RawNode) string { return UUIDFromTXT(n.TXT) }),
)
if resp, err := mdns.NewResponder(instance, noderec.ServiceType, noderec.Domain, noderec.SRVPort, reg.txt()); err != nil {
slog.Warn("mDNS advertising disabled for _nvpair-node", "err", err)
} else {
d.responder = resp
}
return d
}
// run starts the _nvpair-node responder and browser, folding browse events into
// the directory and emitting discovery:node-* until ctx is cancelled.
func (d *daemon) run(ctx context.Context) {
if d.responder != nil {
go func() {
if err := d.responder.Run(ctx); err != nil {
slog.Warn("_nvpair-node responder exited", "err", err)
}
}()
// Keep the advertised ip= tracking the host's live network the same way
// the responder keeps its A records live (both via netmon).
go d.watchAddress(ctx)
}
// Publish our own directory entry from the local registry BEFORE the browse
// loop runs. Self presence is registry-driven, not browse-driven: on Windows
// the host frequently does not loop its own multicast back to the browsing
// socket, so aging self like a peer evicts the local node from its own
// directory. Anchoring self to the identity/services we already own
// makes it present the instant we start and immune to self-multicast loss.
d.publishSelf()
events := make(chan DiscoveryEvent, 32)
go d.browser.Run(ctx, events)
// Converge peer model inventories independently of mDNS change events: the
// browse-driven enrichment above only re-runs when the raw mDNS record
// changes, but models live off the record (engine-manager's em /v1/models),
// so a pull/delete, a late engine start, or a missed announcement would
// otherwise leave a directory entry permanently stale. Stops with ctx.
go d.refreshPeersLoop(ctx)
go d.refreshTelemetryLoop(ctx)
for {
select {
case <-ctx.Done():
if d.modelsClients != nil {
d.modelsClients.CloseIdle()
}
return
case ev, ok := <-events:
if !ok {
return
}
d.onBrowse(ev)
}
}
}
// onBrowse folds one _nvpair-node browse event into the directory (annotating
// trust from the cluster pin set) and emits the matching discovery:node-* event.
func (d *daemon) onBrowse(ev DiscoveryEvent) {
// Never let a browse event touch our own entry. Self is registry-driven
// (publishSelf); folding self browse events in here is exactly what evicted
// the local node on Windows when its own multicast stopped looping back.
// Our record always carries uuid=, so this reliably matches.
if uuid := UUIDFromTXT(ev.Node.TXT); uuid != "" && uuid == d.reg.record().HostUUID {
return
}
trusted := false
if cu := clustertrust.ClusterUUIDFromTXT(ev.Node.TXT); cu != "" {
d.mesh.Refresh()
trusted = d.mesh.HasPin(cu)
}
node, ok := toDirectoryNode(ev.Node, trusted)
if !ok {
// Record without a uuid= — toDirectoryNode already warned. Nothing to
// store, and a removed event for it is a no-op since it was never added.
return
}
var method string
switch ev.Type {
case "removed":
d.dir.remove(node.HostUUID)
d.forget(node.HostUUID)
method = noderec.NotifyNodeRemoved
case "updated":
d.enrich(&node)
d.supersedingUpsert(node)
method = noderec.NotifyNodeUpdated
default: // discovered
d.enrich(&node)
d.supersedingUpsert(node)
method = noderec.NotifyNodeDiscovered
}
d.emit(method, node)
}
// supersedingUpsert folds a browsed peer into the directory, evicting any record
// it is PROVEN to have replaced and emitting the removals BEFORE the caller emits
// the node's own event — so no consumer ever observes the machine under two
// identities at once.
//
// The directory can only nominate (directory.supersedeCandidates). Its signals
// are the arriving record's instance name — an unauthenticated mDNS field
// anything on the LAN can claim — plus a LastSeen gap that a healthy peer's
// frozen timestamp satisfies exactly as readily as a ghost's. Deleting on that
// alone would make mDNS a removal primitive: a record claiming a live peer's
// hostname would evict it, the browser re-reports a node only when its record
// CHANGES so it might never come back, and the loss would carry through to the
// broker's onNodeLost and to workload routing.
//
// So each candidate is confirmed the way the eviction path confirms a
// threshold-missed node: ask the machines involved who they are, and delete only
// on a definite mismatch. A candidate that cannot be confirmed stays, and is not
// left uncovered — when it stops being advertised the ordinary miss-threshold
// path (reachable) puts the same question to it.
func (d *daemon) supersedingUpsert(node noderec.DirectoryNode) {
var proven []noderec.DirectoryNode
for _, old := range d.dir.supersedeCandidates(node, d.reg.record().HostUUID) {
if d.identityReplaced(old, node) {
proven = append(proven, old)
}
}
for _, old := range d.dir.upsertEvicting(node, proven) {
slog.Warn("dropping node record proven superseded by a newer identity on the same host",
"ip", old.IP, "droppedUuid", old.HostUUID, "droppedName", old.Name,
"keptUuid", node.HostUUID, "keptName", node.Name,
"ageSeconds", node.LastSeen-old.LastSeen)
d.forget(old.HostUUID)
d.emit(noderec.NotifyNodeRemoved, old)
}
}
// identityReplaced reports whether old is PROVEN to be a pre-wipe ghost of the
// machine now arriving as replacement — the confirmation supersedingUpsert
// requires before it deletes anything.
//
// It takes two answers, both from node-info, and both are required:
//
// - old must not still be answering for itself. Every address old published is
// asked, at old's own ni port, and any of them naming old keeps it. This is
// what tells a wipe apart from two hosts that merely share a hostname, and it
// is why a record with no ni port to ask is never evicted here: a candidate
// that cannot answer cannot be distinguished from a live namesake, and the
// miss-threshold path will reach it on its own when it stops advertising.
// - the machine at replacement's address must name someone other than old.
// That is the arriving record's own port, which is where the machine is
// answering now.
//
// Anything short of both — no ni port, no answer within the probe budget, a
// response without a hostUuid, or the 403 a clustered peer returns to a plaintext
// caller — leaves the record in place.
func (d *daemon) identityReplaced(old, replacement noderec.DirectoryNode) bool {
oldNI, ok := old.Services[noderec.ServiceNodeInfo]
if !ok || oldNI.Port == 0 {
return false
}
stillItself, _ := d.identityAlive(noderec.NodeRecord{
HostUUID: old.HostUUID,
IP: old.IP,
Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: oldNI.Port},
}, old.CandidateIPs())
if stillItself {
return false
}
ni, ok := replacement.Services[noderec.ServiceNodeInfo]
if !ok || ni.Port == 0 {
return false
}
// Ask the arriving machine at every address it ranked, not just its canonical
// one. Any of them naming old keeps the record, so a wider question can only
// spare a live node; asking one address that this host happens not to reach
// yields no answer at all, and a record with only ips= has no canonical
// address to ask.
replacementIPs := replacement.CandidateIPs()
rec := noderec.NodeRecord{
HostUUID: old.HostUUID,
IP: replacement.IP,
Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: ni.Port},
}
alive, conclusive := d.identityAlive(rec, replacementIPs)
return conclusive && !alive
}
// reloadTrust re-derives everything this daemon caches from the cluster pin set,
// in response to nvpair-cluster-manager reporting that the set moved
// (discovery:reload-trust).
//
// It exists because the browse path is not a sufficient trigger. A peer's trusted
// annotation is computed when its mDNS record is folded in, and the Browser only
// reports a node whose record actually changed — so a peer that was already
// advertising its cluster principal when this node joined is annotated from
// before our own pins existed and never revisited. That left it unroutable for
// the life of the process while the roster and the pins on disk both said
// otherwise.
//
// The announcement comes from the process that writes the pins, and only after
// the write lands, so the directory read here is never older than the event that
// prompted it. Recomputing unconditionally is deliberate: gating on any
// "did it change?" helper is how the previous membership hook silently did
// nothing.
func (d *daemon) reloadTrust() {
d.mesh.Refresh()
// A pin can flip membership on its own (a cluster dir predating admission.json
// is clustered on pins alone), which changes what we advertise, so settle
// identity before annotating peers against it.
d.reconcileIdentity()
d.reconcileTrust()
}
// reconcileIdentity re-advertises this node when live membership no longer
// matches the cluster principal the registry is publishing — the join or leave
// that reloadIdentity exists to pick up. Peers key their pins on our
// cluster-uuid= TXT, so a node that became a member must start advertising it
// without a restart; nothing else tells the fleet.
//
// The comparison is against the registry's current value rather than a change
// signal, so it is safe to call on any event: it is an in-memory string compare,
// and reloadIdentity's file read and interface enumeration only happen when the
// answer actually differs.
func (d *daemon) reconcileIdentity() {
want := ""
if d.mesh.Clustered() {
want = d.mesh.NodeUUID()
}
if d.reg.record().ClusterUUID == want {
return
}
d.reloadIdentity()
}
// reconcileTrust re-evaluates every directory entry's trusted annotation against
// the pin set the mesh currently holds, emitting one discovery:node-updated per
// entry whose answer changed. Entries that agree are left untouched and silent,
// so the steady state is a map lookup per node and no traffic — a peer paired or
// removed since the last pass is the only thing that produces an event.
//
// Self is included: it sits in the directory like any other entry, and
// Mesh.HasPin answers for our own principal through the same self-trust path
// publishSelf uses, so both stay consistent.
//
// The caller refreshes the mesh; this reads whatever it currently holds. It is
// idempotent, so a duplicated or coalesced announcement costs one map lookup per
// known node and emits nothing.
func (d *daemon) reconcileTrust() {
for _, n := range d.dir.snapshot("") {
// nil principal: this pass knows the pin set, not what each peer
// advertises, so it re-derives trust from whatever principal is stored
// when the write lands rather than from the snapshot it is iterating.
node, changed := d.dir.applyClusterIdentity(n.HostUUID, nil, d.mesh.HasPin)
if !changed {
continue
}
slog.Info("node trust changed", "host_uuid", n.HostUUID,
"cluster_uuid", n.ClusterUUID, "trusted", node.Trusted)
d.emit(noderec.NotifyNodeUpdated, node)
}
}
// emit pushes a discovery:node-* event for a directory node. Nil-codec-safe so
// directory-only unit tests can drive the daemon without a wired codec.
func (d *daemon) emit(method string, node noderec.DirectoryNode) {
if d.codec == nil {
return
}
if err := d.codec.Notify(method, noderec.NodeEvent{Node: node}); err != nil {
slog.Warn("failed to emit node event", "method", method, "err", err)
}
}
func (d *daemon) emitTelemetry(telemetry noderec.NodeTelemetry) {
if d.codec == nil {
return
}
if err := d.codec.Notify(noderec.NotifyNodeTelemetry, telemetry); err != nil {
slog.Warn("failed to emit node telemetry", "host_uuid", telemetry.HostUUID, "err", err)
}
}
// reloadIdentity re-resolves this node's advertised identity (hostUuid,
// clusterUuid, ip) and re-advertises if any of it changed. It has three triggers:
//
// - Membership change: the mesh watch calls this when this node joins or leaves
// a cluster, so cluster-uuid= appears in (or disappears from) this node's mDNS
// record within one refresh interval. This is what lets peers key their pins
// on us: without the TXT key a peer treats us as unclustered, skips us in its
// workload broadcast, and drops us as a routing target — while we remain in
// its roster. It must not depend on a process restart.
// - Fresh-host startup order: the scanner spawns before cluster-manager, so it
// mints its own node-id.json and advertises that uuid= before cluster-manager
// writes identity.json. Once cluster-manager is up (the broker triggers a
// reload), the re-resolve picks up identity.json so uuid= converges on the
// cluster principal.
// - Network change: watchAddress calls this on a netmon event so the advertised
// addresses track the host's live network instead of freezing at the startup
// value — matching how the responder keeps its A records live.
//
// Address evidence that arrives later without a network change is handled by
// refreshAdvertisedAddressesOnce, which re-ranks without re-reading identity.
//
// Safe to call concurrently: selfMu serializes it against publishSelf, and
// setIdentity/UpdateTXT are each mutex-guarded and idempotent for unchanged
// fields.
func (d *daemon) reloadIdentity() {
d.selfMu.Lock()
defer d.selfMu.Unlock()
oldHostUUID := d.reg.record().HostUUID
hostUUID := nodeid.Resolve(d.baseDir)
// Re-derive cluster-uuid= from live membership, not keypair presence: a
// keypair persists across a leave/removal, so re-reading the admission record
// here lets a join, a teardown, or a restore converge the advertised cluster
// status without a worker restart. Refresh picks up freshly-written pins too.
d.mesh.Refresh()
var cluUUID string
if d.mesh.Clustered() {
cluUUID = d.mesh.NodeUUID()
}
if !d.reg.setIdentity(hostUUID, cluUUID, d.localAddresses()) {
return
}
if d.responder != nil {
d.responder.UpdateTXT(d.reg.txt())
}
// A changed hostUuid (fresh-host re-stamp onto the cluster principal) leaves
// our old entry under the stale uuid. Drop it (carrying its enrichment cache
// onto the new uuid) so no ghost is served, then republish self under the new
// identity below.
if oldHostUUID != "" && oldHostUUID != hostUUID {
d.dropSelf(oldHostUUID, hostUUID)
}
d.publishSelfLocked()
}
// localAddresses ranks this host's own addresses from the evidence the daemon has
// already gathered, producing the ip= / ips= values it publishes.
//
// Every input is a byproduct of work already done, so this costs no probe of its
// own: the browser records which interfaces its per-scan multicast query failed
// to leave, and the directory already holds the peers that answered — an
// interface whose own subnet contains one of them demonstrably reaches other
// nodes. That is what replaces guessing from subnet numbers, which is what made a
// multi-homed host publish a two-host direct-connect link as its address.
func (d *daemon) localAddresses() []string {
ev := netpick.Evidence{PeerObserved: d.observedAddresses()}
if d.browser != nil {
ev.SendFailed = d.browser.SendFailures()
ev.PeerOnLink = netpick.IfacesFacingPeers(d.peerAddresses())
}
return netpick.LocalCandidates(ev)
}
// setObservedAddresses replaces the peer-proven address set relayed from
// node-info and re-ranks if it changed anything. The set is replaced rather than
// merged so an address peers have stopped reaching stops counting as proof.
func (d *daemon) setObservedAddresses(addrs []string) {
observed := make(map[string]bool, len(addrs))
for _, a := range addrs {
if a != "" {
observed[a] = true
}
}
d.observedMu.Lock()
changed := !maps.Equal(d.observed, observed)
d.observed = observed
d.observedMu.Unlock()
if !changed {
return
}
slog.Debug("peer-observed local addresses updated", "addresses", addrs)
d.refreshAdvertisedAddressesOnce()
}
func (d *daemon) observedAddresses() map[string]bool {
d.observedMu.Lock()
defer d.observedMu.Unlock()
return maps.Clone(d.observed)
}
// peerAddresses returns every address known for other nodes, for the on-link
// inference in localAddresses. Self is excluded: our own record loops back
// through mDNS, and seeing our own address proves nothing about the link.
func (d *daemon) peerAddresses() []string {
self := ""
if d.reg != nil {
self = d.reg.record().HostUUID
}
var out []string
for _, n := range d.dir.snapshot("") {
if n.HostUUID == self {
continue
}
if n.IP != "" {
out = append(out, n.IP)
}
out = append(out, n.IPs...)
}
return out
}
// refreshAdvertisedAddressesOnce re-ranks this host's published addresses. It
// exists because the strongest address evidence is not available at startup: no
// peer has been discovered yet, and no send has failed yet. Without a periodic
// re-rank a node would publish its cold-start guess until the next network change,
// which on a stable host is never.
//
// It deliberately does not go through reloadIdentity: identity needs a file read
// and a trust-store re-read, and neither can change because a peer appeared. The
// common case here is "nothing changed", which must stay cheap.
func (d *daemon) refreshAdvertisedAddressesOnce() {
d.selfMu.Lock()
defer d.selfMu.Unlock()
if !d.reg.setAddresses(d.localAddresses()) {
return
}
rec := d.reg.record()
slog.Info("advertised addresses re-ranked", "ip", rec.IP, "candidates", rec.IPs)
if d.responder != nil {
d.responder.UpdateTXT(d.reg.txt())
}
d.publishSelfLocked()
}
// dropSelf removes this node's stale directory entry after a uuid re-stamp,
// carrying its enrichment cache onto the new uuid so the republish below reuses
// it (and can't blank the card), and emits removed(old) so consumers reconcile.
// The new entry is (re)published by the caller via publishSelfLocked. Must be
// called with selfMu held.
func (d *daemon) dropSelf(oldHostUUID, newHostUUID string) {
node, existed := d.dir.get(oldHostUUID)
d.dir.remove(oldHostUUID)
d.infoMu.Lock()
if info, ok := d.lastInfo[oldHostUUID]; ok {
d.lastInfo[newHostUUID] = info
}
if at, ok := d.lastInfoAt[oldHostUUID]; ok {
d.lastInfoAt[newHostUUID] = at
}
if models, ok := d.lastModels[oldHostUUID]; ok {
d.lastModels[newHostUUID] = models
}
if byEngine, ok := d.lastModelsByEngine[oldHostUUID]; ok {
d.lastModelsByEngine[newHostUUID] = byEngine
}
if loaded, ok := d.lastLoadedByEngine[oldHostUUID]; ok {
d.lastLoadedByEngine[newHostUUID] = loaded
}
delete(d.lastInfo, oldHostUUID)
delete(d.lastInfoAt, oldHostUUID)
delete(d.lastModels, oldHostUUID)
delete(d.lastModelsByEngine, oldHostUUID)
delete(d.lastLoadedByEngine, oldHostUUID)
d.infoMu.Unlock()
if existed {
d.emit(noderec.NotifyNodeRemoved, noderec.DirectoryNode{HostUUID: oldHostUUID, Name: node.Name})
}
}
// publishSelf (re)builds this node's own directory entry from the local registry
// and folds it in, emitting discovered/updated. It is the sole source of the
// self entry: self is anchored to the identity + services we already own, not to
// hearing our own mDNS record, so the local node stays present regardless of
// Windows self-multicast loopback loss. Called at startup, on each
// service (un)register, and after an identity/address change.
func (d *daemon) publishSelf() {
d.selfMu.Lock()
defer d.selfMu.Unlock()
d.publishSelfLocked()
}
// publishSelfLocked is publishSelf's body; the caller must hold selfMu (so a
// registration burst and a concurrent reloadIdentity can't tear the self entry).
func (d *daemon) publishSelfLocked() {
rec := d.reg.record()
if rec.HostUUID == "" {
return // no identity resolved yet — nothing to anchor self on
}
// Local consumers reach our own services over loopback, which is never
// firewalled; fall back to it when the LAN ranker hasn't picked an address
// yet so the self card always has a dialable host.
ip := rec.IP
ips := rec.IPs
if ip == "" {
ip = loopbackHost
ips = []string{loopbackHost}
}
trusted := false
if rec.ClusterUUID != "" {
d.mesh.Refresh()
trusted = d.mesh.HasPin(rec.ClusterUUID)
}
svcs := make(map[noderec.ServiceKey]noderec.ServiceStatus, len(rec.Services))
for k, port := range rec.Services {
svcs[k] = noderec.ServiceStatus{Port: port}
}
node := noderec.DirectoryNode{
HostUUID: rec.HostUUID,
Name: d.instance,
IP: ip,
IPs: ips,
ClusterUUID: rec.ClusterUUID,
Trusted: trusted,
Services: svcs,
LastSeen: time.Now().Unix(),
}
// Enrich over loopback, never the advertised LAN ip=: a first-run Windows
// firewall block on inbound to our own LAN address must not blank the local
// node's metrics/models (the "no metrics" failure mode).
d.enrichAt(&node, loopbackHost)
method := noderec.NotifyNodeUpdated
if isNew := d.dir.upsert(node); isNew {
method = noderec.NotifyNodeDiscovered
}
d.emit(method, node)
}
// watchAddress re-derives the advertised ip= whenever the host's network changes
// (interface or address add/remove — VPN/dock toggles, DHCP re-lease, sleep/wake
// re-IP). Without it the ip= would be frozen at daemon startup and could point at
// an address the node no longer holds; since consumers prefer ip= over the live A
// records, a stale value would misroute them. netmon debounces, so a reload per
// change is cheap. Best-effort: if the monitor can't start, ip= stays static
// (the responder's A records still refresh on their own netmon watch).
func (d *daemon) watchAddress(ctx context.Context) {
mon, err := netmon.Watch(ctx)
if err != nil {
slog.Warn("network monitor unavailable; advertised ip= is static", "err", err)
return
}
for range mon.Subscribe() {
d.reloadIdentity()
}
}
// reachableTimeout bounds each liveness probe attempt (node-info GET or em TCP).
const reachableTimeout = time.Second
// probeIdentityTimeout bounds the liveness probe's node-info read. It is the
// probe's own budget, deliberately not the enrichment client's
// nodeInfoFetchTimeout: enrichment is decorating a card and can afford to wait,
// whereas this runs per threshold-missed node inside the browser's scan cycle
// (shared/discovery). It matches reachableTimeout because the peer
// has already answered a TCP connect by the time we ask — a node that is up and
// listening but cannot serve node-info within a second is one we treat as
// unidentifiable and fall back on, not one worth stalling the scan for.
const probeIdentityTimeout = time.Second
// reachableInfoFreshness is how long a successful node-info enrichment keeps a
// threshold-missed node without a new probe (~2× default discovery scan interval).
const reachableInfoFreshness = 10 * time.Second
// activityFreshness is how long a peer's reported inference traffic vouches for
// it without a probe of its own.
//
// This is the strongest liveness evidence available and the cheapest, because it
// is a by-product of work the node is already doing: the local proxies see
// response bytes arrive from a peer's engine, which proves the peer's OS,
// network path, and engine process are all alive. It matters most in exactly the
// case the probes get wrong — a node saturated by inference is the one least
// able to answer a control-plane probe and the one most certainly not offline.
//
// It is deliberately longer than reachableInfoFreshness. A single long
// generation streams for tens of seconds with gaps between chunks, and the point
// is to cover a whole burst of load rather than to require continuous traffic.
const activityFreshness = 60 * time.Second
// loopbackHost is the address the daemon uses to reach THIS node's own services.
// Loopback is never firewalled, so enriching/probing self over it (rather than
// the advertised LAN ip=) makes the local node immune to a first-run Windows
// firewall block on inbound to its own LAN address.
const loopbackHost = "127.0.0.1"
// reachable reports whether a threshold-missed node is still THERE — an mDNS
// miss is not proof a node is gone, so a missed record is probed before it is
// evicted.
//
// Two kinds of evidence can vouch for a node without dialing it: a fresh
// lastInfo enrichment, and fresh inference traffic reported by the local proxies
// (lastActivityAt). Otherwise it answers in two stages, cheap then precise.
//
// First the TCP sweep: dial node-info (ni) then engine-manager (em) and stop at
// the first that connects. Inference ports (ol/lm) are never dialed — they are
// the promoted proxies and SYN spam there contributed to accept-queue
// exhaustion. Nothing answering means the node is gone, and no further question
// is worth asking — that is the ordinary departure, and it is settled without
// any HTTP.
//
// Something answering is NOT the end of it, and this is the bug that made a
// wiped node permanent. A connect only proves SOMETHING is listening at an
// address, not that it is the node the record describes. A node whose appdata is
// wiped mints a fresh hostUuid but keeps its LAN address and re-binds the same
// fixed service ports, so the sweep reaches the machine's NEW incarnation and
// reports the OLD record alive. The miss counter is reset on every pass and the
// stale record never ages out — the fleet carries a permanent duplicate of one
// machine. So when something does answer, we ask the machine who it is
// (identityAlive) and let a definite answer overrule the connect.
//
// Running the sweep first is what keeps that identity check off the hot path: it
// only runs when the node would otherwise have been KEPT, which is exactly when
// the two cases need telling apart. Ordering the other way would spend a
// node-info timeout on every departed node inside the browser's scan cycle. The
// verdicts are identical either way — a conclusive identity answer
// requires node-info to have responded, so its port is open, so the sweep would
// have succeeded regardless.
//
// A node advertising neither ni nor em (identity-only, or ol/lm alone) has
// nothing to probe and evicts normally.
func (d *daemon) reachable(n RawNode) bool {
rec := noderec.ParseTXT(n.TXT)
candidates := netpick.Candidates(n.TXT, n.Addresses)
if len(candidates) == 0 {
return false
}
if rec.HostUUID != "" && d.reg != nil && rec.HostUUID == d.reg.record().HostUUID {
candidates = []string{loopbackHost}
}
if rec.HostUUID != "" {
d.infoMu.Lock()
at, had := d.lastInfoAt[rec.HostUUID]
d.infoMu.Unlock()
if had && time.Since(at) < reachableInfoFreshness {
return true
}
if since, ok := d.activitySince(rec.HostUUID); ok && since < activityFreshness {
slog.Debug("liveness probe skipped: the node was recently serving inference",
"host_uuid", rec.HostUUID, "last_activity_ago", since)
return true
}
}
answered := false
answeredIP := ""
answeredPort := 0
for _, ip := range candidates {
for _, svc := range []noderec.ServiceKey{noderec.ServiceNodeInfo, noderec.ServiceEngineManager} {
port, ok := rec.Port(svc)
if !ok {
continue
}
conn, err := net.DialTimeout("tcp", net.JoinHostPort(ip, strconv.Itoa(port)), reachableTimeout)
if err == nil {
_ = conn.Close()
answered = true
answeredIP = ip
answeredPort = port
break
}
}
if answered {
break
}
}
if !answered {
return false
}
if alive, conclusive := d.identityAlive(rec, candidates); conclusive {
return alive
}
// Kept on the sweep's word alone. Record which port answered next to the
// node-info port we then failed to read: the pair separates "node-info is
// down but engine-manager answered" from "node-info accepted the connection
// and did not respond in time", which are indistinguishable in a report that
// only says the node flickered.
if niPort, ok := rec.Port(noderec.ServiceNodeInfo); ok {
slog.Debug("liveness probe kept a node without settling its identity",
"dial_ip", answeredIP, "advertised_ip", rec.IP, "answered_port", answeredPort, "ni_port", niPort)
}
return true
}
// identityAlive asks the machines at ips whether any of them is still the node rec
// describes, by reading hostUuid from node-info. In the eviction path it is called
// only after the TCP sweep has already found an address answering, so its job is
// to overrule that verdict when the machine turns out to be someone else.
//
// It walks every address rather than only the record's first because one wrong
// answer is not proof of a wipe: a direct-connect link uses the same subnet on
// every pair of machines that has one, so a record's leading address can resolve
// locally to a different host entirely. An address naming the record's own host
// settles it in the record's favour, and a mismatch counts only once no address
// has.
//
// conclusive reports whether the question could be answered at all. It is false
// — meaning "keep the TCP sweep's verdict" — when the record advertises no ni
// port, when it carries no uuid= to compare against, when no address answers
// within probeIdentityTimeout, or when every answer omits hostUuid. Only a
// node-info response that actually names a host is treated as proof, in either
// direction.
//
// Two properties bound what this can get wrong, and both are worth knowing
// before changing it:
//
// - It only ever concludes for an UNCLUSTERED peer. A clustered node's
// node-info admits pinned mTLS callers only (nvpair-node-info's handler
// refuses everyone else with 403), so this plaintext probe simply fails
// against a member and falls back to the TCP sweep. That is the right
// coverage for this bug: a wiped node is unclustered by definition, so the
// case we need to catch is exactly the case we can see.
// - It assumes node-info's hostUuid and the scanner's advertised uuid= name
// the same identity. They are deliberately kept in agreement — the broker
// passes its resolved node id to node-info, and cluster-manager ADOPTS an
// existing node-id rather than minting a rival one (see its
// TestFirstMintAdoptsExistingNodeUUID). If that invariant is ever broken,
// this probe would read the skew as a wipe and evict a live peer.
func (d *daemon) identityAlive(rec noderec.NodeRecord, ips []string) (alive, conclusive bool) {
if rec.HostUUID == "" || (d.http == nil && d.tlsHTTP == nil) {
return false, false
}
port, ok := rec.Port(noderec.ServiceNodeInfo)
if !ok {
return false, false
}
mismatchedAt, mismatchedUUID := "", ""
for _, ip := range ips {
if ip == "" {
continue
}
info, ok := d.probeIdentity(ip, port)
if !ok || info.HostUUID == "" {
continue
}
if info.HostUUID == rec.HostUUID {
return true, true
}
mismatchedAt, mismatchedUUID = ip, info.HostUUID
}
if mismatchedAt == "" {
return false, false
}
// Every address that answered named someone else, so the record describes a
// node that no longer exists: the machine was reset, or the address was
// reassigned. Warn — this is the signal that a node was wiped, and it is the
// only place the fleet can observe it.
slog.Warn("node record superseded: no address it published still answers for it",
"ip", mismatchedAt, "recordUuid", rec.HostUUID, "actualUuid", mismatchedUUID)
return false, true
}
// probeIdentity reads one address's node-info under the identity probe's own
// budget, which is tighter than enrichment's: this runs inside the browser's scan
// cycle, and each address gets the same allowance so a slow first candidate cannot
// consume the rest of the list's time.
func (d *daemon) probeIdentity(ip string, port int) (NodeInfoResponse, bool) {
ctx, cancel := context.WithTimeout(context.Background(), probeIdentityTimeout)
defer cancel()
return d.fetchNodeInfoWithin(ctx, ip, port)
}
// enrich decorates a node with the data that lives off the mDNS record: its
// node-info inventory (GPU/CPU/memory), fetched over plain HTTP per the transport
// policy, and its engine-manager model list, which is cluster data and so is
// fetched from a peer only over cluster mTLS (see fetchModels). Both are
// independent (a node may advertise one without the other). Peers are dialed at
// each address they published until one answers for the expected identity; self
// is dialed over loopback (see enrichAt).
func (d *daemon) enrich(node *noderec.DirectoryNode) {
hosts := node.CandidateIPs()
d.enrichInfoCandidates(node, hosts)
d.enrichModelsCandidates(node, hosts)
}
// enrichAt is enrich against an explicit dial host. onBrowse enriches a peer at
// its advertised ip=; publishSelf enriches this node over loopbackHost so a
// first-run firewall block on our own LAN address can't blank the local card.
// The enrichment cache stays keyed by hostUuid, so the resilience
// (reuse-last-good on a transient miss) is identical for self and peers.
func (d *daemon) enrichAt(node *noderec.DirectoryNode, host string) {
d.enrichInfoCandidates(node, []string{host})
d.enrichModelsCandidates(node, []string{host})
}
// enrichInfoAt decorates a node that advertises node-info (ni) with its
// GPU/CPU/memory inventory, fetched at the ni port on host. By default this is
// PLAIN HTTP — node-info is plain per the transport policy, and the daemon must
// NEVER infer mTLS from the node-level cluster-uuid= (the subtlest correctness
// requirement in the consolidation). The only way TLS is used is if the operator
// opts in globally via the --client-cert/--client-key/--ca-bundle flags (see
// fetchNodeInfo). A transient fetch failure reuses the node's last successful
// metrics rather than blanking its card; a node never enriched stays un-enriched.
func (d *daemon) enrichInfoAt(node *noderec.DirectoryNode, host string) {
d.enrichInfoCandidates(node, []string{host})
}
// enrichInfoCandidates is enrichInfoAt with address failover. A raw TCP accept
// is not enough here: identically configured direct-connect links can reuse an
// address on different machines, so the response's host UUID is checked before
// the candidate is accepted. Older peers that omit hostUuid remain compatible.
//
// Which address is asked, and in what order, is askRemembered's business: the one
// that answered last sweep by itself, and the rest together only if it stopped.
func (d *daemon) enrichInfoCandidates(node *noderec.DirectoryNode, hosts []string) {
ni, ok := node.Services[noderec.ServiceNodeInfo]
if !ok || len(hosts) == 0 {
return
}
ask := func(host string) (NodeInfoResponse, bool) {
candidate, fetched := d.fetchNodeInfo(host, ni.Port)
if !fetched {
return NodeInfoResponse{}, false
}
if candidate.HostUUID != "" && node.HostUUID != "" && candidate.HostUUID != node.HostUUID {
slog.Debug("node-info candidate answered for a different host; trying next",
"expected_host_uuid", node.HostUUID, "reported_host_uuid", candidate.HostUUID, "ip", host)
return NodeInfoResponse{}, false
}
return candidate, true
}
key := hostKey{hostUUID: node.HostUUID, service: noderec.ServiceNodeInfo}
selectedHost, info, _ := askRemembered(&d.enrichHosts, key, hosts, ask)
endpointHost := selectedHost
if endpointHost == "" {
endpointHost = hosts[0]
}
d.noteNodeInfo(node.HostUUID, net.JoinHostPort(endpointHost, strconv.Itoa(ni.Port)), selectedHost != "")
if selectedHost != "" {
d.infoMu.Lock()
if d.lastInfo == nil {
d.lastInfo = make(map[string]NodeInfoResponse)
}
d.lastInfo[node.HostUUID] = info
if d.lastInfoAt == nil {
d.lastInfoAt = make(map[string]time.Time)
}
d.lastInfoAt[node.HostUUID] = time.Now()
d.infoMu.Unlock()
} else {
d.infoMu.Lock()
cached, had := d.lastInfo[node.HostUUID]
d.infoMu.Unlock()
if !had {
return
}
info = cached
}
node.GPUs = info.GPUs
node.CPU = info.CPU
node.Memory = info.Memory
}
// noteNodeInfo reports a peer's node-info endpoint going quiet, and coming back,
// at a level someone reading the log will see.
//
// Only the transitions are reported. The enrichment sweep repeats every
// peerRefreshInterval for as long as a node is advertised, so a peer whose
// address accepts nothing produces an unbounded run of identical failures; one
// line each time is why this was a debug line, and being a debug line is why an
// address that never once answered looked, in the log, exactly like one that was
// working. A node that is failing is stated once, with where it was asked, and
// its recovery is stated once.
//
// Silence between those two lines does not mean the peer is fine — it means
// nothing has changed and the node's card is still showing the last inventory
// that was actually read.
func (d *daemon) noteNodeInfo(hostUUID, endpoint string, ok bool) {
d.infoMu.Lock()
was := d.nodeInfoDown[hostUUID]
if ok {
delete(d.nodeInfoDown, hostUUID)
} else {
if d.nodeInfoDown == nil {
d.nodeInfoDown = make(map[string]bool)
}
d.nodeInfoDown[hostUUID] = true
}
d.infoMu.Unlock()
switch {
case !ok && !was:
slog.Warn("node-info is not answering; this node's inventory will not update",
"hostUuid", hostUUID, "endpoint", endpoint)
case ok && was:
slog.Info("node-info is answering again", "hostUuid", hostUUID, "endpoint", endpoint)
}
}
// fetchNodeInfo does one GET of a node's /v1/node-info, returning the parsed
// inventory and whether it succeeded. It uses plain HTTP by default; when the
// operator configured a TLS client (--client-cert/--client-key/--ca-bundle) it
// fetches over HTTPS on the same ni port. The choice is purely flag-gated — never
// per-node — so the transport policy stays static.
func (d *daemon) fetchNodeInfo(ip string, port int) (NodeInfoResponse, bool) {
return d.fetchNodeInfoWithin(context.Background(), ip, port)
}
// fetchNodeInfoWithin is fetchNodeInfo bounded by ctx as well as the client's own
// timeout. The liveness probe uses it to hold node-info to a tighter budget than
// enrichment without needing a second HTTP client (which would have to duplicate
// the TLS choice above); every other caller passes a background context and gets
// the client timeout unchanged.
func (d *daemon) fetchNodeInfoWithin(ctx context.Context, ip string, port int) (NodeInfoResponse, bool) {
client, scheme := d.http, "http"
if d.tlsHTTP != nil {
client, scheme = d.tlsHTTP, "https"
}
if client == nil {
return NodeInfoResponse{}, false
}
origin := scheme + "://" + net.JoinHostPort(ip, strconv.Itoa(port))
release, acquired := d.nodeInfoOrigins.acquire(ctx, origin)
if !acquired {
return NodeInfoResponse{}, false
}
defer release()
url := origin + "/v1/node-info"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
slog.Debug("node-info request build failed", "ip", ip, "url", url, "err", err)
return NodeInfoResponse{}, false
}
resp, err := client.Do(req)
if err != nil {
slog.Debug("node-info fetch failed", "ip", ip, "url", url, "err", err)
return NodeInfoResponse{}, false
}
defer func() {
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, nodeInfoResponseDrainLimit))
_ = resp.Body.Close()
}()
if resp.StatusCode != http.StatusOK {
return NodeInfoResponse{}, false
}
var info NodeInfoResponse
if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
return NodeInfoResponse{}, false
}
return info, true
}
// enrichModelsAt decorates a node that advertises engine-manager (em) with its
// model list, fetched at the em port (/v1/models) on host. The list moved off the
// size-limited mDNS TXT onto HTTP so it can grow unbounded and be owned by
// engine-manager (which already owns engine state). A peer is fetched over cluster
// mTLS pinned to the principal it advertises, and skipped when this node holds no
// pin for it — which models a machine holds is cluster data. A transient failure
// reuses the last successful list rather than blanking it; a node advertising no em
// has no model list (nothing is carried in the record). Self is dialed over
// loopback (see enrichAt), which stays plaintext, peers at their advertised address.
func (d *daemon) enrichModelsAt(node *noderec.DirectoryNode, host string) {
d.enrichModelsCandidates(node, []string{host})
}
// enrichModelsCandidates is enrichModelsAt with address failover. For a peer,
// fetchModels performs the pinned mTLS handshake, so a shared address belonging
// to another machine is rejected and the next published address is tried.
func (d *daemon) enrichModelsCandidates(node *noderec.DirectoryNode, hosts []string) {
em, ok := node.Services[noderec.ServiceEngineManager]
if !ok || len(hosts) == 0 {
return
}
ask := func(host string) (modelInventory, bool) {
models, byEngine, loadedByEngine, fetched := d.fetchModels(host, em.Port, node.ClusterUUID)
if !fetched {
return modelInventory{}, false
}
return modelInventory{models: models, byEngine: byEngine, loadedByEngine: loadedByEngine}, true
}
key := hostKey{hostUUID: node.HostUUID, service: noderec.ServiceEngineManager}
if _, inventory, ok := askRemembered(&d.enrichHosts, key, hosts, ask); ok {
d.infoMu.Lock()
if d.lastModels == nil {
d.lastModels = make(map[string][]string)
}
if d.lastModelsByEngine == nil {
d.lastModelsByEngine = make(map[string]map[string][]string)
}
if d.lastLoadedByEngine == nil {
d.lastLoadedByEngine = make(map[string]map[string][]string)
}
d.lastModels[node.HostUUID] = inventory.models
d.lastModelsByEngine[node.HostUUID] = inventory.byEngine
d.lastLoadedByEngine[node.HostUUID] = inventory.loadedByEngine
d.infoMu.Unlock()
node.Models = inventory.models
node.ModelsByEngine = inventory.byEngine
node.LoadedByEngine = inventory.loadedByEngine
return
}
d.infoMu.Lock()
cached, had := d.lastModels[node.HostUUID]
cachedByEngine := d.lastModelsByEngine[node.HostUUID]
cachedLoaded := d.lastLoadedByEngine[node.HostUUID]
d.infoMu.Unlock()
if had {
node.Models = cached
node.ModelsByEngine = cachedByEngine
node.LoadedByEngine = cachedLoaded
}
}
// fetchModels does one GET of a node's /v1/models, returning the
// flat model-name union, the per-engine attribution (engine name -> models),
// the per-engine loaded (in-memory) set, and whether it succeeded. Shape:
// {"models":["name",...],"modelsByEngine":{"ollama":[...]},"loadedByEngine":{"ollama":[...]}}.
// modelsByEngine and loadedByEngine are additive (omitempty on the wire); a peer
// that doesn't send them yields nil maps, and the flat union still enriches the
// node. A present engine key with [] is a successful empty inventory.
// A peer's model inventory is cluster data, so a peer is fetched over cluster
// mTLS keyed on the cluster principal it advertises (clusterUUID, from its
// cluster-uuid= TXT): this node must hold a pin for that principal, or there is
// nothing to fetch and nothing is attempted. THIS node's own engine-manager is
// dialed over loopback (both callers force that for self), which stays plain and
// therefore keeps working when this node belongs to no cluster — a standalone
// machine still has to show its own models.
func (d *daemon) fetchModels(ip string, port int, clusterUUID string) ([]string, map[string][]string, map[string][]string, bool) {
client, scheme, ok := d.modelsClient(ip, clusterUUID)
if !ok {
slog.Debug("model enrichment skipped: peer is not a pinned cluster member",
"ip", ip, "clusterUuid", clusterUUID)
return nil, nil, nil, false
}
url := scheme + "://" + net.JoinHostPort(ip, strconv.Itoa(port)) + "/v1/models"
resp, err := client.Get(url)
if err != nil {
slog.Debug("model enrichment failed", "ip", ip, "url", url, "err", err)
return nil, nil, nil, false
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, nil, nil, false
}
var body struct {
Models []string `json:"models"`
ModelsByEngine map[string][]string `json:"modelsByEngine"`
LoadedByEngine map[string][]string `json:"loadedByEngine"`
}
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
return nil, nil, nil, false
}
return body.Models, body.ModelsByEngine, body.LoadedByEngine, true
}
// modelsClient picks the transport for a model fetch: the plain client over
// loopback (this node's own engine-manager, which serves loopback in plaintext in
// every membership state), or a per-peer pinned mTLS client for a LAN peer. ok is
// false when a peer holds no pin here, which is the client-side cluster gate —
// there is deliberately no plaintext fallback for a peer.
func (d *daemon) modelsClient(ip, clusterUUID string) (*http.Client, string, bool) {
if isLoopbackHost(ip) {
return d.modelsHTTP, "http", true
}
d.mesh.Refresh()
if d.modelsClients == nil {
return nil, "", false
}
client, ok := d.modelsClients.Client(clusterUUID)
if !ok {
return nil, "", false
}
return client, "https", true
}
// isLoopbackHost reports whether a dial host addresses this machine.
func isLoopbackHost(host string) bool {
ip := net.ParseIP(host)
return ip != nil && ip.IsLoopback()
}
// refreshPeersLoop periodically re-reads what each peer reports about itself over
// HTTP and folds any change back into the directory, independent of mDNS browse
// events. It is the convergence mechanism for facts that change without moving
// the raw mDNS record, or whose announcement was simply not received — neither of
// which produces a browse event, so neither reaches the enrichment in onBrowse.
// Stops when ctx is cancelled (the daemon's existing shutdown path).
//
// Cluster membership is refreshed before models on the same tick, because the
// model fetch is pinned mTLS keyed on the peer's principal: reading membership
// first means a peer that just joined or left is dialed under its current
// identity rather than the previous one.
func (d *daemon) refreshPeersLoop(ctx context.Context) {
ticker := time.NewTicker(peerRefreshInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
d.refreshAdvertisedAddressesOnce()
d.mesh.Refresh()
if d.modelsClients != nil {
d.modelsClients.DropUnpinned()
}
d.refreshClusterIdentityOnce()
d.refreshModelsOnce()
}
}
}
func (d *daemon) refreshTelemetryLoop(ctx context.Context) {
hostUUID := ""
if d.reg != nil {
hostUUID = d.reg.record().HostUUID
}
d.runTelemetryLoop(ctx, telemetryIntervalForNode(hostUUID))
}
func telemetryIntervalForNode(hostUUID string) time.Duration {
if hostUUID == "" {
return telemetryRefreshInterval
}
hash := uint32(2166136261)
for i := 0; i < len(hostUUID); i++ {
hash ^= uint32(hostUUID[i])
hash *= 16777619
}
windowMillis := int64(2*telemetryRefreshJitter/time.Millisecond) + 1
offsetMillis := int64(hash%uint32(windowMillis)) - windowMillis/2
return telemetryRefreshInterval + time.Duration(offsetMillis)*time.Millisecond
}
// runTelemetryLoop dispatches compact scheduler telemetry on a fixed cadence.
// Requests may span ticks, but telemetryRetries excludes another request for
// the same node and the loop-owned semaphore bounds total active work. Shutdown
// cancels and drains every worker before returning.
func (d *daemon) runTelemetryLoop(ctx context.Context, interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
sem := make(chan struct{}, telemetryRefreshConcurrency)
var workers sync.WaitGroup
defer workers.Wait()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
d.dispatchTelemetry(ctx, time.Now(), sem, &workers)
}
}
}
// refreshTelemetryOnce synchronously dispatches one telemetry pass. Production
// uses runTelemetryLoop's persistent semaphore and workers; this wrapper keeps
// focused callers and tests able to wait for the pass they requested.
func (d *daemon) refreshTelemetryOnce(ctx context.Context) {
sem := make(chan struct{}, telemetryRefreshConcurrency)
var workers sync.WaitGroup
d.dispatchTelemetry(ctx, time.Now(), sem, &workers)
workers.Wait()
}
// dispatchTelemetry starts every due node without waiting for requests already
// in flight. sem and workers belong to the enclosing loop so work remains
// bounded and drainable across ticks.
func (d *daemon) dispatchTelemetry(
ctx context.Context,
now time.Time,
sem chan struct{},
workers *sync.WaitGroup,
) {
self := ""
if d.reg != nil {
self = d.reg.record().HostUUID
}
for _, n := range d.dir.snapshot(noderec.ServiceNodeInfo) {
ni, ok := n.Services[noderec.ServiceNodeInfo]
if !ok {
continue
}
hosts := append([]string(nil), n.CandidateIPs()...)
if n.HostUUID == self {
hosts = []string{loopbackHost}
}
if len(hosts) == 0 {
continue
}
targetKey := telemetryTargetKey(hosts, ni.Port)
retryToken, due := d.telemetryRetries.claim(n.HostUUID, targetKey, now)
if !due {
continue
}
workers.Add(1)
go func(hostUUID string, hosts []string, port int, token uint64, backoff bool) {
defer workers.Done()
select {
case sem <- struct{}{}:
defer func() { <-sem }()
case <-ctx.Done():
d.telemetryRetries.finish(hostUUID, token, true, time.Now())
return
}
ok := d.refreshNodeTelemetryCandidates(ctx, hostUUID, hosts, port)
if !backoff || ctx.Err() != nil {
ok = true
}
d.telemetryRetries.finish(hostUUID, token, ok, time.Now())
}(n.HostUUID, hosts, ni.Port, retryToken, n.HostUUID != self)
}
}
const maxTelemetryAge = int64(1<<63 - 1)
// refreshNodeTelemetry emits one node's current GPU utilization. Failed fetches
// stay silent so downstream freshness expiry handles temporary unreachability.
// A response naming another host is never attributed to the directory entry
// whose address was queried.
func (d *daemon) refreshNodeTelemetry(ctx context.Context, hostUUID, ip string, niPort int) bool {
return d.refreshNodeTelemetryCandidates(ctx, hostUUID, []string{ip}, niPort)
}
// telemetrySample is one node-info answer plus when the request that produced it
// began. The reported age has to be advanced by that request's own latency, and a
// failover attempt asks several addresses at once, so the time belongs to the
// answer rather than to the sweep.
type telemetrySample struct {
info NodeInfoResponse
startedAt time.Time
}
// refreshNodeTelemetryCandidates is refreshNodeTelemetry with address failover.
//
// Healthy nodes are sampled every two seconds. Failed remote nodes back off, and
// each due attempt goes through the same node-info memory the enrichment sweeps
// use: the address that answered is asked by itself, and the rest of the list
// only once it stops answering. Without that, a peer whose canonical address is
// a link this host cannot reach reported no telemetry at all, and the scheduler
// then assigned it neutral pressure and scheduled it blind.
func (d *daemon) refreshNodeTelemetryCandidates(ctx context.Context, hostUUID string, hosts []string, niPort int) bool {
ask := func(host string) (telemetrySample, bool) {
startedAt := time.Now()
candidate, ok := d.fetchNodeInfoWithin(ctx, host, niPort)
if !ok {
return telemetrySample{}, false
}
if candidate.HostUUID != hostUUID {
slog.Debug("skipping telemetry reported by a different host at this address; trying next",
"expected_host_uuid", hostUUID, "reported_host_uuid", candidate.HostUUID, "ip", host)
return telemetrySample{}, false
}
return telemetrySample{info: candidate, startedAt: startedAt}, true
}
key := hostKey{hostUUID: hostUUID, service: noderec.ServiceNodeInfo}
_, sample, ok := askRemembered(&d.enrichHosts, key, hosts, ask)
if !ok {
return false
}
info, startedAt := sample.info, sample.startedAt
age := int64(0)
if info.TelemetryValid {
age = info.MSSince
if age < 0 {
age = 0
}
elapsed := time.Since(startedAt).Milliseconds()
if elapsed > maxTelemetryAge-age {
age = maxTelemetryAge
} else {
age += elapsed
}
}
var utilization uint32
for i := range info.GPUs {
if info.GPUs[i].UtilizationPercent > utilization {
utilization = info.GPUs[i].UtilizationPercent
}
}
d.emitTelemetry(noderec.NodeTelemetry{
HostUUID: hostUUID,
GPUUtilizationPct: utilization,
TelemetryValid: info.TelemetryValid,
MSSince: age,
})
return true
}
// refreshClusterIdentityOnce re-reads every peer's cluster membership from its
// node-info endpoint and folds any change into the directory.
//
// Membership otherwise reaches this daemon only as the cluster-uuid= key on a
// peer's mDNS record, which is read exactly once per browse *change* event. A
// peer that leaves its cluster and re-advertises produces such an event — but if
// it is not received, nothing asks again: the browse diff has no notion of a fact
// it never saw, and the liveness probe keeps the entry alive precisely because the
// peer is still reachable. The stale principal then outlives the cluster, and a
// client suppresses the invite that would bring that peer back.
//
// node-info is the right place to ask because it is the one inter-node surface
// deliberately kept plain (see the broker's spawnNodeInfo), so it answers whether
// or not this node and the peer share a cluster — and it is reached over TCP at
// the address the peer advertised, sharing nothing with multicast reception.
//
// Self is excluded: this node's own entry is registry-driven (publishSelf owns its
// principal), not something to learn from a loopback HTTP read.
func (d *daemon) refreshClusterIdentityOnce() {
self := ""
if d.reg != nil {
self = d.reg.record().HostUUID
}
// One refresh for the whole sweep: every peer is then annotated against the
// same pin set, and 16 concurrent re-reads of the trust store are avoided.
d.mesh.Refresh()
sem := make(chan struct{}, modelsRefreshConcurrency)
var wg sync.WaitGroup
for _, n := range d.dir.snapshot(noderec.ServiceNodeInfo) {
ni, ok := n.Services[noderec.ServiceNodeInfo]
if !ok || n.IP == "" || n.HostUUID == self {
continue
}
wg.Add(1)
sem <- struct{}{}
hosts := append([]string(nil), n.CandidateIPs()...)
go func(hostUUID string, hosts []string, port int) {
defer wg.Done()
defer func() { <-sem }()
d.refreshNodeClusterIdentityCandidates(hostUUID, hosts, port)
}(n.HostUUID, hosts, ni.Port)
}
wg.Wait()
}
// refreshNodeClusterIdentity reads one peer's reported membership and applies it,
// emitting one discovery:node-updated when it moved. Returns whether it emitted
// (used by tests).
//
// Two answers mean "leave the annotation alone": a failed fetch, and a peer that
// reports no clusterUuid field at all because it predates it. Only the peer's own
// explicit value is acted on — inferring "unclustered" from silence would mark a
// clustered peer invitable, which is the failure this annotation exists to
// prevent.
//
// A third: the machine answering may not be the one this entry describes. An
// address outlives its occupant — a wiped and reinstalled node, or a reused DHCP
// lease — and node-info names who it really is, so a mismatched hostUuid means
// writing a stranger's membership onto someone else's entry. Skipped rather than
// corrected here: the entry ages out through the same identity check in the
// liveness probe (see identityAlive), which owns that decision. A peer reporting
// no hostUuid predates the field and cannot be checked either way, so it is taken
// at its word — the same fallback that probe applies to the same answer.
func (d *daemon) refreshNodeClusterIdentity(hostUUID, ip string, niPort int) bool {
return d.refreshNodeClusterIdentityCandidates(hostUUID, []string{ip}, niPort)
}
// refreshNodeClusterIdentityCandidates is refreshNodeClusterIdentity with
// address failover. A response from a different host is not authoritative for
// this entry, but it is also not the end of the candidate list: shared
// direct-connect addresses can legitimately resolve to a different machine.
func (d *daemon) refreshNodeClusterIdentityCandidates(hostUUID string, hosts []string, niPort int) bool {
ask := func(host string) (NodeInfoResponse, bool) {
candidate, ok := d.fetchNodeInfo(host, niPort)
if !ok {
return NodeInfoResponse{}, false
}
if candidate.HostUUID != "" && candidate.HostUUID != hostUUID {
slog.Debug("skipping cluster membership reported by a different host at this address; trying next",
"expected_host_uuid", hostUUID, "reported_host_uuid", candidate.HostUUID, "ip", host)
return NodeInfoResponse{}, false
}
return candidate, true
}
// Shares the node-info memory with the enrichment sweep, so the two node-info
// callers converge on one address per peer rather than each finding it again.
key := hostKey{hostUUID: hostUUID, service: noderec.ServiceNodeInfo}
_, info, _ := askRemembered(&d.enrichHosts, key, hosts, ask)
if info.ClusterUUID == nil {
return false
}
node, changed := d.dir.applyClusterIdentity(hostUUID, info.ClusterUUID, d.mesh.HasPin)
if !changed {
return false
}
slog.Info("peer cluster membership changed", "host_uuid", hostUUID,
"clustered", node.Clustered(), "trusted", node.Trusted)
d.emit(noderec.NotifyNodeUpdated, node)
return true
}
// refreshModelsOnce snapshots every directory node advertising engine-manager
// (em) with a usable IP and refreshes each concurrently, bounded by
// modelsRefreshConcurrency so an unexpectedly large network can't spawn an
// unbounded burst of fetches. Running the sweep to completion (wg.Wait) before
// the loop reads the next tick makes overlapping sweeps impossible, and each
// per-node goroutine uses modelsHTTP's own timeout, so one slow peer never
// consumes another's budget.
func (d *daemon) refreshModelsOnce() {
sem := make(chan struct{}, modelsRefreshConcurrency)
var wg sync.WaitGroup
for _, n := range d.dir.snapshot(noderec.ServiceEngineManager) {
em, ok := n.Services[noderec.ServiceEngineManager]
if !ok || n.IP == "" {
continue
}
wg.Add(1)
sem <- struct{}{}
hosts := append([]string(nil), n.CandidateIPs()...)
go func(hostUUID, guardIP string, hosts []string, port int, clusterUUID string) {
defer wg.Done()
defer func() { <-sem }()
d.refreshNodeModelsCandidates(hostUUID, guardIP, hosts, port, clusterUUID)
}(n.HostUUID, n.IP, hosts, em.Port, n.ClusterUUID)
}
wg.Wait()
}
// refreshNodeModels fetches one peer's /v1/models and, on success, applies it to
// the directory under the removed/re-addressed guard (directory.applyModels).
// A failed fetch retains the last successful inventory: no cache write, no
// directory change, no event. On a change it refreshes the enrichment caches
// (so a later browse-driven fallback can't revert the directory to a stale
// value) and emits exactly one discovery:node-updated carrying the full node.
// Returns whether it emitted an update (used by tests).
//
// The fetch dials our OWN engine-manager over loopback, never the advertised
// LAN ip=: this loop — not the registry-driven publishSelf — is what converges
// model-only changes (a pull/delete, a late engine start) that don't move the
// registry, so on the Windows self-LAN firewall block this daemon guards
// against, a LAN dial would make self's model list freeze at its
// startup value. The stored ip is still passed to applyModels as the
// stale-address guard, so an in-flight result can't overwrite a re-addressed
// node. Peers are still dialed at their advertised address.
func (d *daemon) refreshNodeModels(hostUUID, ip string, emPort int, clusterUUID string) bool {
return d.refreshNodeModelsCandidates(hostUUID, ip, []string{ip}, emPort, clusterUUID)
}
// refreshNodeModelsCandidates is refreshNodeModels with address failover. The
// canonical IP remains the stale-result guard even when a later address is the
// one that answered.
//
// Which address is asked is askRemembered's business — the one that answered last
// sweep, alone — and it shares that memory with the enrichment path, so both
// converge on one address per peer instead of each rediscovering it.
func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []string, emPort int, clusterUUID string) bool {
if d.reg != nil && hostUUID == d.reg.record().HostUUID {
hosts = []string{loopbackHost}
}
ask := func(host string) (modelInventory, bool) {
models, byEngine, loadedByEngine, fetched := d.fetchModels(host, emPort, clusterUUID)
if !fetched {
return modelInventory{}, false
}
return modelInventory{models: models, byEngine: byEngine, loadedByEngine: loadedByEngine}, true
}
key := hostKey{hostUUID: hostUUID, service: noderec.ServiceEngineManager}
_, inventory, fetched := askRemembered(&d.enrichHosts, key, hosts, ask)
if !fetched {
return false
}
models, byEngine, loadedByEngine := inventory.models, inventory.byEngine, inventory.loadedByEngine
// Refresh the enrichment cache BEFORE touching the directory: the caches
// (infoMu) and the directory (dir.mu) are guarded by different locks, so if
// the directory were updated first, a concurrent onBrowse->enrichModelsAt whose
// own fetch fails would fall back to a still-stale lastModels and upsert the
// old inventory, reverting this update. Writing the cache first means that
// fallback reads the fresh value. The only side effect — caching a value for
// a node applyModels then finds removed — is benign (it is the last-known-good
// value the cache exists to preserve, and forget() clears it on eviction).
d.infoMu.Lock()
if d.lastModels == nil {
d.lastModels = make(map[string][]string)
}
if d.lastModelsByEngine == nil {
d.lastModelsByEngine = make(map[string]map[string][]string)
}
if d.lastLoadedByEngine == nil {
d.lastLoadedByEngine = make(map[string]map[string][]string)
}
d.lastModels[hostUUID] = models
d.lastModelsByEngine[hostUUID] = byEngine
d.lastLoadedByEngine[hostUUID] = loadedByEngine
d.infoMu.Unlock()
node, changed, valid := d.dir.applyModels(hostUUID, guardIP, emPort, models, byEngine, loadedByEngine)
if !valid {
return false
}
if changed {
d.emit(noderec.NotifyNodeUpdated, node)
}
return changed
}
// forget drops a node's cached enrichment when it's evicted, so a later
// re-discovery re-probes from scratch rather than resurrecting stale metrics.
func (d *daemon) forget(hostUUID string) {
d.telemetryRetries.forget(hostUUID)
d.infoMu.Lock()
delete(d.lastInfo, hostUUID)
delete(d.lastInfoAt, hostUUID)
delete(d.lastModels, hostUUID)
delete(d.lastModelsByEngine, hostUUID)
delete(d.lastLoadedByEngine, hostUUID)
delete(d.nodeInfoDown, hostUUID)
d.infoMu.Unlock()
d.activityMu.Lock()
delete(d.lastActivityAt, hostUUID)
d.activityMu.Unlock()
}
// noteActivity records that a peer returned inference response bytes msSince
// milliseconds ago. The broker relays this from whichever proxy saw the bytes;
// an age rather than a timestamp keeps the two processes off each other's clock.
//
// Our own uuid is dropped. A proxy identifies targets by URL and port and cannot
// tell which uuid is this machine's, so it reports every node it streamed from
// and the filtering happens here, where identity actually lives. Self is never
// evicted by a browse miss anyway (onBrowse ignores our own uuid), so recording
// it would only grow the map.
func (d *daemon) noteActivity(hostUUID string, msSince int64) {
if hostUUID == "" {
return
}
if d.reg != nil && hostUUID == d.reg.record().HostUUID {
return
}
// Bound the age at both ends before it becomes a duration. A negative value,
// or one large enough to overflow the multiplication, would place the
// observation in the future — and a future observation never expires, so the
// node could never be probed again. This is the consumer, so it validates the
// number rather than assuming the relay already did.
if msSince < 0 {
msSince = 0
}
if maxMS := int64(activityFreshness / time.Millisecond); msSince > maxMS {
msSince = maxMS
}
at := time.Now().Add(-time.Duration(msSince) * time.Millisecond)
d.activityMu.Lock()
defer d.activityMu.Unlock()
// Reports from two proxies can arrive out of order; keep the newest.
if prev, ok := d.lastActivityAt[hostUUID]; ok && prev.After(at) {
return
}
d.lastActivityAt[hostUUID] = at
}
// activitySince reports how long ago a peer last returned inference bytes, and
// whether anything has ever been reported for it.
func (d *daemon) activitySince(hostUUID string) (time.Duration, bool) {
d.activityMu.Lock()
defer d.activityMu.Unlock()
at, ok := d.lastActivityAt[hostUUID]
if !ok {
return 0, false
}
return time.Since(at), true
}
// handle dispatches the discovery:* methods, returning true if msg was one of
// them. register/unregister/update-txt update the registry and re-advertise on a
// real change; register is acked when sent as a request. get-nodes returns the
// filtered directory snapshot.
func (d *daemon) handle(msg *Message) bool {
switch msg.Method {
case noderec.MethodRegister, noderec.MethodUpdateTXT:
var p noderec.RegisterParams
if err := json.Unmarshal(msg.Params, &p); err != nil {
if msg.IsRequest() {
_ = d.codec.RespondError(msg.ID, -32602, err.Error())
}
return true
}
if d.reg.register(p) {
if d.responder != nil {
d.responder.UpdateTXT(d.reg.txt())
}
// Reflect the new service set into our own directory entry. Async so a
// registration burst at startup can't serialize the read loop behind
// the loopback enrichment fetch.
go d.publishSelf()
}
if msg.IsRequest() {
_ = d.codec.Respond(msg.ID, map[string]bool{"ok": true})
}
return true
case noderec.MethodUnregister:
var p noderec.UnregisterParams
if err := json.Unmarshal(msg.Params, &p); err != nil {
if msg.IsRequest() {
_ = d.codec.RespondError(msg.ID, -32602, err.Error())
}
return true
}
if d.reg.unregister(p.Service) {
if d.responder != nil {
d.responder.UpdateTXT(d.reg.txt())
}
// Drop the service from our own directory entry (e.g. an engine going
// away). Self stays present; only its service set shrinks.
go d.publishSelf()
}
if msg.IsRequest() {
_ = d.codec.Respond(msg.ID, map[string]bool{"ok": true})
}
return true
case noderec.MethodGetNodes:
var p noderec.GetNodesParams
_ = json.Unmarshal(msg.Params, &p) // an unparseable/empty filter returns all
if msg.IsRequest() {
_ = d.codec.Respond(msg.ID, noderec.GetNodesResult{Nodes: d.dir.snapshot(p.Service)})
}
return true
case noderec.MethodReloadIdentity:
d.reloadIdentity()
if msg.IsRequest() {
_ = d.codec.Respond(msg.ID, map[string]bool{"ok": true})
}
return true
case noderec.MethodReloadTrust:
d.reloadTrust()
if msg.IsRequest() {
_ = d.codec.Respond(msg.ID, map[string]bool{"ok": true})
}
return true
case noderec.MethodSetObservedAddresses:
var p noderec.ObservedAddressesParams
if err := json.Unmarshal(msg.Params, &p); err != nil {
if msg.IsRequest() {
_ = d.codec.RespondError(msg.ID, -32602, err.Error())
}
return true
}
d.setObservedAddresses(p.Addresses)
if msg.IsRequest() {
_ = d.codec.Respond(msg.ID, map[string]bool{"ok": true})
}
return true
case noderec.MethodNodeActivity:
var p noderec.NodeActivityParams
if err := json.Unmarshal(msg.Params, &p); err != nil {
if msg.IsRequest() {
_ = d.codec.RespondError(msg.ID, -32602, err.Error())
}
return true
}
d.noteActivity(p.HostUUID, p.MSSince)
if msg.IsRequest() {
_ = d.codec.Respond(msg.ID, map[string]bool{"ok": true})
}
return true
}
return false
}