Router that virtually distributes inference across connected devices in the home.
Go
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package main
import (
"context"
"encoding/json"
"net"
"sync"
"testing"
"time"
"nvpair-shared/noderec"
"nvpair-shared/schedulerwire"
"nvpair-ui-broker/workloadstore"
)
// TestSchedulerFeedBaselinePrecedesConcurrentLiveWorkload exercises the
// scheduler-spawn ordering: active workload upserts are queued first, followed
// by telemetry and discovery, and a concurrent live transition can only follow
// all three. Terminal history must never be replayed.
func TestSchedulerFeedBaselinePrecedesConcurrentLiveWorkload(t *testing.T) {
brokerSide, schedulerSide := net.Pipe()
t.Cleanup(func() {
_ = brokerSide.Close()
_ = schedulerSide.Close()
})
worker := &rpcWorker{peer: NewPeer(NewCodec(brokerSide))}
b := &Broker{workloads: workloadstore.New(), telemetry: newTelemetryCache()}
b.workloads.Apply(storeIncoming("active", "host", "ollama", "run", "running", "a"))
b.workloads.Apply(storeIncoming("historic", "host", "ollama", "run", "completed", "a"))
b.telemetry.Upsert(sourceScanner, noderec.NodeTelemetry{
HostUUID: "a",
GPUUtilizationPct: 50,
TelemetryValid: true,
}, time.Now())
feedLocked := make(chan struct{})
initDone := make(chan int, 1)
go func() {
// This is the same lock order and baseline sequence used by
// spawnJobScheduler.
b.workloadEmitMu.Lock()
b.schedulerFeedMu.Lock()
b.setScheduler(worker)
close(feedLocked)
replayed := b.replayActiveWorkloadsToScheduler(worker)
b.replayTelemetryToScheduler(worker)
_ = worker.Notify("discovery:nodes-changed", []AvailableNode{{HostUUID: "a"}})
b.schedulerFeedMu.Unlock()
b.workloadEmitMu.Unlock()
initDone <- replayed
}()
<-feedLocked
liveInfo := storeIncoming("live", "host", "lmstudio", "run", "queued", "a").Info
liveParams, err := json.Marshal(map[string]json.RawMessage{"workloadInfo": liveInfo})
if err != nil {
t.Fatalf("marshal live workload: %v", err)
}
liveDone := make(chan struct{})
go func() {
b.emitWorkloadEvent("workloads:upsert", liveParams)
close(liveDone)
}()
if err := schedulerSide.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
t.Fatalf("set read deadline: %v", err)
}
codec := NewCodec(schedulerSide)
first := readSchedulerTestMessage(t, codec)
second := readSchedulerTestMessage(t, codec)
third := readSchedulerTestMessage(t, codec)
fourth := readSchedulerTestMessage(t, codec)
if first.Method != "workloads:upsert" || workloadIDFromParams(t, first.Params) != "active" {
t.Fatalf("first scheduler frame = %s %s, want active baseline upsert", first.Method, first.Params)
}
if second.Method != schedulerwire.MethodTelemetry {
t.Fatalf("second scheduler frame = %s, want telemetry baseline", second.Method)
}
if third.Method != "discovery:nodes-changed" {
t.Fatalf("third scheduler frame = %s, want discovery baseline", third.Method)
}
if fourth.Method != "workloads:upsert" || workloadIDFromParams(t, fourth.Params) != "live" {
t.Fatalf("fourth scheduler frame = %s %s, want concurrent live upsert", fourth.Method, fourth.Params)
}
select {
case replayed := <-initDone:
if replayed != 1 {
t.Fatalf("replayed = %d, want 1 active record", replayed)
}
case <-time.After(2 * time.Second):
t.Fatal("scheduler initialization did not finish")
}
select {
case <-liveDone:
case <-time.After(2 * time.Second):
t.Fatal("live workload fanout did not finish")
}
}
func readSchedulerTestMessage(t *testing.T, codec *Codec) *Message {
t.Helper()
msg, err := codec.Read()
if err != nil {
t.Fatalf("read scheduler frame: %v", err)
}
return msg
}
func workloadIDFromParams(t *testing.T, params json.RawMessage) string {
t.Helper()
var env struct {
WorkloadInfo struct {
ID string `json:"id"`
} `json:"workloadInfo"`
}
if err := json.Unmarshal(params, &env); err != nil {
t.Fatalf("decode workload params: %v", err)
}
return env.WorkloadInfo.ID
}
func TestDeliverPrioritySkipsStaleGenerationsAndPreservesNewest(t *testing.T) {
b := &Broker{}
oldInput := schedulerwire.Priority{
Nodes: []string{"old"},
Ranks: []schedulerwire.NodeRank{{ID: "old", Pending: 1}},
}
oldGeneration, _ := b.cachePrioritySnapshot(oldInput)
oldInput.Nodes[0] = "mutated-after-cache"
oldInput.Ranks[0].Pending = 99
var appliedMu sync.Mutex
var applied []schedulerwire.Priority
record := func(priority schedulerwire.Priority) {
appliedMu.Lock()
applied = append(applied, priority.Clone())
appliedMu.Unlock()
}
oldEntered := make(chan struct{})
releaseOld := make(chan struct{})
oldDone := make(chan struct{})
go func() {
b.deliverPrioritySnapshot(oldGeneration, func(priority schedulerwire.Priority) {
record(priority)
close(oldEntered)
<-releaseOld
})
close(oldDone)
}()
waitSchedulerTestChannel(t, oldEntered, "old delivery did not start")
middleGeneration, _ := b.cachePrioritySnapshot(schedulerwire.Priority{
Nodes: []string{"middle"},
Ranks: []schedulerwire.NodeRank{{ID: "middle", Pending: 2}},
})
middleDone := make(chan struct{})
go func() {
b.deliverPrioritySnapshot(middleGeneration, record)
close(middleDone)
}()
newPriority := schedulerwire.Priority{
Nodes: []string{"new"},
Ranks: []schedulerwire.NodeRank{{ID: "new", Pending: 3}},
}
newGeneration, _ := b.cachePrioritySnapshot(newPriority)
newDone := make(chan struct{})
go func() {
b.deliverPrioritySnapshot(newGeneration, record)
close(newDone)
}()
close(releaseOld)
waitSchedulerTestChannel(t, oldDone, "old delivery did not finish")
waitSchedulerTestChannel(t, middleDone, "stale middle delivery did not finish")
waitSchedulerTestChannel(t, newDone, "new delivery did not finish")
appliedMu.Lock()
defer appliedMu.Unlock()
if len(applied) != 2 {
t.Fatalf("applied snapshots = %v, want old then new (middle skipped)", applied)
}
wantOld := schedulerwire.Priority{
Nodes: []string{"old"},
Ranks: []schedulerwire.NodeRank{{ID: "old", Pending: 1}},
}
// Ranking rather than DeepEqual: a cached snapshot also carries the
// generation the broker stamped on it, which is not part of what the
// scheduler produced.
if !applied[0].SameRanking(wantOld) {
t.Fatalf("first applied snapshot = %#v, want cached copy %#v", applied[0], wantOld)
}
if applied[0].Generation != oldGeneration {
t.Fatalf("first applied generation = %d, want %d", applied[0].Generation, oldGeneration)
}
if !applied[1].SameRanking(newPriority) {
t.Fatalf("last applied snapshot = %#v, want newest %#v", applied[1], newPriority)
}
if applied[1].Generation != newGeneration {
t.Fatalf("last applied generation = %d, want %d", applied[1].Generation, newGeneration)
}
}
// The scheduler computes one node-wide ranking and emits it once per engine, so
// the broker sees the same content twice per recompute. The duplicate has to be
// dropped before it mints a generation: a second delivery of one recompute
// clears the proxy's optimistic reservations a second time, discarding the
// dispatches the first delivery's pending counts do not yet include.
func TestDuplicatePerEngineEmissionDoesNotMintAGeneration(t *testing.T) {
b := &Broker{}
ranking := schedulerwire.Priority{
Nodes: []string{"b", "a"},
Ranks: []schedulerwire.NodeRank{{ID: "b", Pending: 1}, {ID: "a", Pending: 4}},
}
first, fresh := b.cachePrioritySnapshot(ranking)
if !fresh {
t.Fatal("first emission of a ranking was treated as a duplicate")
}
second, fresh := b.cachePrioritySnapshot(ranking)
if fresh {
t.Fatal("the sibling engine's identical emission minted a second generation")
}
if second != first {
t.Fatalf("duplicate emission moved the generation from %d to %d", first, second)
}
// A genuinely changed ranking still advances, so the dedupe is not just
// swallowing everything after the first.
changed, fresh := b.cachePrioritySnapshot(schedulerwire.Priority{
Nodes: []string{"a", "b"},
Ranks: []schedulerwire.NodeRank{{ID: "a", Pending: 0}, {ID: "b", Pending: 5}},
})
if !fresh || changed <= first {
t.Fatalf("changed ranking = generation %d fresh %v, want a newer generation", changed, fresh)
}
}
// An empty ranking is a legitimate state — it means the scheduler has no
// influence — and must be distinguishable from having no ranking at all, or a
// repush would either send nothing or send an uncached zero value.
func TestEmptyRankingIsCachedNotTreatedAsAbsent(t *testing.T) {
b := &Broker{}
if _, fresh := b.cachePrioritySnapshot(schedulerwire.Priority{}); !fresh {
t.Fatal("first empty ranking was treated as a duplicate")
}
b.schedMu.Lock()
have := b.havePriority
b.schedMu.Unlock()
if !have {
t.Fatal("an empty ranking did not mark a ranking as cached, so repush would skip it")
}
}
func TestRepushPriorityDeliversCompleteSnapshotToReplacementProxy(t *testing.T) {
brokerSide, proxySide := net.Pipe()
t.Cleanup(func() {
_ = brokerSide.Close()
_ = proxySide.Close()
})
proxy := &proxyProcess{peer: NewPeer(NewCodec(brokerSide))}
go proxy.peer.Serve(nil, nil)
b := &Broker{}
want := schedulerwire.Priority{
Nodes: []string{"b", "a"},
Ranks: []schedulerwire.NodeRank{
{ID: "b", Pending: 1, Rank: 0},
{ID: "a", Pending: 4, Rank: 1},
},
}
b.cachePrioritySnapshot(want)
b.setProxy(proxy) // replacement proxy starts with no scheduler state
replayed := make(chan struct{})
go func() {
b.repushPriority(context.Background())
close(replayed)
}()
if err := proxySide.SetDeadline(time.Now().Add(2 * time.Second)); err != nil {
t.Fatalf("set proxy pipe deadline: %v", err)
}
codec := NewCodec(proxySide)
request := readSchedulerTestMessage(t, codec)
if request.Method != "node/set-priority" {
t.Fatalf("replacement request method = %q, want node/set-priority", request.Method)
}
var got schedulerwire.Priority
if err := json.Unmarshal(request.Params, &got); err != nil {
t.Fatalf("decode replacement priority: %v", err)
}
if !got.SameRanking(want) {
t.Fatalf("replacement priority = %#v, want %#v", got, want)
}
// The replay reuses the cached generation rather than minting one, so the
// child can recognize a redelivery it has already applied. A repush that
// arrived unversioned would clear reservations on every proxy ready.
if got.Generation == 0 {
t.Fatal("replayed snapshot carried no generation; the child cannot detect a redelivery")
}
if err := codec.Respond(request.ID, map[string]int{"count": len(got.Nodes)}); err != nil {
t.Fatalf("respond to replacement priority: %v", err)
}
waitSchedulerTestChannel(t, replayed, "priority replay did not finish")
}
func waitSchedulerTestChannel(t *testing.T, ch <-chan struct{}, failure string) {
t.Helper()
select {
case <-ch:
case <-time.After(2 * time.Second):
t.Fatal(failure)
}
}