Router that virtually distributes inference across connected devices in the home.
Go
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"net/http/httptest"
"strconv"
"sync"
"testing"
"time"
"nvpair-shared/clustertrust"
)
// codecNop is a throwaway io.ReadWriter for a Manager whose broker side is
// never exercised.
type codecNop struct{}
func (codecNop) Read([]byte) (int, error) { return 0, io.EOF }
func (codecNop) Write(p []byte) (int, error) { return len(p), nil }
type jSequence struct {
Seq int `json:"seq"`
}
type jParams struct {
Params jSequence `json:"params"`
}
func assertFrame(t *testing.T, body []byte, want int) {
t.Helper()
var frame jParams
if err := json.Unmarshal(body, &frame); err != nil {
t.Fatalf("decode frame %d: %v", want, err)
}
if frame.Params.Seq != want {
t.Fatalf("frame %d arrived with sequence %d", want, frame.Params.Seq)
}
}
func newBroadcastManagerForPeer(t *testing.T, selfDir string, peer *httptest.Server) *Manager {
t.Helper()
host, portStr, err := net.SplitHostPort(peer.Listener.Addr().String())
if err != nil {
t.Fatalf("split test peer address: %v", err)
}
port, err := strconv.Atoi(portStr)
if err != nil {
t.Fatalf("parse test peer port: %v", err)
}
m := NewManager(NewCodec(codecNop{}), 0, "uuid-self", selfDir)
m.peers.Replace([]PeerNode{{
ID: "peer-1", Addresses: []string{host}, Port: port,
TXT: []string{"cluster-uuid=uuid-peer"},
}})
return m
}
// TestManager_BroadcastPreservesEnqueueOrder checks that frames queued in order
// reach a peer in the same order over the cluster-mTLS broadcast path.
func TestManager_BroadcastPreservesEnqueueOrder(t *testing.T) {
selfDir, peerDir := newPinnedPeerDirs(t)
peerMesh := clustertrust.Open(peerDir)
const frameCount = 25
received := make(chan []byte, frameCount)
mux := http.NewServeMux()
mux.HandleFunc(eventsPath, func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
if err != nil {
http.Error(w, "failed to read frame", http.StatusBadRequest)
return
}
received <- body
w.WriteHeader(http.StatusOK)
})
ts := httptest.NewUnstartedServer(mux)
ts.TLS = peerMesh.ServerTLSConfig()
ts.StartTLS()
t.Cleanup(ts.Close)
m := newBroadcastManagerForPeer(t, selfDir, ts)
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
go m.broadcastLoop(ctx)
for i := 0; i < frameCount; i++ {
m.broadcastFrame("workload:started", []byte(fmt.Sprintf(`{"seq":%d}`, i)))
}
for want := 0; want < frameCount; want++ {
select {
case body := <-received:
assertFrame(t, body, want)
case <-ctx.Done():
t.Fatalf("peer received %d of %d frames: %v", want, frameCount, ctx.Err())
}
}
}
// snapshotPauseHandler stops broadcastSnapshot after it copies activeLocal but
// before it queues the copied frames. This makes the remove race reproducible.
type snapshotPauseHandler struct {
paused chan struct{}
release <-chan struct{}
once sync.Once
}
func (*snapshotPauseHandler) Enabled(context.Context, slog.Level) bool { return true }
func (h *snapshotPauseHandler) Handle(_ context.Context, record slog.Record) error {
if record.Message == "snapshot copied before removal" {
h.once.Do(func() { close(h.paused) })
<-h.release
}
return nil
}
func (h *snapshotPauseHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
func (h *snapshotPauseHandler) WithGroup(string) slog.Handler { return h }
func TestManager_SnapshotCannotResurrectRemovedWorkload(t *testing.T) {
m := &Manager{
activeLocal: make(map[workloadKey]workloadEvent),
broadcastCh: make(chan []byte, 2),
peers: newPeerSet(0),
}
workload := &Workload{
ID: "7", Model: "m", Engine: "ollama", RunID: "r1",
State: StateRunning, OriginatedFrom: "node-a", CreatedAt: 1,
}
lifecycle, err := json.Marshal(lifecycleParams{WorkloadInfo: workload})
if err != nil {
t.Fatalf("marshal lifecycle: %v", err)
}
m.trackActive(workloadKey{origin: "node-a", engine: "ollama", runID: "r1", id: "7"}, MethodStarted, lifecycle, StateRunning)
paused := make(chan struct{})
release := make(chan struct{})
var releaseOnce sync.Once
releaseSnapshot := func() { releaseOnce.Do(func() { close(release) }) }
defer releaseSnapshot()
previousLogger := slog.Default()
slog.SetDefault(slog.New(&snapshotPauseHandler{paused: paused, release: release}))
t.Cleanup(func() { slog.SetDefault(previousLogger) })
snapshotDone := make(chan struct{})
go func() {
m.broadcastSnapshot("snapshot copied before removal")
close(snapshotDone)
}()
select {
case <-paused:
case <-time.After(2 * time.Second):
t.Fatal("snapshot did not pause after copying the workload")
}
removal, err := json.Marshal(removeParams{WorkloadID: "7", OriginatedFrom: "node-a"})
if err != nil {
t.Fatalf("marshal removal: %v", err)
}
removeDone := make(chan struct{})
go func() {
m.handleLocalRemove(&Message{Method: MethodRemove, Params: removal})
close(removeDone)
}()
// The current implementation queues the removal while the snapshot is
// paused. An implementation that serializes both operations may instead
// hold the removal until the snapshot resumes.
select {
case <-removeDone:
case <-time.After(100 * time.Millisecond):
}
releaseSnapshot()
for _, done := range []<-chan struct{}{snapshotDone, removeDone} {
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("snapshot or removal did not finish")
}
}
// The peer already has the running workload. Apply the queued frames in
// their actual delivery order through the receiver's handlers.
present := true
removes := 0
receiver := NewServer(0, newDedupIndex(8), nil,
func(*Workload) error { present = true; return nil },
func(string, string) error { present = false; removes++; return nil },
)
for len(m.broadcastCh) > 0 {
var frame Message
if err := json.Unmarshal(<-m.broadcastCh, &frame); err != nil {
t.Fatalf("decode queued frame: %v", err)
}
response := httptest.NewRecorder()
switch frame.Method {
case MethodStarted:
receiver.handleLifecycle(response, &frame)
case MethodRemove:
receiver.handleRemove(response, &frame)
default:
t.Fatalf("unexpected queued method %q", frame.Method)
}
if response.Code != http.StatusOK {
t.Fatalf("peer handled %s with HTTP %d", frame.Method, response.Code)
}
}
if removes != 1 {
t.Fatalf("peer received %d removals, want 1", removes)
}
if present {
t.Fatal("stale snapshot upsert resurrected the workload after its removal")
}
}