Router that virtually distributes inference across connected devices in the home.
Go
Retry jobs when a node stalls or drops out
Noah (Terve) Tervalone0f75ddc841a
diff --git a/desktop/src/shared/constants/workloads.ts b/desktop/src/shared/constants/workloads.tsindex cc5816a..6e1cac7 100644--- a/desktop/src/shared/constants/workloads.ts+++ b/desktop/src/shared/constants/workloads.ts@@ -1,4 +1,4 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-export const WorkloadStates = ['initializing', 'queued', 'running', 'completed', 'failed'] as const+export const WorkloadStates = ['queued', 'running', 'completed', 'failed', 'cancelled'] as constdiff --git a/desktop/src/ui/components/JobsFilter.tsx b/desktop/src/ui/components/JobsFilter.tsxindex 3787576..bba1a96 100644--- a/desktop/src/ui/components/JobsFilter.tsx+++ b/desktop/src/ui/components/JobsFilter.tsx@@ -8,11 +8,7 @@ import { JobsFilterType } from '@/ui/types/types'interface JobsFilterProps {setFilter: (type: JobsFilterType, checked: boolean) => void- values: {- active: { checked: boolean; count: number }- completed: { checked: boolean; count: number }- failed: { checked: boolean; count: number }- }+ values: Record<JobsFilterType, { checked: boolean; count: number }>}function JobFilterItem({ id, count }: { id: JobsFilterType; count: number }) {@@ -28,28 +24,19 @@ function JobFilterItem({ id, count }: { id: JobsFilterType; count: number }) {)}+// Dropdown order, listed once so a new bucket needs one entry rather than a+// parallel edit in three places.+const FILTER_ORDER: JobsFilterType[] = ['active', 'completed', 'failed', 'cancelled']+export default function JobsFilter({ setFilter, values }: JobsFilterProps) {const dropdownItems: DropdownEntry[] = useMemo(- () => [- {- kind: 'checkbox',- checked: values.active.checked,- onCheckedChange: checked => setFilter('active', checked === true),- children: <JobFilterItem id="active" count={values.active.count} />- },- {- kind: 'checkbox',- checked: values.completed.checked,- onCheckedChange: checked => setFilter('completed', checked === true),- children: <JobFilterItem id="completed" count={values.completed.count} />- },- {+ () =>+ FILTER_ORDER.map(id => ({kind: 'checkbox',- checked: values.failed.checked,- onCheckedChange: checked => setFilter('failed', checked === true),- children: <JobFilterItem id="failed" count={values.failed.count} />- }- ],+ checked: values[id].checked,+ onCheckedChange: checked => setFilter(id, checked === true),+ children: <JobFilterItem id={id} count={values[id].count} />+ })),[setFilter, values])diff --git a/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx b/desktop/src/ui/components/Workloads/WorkloadItemCard.tsxindex 2fa7b47..799de94 100644--- a/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx+++ b/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx@@ -65,7 +65,10 @@ function WorkloadItemCard({ workload }: { workload: Workload }) {switch (state) {case 'queued':value = workload.createdAt- label = 'Queued at'+ // Not "Queued at": PAIR runs no queue of its own, and the+ // elapsed wait is the useful part for a job that has been+ // accepted but is not generating yet.+ label = 'Waiting since'breakcase 'running':value = workload.startedAt ?? 0@@ -79,6 +82,10 @@ function WorkloadItemCard({ workload }: { workload: Workload }) {value = workload.completedAt ?? 0label = 'Failed at'break+ case 'cancelled':+ value = workload.completedAt ?? 0+ label = 'Cancelled at'+ break}return (@@ -93,18 +100,16 @@ function WorkloadItemCard({ workload }: { workload: Workload }) {)}, [workload.state, workload.createdAt, workload.startedAt, workload.completedAt])- // const showBadge = useMemo(- // () =>- // workload.state === 'initializing' ||- // workload.state === 'queued' ||- // workload.state === 'running',- // [workload.state]- // )-const className = useMemo(() => {let result = `h-auto dir-ltr max-w-full min-w-0 pair-paper workload-card workload-card-${workload.state}`- if (workload.state === 'failed' || workload.state === 'completed') {+ // Terminal states read as history and are dimmed; only live work keeps+ // full contrast.+ if (+ workload.state === 'failed' ||+ workload.state === 'completed' ||+ workload.state === 'cancelled'+ ) {result += ' card-low'}diff --git a/desktop/src/ui/components/Workloads/WorkloadListView.tsx b/desktop/src/ui/components/Workloads/WorkloadListView.tsxindex bfcaa28..2832319 100644--- a/desktop/src/ui/components/Workloads/WorkloadListView.tsx+++ b/desktop/src/ui/components/Workloads/WorkloadListView.tsx@@ -8,13 +8,14 @@ import type { PartialOptions } from 'overlayscrollbars'import { Flipper, Flipped } from 'react-flip-toolkit'import {useActiveWorkloads,+ useCancelledWorkloads,useCompletedWorkloads,useFailedWorkloads} from '@/ui/stores/workloads.store'import WorkloadItemCard from './WorkloadItemCard'import JobsFilter from '@/ui/components/JobsFilter'import { CONNECTIONS_WIDTH, MAX_HISTORY_ITEMS } from '@/ui/constants/app'-import type { Workload } from '@/shared/types/workloads'+import type { Workload, WorkloadState } from '@/shared/types/workloads'import { workloadKey } from '@/shared/utils/workloads'import type { JobsFilterType } from '@/ui/types/types'@@ -30,18 +31,32 @@ interface WorkloadListViewProps {setFilter: (type: JobsFilterType, checked: boolean) => void}+// Which filter bucket a finished job belongs to. Exhaustive over WorkloadState+// so adding a state is a compile error here rather than a job that silently+// stops appearing: the history list used to index the filter record by the raw+// state string, and any state without a matching bucket read as unchecked.+const HISTORY_BUCKET: Record<WorkloadState, JobsFilterType | null> = {+ queued: null,+ running: null,+ completed: 'completed',+ failed: 'failed',+ cancelled: 'cancelled'+}+function WorkloadListView({ filter, setFilter }: WorkloadListViewProps) {const activeWorkloads = useActiveWorkloads()const completedWorkloads = useCompletedWorkloads()const failedWorkloads = useFailedWorkloads()+ const cancelledWorkloads = useCancelledWorkloads()const jobValues = useMemo(() => ({active: { checked: filter.active, count: activeWorkloads.length },completed: { checked: filter.completed, count: completedWorkloads.length },- failed: { checked: filter.failed, count: failedWorkloads.length }+ failed: { checked: filter.failed, count: failedWorkloads.length },+ cancelled: { checked: filter.cancelled, count: cancelledWorkloads.length }}),- [filter, activeWorkloads, completedWorkloads, failedWorkloads]+ [filter, activeWorkloads, completedWorkloads, failedWorkloads, cancelledWorkloads])const currentWorkloads = useMemo(() => {@@ -51,13 +66,16 @@ function WorkloadListView({ filter, setFilter }: WorkloadListViewProps) {result = activeWorkloads}- const rest = [...completedWorkloads, ...failedWorkloads]- .filter(w => filter[w.state as JobsFilterType])+ const rest = [...completedWorkloads, ...failedWorkloads, ...cancelledWorkloads]+ .filter(w => {+ const bucket = HISTORY_BUCKET[w.state]+ return bucket !== null && filter[bucket]+ }).sort((a, b) => (b.completedAt ?? 0) - (a.completedAt ?? 0)).slice(0, MAX_HISTORY_ITEMS)return result.concat(rest)- }, [filter, completedWorkloads, failedWorkloads, activeWorkloads])+ }, [filter, completedWorkloads, failedWorkloads, cancelledWorkloads, activeWorkloads])const flipKey = useMemo(() => {const filterKey = Object.entries(filter)diff --git a/desktop/src/ui/constants/app.ts b/desktop/src/ui/constants/app.tsindex 2afc49c..16bb356 100644--- a/desktop/src/ui/constants/app.ts+++ b/desktop/src/ui/constants/app.ts@@ -10,9 +10,9 @@ export const MAX_HISTORY_ITEMS = 30export const stateOrder: Record<WorkloadState, number> = {running: 0,queued: 1,- initializing: 2,- completed: 3,- failed: 4+ completed: 2,+ failed: 3,+ cancelled: 4}export const ContentBaseClass = 'grow w-full min-h-0 min-w-0'diff --git a/desktop/src/ui/stores/workloads.store.ts b/desktop/src/ui/stores/workloads.store.tsindex 9b2799a..7bb9082 100644--- a/desktop/src/ui/stores/workloads.store.ts+++ b/desktop/src/ui/stores/workloads.store.ts@@ -144,8 +144,7 @@ export function useActiveWorkloads(): Workload[] {useShallow(state => {const result: Workload[] = []for (const w of state.workloads.values()) {- if (w.state === 'running' || w.state === 'queued' || w.state === 'initializing')- result.push(w)+ if (w.state === 'running' || w.state === 'queued') result.push(w)}return result.sort((a, b) => {const stateDiff = stateOrder[a.state] - stateOrder[b.state]@@ -178,16 +177,16 @@ export function useAssignedWorkloads(): Workload[] {)}-// Number of active (running/queued/initializing) workloads scheduled on a node.-// Returns a primitive so a subscribing node card only re-renders when its own-// count changes, instead of on every workload event across the cluster.+// Number of active (running/queued) workloads scheduled on a node. Returns a+// primitive so a subscribing node card only re-renders when its own count+// changes, instead of on every workload event across the cluster.export function useNodeActiveJobCount(nodeId: string): number {return useWorkloadsStore(state => {let count = 0for (const w of state.workloads.values()) {if (workloadExecutionNodeId(w) === nodeId &&- (w.state === 'running' || w.state === 'queued' || w.state === 'initializing')+ (w.state === 'running' || w.state === 'queued'))count++}@@ -222,3 +221,20 @@ export function useFailedWorkloads(): Workload[] {}))}++// Jobs the requester stopped waiting for. Kept out of useFailedWorkloads so+// the failed bucket only holds outcomes a user might act on, rather than also+// collecting every time somebody pressed stop.+export function useCancelledWorkloads(): Workload[] {+ return useWorkloadsStore(+ useShallow(state => {+ const result: Workload[] = []+ for (const w of state.workloads.values()) {+ if (w.state === 'cancelled') result.push(w)+ }+ return result+ .sort((a, b) => (b.completedAt ?? 0) - (a.completedAt ?? 0))+ .slice(0, MAX_HISTORY_ITEMS)+ })+ )+}diff --git a/desktop/src/ui/types/types.ts b/desktop/src/ui/types/types.tsindex fceed83..de24e4e 100644--- a/desktop/src/ui/types/types.ts+++ b/desktop/src/ui/types/types.ts@@ -6,6 +6,6 @@ export interface PerformanceMetric {value: number}-export type JobsFilterType = 'active' | 'completed' | 'failed'+export type JobsFilterType = 'active' | 'completed' | 'failed' | 'cancelled'export type BadgeColor = 'yellow' | 'blue' | 'green' | 'gray' | 'red' | 'teal' | 'purple'diff --git a/desktop/src/ui/utils/colors.ts b/desktop/src/ui/utils/colors.tsindex a220117..ad6b18c 100644--- a/desktop/src/ui/utils/colors.ts+++ b/desktop/src/ui/utils/colors.ts@@ -7,8 +7,6 @@ import { GPU_COLOR_PALETTE, VRAM_COLOR_PALETTE, WORKLOAD_COLOR_MAP } from '@/ui/export function getWorkloadStateColor(state: WorkloadState): BadgeColor {switch (state) {- case 'initializing':- return 'gray'case 'queued':return 'yellow'case 'running':@@ -17,6 +15,11 @@ export function getWorkloadStateColor(state: WorkloadState): BadgeColor {return 'gray'case 'failed':return 'red'+ // A job the requester stopped waiting for is not an error: it reads as+ // neutral history, so it does not compete with real failures for+ // attention.+ case 'cancelled':+ return 'gray'default:return 'gray'}@@ -24,8 +27,6 @@ export function getWorkloadStateColor(state: WorkloadState): BadgeColor {export function getWorkloadColorBar(state: WorkloadState): string {switch (state) {- case 'initializing':- return WORKLOAD_COLOR_MAP.graycase 'queued':return WORKLOAD_COLOR_MAP.yellowcase 'running':@@ -34,6 +35,8 @@ export function getWorkloadColorBar(state: WorkloadState): string {return WORKLOAD_COLOR_MAP.greencase 'failed':return WORKLOAD_COLOR_MAP.red+ case 'cancelled':+ return WORKLOAD_COLOR_MAP.graydefault:return WORKLOAD_COLOR_MAP.gray}diff --git a/desktop/src/ui/utils/jobs-filter-storage.ts b/desktop/src/ui/utils/jobs-filter-storage.tsindex 1653bb7..704c238 100644--- a/desktop/src/ui/utils/jobs-filter-storage.ts+++ b/desktop/src/ui/utils/jobs-filter-storage.ts@@ -9,10 +9,11 @@ export const JOBS_FILTER_STORAGE_KEY = 'nvpair.jobs.filter'export const DEFAULT_JOBS_FILTER: Record<JobsFilterType, boolean> = {active: true,completed: true,- failed: true+ failed: true,+ cancelled: true}-const KEYS: JobsFilterType[] = ['active', 'completed', 'failed']+const KEYS: JobsFilterType[] = ['active', 'completed', 'failed', 'cancelled']/** Merge stored booleans with defaults so missing/invalid keys do not break the UI. */export function parseJobsFilterRecord(raw: unknown): Record<JobsFilterType, boolean> {diff --git a/services/nvpair-job-scheduler/schedule_test.go b/services/nvpair-job-scheduler/schedule_test.goindex 4bbfbab..6c1a402 100644--- a/services/nvpair-job-scheduler/schedule_test.go+++ b/services/nvpair-job-scheduler/schedule_test.go@@ -269,6 +269,22 @@ func TestRank_TerminalStatesExcluded(t *testing.T) {assertStrs(t, order, []string{"b", "a"}) // b=0 wins; a would tie only if completed counted}+// TestRank_CancelledExcluded pins that isPending is an allow-list, which is+// what keeps a new terminal state from silently counting as load. "cancelled"+// is not queued and not running, so it must contribute nothing — a job the+// requester walked away from is not occupying its node.+func TestRank_CancelledExcluded(t *testing.T) {+ m := mgrWith(nopRW{}, []string{"a", "b"},+ wl("1", "ollama", "running", "x", "a"), // a = 1+ wl("2", "ollama", "cancelled", "x", "b"), // must not count toward b+ )+ order, ranks := m.rank()+ assertStrs(t, order, []string{"b", "a"})+ if got := pendingOf(ranks, "b"); got != 0 {+ t.Fatalf("pending for b = %d, want 0 (cancelled must not count as pending)", got)+ }+}+// TestRank_NodeWideMixedEngineSynthetic verifies the requested synthetic// ranking: A has three mixed-engine pending jobs, B has one, and C has none.// Both engine outputs must therefore receive C,B,A.diff --git a/services/nvpair-job-scheduler/spec.md b/services/nvpair-job-scheduler/spec.mdindex c461e92..9cab523 100644--- a/services/nvpair-job-scheduler/spec.md+++ b/services/nvpair-job-scheduler/spec.md@@ -112,9 +112,10 @@ capacity, model locality, latency, and affinity can grow later behind the same`scheduler:set-interval`, clamped to a **200 ms** floor (§7.5).Workload and discovery events do not wait for this timer; emit-only-on-snapshot-change plus the stable tie-break keep timer recomputes from churning.-- **Pending-count (decided — node-wide).** *Pending* = `queued` or `running`- (`completed`/`failed` excluded; `initializing` never transmitted), regardless of- engine. Counted by **`scheduledOn`** (where work runs), not `originatedFrom`. A+- **Pending-count (decided — node-wide).** *Pending* = `queued` or `running`,+ regardless of engine. The test is an allow-list, not a terminal-state+ exclusion list, so `completed`, `failed`, `cancelled`, and any state added+ later count as zero load without needing a change here. Counted by **`scheduledOn`** (where work runs), not `originatedFrom`. Aworkload with no `scheduledOn` is unplaced and counts toward no node.- **Identity alignment (decided).** Discovery `hostUuid`, proxy candidate ids, andworkload `scheduledOn` use the same stable UUID namespace. A malformed or legacydiff --git a/services/nvpair-proxy/README.md b/services/nvpair-proxy/README.mdindex 731e6b9..6bcdc50 100644--- a/services/nvpair-proxy/README.md+++ b/services/nvpair-proxy/README.md@@ -201,6 +201,10 @@ the local machine is unaffected.proxy falls back to auto-select and emits `node/selection-changed`. Atransport error or retryable status, including a model `404` from anadvertised owner with stale inventory, steps to the next eligible owner.+ Once a round's owners are used up, an inference request re-resolves and keeps+ trying under a dispatch budget and a wall-clock deadline, and it commits to a+ node at the first byte of response body rather than at its headers. `spec.md`+ §5.1–§5.4 is normative for the bounds, the commit point, and the statuses.### IPC Transportdiff --git a/services/nvpair-proxy/discovery.go b/services/nvpair-proxy/discovery.goindex 87313a4..a31213b 100644--- a/services/nvpair-proxy/discovery.go+++ b/services/nvpair-proxy/discovery.go@@ -102,6 +102,19 @@ func (d *Discovery) Nodes() []Node {return out}+// Has reports whether a node id is still present in either overlay. Used to+// watch an in-flight request's target: a node that has dropped out of discovery+// is not coming back to answer the attempt already sent to it.+func (d *Discovery) Has(id string) bool {+ d.mu.RLock()+ defer d.mu.RUnlock()+ if _, ok := d.subscribedNodes[id]; ok {+ return true+ }+ _, ok := d.manualNodes[id]+ return ok+}+// SetSubscribed replaces the relay-fed routing overlay with the given set and// reports what changed versus the previous set (keyed by node ID): nodes newly// present, nodes whose routable details changed, and nodes that dropped out. Thediff --git a/services/nvpair-proxy/firstbody_test.go b/services/nvpair-proxy/firstbody_test.gonew file mode 100644index 0000000..294b14f--- /dev/null+++ b/services/nvpair-proxy/firstbody_test.go@@ -0,0 +1,266 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++package main++import (+ "io"+ "net/http"+ "net/http/httptest"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++// Commit is the first byte of response body, not the response headers, because+// headers can mean nothing more than "accepted" (spec §5.2 for the engine+// measurements behind that). These bodies pin the boundary: what fails over,+// what commits, what terminates, and what is left alone.+//+// firstBodyTimeout stands in at 200ms for its production 120s throughout, so a+// body that drives a node to its five-dispatch budget costs a second of wall+// clock rather than ten minutes.+const testFirstBodyTimeout = 200 * time.Millisecond++// stalledEngine is an upstream that sends 200 and headers immediately and then+// produces no body until released, which is the shape an engine presents for a+// streaming request queued behind other work.+type stalledEngine struct {+ url string+ // hits counts dispatches, so a test can show the retry budget was spent+ // rather than inferring it from timing.+ hits func() int+ release func()+}++func newStalledEngine(t *testing.T) stalledEngine {+ t.Helper()+ var n atomic.Int32+ releaseCh := make(chan struct{})+ var once sync.Once+ release := func() { once.Do(func() { close(releaseCh) }) }++ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ n.Add(1)+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusOK)+ if f, ok := w.(http.Flusher); ok {+ f.Flush() // headers on the wire, body still empty+ }+ select {+ case <-releaseCh:+ case <-r.Context().Done():+ }+ }))+ // Cleanup is LIFO, so release runs before Close. Close waits for the+ // handler to return and the handler is parked on release, so the other+ // order deadlocks the end of the test.+ t.Cleanup(srv.Close)+ t.Cleanup(release)++ return stalledEngine{+ url: srv.URL,+ hits: func() int { return int(n.Load()) },+ release: release,+ }+}++// TestHandleHTTP_HeadersWithoutContentFailsOver is the regression for the+// engine that accepts and stalls. The first owner sends headers and no content;+// the request must move to the next owner rather than binding itself to a node+// that has not started generating.+func TestHandleHTTP_HeadersWithoutContentFailsOver(t *testing.T) {+ setForTest(t, &firstBodyTimeout, testFirstBodyTimeout)++ forEachEngine(t, func(t *testing.T, tc engineCase) {+ stalled := newStalledEngine(t)++ var servedBody string+ good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ b, _ := io.ReadAll(r.Body)+ servedBody = string(b)+ w.WriteHeader(http.StatusOK)+ io.WriteString(w, `{"done":true}`)+ }))+ defer good.Close()++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "stalled", stalled.url, tc.advertisedModel))+ disc.AddManual(nodeForModel(t, "good", good.URL, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)+ p.soleFacade().SetSelected("stalled") // deterministic: the staller is tried first++ rec := httptest.NewRecorder()+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())++ if rec.Code != http.StatusOK {+ t.Fatalf("status = %d, want 200 (should have failed over past the stalled engine)", rec.Code)+ }+ if got := decodeJSONBody(t, rec.Body.String()); got["done"] != true {+ t.Fatalf(`body has done = %v, want true: the response came from the wrong node`, got["done"])+ }+ if servedBody != tc.inferenceBody() {+ t.Errorf("failover node got body %q, want the original request body", servedBody)+ }+ })+}++// TestHandleHTTP_NoContentOnFinalAttemptTerminates covers the exhausted case,+// which is the one that could hang: a first-content timeout must not commit+// even with no attempt left, because the peek reader is still blocked on the+// upstream body (spec §5.2).+//+// 504 rather than 502: the node answered, it just never produced anything.+func TestHandleHTTP_NoContentOnFinalAttemptTerminates(t *testing.T) {+ setForTest(t, &firstBodyTimeout, testFirstBodyTimeout)++ forEachEngine(t, func(t *testing.T, tc engineCase) {+ // The only owner, so every attempt in the budget goes to it and the+ // last one has nowhere to fail over to.+ stalled := newStalledEngine(t)++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "silent", stalled.url, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)++ rec := httptest.NewRecorder()+ done := make(chan struct{})+ go func() {+ defer close(done)+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())+ }()++ select {+ case <-done:+ // Five attempts at the shortened budget is about a second. Any finite+ // guard catches the regression this test exists for, since a committed+ // silent upstream blocks forever.+ case <-time.After(5 * time.Second):+ t.Fatal("handleHTTP never returned: a silent upstream committed and hung the request")+ }++ if got := stalled.hits(); got != maxDispatchAttempts {+ t.Errorf("upstream saw %d dispatches, want the full budget of %d spent on the only owner", got, maxDispatchAttempts)+ }+ if rec.Code != http.StatusGatewayTimeout {+ t.Fatalf("status = %d, want 504 for an upstream that answered but produced no content", rec.Code)+ }+ want := "upstream error: " + errFirstBodyTimeout.Error()+ if got := decodeJSONBody(t, rec.Body.String()); got["error"] != want {+ t.Fatalf("body error = %v, want %q", got["error"], want)+ }+ })+}++// TestHandleHTTP_FirstBytePreservedOnCommit guards the splice: the byte+// awaitFirstBody consumes to detect the commit has to be handed back, or every+// committed response would be served a byte short. Compared bytewise rather+// than as JSON, since losing the first byte is exactly what would still parse+// as something.+func TestHandleHTTP_FirstBytePreservedOnCommit(t *testing.T) {+ forEachEngine(t, func(t *testing.T, tc engineCase) {+ const payload = `{"response":"whole body intact","done":true}`+ good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusOK)+ io.WriteString(w, payload)+ }))+ defer good.Close()++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "good", good.URL, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)++ rec := httptest.NewRecorder()+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())++ if rec.Code != http.StatusOK {+ t.Fatalf("status = %d, want 200", rec.Code)+ }+ if got := rec.Body.String(); got != payload {+ t.Fatalf("body = %q, want %q (the peeked first byte was dropped)", got, payload)+ }+ })+}++// TestHandleHTTP_EmptyBodyCommits: a 200 with no body at all commits rather+// than burning the budget waiting for content that is never coming (spec §5.2).+func TestHandleHTTP_EmptyBodyCommits(t *testing.T) {+ // Long, so a wait would be unmistakable: if the empty body were treated as+ // a stall, the guard below would trip well before this expires.+ setForTest(t, &firstBodyTimeout, 30*time.Second)++ forEachEngine(t, func(t *testing.T, tc engineCase) {+ empty := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusOK)+ }))+ defer empty.Close()++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "empty", empty.URL, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)++ done := make(chan int, 1)+ go func() {+ rec := httptest.NewRecorder()+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())+ done <- rec.Code+ }()++ select {+ case code := <-done:+ if code != http.StatusOK {+ t.Fatalf("status = %d, want 200", code)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("an empty 200 body blocked on the first-byte wait instead of committing")+ }+ })+}++// TestHandleHTTP_NonInferenceCommitsOnHeaders: the gate is scoped to inference.+// A control endpoint's headers mean what they say, and some of them (a pull's+// progress stream) legitimately produce their first byte late, so waiting there+// would add latency for no benefit.+func TestHandleHTTP_NonInferenceCommitsOnHeaders(t *testing.T) {+ setForTest(t, &firstBodyTimeout, testFirstBodyTimeout)++ forEachEngine(t, func(t *testing.T, tc engineCase) {+ stalled := newStalledEngine(t)++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "stalled", stalled.url, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)++ rec := httptest.NewRecorder()+ done := make(chan struct{})+ go func() {+ req := httptest.NewRequest(http.MethodGet, tc.nonInferencePath, nil)+ p.soleFacade().handleHTTP(rec, req)+ close(done)+ }()++ // Still blocked on the upstream body well past the first-content+ // budget, which is the whole assertion: if the gate applied here the+ // handler would have given up at firstBodyTimeout and returned 504,+ // and simply waiting for it to finish would pass either way.+ blocked := 2 * testFirstBodyTimeout+ time.Sleep(blocked)+ select {+ case <-done:+ t.Fatalf("handler returned after %v with status %d: the first-content gate was applied to a non-inference route",+ blocked, rec.Code)+ default:+ }++ stalled.release()+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("a non-inference request did not complete after the upstream released")+ }+ if rec.Code != http.StatusOK {+ t.Fatalf("status = %d, want 200: the request committed on headers and streamed normally", rec.Code)+ }+ })+}diff --git a/services/nvpair-proxy/proxy.go b/services/nvpair-proxy/proxy.goindex 33a32bd..ff89dbe 100644--- a/services/nvpair-proxy/proxy.go+++ b/services/nvpair-proxy/proxy.go@@ -24,6 +24,7 @@ import ("strconv""strings""sync"+ "sync/atomic""time""nvpair-shared/applog"@@ -150,6 +151,7 @@ type RequestEvent struct {// workload:submitted (the proxy never queues — it forwards immediately) or// workloads:remove (retirement is a broker concern).const (+ workloadSubmittedMethod = "workload:submitted"workloadStartedMethod = "workload:started"workloadCompletedMethod = "workload:completed"workloadErroredMethod = "workload:errored"@@ -189,6 +191,20 @@ type Workload struct {CompletedAt *int64 `json:"completedAt"`Error *string `json:"error"`RequesterID *string `json:"requesterId"`+ // Seq counts this workload's events, from 1, in emission order.+ //+ // The workload-manager's inter-node dedup is a permanent set, so an event+ // that repeats an earlier (state, scheduledOn) pair is indistinguishable+ // from a redelivery and is dropped by every peer. The retry loop produces+ // exactly that routinely: queued on A, placement cleared between attempts,+ // then queued on A again. Peers kept the interim unplaced record while the+ // job was running on A, so their schedulers stopped counting it against the+ // node actually doing the work.+ //+ // Sequencing each event makes the dedup exact without weakening it: a+ // broadcast retry resends an identical frame, sequence included, so a true+ // redelivery is still suppressed.+ Seq int64 `json:"seq"`}// workloadParams is the params envelope for a workload:* notification@@ -503,6 +519,210 @@ func (p *Proxy) Run(ctx context.Context) error {return err}+// Retry bounds for a single inference request.+//+// The two are not redundant, because they bound different things. An attempt is+// consumed by a dispatch, never by a resolution: finding no eligible owner+// sends nothing, so maxDispatchAttempts governs dispatch failures while+// jobDeadline governs the wait for an owner to exist at all — a wait that costs+// no attempts and would otherwise be unbounded.+//+// Read only against dispatch failures the deadline looks unreachable, and the+// arithmetic says so: five attempts at a 120s first-content cap plus 15s of+// cumulative backoff span 615s, so every reschedule check before the last sees+// a clock under 600s. It is not dead — it is the only bound on the no-owner+// wait. Do not remove it as unreachable.+const maxDispatchAttempts = 5++// jobDeadline is a var (not a const) only so a test can shorten it; production+// never reassigns it.+var jobDeadline = 10 * time.Minute++// targetWatchInterval is how often an in-flight attempt rechecks that its+// target is still in discovery. It bounds detection latency only: the effect is+// to replace a full first-content budget spent waiting on a node that is+// already gone with a sub-second abort.+//+// A var (not a const) only so a test can shorten it; production never+// reassigns it.+var targetWatchInterval = 500 * time.Millisecond++// defaultRetryBackoff is the delay before a retry, indexed by the number of+// dispatches already made and clamped to the last entry. It also paces the poll+// while no owner is available.+var defaultRetryBackoff = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second}++// retryBackoff is the schedule in force. A var (not a const) only so a test can+// shorten it; production never reassigns it. Naming the default separately lets+// a test that asserts the schedule itself put the real one back.+var retryBackoff = defaultRetryBackoff++// backoffFor returns the delay before the next dispatch, jittered and never+// longer than the time left before the deadline.+func backoffFor(dispatches int, remaining time.Duration) time.Duration {+ if len(retryBackoff) == 0 || remaining <= 0 {+ return 0+ }+ i := dispatches - 1+ if i < 0 {+ i = 0+ }+ if i >= len(retryBackoff) {+ i = len(retryBackoff) - 1+ }+ d := retryBackoff[i]+ // Jitter by up to ±25% so a burst of requests that all failed against the+ // same node does not retry in lockstep and re-collide. The clock's+ // nanosecond low bits are the entropy source: they differ between+ // concurrent handlers, and `rand` here is crypto/rand, whose interface is+ // far heavier than spreading a backoff warrants.+ if spread := int64(d) / 2; spread > 0 {+ d = time.Duration(int64(d) - spread/2 + time.Now().UnixNano()%(spread+1))+ }+ if d > remaining {+ d = remaining+ }+ return d+}++// waitBeforeRetry sleeps d, reporting false if the request was abandoned while+// waiting. A client that has gone away should not keep a retry budget alive:+// nobody is left to receive the answer, so the remaining attempts are better+// spent on work someone is waiting for.+func waitBeforeRetry(ctx context.Context, d time.Duration) bool {+ if ctx.Err() != nil {+ return false+ }+ if d <= 0 {+ return true+ }+ t := time.NewTimer(d)+ defer t.Stop()+ select {+ case <-t.C:+ return true+ case <-ctx.Done():+ return false+ }+}++// errFirstBodyTimeout is returned by awaitFirstBody when an upstream sent its+// headers but produced no content within the budget.+var errFirstBodyTimeout = stderrors.New("upstream sent no response body within the first-byte budget")++// errTargetGone is the reason an attempt was abandoned because its target left+// discovery. It replaces the bare "context canceled" the transport reports,+// which in a log is indistinguishable from the client hanging up.+var errTargetGone = stderrors.New("target node left discovery mid-request")++// Attempt outcomes for attemptClaim.+const (+ attemptPending int32 = iota+ attemptCommitted+ attemptAbandoned+)++// attemptClaim resolves the race between committing an attempt and abandoning+// it because its target left discovery.+//+// Both can become true at once: the watcher's discovery check can pass while+// the commit path is already running, and between the first body byte arriving+// and the commit finishing that path does a synchronous notification write and+// takes two mutexes. A pair of channels cannot express which happened first —+// closing one says an event occurred, not that it won — so the outcome is one+// value that each side claims by compare-and-swap.+//+// Exactly one of commit and abandon succeeds, and callers must honor the+// result: a false from commit means this attempt is being cancelled and must+// not be served, and a false from abandon means the response is already+// committed and must not be torn down.+type attemptClaim struct {+ state atomic.Int32+}++// commit claims the attempt for the response. Idempotent, so an already+// committed attempt still reports true.+func (a *attemptClaim) commit() bool {+ return a.state.CompareAndSwap(attemptPending, attemptCommitted) ||+ a.state.Load() == attemptCommitted+}++// abandon claims the attempt for cancellation, reporting false when the+// response has already committed.+//+// A committed first byte wins on purpose. The byte is direct evidence that this+// node is serving this request, whereas an absence from discovery can be a+// transient announcement gap; cancelling then would truncate a working stream.+func (a *attemptClaim) abandon() bool {+ return a.state.CompareAndSwap(attemptPending, attemptAbandoned)+}++// settled reports whether either side has claimed the outcome.+func (a *attemptClaim) settled() bool {+ return a.state.Load() != attemptPending+}++// abandoned reports whether cancellation won.+func (a *attemptClaim) abandoned() bool {+ return a.state.Load() == attemptAbandoned+}++// bodyWithPeek re-attaches an already-read first byte ahead of the rest of an+// upstream body, while keeping Close bound to the original so the connection+// is still released.+type bodyWithPeek struct {+ io.Reader+ io.Closer+}++// awaitFirstBody blocks until the upstream produces the first byte of its+// response body, and returns that byte spliced back onto the front of the+// stream.+//+// It exists because response headers are not a reliable signal that an engine+// has started work, and the difference is engine-specific. LM Studio answers a+// streaming request with 200 and headers ~12ms after accepting it and then+// holds the connection open, silent, for the entire time the request waits+// behind other work — measured at 128s behind a single-slot generation. Ollama+// withholds headers until generation begins. Committing on headers therefore+// meant that on LM Studio the proxy bound itself to a node that had merely+// accepted the request: the header timeout could never fire, failover was+// impossible, and a node dying during the wait took the job with it.+//+// The first content byte is the signal both engines agree on, and it is a real+// one — no SSE keepalive or empty role delta arrives during the wait, so the+// byte means generation actually started.+//+// A read that ends in EOF with no bytes is an empty body, which is a complete+// (if unusual) response and commits. A genuine read error before any content+// is retryable, like any other pre-commit upstream failure.+func awaitFirstBody(body io.ReadCloser, budget time.Duration) (io.ReadCloser, error) {+ type peeked struct {+ b []byte+ err error+ }+ // Buffered so the reader goroutine can always finish. On the timeout path+ // ReverseProxy closes resp.Body, which unblocks the pending Read.+ ch := make(chan peeked, 1)+ go func() {+ var one [1]byte+ n, err := body.Read(one[:])+ ch <- peeked{b: one[:n], err: err}+ }()++ timer := time.NewTimer(budget)+ defer timer.Stop()+ select {+ case p := <-ch:+ if len(p.b) == 0 && p.err != nil && !stderrors.Is(p.err, io.EOF) {+ return nil, p.err+ }+ return bodyWithPeek{Reader: io.MultiReader(bytes.NewReader(p.b), body), Closer: body}, nil+ case <-timer.C:+ return nil, errFirstBodyTimeout+ }+}+// Timeouts for upstream connections. Logged at startup so they're always// present in any captured log for post-mortem analysis.const (@@ -532,6 +752,18 @@ const (// deadline against a real socket; production never reassigns it.var idleClientWriteTimeout = 30 * time.Second+// firstBodyTimeout bounds how long awaitFirstBody waits for an engine to emit+// its first content byte. It matches proxyResponseTimeout, the budget the+// transport applies to headers, because from the caller's point of view the two+// bound the same thing: how long we wait for a node to start the work. Keeping+// it generous is deliberate — it also covers a cold model load, which is real+// work rather than a stall, and which cannot be told apart from a queue wait+// from outside the engine.+//+// It is a var (not a const) only so a test can shorten it; production never+// reassigns it.+var firstBodyTimeout = proxyResponseTimeout+var modelListClient = &http.Client{Transport: &http.Transport{DialContext: (&net.Dialer{@@ -969,28 +1201,36 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {wl *Workload)- // Emit workload:started up front, the moment we begin forwarding, naming- // the first candidate we'll try. A burst of concurrent inference requests- // must surface as job cards immediately; the upstream engine serializes- // concurrent requests on a single GPU slot, so gating "started" on the- // upstream response headers (the commit point) left every queued-but-- // forwarded job invisible until the node dequeued it — only one card at a- // time (a prior regression). If failover later commits a different- // node, the commit block re-points scheduledOn; the terminal- // completed/errored transition is emitted once at the end regardless.+ // wlSeq numbers this workload's events. Every mutation of wl below happens+ // either before the watcher goroutine exists or under wlMu, so a plain+ // counter is enough; see Workload.Seq for why the sequence is needed.+ var wlSeq int64+ nextWlSeq := func() int64 { wlSeq++; return wlSeq }++ // Emit workload:submitted the moment the request is admitted, before any+ // dispatch. A burst of concurrent inference requests must surface as job+ // cards immediately — the upstream engine serializes work on a single GPU+ // slot, so a card that waited for the upstream response would leave every+ // queued job invisible until the node dequeued it, one at a time (a prior+ // regression). "queued" is what makes that visible without claiming the+ // engine is generating, which only becomes true at the commit point below.+ //+ // scheduledOn is left empty here: nothing has been dispatched yet, and the+ // scheduler counts pending work by scheduledOn, so naming a node we have+ // not tried would inflate its load. Each dispatch and each gap between+ // attempts re-points it.if isInf && model != "" {createdMs := start.UnixMilli()wl = &Workload{- ID: reqID,- Model: model,- Engine: f.profile.Name,- RunID: p.runID,- State: "running",- ScheduledOn: candidates[0].id,- CreatedAt: createdMs,- StartedAt: &createdMs,+ ID: reqID,+ Model: model,+ Engine: f.profile.Name,+ RunID: p.runID,+ State: "queued",+ CreatedAt: createdMs,+ Seq: nextWlSeq(),}- p.emitWorkload(workloadStartedMethod, *wl)+ p.emitWorkload(workloadSubmittedMethod, *wl)}// The terminal workload transition (completed/errored) can be reached from@@ -1017,8 +1257,14 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {if errMsg != "" {wl.Error = &errMsg}+ wl.Seq = nextWlSeq()snapshot := *wlwlMu.Unlock()+ // Only "completed" gets its own method; every other terminal state,+ // including "cancelled", rides workload:errored. The workload+ // manager does not validate method against state and consumers read+ // the state out of the payload, so a new terminal state needs no new+ // method on the wire.method := workloadCompletedMethodif state != "completed" {method = workloadErroredMethod@@ -1043,7 +1289,7 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {go func() {select {case <-reqCtx.Done():- emitTerminal("failed", "client disconnected before completion")+ emitTerminal("cancelled", "client disconnected before completion")case <-finished:}}()@@ -1051,21 +1297,313 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {// committedSC is the statusCapture of the candidate we committed to// streaming; its wroteErr tells us after the fact whether the client write- // failed (dead/half-open client) so we can mark the workload failed.+ // failed (dead/half-open client) so we can mark the workload failed. It is+ // assigned at the commit point as well as after the loop, because the+ // post-loop assignment does not always run — see finalize below.var committedSC *statusCapture- // Failover loop: try candidates in order until one returns a- // usable response or the list is exhausted. We can only retry before the- // first byte reaches the client; once a response starts streaming we're- // committed. proxy/request-started fires at that commit point so it names- // the node that actually serves the request, not one we failed over from;- // workload:started was already emitted above (and is re-pointed there on a- // failover). The self-forward guard lives in resolveCandidates.- for i := range candidates {- cand := candidates[i]- last := i == len(candidates)-1+ // finalize reports this request's outcome, and it is deferred because a+ // mid-copy error is never returned to us: ReverseProxy converts one into+ // panic(http.ErrAbortHandler) whenever it detects a real server, and that+ // panic unwinds straight past the end of this function. Reported inline,+ // both the terminal workload event and the proxy/request completion were+ // lost whenever an upstream died mid-stream while the client was healthy,+ // leaving the job "running" for the life of the broker. That is worse than+ // a stuck card: the scheduler counts queued and running by scheduledOn, so+ // the ghost permanently inflated that node's pending count and biased+ // routing away from a node whose only fault was a crashed engine.+ //+ // Client-side failures never had this problem. net/http cancels the+ // request context synchronously before a write error propagates+ // (checkConnErrorWriter), so the watcher above has always emitted by the+ // time the panic arrives. The upstream-died case is the one that had no+ // reporter at all.+ //+ // Registered after the reservation release above, so on unwind this runs+ // first and the node is still counted as loaded while we report.+ defer func() {+ aborted := recover()+ // An aborted copy is operational detail, not a verdict on the job. A+ // stream that already committed a 2xx is a completion whether or not+ // it reached its end, so this never feeds the terminal state below,+ // and proxyErr is left untouched for that reason.+ reportErr := proxyErr+ if aborted != nil && reportErr == "" {+ reportErr = "upstream stream aborted mid-response"+ }+ if committedSC != nil && finalStatus == 0 {+ // The panic skipped the post-loop assignment, but the committed+ // capture still carries the status the upstream sent.+ finalStatus = committedSC.status+ }++ slog.Debug("proxy request complete",+ "id", reqID,+ "node_id", servedNodeID,+ "method", r.Method,+ "path", r.URL.Path,+ "target", servedTarget,+ "status", finalStatus,+ "duration_ms", time.Since(start).Milliseconds(),+ "ttfb_ms", ttfbMs,+ "err", reportErr,+ )++ _ = f.notify("proxy/request", RequestEvent{+ ID: reqID,+ NodeID: servedNodeID,+ Method: r.Method,+ Path: r.URL.Path,+ Target: servedTarget,+ Status: finalStatus,+ Duration: time.Since(start).Milliseconds(),+ TTFB: ttfbMs,+ Error: reportErr,+ })++ // Terminal workload transition pairs with the workload:started emitted+ // at forward time above. Cancellation, an upstream/transport error, a+ // failed client write (dead/half-open client), or any non-2xx status is+ // a failure; a clean 2xx is a completion. The Workload carries the same+ // id so the broker (and peers) can collapse the start/finish pair.+ // Routed through emitTerminal so the disconnect watcher and this path+ // emit exactly once.+ if wl != nil {+ switch {+ case r.Context().Err() != nil:+ // The request was cancelled before it finished — either the+ // client disconnected or, on shutdown, we cancelled it to stop+ // the in-flight inference. A mid-stream cancel never reaches+ // ErrorHandler (the 200 headers are already sent), so without+ // this branch it would be misreported as completed. (The+ // watcher above usually beats us to it; emitTerminal makes that+ // a no-op.) Cancelled rather than failed: nothing went wrong+ // here, the requester stopped waiting.+ emitTerminal("cancelled", "request cancelled before completion")+ case committedSC != nil && committedSC.wroteErr != nil:+ // The response committed but a write to (or flush toward) the+ // client failed — typically the idle deadline tripping on a+ // dead/half-open client. The same event as the branch above,+ // differing only in how we noticed: a client that vanished+ // without a FIN never cancels the context, so the write+ // deadline is what surfaces it. Classified identically.+ emitTerminal("cancelled", "client connection lost: "+committedSC.wroteErr.Error())+ case proxyErr != "" || finalStatus >= http.StatusBadRequest:+ msg := proxyErr+ if msg == "" {+ msg = fmt.Sprintf("upstream returned HTTP %d", finalStatus)+ }+ emitTerminal("failed", msg)+ default:+ emitTerminal("completed", "")+ }+ }++ if aborted != nil {+ // Preserve ReverseProxy's contract with http.Server, which recovers+ // ErrAbortHandler silently and closes the connection. Swallowing it+ // here would leave the client waiting on a response that will never+ // be finished or closed.+ panic(aborted)+ }+ }()++ // repointWorkload records where the job is currently placed: a node id while+ // an attempt is in flight, empty between attempts. The scheduler counts+ // pending work by scheduledOn, so clearing it is what stops a node we have+ // given up on from still looking busy. The state stays "queued" throughout,+ // because "running" means the engine is generating and the broker's store+ // would reject a return to "queued" as a backwards transition.+ repointWorkload := func(nodeID string) {+ if wl == nil {+ return+ }+ wlMu.Lock()+ if terminated || wl.ScheduledOn == nodeID {+ wlMu.Unlock()+ return+ }+ wl.ScheduledOn = nodeID+ wl.Seq = nextWlSeq()+ snapshot := *wl+ wlMu.Unlock()+ p.emitWorkload(workloadSubmittedMethod, snapshot)+ }++ // releaseHeld drops the capacity claim. Called before every backoff: the+ // reservation map is process-wide and shared with the other engine's+ // facade, so holding a claim on a node we have stopped using would push+ // that engine's traffic away from a node that is in fact free. It is the+ // same reasoning as clearing scheduledOn, applied to the proxy's own+ // estimate rather than the scheduler's.+ releaseHeld := func() {+ resMu.Lock()+ defer resMu.Unlock()+ p.releaseReservation(held)+ held = reservation{}+ }+ // takeHeld re-reserves for a fresh round, replacing any claim still held.+ // reserveCandidate both orders the round and takes the claim, so a retry+ // that re-resolves has to go back through it.+ takeHeld := func(cands []candidate) []candidate {+ resMu.Lock()+ defer resMu.Unlock()+ p.releaseReservation(held)+ var out []candidate+ out, held = p.reserveCandidate(f, cands)+ return out+ }++ // Retry loop. Each iteration dispatches to one candidate; when a round's+ // candidates are exhausted it backs off and re-resolves, so a node that+ // recovered or a model that finished pulling becomes eligible mid-retry.+ //+ // Bounded by maxDispatchAttempts for dispatch failures and by jobDeadline+ // for the whole pre-commit life. Both bounds are checked before every+ // dispatch — the deadline never truncates an attempt already in flight,+ // because killing one seconds from its first token and then failing the job+ // for being out of time would spend the entire wait and discard the result.+ //+ // Retrying is only possible before the first byte reaches the client; past+ // the commit point we are bound to that node, and a stream truncated from+ // there is the caller's to resume. proxy/request-started fires at the commit+ // so it names the node that actually served. The self-forward guard lives in+ // resolveCandidates.+ // The budget is for inference only. Everything not in the engine's route+ // table is forwarded verbatim, and some of those are state-changing: a+ // POST /api/pull starts a model download. shouldRetry treats any 5xx as+ // retryable regardless of route, so without this split a failed pull would+ // be replayed up to five times — including against the node that just took+ // the work, since re-resolution can pick it again.+ //+ // Non-inference therefore gets one pass over the resolved candidates: no+ // re-resolution, no backoff, no deadline.+ maxDispatches := maxDispatchAttempts+ retryRounds := true+ if !isInf {+ maxDispatches = len(candidates)+ retryRounds = false+ }++ deadline := start.Add(jobDeadline)+ dispatches := 0+ committed := false+ respondedWithError := false+ round := candidates+ next := 0++ for !committed {+ if dispatches >= maxDispatches {+ break+ }+ if retryRounds && !time.Now().Before(deadline) {+ break+ }+ // Abandon-now: the requester has gone, so there is nobody left to+ // receive an answer and the remaining attempts are better spent on work+ // someone is waiting for. The watcher has already emitted the terminal+ // as cancelled; the guard on the exhaustion response below keeps this+ // from also inventing a reason for a client that stopped listening.+ if r.Context().Err() != nil {+ break+ }+ if next >= len(round) {+ if !retryRounds {+ // Single pass: the candidate list is the whole budget.+ break+ }+ // Round exhausted, or no owner was available. Either way the job is+ // on no node right now, so drop both the placement and the capacity+ // claim before waiting.+ repointWorkload("")+ releaseHeld()+ if !waitBeforeRetry(r.Context(), backoffFor(dispatches, time.Until(deadline))) {+ break+ }+ round = f.resolveCandidates(routingModel)+ if isInf && model != "" {+ round = takeHeld(round)+ }+ next = 0+ if len(round) == 0 {+ // Every owner went away mid-flight. Not a rejection: keep+ // waiting for one to come back. This consumes no attempt, which+ // is why jobDeadline is the only bound on it.+ continue+ }+ }+ cand := round[next]+ next+++ dispatches+++ // lastPermitted means no further dispatch can happen, so this attempt's+ // failure is the client's answer. It is not "last candidate in the+ // round" any more: another round may follow.+ lastPermitted := dispatches >= maxDispatches ||+ (retryRounds && !time.Now().Before(deadline))+ last := lastPermitted+ repointWorkload(cand.id)+ // The claim follows the node we are about to try, not the one+ // reserveCandidate happened to pick for the round — an attempt can hold+ // a node for the whole first-content budget, and for that time it is+ // the node under load. A same-node move is a no-op.+ moveHeld(cand.id)++ // Each attempt runs on its own context, a child of the request's, so we+ // can abandon this dispatch without touching the request itself. The+ // distinction is load-bearing: the disconnect watcher and the terminal+ // classification both read r.Context(), the parent, so cancelling a+ // child cannot be mistaken for the client going away.+ attemptCtx, cancelAttempt := context.WithCancel(r.Context())+ // claim decides, once, whether this attempt commits or is abandoned;+ // see attemptClaim for why a pair of channels could not.+ claim := &attemptClaim{}++ // Watch the target while the attempt is pre-commit. A node that has+ // dropped out of discovery is not coming back to answer a request+ // already sent to it, and without this the attempt sat out the whole+ // first-content budget before failing over — while the broker had+ // already stamped the record failed from its own node-loss sweep, so+ // the UI said failed while the client still waited.+ //+ // Polling rather than subscribing keeps this out of Discovery's+ // contract, and the interval only bounds detection latency. Watching+ // stops at the commit point, because cancelling after that would kill a+ // stream that is working.+ //+ // Read the interval once, on this goroutine: the watcher must not+ // depend on a value that could change under it mid-attempt.+ watchEvery := targetWatchInterval+ go func() {+ t := time.NewTicker(watchEvery)+ defer t.Stop()+ for {+ select {+ case <-attemptCtx.Done():+ return+ case <-t.C:+ // The commit path may have claimed the outcome since the+ // last tick, in which case this attempt is no longer ours+ // to cancel.+ if claim.settled() {+ return+ }+ if f.discovery.Has(cand.id) {+ continue+ }+ if claim.abandon() {+ cancelAttempt()+ }+ return+ }+ }+ }()++ // ReverseProxy derives the outbound request from the one it is given,+ // so the attempt context has to travel on a copy. The replayed body+ // goes on the copy for the same reason.+ attemptReq := r.WithContext(attemptCtx)if bodyBytes != nil {- r.Body = io.NopCloser(bytes.NewReader(bodyBytes))+ attemptReq.Body = io.NopCloser(bytes.NewReader(bodyBytes))}retry := falsesc := &statusCapture{ResponseWriter: w, status: http.StatusOK, idle: idleClientWriteTimeout}@@ -1089,6 +1627,65 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {retry = truereturn retrySignal{}}+ // Hold the commit until the engine actually produces content.+ // Only for inference: a control endpoint's headers mean what+ // they say, and the model-list fan-out has its own client.+ // Waiting here costs a non-streaming response nothing, since+ // its headers and body arrive together, so the request's own+ // stream flag never has to be consulted — which also avoids+ // guessing per-dialect defaults (Ollama streams by default,+ // the OpenAI routes do not).+ var peeked io.ReadCloser+ if isInf {+ body, err := awaitFirstBody(resp.Body, firstBodyTimeout)+ if err != nil {+ // This response must not commit, whether or not another+ // attempt is permitted. awaitFirstBody's reader is still+ // blocked on resp.Body, so committing would put two+ // readers on one stream — the peek could swallow a byte+ // the client never sees — and against a silent upstream+ // the copy would then block with nothing left to+ // interrupt it, hanging the request past every bound we+ // set.+ //+ // Returning an error is what releases that reader:+ // ReverseProxy closes resp.Body and calls ErrorHandler.+ proxyErr = err.Error()+ if !last {+ retry = true+ slog.Warn("proxy upstream produced no content, failing over",+ "id", reqID, "node_id", cand.id, "target", cand.url.Host,+ "path", r.URL.Path, "err", err)+ return retrySignal{}+ }+ slog.Warn("proxy upstream produced no content, retries exhausted",+ "id", reqID, "node_id", cand.id, "target", cand.url.Host,+ "method", r.Method, "path", r.URL.Path,+ "duration_ms", time.Since(start).Milliseconds(), "err", err)+ return err+ }+ peeked = body+ }+ // Claim the attempt before any commit side effect. Losing the+ // claim means the watcher has already cancelled this attempt,+ // so committing would stream through a context that is about to+ // die and truncate the response we just started.+ if !claim.commit() {+ proxyErr = errTargetGone.Error()+ if !last {+ retry = true+ slog.Warn("proxy target left discovery as the response committed, failing over",+ "id", reqID, "node_id", cand.id, "target", cand.url.Host,+ "path", r.URL.Path)+ return retrySignal{}+ }+ return errTargetGone+ }+ // The peeked byte goes back only once the commit is ours, so a+ // lost claim leaves the body untouched for ReverseProxy to close.+ if peeked != nil {+ resp.Body = peeked+ }// Prefer an engine-declared preflight policy so an exact origin plus// Allow-Credentials can pass a credentialed browser fetch. Engines// that publish no policy retain the proxy's permissive 204 fallback.@@ -1125,21 +1722,31 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {Path: r.URL.Path,Target: cand.url.Host,})- // workload:started was already emitted up front naming the- // first candidate. If failover landed us on a different- // node, re-point scheduledOn so the card — and the terminal- // completed/errored event, which carries the same wl — name- // the node that actually served. Guarded by wlMu against the- // disconnect watcher, and skipped once terminated so a late- // re-point can't resurrect a workload we've already failed.- // Move the capacity claim to the node that actually served,- // for the same reason: the node that refused the request is- // not doing the work and must not keep counting as loaded.+ // The claim already follows each dispatch, so this is a+ // no-op unless something reordered underneath us; it stays+ // as the belt-and-braces guarantee that the node that+ // actually served is the one counted as loaded.moveHeld(cand.id)+ // Record the commit here, not just after the loop: a copy+ // error aborts this handler by panic, and finalize needs to+ // know the response had committed (and on which capture) to+ // classify it.+ committedSC = sc+ // queued -> running: the engine is producing content, so+ // this is the first moment "running" is true. It also fixes+ // the placement on the node that actually served, which may+ // differ from the last node the queued updates named.+ // Guarded by wlMu against the disconnect watcher, and+ // skipped once terminated so a late transition cannot+ // resurrect a workload we have already finalized.if wl != nil {wlMu.Lock()- if !terminated && wl.ScheduledOn != cand.id {+ if !terminated {+ startedMs := time.Now().UnixMilli()+ wl.State = "running"+ wl.StartedAt = &startedMswl.ScheduledOn = cand.id+ wl.Seq = nextWlSeq()snapshot := *wlwlMu.Unlock()p.emitWorkload(workloadStartedMethod, snapshot)@@ -1169,7 +1776,8 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {"path", r.URL.Path, "err", err)return}- // Last candidate failed at the transport: terminal, surface it.+ // No further dispatch is permitted: terminal, surface it.+ respondedWithError = trueservedNodeID = cand.idservedTarget = cand.url.Hostif cors.WritePreflight(ew, r) {@@ -1177,7 +1785,7 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {return}proxyErr = err.Error()- slog.Warn("proxy upstream error, candidates exhausted",+ slog.Warn("proxy upstream error, retries exhausted","id", reqID, "node_id", cand.id, "target", cand.url.Host,"method", r.Method, "path", r.URL.Path,"duration_ms", time.Since(start).Milliseconds(), "err", err)@@ -1187,79 +1795,79 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {if mErr != nil {body = []byte(`{"error":"upstream error"}`)}+ // A node that answered but never produced content timed out+ // rather than failed to be reached, and 504 says so.+ status := http.StatusBadGateway+ if stderrors.Is(err, errFirstBodyTimeout) {+ status = http.StatusGatewayTimeout+ }cors.Apply(ew.Header())ew.Header().Set("Content-Type", "application/json")ew.Header().Set("X-Content-Type-Options", "nosniff")- ew.WriteHeader(http.StatusBadGateway)+ ew.WriteHeader(status)ew.Write(body)},}- proxy.ServeHTTP(sc, r)+ proxy.ServeHTTP(sc, attemptReq)+ cancelAttempt()+ if claim.abandoned() {+ // Name the real reason rather than the bare "context canceled" the+ // transport reports, which reads identically to a client hangup.+ proxyErr = errTargetGone.Error()+ }if !retry {finalStatus = sc.statuscommittedSC = sc- break+ committed = true}}- slog.Debug("proxy request complete",- "id", reqID,- "node_id", servedNodeID,- "method", r.Method,- "path", r.URL.Path,- "target", servedTarget,- "status", finalStatus,- "duration_ms", time.Since(start).Milliseconds(),- "ttfb_ms", ttfbMs,- "err", proxyErr,- )-- _ = f.notify("proxy/request", RequestEvent{- ID: reqID,- NodeID: servedNodeID,- Method: r.Method,- Path: r.URL.Path,- Target: servedTarget,- Status: finalStatus,- Duration: time.Since(start).Milliseconds(),- TTFB: ttfbMs,- Error: proxyErr,- })-- // Terminal workload transition pairs with the workload:started emitted at- // the commit point above. Cancellation, an upstream/transport error, a- // failed client write (dead/half-open client), or any non-2xx status is a- // failure; a clean 2xx is a completion. The Workload carries the same id so- // the broker (and peers) can collapse the start/finish pair. Routed through- // emitTerminal so the disconnect watcher and this path emit exactly once.- if wl != nil {+ // Nothing committed and no attempt wrote the client's answer, so the loop+ // ended between attempts: either the budget ran out or, more usually, every+ // owner went away and none came back before the deadline. The dispatch+ // paths answer for themselves — a final transport failure writes its own+ // 502 and a non-retryable status is passed through — so this is the only+ // outcome left without a response.+ //+ // An abandoned request is excluded: writing to a client that has gone+ // achieves nothing, and attributing its end to a missing node would be a+ // fabricated reason for something that was the requester's own choice.+ if !committed && !respondedWithError && r.Context().Err() == nil {+ // Name what the loop was actually doing when it ran out, which is what+ // the caller needs to know. The three cases are distinguishable: the+ // budget is gone; or the deadline passed while an owner was still there+ // to try, because slow attempts can reach 10 minutes before they reach+ // five dispatches; or the deadline passed with the candidate set empty,+ // which is the wait for an owner to come back.+ reason := "no node advertising the requested model became available before the retry deadline"switch {- case r.Context().Err() != nil:- // The request was cancelled before it finished — either the- // client disconnected or, on shutdown, we cancelled it to stop- // the in-flight inference. A mid-stream cancel never reaches- // ErrorHandler (the 200 headers are already sent), so without this- // branch it would be misreported as completed. (The watcher above- // usually beats us to it; emitTerminal makes that a no-op.)- emitTerminal("failed", "request cancelled before completion")- case committedSC != nil && committedSC.wroteErr != nil:- // The response committed but a write to (or flush toward) the- // client failed — typically the idle deadline tripping on a- // dead/half-open client. The stream is truncated, so this is a- // failure, not the completion the 200 status would otherwise- // suggest.- emitTerminal("failed", "client connection lost: "+committedSC.wroteErr.Error())- case proxyErr != "" || finalStatus >= http.StatusBadRequest:- msg := proxyErr- if msg == "" {- msg = fmt.Sprintf("upstream returned HTTP %d", finalStatus)- }- emitTerminal("failed", msg)- default:- emitTerminal("completed", "")+ case dispatches >= maxDispatches:+ reason = "every dispatch attempt failed"+ case len(round) > 0:+ reason = "the retry deadline passed while dispatch attempts were still failing"+ }+ proxyErr = reason+ finalStatus = http.StatusServiceUnavailable+ slog.Warn("proxy request exhausted its retry budget",+ "id", reqID, "method", r.Method, "path", r.URL.Path,+ "dispatches", dispatches, "duration_ms", time.Since(start).Milliseconds(),+ "reason", reason)+ body, mErr := json.Marshal(map[string]string{"error": reason})+ if mErr != nil {+ body = []byte(`{"error":"retry budget exhausted"}`)}+ cors.Apply(w.Header())+ w.Header().Set("Content-Type", "application/json")+ w.Header().Set("X-Content-Type-Options", "nosniff")+ // Advisory only: the caller knows its own patience better than we do.+ w.Header().Set("Retry-After", "1")+ w.WriteHeader(http.StatusServiceUnavailable)+ w.Write(body)}++ // Reporting happens in the deferred finalize above, so it survives the+ // ErrAbortHandler panic that a mid-copy error raises.}// resolveCandidates returns the ordered list of nodes to try for the currentdiff --git a/services/nvpair-proxy/retry_test.go b/services/nvpair-proxy/retry_test.gonew file mode 100644index 0000000..f3634f4--- /dev/null+++ b/services/nvpair-proxy/retry_test.go@@ -0,0 +1,586 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++package main++import (+ "context"+ "encoding/json"+ "io"+ "net/http"+ "net/http/httptest"+ "os"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++// TestMain shortens the retry backoff for the whole package. Production waits+// seconds between attempts, and a body that drives a node to its dispatch+// budget would otherwise pay all of it in wall clock — one all-nodes-down test+// alone went from under a second to nineteen. The schedule itself is asserted+// directly in TestBackoffFor, so shortening it here costs no coverage.+func TestMain(m *testing.M) {+ retryBackoff = []time.Duration{time.Millisecond}+ os.Exit(m.Run())+}++// newCountingServer is an upstream that answers every request the same way and+// counts how many it saw, which is the quantity the retry tests assert on: not+// what a node replied, but how many times it was contacted.+//+// The count comes back as a func so a caller cannot reset it, and reading it+// while requests are still in flight is safe.+func newCountingServer(t *testing.T, status int, body string) (serverURL string, hits func() int) {+ t.Helper()+ var n atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ n.Add(1)+ w.WriteHeader(status)+ io.WriteString(w, body)+ }))+ t.Cleanup(srv.Close)+ return srv.URL, func() int { return int(n.Load()) }+}++// waitForCond polls until cond holds, failing the test if it never does.+//+// The retry tests need to act at a point in a request's life — "the dispatch+// has reached the upstream", not "50ms have passed". On a loaded runner those+// are different statements, and a fixed sleep that loses the race usually makes+// the test pass while proving nothing.+func waitForCond(t *testing.T, timeout time.Duration, what string, cond func() bool) {+ t.Helper()+ deadline := time.Now().Add(timeout)+ for time.Now().Before(deadline) {+ if cond() {+ return+ }+ time.Sleep(time.Millisecond)+ }+ t.Fatalf("timed out after %v waiting for %s", timeout, what)+}++// decodeJSONBody parses a response body as a JSON object so an assertion can+// name the field it cares about, instead of substring-matching a serialization+// that a reordered marshal or a changed quote style would break.+func decodeJSONBody(t *testing.T, body string) map[string]any {+ t.Helper()+ var got map[string]any+ if err := json.Unmarshal([]byte(body), &got); err != nil {+ t.Fatalf("response body is not a JSON object: %v (body %q)", err, body)+ }+ return got+}++// setForTest overrides a package-level knob for one test and restores it after.+//+// The retry bounds are vars precisely so tests can shorten them. Registering+// the restore with t.Cleanup keeps it on the same line as the override and runs+// it even when the body ends in t.Fatal, which a hand-written defer pair in+// each test does not.+func setForTest[A any](t *testing.T, target *A, value A) {+ t.Helper()+ orig := *target+ t.Cleanup(func() { *target = orig })+ *target = value+}++// TestBackoffFor covers the production schedule, which TestMain has replaced+// for every other body in the package, so it puts the real one back first.+func TestBackoffFor(t *testing.T) {+ setForTest(t, &retryBackoff, defaultRetryBackoff)++ // Indexed by dispatches already made, so the wait after the first dispatch+ // is the first entry. A deadline an hour out keeps the remaining-time clamp+ // out of it. Jitter is up to ±25%, so assert a band.+ for _, tc := range []struct {+ dispatches int+ want time.Duration+ }{+ {dispatches: 1, want: time.Second},+ {dispatches: 2, want: 2 * time.Second},+ {dispatches: 3, want: 4 * time.Second},+ {dispatches: 4, want: 8 * time.Second},+ {dispatches: 9, want: 8 * time.Second}, // clamped to the last entry+ } {+ got := backoffFor(tc.dispatches, time.Hour)+ lo := tc.want - tc.want/4+ hi := tc.want + tc.want/4+ if got < lo || got > hi {+ t.Errorf("backoffFor(%d) = %v, want within ±25%% of %v", tc.dispatches, got, tc.want)+ }+ }+}++// TestBackoffFor_NeverOutlastsTheDeadline: a wait longer than the time left+// would guarantee the bounds check after it fails, so the delay would buy+// latency and no attempt.+func TestBackoffFor_NeverOutlastsTheDeadline(t *testing.T) {+ setForTest(t, &retryBackoff, defaultRetryBackoff)++ // Dispatch 3 alone would wait 4s.+ if got := backoffFor(3, 100*time.Millisecond); got > 100*time.Millisecond {+ t.Errorf("backoffFor with 100ms left = %v, want no more than 100ms", got)+ }+ if got := backoffFor(1, 0); got != 0 {+ t.Errorf("backoffFor with no time left = %v, want 0", got)+ }+}++// Committing an attempt and abandoning it because its target left discovery can+// become true at once, so the outcome is a single claim rather than a pair of+// signals (spec §5.3). These cover the claim directly, because the race window+// in the handler is not reachable deterministically.++func TestAttemptClaim_CommitWinsATie(t *testing.T) {+ var claim attemptClaim++ if !claim.commit() {+ t.Fatal("commit on a pending claim should win")+ }+ if claim.abandon() {+ t.Fatal("abandon after commit must lose: a delivered first byte is evidence the node is serving, and cancelling would truncate a working stream")+ }+ if claim.abandoned() {+ t.Fatal("a committed claim must not report as abandoned")+ }+}++func TestAttemptClaim_AbandonBlocksALaterCommit(t *testing.T) {+ var claim attemptClaim++ if !claim.abandon() {+ t.Fatal("abandon on a pending claim should win")+ }+ if claim.commit() {+ t.Fatal("commit after abandon must lose: the attempt is already being cancelled, so serving it would stream through a dying context")+ }+ if !claim.abandoned() {+ t.Fatal("an abandoned claim must report as abandoned so the caller can name the real reason")+ }+}++func TestAttemptClaim_CommitIsIdempotent(t *testing.T) {+ var claim attemptClaim++ if !claim.commit() || !claim.commit() {+ t.Fatal("a second commit from the same committed attempt should still report true")+ }+}++func TestAttemptClaim_SettledOnlyAfterAClaim(t *testing.T) {+ var pending, committed, abandoned attemptClaim+ committed.commit()+ abandoned.abandon()++ if pending.settled() {+ t.Error("an unclaimed attempt must read as unsettled, or the watcher would stop watching it")+ }+ if !committed.settled() || !abandoned.settled() {+ t.Error("a claimed attempt must read as settled so the watcher stops")+ }+}++// TestAttemptClaim_ExactlyOneWinnerUnderContention is the property the type+// exists for: with both sides racing, one and only one succeeds.+func TestAttemptClaim_ExactlyOneWinnerUnderContention(t *testing.T) {+ for range 500 {+ var claim attemptClaim+ var commits, abandons atomic.Int32++ start := make(chan struct{})+ var wg sync.WaitGroup+ wg.Add(2)+ go func() {+ defer wg.Done()+ <-start+ if claim.commit() {+ commits.Add(1)+ }+ }()+ go func() {+ defer wg.Done()+ <-start+ if claim.abandon() {+ abandons.Add(1)+ }+ }()+ close(start)+ wg.Wait()++ if got := commits.Load() + abandons.Load(); got != 1 {+ t.Fatalf("winners = %d, want exactly 1 (commit=%d abandon=%d)",+ got, commits.Load(), abandons.Load())+ }+ }+}++// TestHandleHTTP_RetriesTheOnlyOwner is the case the old loop could not serve.+// Bounded by the candidate count, a single-owner cluster got exactly one+// attempt and then a terminal error — which is the common single-desktop+// shape. The budget is a dispatch count now, so the same node is tried again.+func TestHandleHTTP_RetriesTheOnlyOwner(t *testing.T) {+ forEachEngine(t, func(t *testing.T, tc engineCase) {+ var hits atomic.Int32+ flaky := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if hits.Add(1) <= 2 {+ w.WriteHeader(http.StatusServiceUnavailable)+ io.WriteString(w, `{"error":"loading model"}`)+ return+ }+ w.WriteHeader(http.StatusOK)+ io.WriteString(w, `{"done":true}`)+ }))+ defer flaky.Close()++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "only", flaky.URL, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)++ rec := httptest.NewRecorder()+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())++ if rec.Code != http.StatusOK {+ t.Fatalf("status = %d, want 200 after retrying the only owner", rec.Code)+ }+ if got := hits.Load(); got != 3 {+ t.Fatalf("upstream saw %d dispatches, want 3 (two failures then a success)", got)+ }+ })+}++// TestHandleHTTP_StopsAtDispatchBudget: retrying is bounded. An owner that+// always fails is dispatched to exactly maxDispatchAttempts times, and the+// final attempt's own response is what the client gets — the proxy does not+// substitute a generated status when the budget is what ran out.+//+// The upstream answers 502 rather than 503 so those two outcomes are+// distinguishable: 503 is what the proxy writes when the loop ends between+// attempts (spec §5.4), so an upstream 503 here would match either behavior.+func TestHandleHTTP_StopsAtDispatchBudget(t *testing.T) {+ forEachEngine(t, func(t *testing.T, tc engineCase) {+ const upstreamBody = `{"error":"still loading"}`+ brokenURL, hits := newCountingServer(t, http.StatusBadGateway, upstreamBody)++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "broken", brokenURL, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)++ rec := httptest.NewRecorder()+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())++ if got := hits(); got != maxDispatchAttempts {+ t.Fatalf("upstream saw %d dispatches, want exactly %d", got, maxDispatchAttempts)+ }+ if rec.Code != http.StatusBadGateway {+ t.Fatalf("status = %d, want the final attempt's own 502 (503 would mean the proxy answered for it)", rec.Code)+ }+ if got := rec.Body.String(); got != upstreamBody {+ t.Fatalf("body = %q, want the upstream's %q passed through unchanged", got, upstreamBody)+ }+ })+}++// TestHandleHTTP_NonInferenceIsNotRetriedBeyondItsCandidates keeps the retry+// budget off state-changing control-plane routes (spec §5.1). A 5xx is+// retryable regardless of route, so without the bound a failed POST /api/pull+// would be replayed into repeated model downloads. One owner means one+// dispatch.+func TestHandleHTTP_NonInferenceIsNotRetriedBeyondItsCandidates(t *testing.T) {+ forEachEngine(t, func(t *testing.T, tc engineCase) {+ failingURL, hits := newCountingServer(t, http.StatusInternalServerError, `{"error":"pull failed"}`)++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "only", failingURL, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)++ rec := httptest.NewRecorder()+ req := httptest.NewRequest(http.MethodPost, tc.nonInferencePath, strings.NewReader(`{"name":"m"}`))+ p.soleFacade().handleHTTP(rec, req)++ if got := hits(); got != 1 {+ t.Fatalf("upstream saw %d dispatches for a non-inference POST, want exactly 1 (the retry budget must not replay a state-changing route)", got)+ }+ if rec.Code != http.StatusInternalServerError {+ t.Fatalf("status = %d, want the upstream 500 passed through", rec.Code)+ }+ })+}++// TestHandleHTTP_ReservationReleasedBetweenAttempts is the shared-scheduler+// half of the retry policy, and the reason it is not just a loop.+//+// The reservation map is process-wide: both engine facades estimate load from+// the same counts. A retry that kept its claim while sitting in a backoff would+// leave a node looking busy when nothing is running on it, and would push the+// OTHER engine's traffic away from a node that is in fact free. So the claim is+// dropped before every wait and re-taken on the next dispatch, which is the+// same reasoning that clears scheduledOn — applied to the proxy's own estimate+// rather than the scheduler's.+//+// Two failing owners, so the claim has to move as well as clear. A leak on the+// first node would make it look permanently loaded, which is observable twice:+// the count it is left holding, and the skew in how the five dispatches were+// split between two nodes that are equally broken.+func TestHandleHTTP_ReservationReleasedBetweenAttempts(t *testing.T) {+ tc := anyCase(t)++ firstURL, firstHits := newCountingServer(t, http.StatusServiceUnavailable, "")+ secondURL, secondHits := newCountingServer(t, http.StatusServiceUnavailable, "")++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "broken-a", firstURL, tc.advertisedModel))+ disc.AddManual(nodeForModel(t, "broken-b", secondURL, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)+ // A snapshot so both nodes are scheduler-listed and therefore reservable.+ p.SetPriority([]string{"broken-a", "broken-b"})++ rec := httptest.NewRecorder()+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())++ if got := firstHits() + secondHits(); got != maxDispatchAttempts {+ t.Fatalf("dispatches = %d (broken-a %d, broken-b %d), want %d in total",+ got, firstHits(), secondHits(), maxDispatchAttempts)+ }+ if firstHits() == 0 || secondHits() == 0 {+ t.Fatalf("dispatches = broken-a %d, broken-b %d, want both owners tried: a claim stuck on one node steers every later attempt away from it",+ firstHits(), secondHits())+ }++ p.priorityMu.Lock()+ leftover := map[string]int{+ "broken-a": p.priorityReservations["broken-a"],+ "broken-b": p.priorityReservations["broken-b"],+ }+ p.priorityMu.Unlock()+ for id, n := range leftover {+ if n != 0 {+ t.Errorf("reservations left on %s after the request ended = %d, want 0: a retry leaked its claim, so the node looks loaded to both engines forever", id, n)+ }+ }+}++// TestHandleHTTP_WaitsForAnOwnerWithoutSpendingAttempts pins the split between+// the two bounds. When every owner disappears mid-flight the request waits for+// one to come back, and that wait consumes no dispatch attempt — so the only+// thing that ends it is the deadline. Read against dispatch failures alone the+// deadline looks unreachable, which is exactly why this path needs a test: it+// is the reason the deadline is not dead configuration.+func TestHandleHTTP_WaitsForAnOwnerWithoutSpendingAttempts(t *testing.T) {+ setForTest(t, &jobDeadline, 1500*time.Millisecond)+ setForTest(t, &retryBackoff, []time.Duration{150 * time.Millisecond})++ tc := anyCase(t)++ // Present at admission so the request is accepted, but its server is+ // already closed, so the one dispatch it gets fails at the transport.+ dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))+ deadURL := dead.URL+ dead.Close()++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "gone", deadURL, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)++ rec := httptest.NewRecorder()+ start := time.Now()+ done := make(chan struct{})+ go func() {+ defer close(done)+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())+ }()++ // Drop the owner once that first dispatch has failed, so every later+ // re-resolution finds nothing at all.+ time.Sleep(75 * time.Millisecond)+ disc.RemoveManual("gone")++ select {+ case <-done:+ case <-time.After(10 * time.Second):+ t.Fatal("handleHTTP never returned")+ }+ elapsed := time.Since(start)++ if rec.Code != http.StatusServiceUnavailable {+ t.Fatalf("status = %d, want 503 once the deadline passed with no owner", rec.Code)+ }+ // The distinguishing evidence: the budget was never spent, so the reason is+ // the missing owner rather than exhausted dispatches.+ const want = "no node advertising the requested model became available before the retry deadline"+ if got := decodeJSONBody(t, rec.Body.String()); got["error"] != want {+ t.Fatalf("body error = %v, want %q (attempts must not have been consumed by empty resolutions)", got["error"], want)+ }+ // It waited rather than giving up at once, which is the point of the wait.+ if elapsed < jobDeadline/2 {+ t.Fatalf("returned after %v, want it to keep waiting for an owner until close to the %v deadline", elapsed, jobDeadline)+ }+}++// TestHandleHTTP_DeadlineWithAnOwnerPresentSaysSo is the other way to run out+// of time, and the one whose reason is easy to get wrong.+//+// A slow attempt can spend 240s — the header cap and the first-content cap back+// to back — so a job can reach the 10-minute deadline before it reaches five+// dispatches. The owner is still right there, so reporting that no node became+// available would blame a missing node for attempts that were made and failed.+func TestHandleHTTP_DeadlineWithAnOwnerPresentSaysSo(t *testing.T) {+ // A deadline the first slow attempt outlives, so the loop breaks on time+ // rather than on the dispatch budget, with the owner still advertising.+ setForTest(t, &firstBodyTimeout, 300*time.Millisecond)+ setForTest(t, &jobDeadline, 150*time.Millisecond)++ tc := anyCase(t)+ stalled := newStalledEngine(t)++ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "slow", stalled.url, tc.advertisedModel))+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)++ rec := httptest.NewRecorder()+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())++ if got := stalled.hits(); got == 0 || got >= maxDispatchAttempts {+ t.Fatalf("upstream saw %d dispatches, want between 1 and %d: the deadline, not the budget, must be what ended this",+ got, maxDispatchAttempts-1)+ }+ if rec.Code != http.StatusServiceUnavailable {+ t.Fatalf("status = %d, want 503", rec.Code)+ }+ const want = "the retry deadline passed while dispatch attempts were still failing"+ if got := decodeJSONBody(t, rec.Body.String()); got["error"] != want {+ t.Fatalf("body error = %v, want %q: an owner was advertising throughout, so a missing-node reason would be fabricated",+ got["error"], want)+ }+}++// TestHandleHTTP_TargetLeavingDiscoveryAbortsAttempt covers the in-flight+// abort, and with it the distinction the whole mechanism rests on.+//+// Without it, a node that vanished mid-request was noticed only when the+// first-content budget expired: the attempt sat for the full 120s while the+// broker's own node-loss sweep had already stamped the record failed, so the UI+// reported a failure the client was still waiting on. Here the budget is left+// long on purpose, so finishing quickly is only possible via the discovery+// watcher.+//+// The second assertion is the subtle half. Aborting an attempt cancels a+// context, and so does a client hanging up; if the two were confused, either a+// retarget would be reported as cancelled — losing a job that went on to+// succeed — or a real disconnect would keep burning attempts. The attempt+// context is a child, and both the disconnect watcher and the terminal+// classification read the parent, which is what keeps them apart.+func TestHandleHTTP_TargetLeavingDiscoveryAbortsAttempt(t *testing.T) {+ // Left long on purpose: if the watcher does not end the attempt, nothing+ // else will inside the test's patience.+ setForTest(t, &firstBodyTimeout, 30*time.Second)+ setForTest(t, &targetWatchInterval, 25*time.Millisecond)++ tc := anyCase(t)++ stalled := newStalledEngine(t)++ good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusOK)+ io.WriteString(w, `{"done":true}`)+ }))+ defer good.Close()++ rec := &recRW{}+ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "vanishing", stalled.url, tc.advertisedModel))+ disc.AddManual(nodeForModel(t, "survivor", good.URL, tc.advertisedModel))+ p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)+ p.soleFacade().SetSelected("vanishing") // deterministic: the doomed node first++ resp := httptest.NewRecorder()+ start := time.Now()+ done := make(chan struct{})+ go func() {+ defer close(done)+ p.soleFacade().handleHTTP(resp, tc.inferenceRequest())+ }()++ // Take the node away only once the attempt has actually reached the stalled+ // engine. Removing it first would leave the survivor as the only owner at+ // resolution time, and the request would succeed through it without the+ // watcher ever being involved — passing while proving nothing.+ waitForCond(t, 5*time.Second, "the first dispatch to reach the stalled engine",+ func() bool { return stalled.hits() > 0 })+ disc.RemoveManual("vanishing")++ select {+ case <-done:+ case <-time.After(20 * time.Second):+ t.Fatal("handleHTTP never returned: the target leaving discovery did not abort the attempt")+ }+ elapsed := time.Since(start)++ if elapsed > 10*time.Second {+ t.Fatalf("took %v: the attempt waited out the first-content budget instead of aborting on node loss", elapsed)+ }+ if resp.Code != http.StatusOK {+ t.Fatalf("status = %d, want 200 from the surviving node", resp.Code)+ }+ if !strings.Contains(resp.Body.String(), `"done":true`) {+ t.Fatalf("body came from the wrong node: %q", resp.Body.String())+ }+ if !rec.has(`"state":"completed"`) {+ t.Fatal("the job must complete: aborting an attempt is not the client disconnecting")+ }+ if rec.has(`"state":"cancelled"`) {+ t.Fatal("a retarget was misreported as cancelled: the attempt cancel was mistaken for the client leaving")+ }+}++// TestHandleHTTP_AbandonedRequestStopsRetrying: a client that has gone away+// should not keep a retry budget alive. Nobody is left to receive the answer,+// so the remaining attempts belong to work someone is waiting for.+func TestHandleHTTP_AbandonedRequestStopsRetrying(t *testing.T) {+ // Long enough that the cancel below lands inside the wait rather than+ // between two dispatches.+ setForTest(t, &retryBackoff, []time.Duration{300 * time.Millisecond})++ tc := anyCase(t)++ brokenURL, hits := newCountingServer(t, http.StatusServiceUnavailable, "")++ rec := &recRW{}+ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "broken", brokenURL, tc.advertisedModel))+ p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)++ ctx, cancel := context.WithCancel(context.Background())+ req := tc.inferenceRequest().WithContext(ctx)++ done := make(chan struct{})+ go func() {+ defer close(done)+ p.soleFacade().handleHTTP(httptest.NewRecorder(), req)+ }()++ // Abandon the request once the first dispatch has been made, which puts the+ // cancel inside the backoff before the second. Waiting on the count rather+ // than the clock is what makes the assertion below mean something: with no+ // dispatch yet observed there would be nothing for a retry to exceed.+ waitForCond(t, 5*time.Second, "the first dispatch to reach the upstream",+ func() bool { return hits() > 0 })+ afterFirst := hits()+ cancel()++ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("handleHTTP did not return after the client abandoned the request")+ }++ if got := hits(); got > afterFirst {+ t.Fatalf("upstream saw %d dispatches after the client left (was %d): an abandoned request must stop retrying", got, afterFirst)+ }+ if !rec.has(`"state":"cancelled"`) {+ t.Fatal("an abandoned request must terminate as cancelled")+ }+}diff --git a/services/nvpair-proxy/spec.md b/services/nvpair-proxy/spec.mdindex 69d486f..92bce19 100644--- a/services/nvpair-proxy/spec.md+++ b/services/nvpair-proxy/spec.md@@ -17,10 +17,62 @@ engine's own dialect, filtering to nodes that advertise the requested model,choosing among them, and failing over when one refuses.It is a data plane, not a policy component. Ranking belongs to-`nvpair-job-scheduler`, membership and trust to `nvpair-cluster-manager`,-lifecycle to `nvpair-engine-manager`. What this process owns is the choice of-*which advertised owner receives this request*, and the local accounting that-makes that choice good under bursts.+`nvpair-job-scheduler`, and cluster membership and trust to+`nvpair-cluster-manager`. `nvpair-engine-manager` owns an engine's *install and+process* lifecycle; the per-request *workload* lifecycle events in §5.5 are this+process's own. What it owns beyond those events is the choice of *which+advertised owner receives this request*, and the local accounting that makes+that choice good under bursts.++### 1.1 What a caller is promised++The proxy aims to get every inference request **started** on some node that can+serve it, and spends a bounded amount of effort doing so. Stated precisely:++> While budget remains (§5.1), a failure that happens *before* the engine begins+> producing output is retried rather than returned. Once output begins, the+> request is committed to that node and is the caller's to finish with.++Failures absorbed that way include a retryable status, a transport error, an+engine that answers its headers and then produces nothing, and a target that+leaves discovery mid-request. Waiting for an owner to *exist* costs no attempt+at all, so a node blinking out does not spend the budget.++Four things bound that aim, and a caller should design for them.++**Effort is bounded, and which bound ran out changes what the caller gets.**+Absorption holds only while budget remains. Past that there are two shapes, both+detailed in §5.4:++- **The dispatch budget ran out.** The final permitted attempt is never+ retried, so its own failure is the answer: the upstream's status and body+ passed through unchanged, or a `502` if it failed at the transport and a+ `504` if it answered and produced nothing.+- **The deadline passed between attempts, or no owner came back.** Nothing is+ in flight to answer, so the proxy generates a `503` with `Retry-After` that+ names which of those it was.++Either way this is a persistent, bounded attempt to start — not an assurance+that every submission eventually runs.++**"Started" is the guarantee, not "finished".** Commitment ends at the first+byte of response body (§5.2), so `completed` means the engine began generating,+not that a whole answer arrived.++**A truncated stream still reports `completed`**, for two reasons that compound.+To the caller it is the same situation either way — it holds partial output and+is the only party that can decide what to do with it, which is what the+retrying harness above this layer is for. And the proxy genuinely cannot tell a+truncated stream from a complete one: a streaming response carries no+`Content-Length`, and the end-of-turn marker is engine-specific, so recognising+it would mean parsing response bodies, which §9 forbids.++**A non-streaming request is the weakest case.** Its headers arrive only when+the response is complete, so "queued inside the engine" and "generating right+now" look identical from outside. The attempt cap therefore has to cover whole+generation, and when it expires the proxy cannot tell whether it abandoned a+stalled request or a working one — so retrying may discard real progress. That+is a genuine limitation, not a safety margin (§5.6).## 2. Scope@@ -54,8 +106,10 @@ single-valued flag carries only one plan.Between scheduler snapshots a facade takes short-lived **reservations** for workit has dispatched but that the scheduler has not yet observed. Those live in the-process. With a process per engine each held half the picture, so simultaneous-Ollama and LM Studio bursts could both pick the same node believing it idle.+process, and every facade shares them. If each engine had its own proxy+process, neither could see the other's reservations, so simultaneous bursts of+Ollama and LM Studio requests could both select the same node, each incorrectly+believing it was idle.Sharing the map is the reason the engines share a process.### 3.2 What sharing costs@@ -72,8 +126,10 @@ Inside the process the boundaries are finer, and deliberately so:| Panic handling a request | That request only; logged with stack, answered as an internal error || Process crash or exit | Every facade |-The first three used to be process-fatal, which was acceptable when the process-was one engine. They are contained now because they are not.+The first three failures are contained within their listed boundaries. Only a+process crash or exit takes down every facade. Containment matters here in a way+it would not for a single-engine process: one engine's bind race must not take+another engine's working listener with it.## 4. Addressing@@ -116,22 +172,340 @@ For a model-bearing inference request:4. Forward, failing over through the remaining candidates on a retryable statusor transport error. An upstream model `404` counts as retryable, becausepositive inventory can be stale.+5. When a round's candidates are exhausted, back off and re-resolve (§5.1),+ until the dispatch budget or the deadline runs out.An ineligible manual selection cannot override the capability gate, and failovernever broadens to an excluded node.+### 5.1 Retry bounds++| Bound | Value | Governs |+| --- | --- | --- |+| `maxDispatchAttempts` | 5 | how many dispatches may fail |+| `jobDeadline` | 10 minutes from request creation | how long new attempts may keep being started, including while waiting for an owner to exist |+| `firstBodyTimeout` | `proxyResponseTimeout` (120s) | one attempt's wait for first content, measured from the arrival of headers |++The retry budget applies only to inference requests. Routes not classified in+the engine's route table are still forwarded verbatim, and some perform+state-changing operations; for example, `POST /api/pull` starts a model+download. Since any `5xx` is retryable regardless of route, re-resolving and+redispatching such a request could start the operation several times, which is+both non-obvious and wasteful. For a non-inference request the proxy therefore+resolves candidates once and tries each at most once. It does not re-resolve+candidates, back off between rounds, or apply the inference deadline.++**An attempt is consumed by a dispatch, never by a resolution.** Finding no+eligible owner sends nothing, so a job whose only owner briefly drops out does+not burn its budget while the node comes back.++**Elapsed time per attempt is the sum of three budgets, not one.** They run in+sequence: the dial (`proxyDialTimeout`, 10s), then headers+(`proxyResponseTimeout`, 120s, enforced by the transport), then first content+(`firstBodyTimeout`, 120s, which starts only once headers arrive). One attempt+can therefore occupy around 240s — the two 120s caps back to back, since a slow+dial is a LAN rarity — and nothing caps their sum.++Both bounds are checked before every dispatch and **never truncate an attempt+already in flight**. Killing one seconds from its first token and then failing+the job for being out of time would spend the entire wait and discard the+result. `jobDeadline` is therefore a "no new attempts after" line rather than a+ceiling on elapsed time, and the worst case is the deadline plus one whole+attempt: a last dispatch starting just under 600s and running its full 240s+puts time to first content near fourteen minutes.++Which bound ends a job depends on how its attempts fail, and all three cases+are real:++- **Fast failures** — a refused connection, a prompt `503` — cost about a+ second, so all five attempts fit well inside the deadline and the dispatch+ budget is what ends the job. This is the common case.+- **Slow failures** — an engine that accepts and then withholds everything —+ cost close to a whole attempt each, so the deadline arrives first. At 240s+ per attempt the fourth dispatch would start past 600s, so the job ends after+ three attempts with budget unspent.+- **No owner at all** costs no attempt, so the deadline is the only thing that+ can end the wait.++The last case is why the deadline cannot be dropped as redundant with the+attempt count, and the middle one is why five attempts is a ceiling rather than+a promise. **Do not remove either bound as unreachable.**++#### Waiting for an owner++There is no memory of nodes seen in the past: every resolution runs against the+live discovery snapshot, so "eligible owner" always means "advertising this+model right now". What differs is the position in the request's life.++- **No owner at admission** — nothing has been dispatched, so there is nothing+ to wait for being interrupted. Return `502` immediately (§5.4). A model+ nobody advertises will not become servable by waiting.+- **No owner after at least one dispatch** — an owner existed a moment ago and+ went away, so the proxy waits for one to reappear, re-resolving on the backoff+ schedule. This costs no attempt, so `jobDeadline` is the only thing that ends+ it: the deadline *is* the grace period for a node coming back.++The asymmetry is deliberate. Waiting is justified by evidence that a capable+owner existed within this request's lifetime, and nothing weaker.++Candidates are re-resolved every round, so a node that recovered or a model that+finished pulling becomes eligible mid-retry. The same node may be retried — with+one owner, all five attempts go to it.++A requester that has gone away ends the retry immediately: nobody is left to+receive the answer, so the remaining attempts belong to work someone is waiting+for. Its end is reported as `cancelled` and is never attributed to a missing+node.++### 5.2 The commit point++**Commit is the first byte of response body, not the response headers.**++Headers are not a valid commit signal: they may show only that the engine+accepted the request. Treating them as commitment would disable the header+timeout and prevent failover while the request remains queued inside the+engine. The first content byte is the first engine-independent evidence that+generation has begun.++The difference is engine-specific and large. Measured against LM Studio 0.4.x+with a model loaded at `--parallel 1`, probing while a long generation held the+single slot:++| Request | Headers | First body byte |+| --- | --- | --- |+| streaming | 12ms, on accept | 35.5s, when the slot freed |+| non-streaming | 38.9s, withheld until complete | with the headers |++Nothing arrives during that wait — no SSE keepalive, no empty role delta — so a+byte is real content rather than a heartbeat.++**Only the existence of the byte matters, never its value.** The proxy does not+look at it: inspecting content would make the gate engine-specific, and it+would mean reading response data §9 forbids. Whatever the byte is, it is+spliced back onto the front of the stream so the client receives the response+intact.++`EOF` with no bytes counts as a commit. This is defensive rather than expected+— an inference route answering `200` with an empty body is an engine+misbehaving — but treating it as a stall would spend the whole budget waiting+for content that is never coming and then fail a request the engine considered+answered.++A first-content timeout **never** commits, including on the final permitted+attempt, where it is terminal with `504`. Committing there would leave the peek+reader blocked on the same body, racing two readers on one stream, and against+a silent upstream the copy would block with nothing left to interrupt it.++The gate is scoped to inference, and costs a non-streaming response nothing,+since its headers and body arrive together. That is why the request's own+`stream` flag never has to be consulted and no per-dialect default has to be+guessed, and it is what makes the boundary engine-independent — correct whether+an engine withholds headers or sends them on accept.++**After commit there is no retry.** A stream truncated from there is the+caller's to finish with (§1.1).++#### Known gap: a silent upstream after commit++There is no upstream read or idle deadline. If a committed engine keeps the+connection open and stops producing, nothing in this process ends the request:+the client write deadline needs a write to trip, and a silent upstream produces+nothing to write. A *crashed* engine is handled — the copy fails and §5.4's+deferred reporter emits the terminal — but a *frozen* one holds the request+until the client gives up.++Closing it needs a read deadline refreshed on every chunk, and a value chosen+so a slow-but-healthy generation is not cut off mid-answer. That judgement is+not made here.++### 5.3 Per-attempt cancellation++A watcher cancels an attempt promptly when its target leaves discovery.+Otherwise the attempt could stay blocked until the first-content timeout while+the broker's node-loss sweep had already marked the workload failed, so the UI+would report failure while the client was still waiting.++Each attempt runs on a context derived from the request's, and that child+relationship is load-bearing: the disconnect watcher and the outcome+classification both read the *parent*, so cancelling a child cannot be mistaken+for the requester leaving. Confusing the two would either report a job that went+on to succeed as `cancelled`, or let a real disconnect keep burning attempts.++**Commit and cancellation are mutually exclusive, decided by one atomic+claim.** Both can become true at once — the watcher's discovery check can pass+while the commit path is already running, and the work between the arrival of+the first byte and the end of the commit includes a synchronous notification+write and two mutex acquisitions. A pair of signals cannot express which+happened first, so the attempt has a single state that each side claims by+compare-and-swap; the loser becomes a no-op.++**A committed first byte wins that claim.** A delivered byte is direct evidence+that this node is serving this request, whereas an absence from discovery can be+a transient announcement gap. If the watcher claims first the attempt is+abandoned without committing, so the request retries rather than being served a+stream the proxy then truncates itself.++### 5.4 Status handling and outcome classification++Whether a status is retried depends on *when* it arrives, not only on its class.+Before commit the proxy may try elsewhere; at commit the status becomes the+client's answer.++| Upstream status | Pre-commit, budget remaining | Workload state if it is the answer |+| --- | --- | --- |+| `2xx` | commits (§5.2) | `completed` |+| `3xx` | not retried; not expected from an engine, and forwarded verbatim if one sends it | `failed` |+| `4xx` except `404`/`408`/`429` | not retried — the same request fails identically everywhere, so retrying only spends budget | `failed` |+| `404` | retried on an inference route only: positive model inventory can be stale | `failed` |+| `408`, `429` | retried | `failed` |+| `5xx` | retried | `failed` |++Anything not retried commits and becomes the client's answer immediately, with+no further dispatch, whatever budget is left.++These are the statuses the proxy itself returns:++| Condition | Status |+| --- | --- |+| No owner advertises the model, nothing dispatched | `502` |+| Final permitted attempt returned a status | that status and body, unchanged |+| Final permitted attempt failed at the transport | `502` |+| Final permitted attempt answered but produced no content | `504` |+| The loop ended between attempts, nothing in flight | `503` with `Retry-After` |++The final permitted attempt is never retried, so it answers for itself. That is+why exhausting the dispatch budget does not produce a `503`: the fifth attempt+is the answer. The `503` is for a loop that ended with no attempt to answer,+which means time ran out.++Its body names which, because the two call for different responses from the+caller:++- **the deadline passed while attempts were still failing** — an owner was+ present throughout, and slow attempts reached 10 minutes before they reached+ five dispatches (§5.1);+- **no owner became available** — the candidate set was empty and the wait for+ a node to come back timed out.++A third reason, **every dispatch attempt failed**, guards the case where the+loop ends on the count with nothing having answered. The paths above make that+unreachable today; it is cheap to keep and wrong to rely on.++The resulting workload state:++| Outcome | State |+| --- | --- |+| committed 2xx, streamed to the end | `completed` |+| committed 2xx, truncated by the node dying | `completed` |+| committed non-2xx | `failed` |+| client disconnected, or dead-client write deadline tripped | `cancelled` |+| cancelled by our own shutdown | `cancelled` |+| budget or deadline exhausted pre-commit | `failed` |++`cancelled` distinguishes requester-initiated termination from an actionable+engine, node, or routing failure. Client disconnects and shutdown cancellation+therefore do not appear in the failed bucket.++**Reporting is deferred.** A mid-copy error is never returned to the handler:+`httputil.ReverseProxy` converts it into `panic(http.ErrAbortHandler)` when+running under a real server. That panic bypasses ordinary code following the+proxy call, so terminal workload and request reporting must run in a `defer`.+The deferred reporter records the abort as operational detail, emits the+terminal event, and re-panics so `http.Server` still closes the connection.+Reservation release is also deferred so it survives the same unwind.++That panic is not the failure mode §3.2 describes, despite the shared word.+`http.ErrAbortHandler` is a sentinel `net/http` defines to mean "abandon this+response without logging a stack", and `http.Server` recovers it silently by+contract; re-panicking is how the connection gets closed, and swallowing it+would leave the client waiting on a response that will never be finished.+§3.2's entry covers an *unexpected* panic — a bug — which is logged with its+stack and answered as an internal error.++### 5.5 Workload lifecycle++| Event | When | State | `scheduledOn` |+| --- | --- | --- | --- |+| `workload:submitted` | admitted | `queued` | empty |+| `workload:submitted` | each dispatch | `queued` | the target |+| `workload:submitted` | between attempts | `queued` | empty |+| `workload:started` | commit | `running` | the node that served |+| `workload:completed` / `workload:errored` | terminal | per §5.4 | unchanged |++`submitted` covers the whole pre-commit life; `started` means the engine is+producing content. That split is forced as well as honest: the broker's store+merges by state rank and rejects a lower one, so a job that claimed `running` at+dispatch time could not return to `queued` for its next attempt and the retry+would be invisible. Keeping the loop inside one state, with `scheduledOn`+carrying the detail, is the only shape that store can represent.++Every event carries `seq`, the producer's event counter from 1. The+workload-manager's inter-node dedup is a permanent set, so a retry returning to+a placement it already used — `queued` on A, cleared, `queued` on A again, which+this loop produces routinely — would otherwise be indistinguishable from a+redelivery and dropped by every peer.++A job admitted but not yet dispatched has no execution node. Consumers must+treat an absent `scheduledOn` as "not placed" rather than assuming a node.++### 5.6 Coverage gap: non-streaming requests++A non-streaming request is the case the retry policy serves worst, and the+weakness is the opposite of too little retrying.++Its headers arrive only when the response is complete, so from outside the+engine "queued behind other work" and "generating right now" are+indistinguishable. The 120s header cap therefore has to cover whole generation+— the first-content wait adds nothing here, since the body arrives with the+headers — and when it expires the proxy retries without being able to tell+whether it just abandoned a stalled request or a working one. A generation+legitimately slower than that cap is restarted elsewhere, discarding real+progress, and the restart faces the same cap.++Nothing in the current engine surface fixes this: there is no progress signal+before the response is complete. A caller that cares should prefer a streaming+request, where the first content byte gives the proxy the evidence it lacks+here.++### 5.7 Accepted behavior++**An abandoned attempt may still run inside the engine.** Neither engine PAIR+supports today exposes a way to withdraw a request it has accepted, and both+queue internally, so a retried job can have more than one copy generating while+the abandoned copy keeps its node busy.++Whether closing the connection cancels anything upstream is **unverified** —+plausible, engine-specific, and never measured. The behavior above is stated as+the pessimistic case for that reason.++This is accepted rather than overlooked: those internal queues are invisible and+un-retargetable, which is the problem the retry policy works around rather than+a mechanism it can build on. Do not reach for engine environment bounds or+occupancy probes to "fix" it — LM Studio exposes no such signal at all, and an+adopted engine keeps its own environment regardless. An engine that did offer a+real cancellation API would change this calculus, and the claim should be+revisited per engine rather than assumed to hold forever.+## 6. ReservationsA reservation is one in-flight dispatch this process has made since the lastscheduler snapshot. Estimated load for a node is`pending + gpuPressure + reservations`, the first two from the scheduler.-- **Taken** when a candidate is chosen, unless an eligible manual pin applies.-- **Moved** to the node a failover actually lands on, because the node that- refused the request is not doing the work.+- **Taken** when a candidate is chosen, unless an eligible manual pin applies,+ and re-taken through the same path on each retry round (§5.1).+- **Moved** to the node an attempt actually targets, because the node that+ refused the request is not doing the work. An attempt can hold a node for the+ whole first-content budget, so the claim follows each dispatch rather than+ waiting for the commit.- **Released** when the request ends, so a node stops counting as loaded as soonas it stops working. Without release a trickle of short requests makes an idlenode read as busy for the rest of the snapshot interval.+- **Released before every retry backoff**, for the same reason applied to a+ waiting job: between attempts nothing is running, and because the map is+ process-wide a stale claim would push the *other* engine's traffic away from a+ node that is in fact free. This mirrors clearing `scheduledOn`, one being the+ proxy's own estimate and the other the scheduler's.Each is stamped with the snapshot **generation** it was counted against. Asnapshot supersedes every reservation taken before it — its pending countsdiff --git a/services/nvpair-proxy/workload_test.go b/services/nvpair-proxy/workload_test.goindex 07acdd8..a0dbe71 100644--- a/services/nvpair-proxy/workload_test.go+++ b/services/nvpair-proxy/workload_test.go@@ -37,19 +37,24 @@ func (r *recRW) has(s string) bool {return strings.Contains(string(r.b), s)}-// TestHandleHTTP_WorkloadStartedAtForward is a regression test: a job's-// workload:started must be emitted when the request is FORWARDED, not when the-// upstream first responds. Ollama serializes inference on a single GPU, so a-// burst of parallel submissions is forwarded concurrently but streams back one-// at a time; emitting at first byte made the queued submissions invisible-// (they appeared one card at a time). Here the upstream accepts the connection-// but never sends a byte until released, so a first-byte emission could never-// fire — yet workload:started must already be on the wire.+// TestHandleHTTP_WorkloadVisibleBeforeFirstByte is a regression test: a job+// must be on the wire as soon as it is admitted, not when the upstream first+// responds. An engine serializes inference on a single GPU slot, so a burst of+// parallel submissions is dispatched concurrently but streams back one at a+// time; emitting only at first byte made the waiting submissions invisible and+// they appeared one card at a time. Here the upstream accepts the connection+// but never sends a byte until released, so a first-byte emission could not+// have fired — yet the job must already be visible.//-// proxy/request-started intentionally stays at the commit point (so it names-// the node that actually served after any failover); only workload:started-// moves to forward time, so this test asserts only that.-func TestHandleHTTP_WorkloadStartedAtForward(t *testing.T) {+// The event is workload:submitted, carrying state "queued". "running" is+// reserved for the commit point, when the engine is actually producing content:+// a job waiting its turn inside the engine is not running, and the broker's+// store rejects a backwards transition, so a job that claimed "running" at+// dispatch time could never return to "queued" for a retry.+//+// proxy/request-started also stays at the commit point, so it names the node+// that actually served after any failover.+func TestHandleHTTP_WorkloadVisibleBeforeFirstByte(t *testing.T) {forEachEngine(t, func(t *testing.T, tc engineCase) {received := make(chan struct{}, 1)release := make(chan struct{})@@ -92,13 +97,28 @@ func TestHandleHTTP_WorkloadStartedAtForward(t *testing.T) {}// The upstream has the request but has sent no response byte, so a- // first-byte emission could not have fired. workload:started must- // still be on the wire — that is the forward-time emission.- if !waitFor(t, rec, "workload:started") {- t.Fatal("workload:started not emitted at forward time")+ // first-byte emission could not have fired. The job must still be+ // visible, as queued.+ if !waitFor(t, rec, "workload:submitted") {+ t.Fatal("workload:submitted not emitted when the request was admitted")+ }+ if !rec.has(`"state":"queued"`) {+ t.Fatal("an admitted job must be queued until the engine produces content")+ }+ if rec.has("workload:started") {+ t.Fatal("workload:started emitted before the engine produced any content")}doRelease()<-done++ // Released: the engine produced content, so the job is now running and+ // then completes. queued -> running -> completed, never backwards.+ if !waitFor(t, rec, "workload:started") {+ t.Fatal("workload:started not emitted once the engine produced content")+ }+ if !rec.has(`"state":"running"`) {+ t.Fatal("the commit point must transition the job to running")+ }})}diff --git a/services/nvpair-proxy/zombie_test.go b/services/nvpair-proxy/zombie_test.goindex 372d8f3..306bfff 100644--- a/services/nvpair-proxy/zombie_test.go+++ b/services/nvpair-proxy/zombie_test.go@@ -64,14 +64,14 @@ func (c *clientGoneWriter) Write(b []byte) (int, error) {func (c *clientGoneWriter) Flush() {}-// TestHandleHTTP_ClientWriteError_MarksFailed is the zombie-job regression: a+// TestHandleHTTP_ClientWriteError_MarksCancelled is the zombie-job regression: a// streaming inference response that has committed (200 headers sent) but whose// body write to the client fails — the signature of a killed / half-open client-// whose write deadline tripped — must terminate the workload as FAILED, not be-// silently reported completed. Before the fix the terminal event fired only in-// the post-handler switch, whose default arm (no ctx cancel, no upstream error,-// 2xx status) marked it completed, so the truncated stream looked successful.-func TestHandleHTTP_ClientWriteError_MarksFailed(t *testing.T) {+// whose write deadline tripped — must terminate the workload as `cancelled`,+// not be silently reported completed. A default arm that sees no ctx cancel, no+// upstream error, and a 2xx status would call this a success and leave a+// truncated stream looking clean.+func TestHandleHTTP_ClientWriteError_MarksCancelled(t *testing.T) {upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {w.Header().Set("Content-Type", "application/json")w.WriteHeader(http.StatusOK)@@ -106,7 +106,10 @@ func TestHandleHTTP_ClientWriteError_MarksFailed(t *testing.T) {t.Fatalf("workload:errored emitted %d times, want exactly 1", got)}if rec.has("workload:completed") {- t.Fatal("workload:completed emitted for a request whose client write failed (should be failed)")+ t.Fatal("workload:completed emitted for a request whose client write failed")+ }+ if !rec.has(`"state":"cancelled"`) {+ t.Fatal("a client that vanished mid-stream must terminate as cancelled, not failed")}}@@ -409,7 +412,10 @@ func TestHandleHTTP_RealSocketWriteDeadline(t *testing.T) {t.Fatal("workload never terminated after the client stopped reading (zombie: write deadline did not trip / no terminal emitted)")}if rec.has("workload:completed") {- t.Fatal("workload:completed emitted for a client that stopped reading (should be failed)")+ t.Fatal("workload:completed emitted for a client that stopped reading")+ }+ if !rec.has(`"state":"cancelled"`) {+ t.Fatal("a client that stopped reading must terminate as cancelled, not failed")}}@@ -507,6 +513,117 @@ func TestHandleHTTP_RealSocketFlushDeadline(t *testing.T) {t.Fatal("workload never terminated after the client stopped reading (flush path not deadline-aware)")}if rec.has("workload:completed") {- t.Fatal("workload:completed emitted for a stalled client (should be failed)")+ t.Fatal("workload:completed emitted for a stalled client")+ }+ if !rec.has(`"state":"cancelled"`) {+ t.Fatal("a stalled client must terminate as cancelled, not failed")+ }+}++// TestHandleHTTP_UpstreamDiesMidStream_EmitsTerminal is the other half of the+// zombie-job story, and the half no in-process test can express. Here the+// CLIENT stays healthy — it reads everything the proxy sends — and the+// UPSTREAM dies mid-body instead.+//+// That combination defeats both existing reporters. Nothing cancels the+// request context, because no write to the client ever fails, so the+// disconnect watcher never fires. And the copy error is never returned to us:+// ReverseProxy converts a mid-copy failure into panic(http.ErrAbortHandler)+// whenever it detects a real server, and that panic unwinds straight past the+// end of handleHTTP. A terminal emitted inline is therefore lost outright and+// the workload stays "running" for the life of the broker. The cost is not+// just a stuck card: the scheduler counts queued and running by scheduledOn,+// so the ghost permanently inflates that node's pending count and biases+// routing away from a node whose only sin was a crashed engine.+//+// A real http.Server is essential. shouldPanicOnCopyError keys off+// ServerContextKey, so calling handleHTTP directly — as the in-process tests+// above do — suppresses the panic and cannot reach this path at all.+//+// The response committed a 2xx before truncating, so the terminal is a+// COMPLETION: a cut-off stream is a success the caller resumes from whatever+// it received, not a cluster failure.+func TestHandleHTTP_UpstreamDiesMidStream_EmitsTerminal(t *testing.T) {+ // A raw listener rather than httptest, so the abrupt close is exact: a+ // chunked response that stops without its terminating zero-length chunk,+ // which is what the proxy sees when an engine is killed mid-generation.+ // Accepting in a loop keeps the address probe in targetURL from consuming+ // the one connection the request needs.+ upLn, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen upstream: %v", err)+ }+ defer upLn.Close()+ go func() {+ for {+ conn, err := upLn.Accept()+ if err != nil {+ return+ }+ go func(c net.Conn) {+ br := bufio.NewReader(c)+ for {+ line, err := br.ReadString('\n')+ if err != nil {+ c.Close()+ return+ }+ if line == "\r\n" {+ break+ }+ }+ chunk := `{"response":"partial tokens","done":false}`+ io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nTransfer-Encoding: chunked\r\n\r\n")+ fmt.Fprintf(c, "%x\r\n%s\r\n", len(chunk), chunk)+ c.Close()+ }(conn)+ }+ }()++ tc := anyCase(t)+ rec := &recRW{}+ disc := NewDiscovery()+ disc.AddManual(nodeForModel(t, "node-a", "http://"+upLn.Addr().String(), tc.advertisedModel))+ p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)++ ln, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatalf("listen proxy: %v", err)+ }+ srv := &http.Server{Handler: http.HandlerFunc(p.soleFacade().handleHTTP)}+ go func() { _ = srv.Serve(ln) }()+ defer srv.Close()++ conn, err := net.Dial("tcp", ln.Addr().String())+ if err != nil {+ t.Fatalf("dial proxy: %v", err)+ }+ defer conn.Close()++ body := tc.inferenceBody()+ reqText := fmt.Sprintf("POST %s HTTP/1.1\r\n", tc.inferencePath) ++ "Host: localhost\r\n" ++ "Content-Type: application/json\r\n" ++ fmt.Sprintf("Content-Length: %d\r\n", len(body)) ++ "\r\n" + body+ if _, err := conn.Write([]byte(reqText)); err != nil {+ t.Fatalf("write request: %v", err)+ }++ // Keep the client healthy by draining until the proxy hangs up. A client+ // that stopped reading would trip the write deadline and cancel the+ // context, which is the case the tests above cover and would mask this one.+ go func() { _, _ = io.Copy(io.Discard, conn) }()++ deadline := time.Now().Add(5 * time.Second)+ terminals := func() int { return rec.count("workload:completed") + rec.count("workload:errored") }+ for time.Now().Before(deadline) && terminals() == 0 {+ time.Sleep(10 * time.Millisecond)+ }+ if got := terminals(); got != 1 {+ t.Fatalf("terminal workload events = %d, want exactly 1 (zombie: the ErrAbortHandler panic skipped the terminal)", got)+ }+ if !rec.has("workload:completed") {+ t.Fatal("a truncated stream that had already committed 2xx must terminate as completed, not failed")}}diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.goindex cf43461..3dcb6f7 100644--- a/services/nvpair-ui-broker/broker.go+++ b/services/nvpair-ui-broker/broker.go@@ -1011,7 +1011,12 @@ func stateReplayMethod(state string) string {return "workload:started"case "completed":return "workload:completed"- case "failed":+ // "cancelled" rides workload:errored, the same frame the proxy emits it on:+ // the manager does not validate method against state, and consumers read+ // the state out of the payload. Every state the store can hold has to+ // appear here, because an omitted one is silently left out of the replay+ // and drops off the manager's re-sync set across a supervised restart.+ case "failed", "cancelled":return "workload:errored"default:return ""diff --git a/services/nvpair-ui-broker/workloadstore/store.go b/services/nvpair-ui-broker/workloadstore/store.goindex 154d56b..4af3c49 100644--- a/services/nvpair-ui-broker/workloadstore/store.go+++ b/services/nvpair-ui-broker/workloadstore/store.go@@ -548,21 +548,26 @@ func recordFrom(in Incoming, now time.Time) Record {}}-// isTerminal reports whether a state ends a workload's life.+// isTerminal reports whether a state ends a workload's life. A state missing+// from here is treated as live forever: it would keep counting toward the+// scheduler's pending total for its node and stay exposed to the staleness+// sweeps, so every terminal state must be listed.func isTerminal(state string) bool {- return state == "completed" || state == "failed"+ return state == "completed" || state == "failed" || state == "cancelled"}// rank orders lifecycle states so state only moves forward within a-// generation: queued < running < {completed, failed}. "initializing" and any-// unknown value rank below queued (they're never transmitted; see spec §4).+// generation: queued < running < {completed, failed, cancelled}. An unknown+// value ranks below queued, which means a state missing from here is not+// merely mis-sorted — applyLocked rejects every event carrying it, so the+// transition is dropped and never reaches the scheduler or any client.func rank(state string) int {switch state {case "queued":return 1case "running":return 2- case "completed", "failed":+ case "completed", "failed", "cancelled":return 3default:return 0diff --git a/services/nvpair-ui-broker/workloadstore/store_test.go b/services/nvpair-ui-broker/workloadstore/store_test.goindex 5ee5bb3..d515e88 100644--- a/services/nvpair-ui-broker/workloadstore/store_test.go+++ b/services/nvpair-ui-broker/workloadstore/store_test.go@@ -118,6 +118,38 @@ func TestApplyEqualRankReemit(t *testing.T) {}}+// TestApplyCancelledIsTerminal covers the two gates a new terminal state has+// to pass, both of which fail silently and in opposite directions. Omitted from+// rank() it sorts below queued, so applyLocked rejects the event outright and+// the transition never reaches the scheduler or any client. Omitted from+// isTerminal() it is accepted but treated as live forever, so it keeps counting+// toward its node's pending total and stays exposed to the staleness sweeps.+func TestApplyCancelledIsTerminal(t *testing.T) {+ s := New()+ if !s.Apply(mkIn("1", "a", "running", "node1", 100)) {+ t.Fatal("initial running should be accepted")+ }+ // Ranks above running, so the transition is accepted rather than dropped.+ if !s.Apply(mkIn("1", "a", "cancelled", "node1", 100)) {+ t.Fatal("cancelled after running should be accepted (missing from rank(): sorts below queued and is rejected)")+ }+ // Terminal, so a late running cannot resurrect it.+ if s.Apply(mkIn("1", "a", "running", "node1", 100)) {+ t.Fatal("running after cancelled should be rejected as backwards")+ }+ // Terminal, so it drops out of the active set rather than counting as load.+ r, ok := s.Get("a", "1")+ if !ok {+ t.Fatal("record should still be stored")+ }+ if !r.Terminal {+ t.Fatal("cancelled must be terminal (missing from isTerminal(): counts as pending forever and stays exposed to the staleness sweeps)")+ }+ if got := len(s.ActiveSnapshot()); got != 0 {+ t.Fatalf("active snapshot has %d entries, want 0 for a cancelled workload", got)+ }+}+// TestApplyCrossNodeIsolation: the same numeric id from two origins is two// distinct workloads and must never merge against each other.func TestApplyCrossNodeIsolation(t *testing.T) {diff --git a/services/nvpair-workload-manager/README.md b/services/nvpair-workload-manager/README.mdindex 474d938..3a76cfd 100644--- a/services/nvpair-workload-manager/README.md+++ b/services/nvpair-workload-manager/README.md@@ -81,13 +81,14 @@ Defined in [`workload.go`](workload.go):| --- | --- || `id` | Stable workload identifier || `model`, `engine` | What was requested and by which engine |-| `runId` | Optional grouping key |-| `state` | `initializing`, `queued`, `running`, `completed`, or `failed` |+| `runId` | Producing process's nonce, minted at proxy startup. Part of the dedup key: `id` is a per-process counter that both engine proxies start at 1 and that resets on restart, so `runId` keeps a reused id from colliding with an older workload |+| `state` | `queued`, `running`, `completed`, `failed`, or `cancelled` || `originatedFrom` | Node the request entered the cluster on || `scheduledOn` | Node it was routed to; absent until a target is chosen || `createdAt`, `startedAt`, `completedAt` | Epoch milliseconds; the last two are nullable || `error` | Normalized failure text, nullable || `requesterId` | Optional client attribution, nullable |+| `seq` | Producer's event counter, from 1. Part of the dedup key, so a workload that returns to a placement it already had is not mistaken for a redelivery |Optional and nullable fields use pointers so a peer's payload round-trips withoutinventing zero values.diff --git a/services/nvpair-workload-manager/dedup.go b/services/nvpair-workload-manager/dedup.goindex c265a62..4d44dd5 100644--- a/services/nvpair-workload-manager/dedup.go+++ b/services/nvpair-workload-manager/dedup.go@@ -5,6 +5,7 @@ package mainimport ("container/list"+ "strconv""sync")@@ -19,12 +20,14 @@ const defaultDedupCapacity = 10000// request.//// Keys are opaque strings built by the caller: lifecycle events key on-// nodeId + Workload.id + state, removals key on workloadId (see keyLifecycle-// / keyRemove). nodeId is part of the lifecycle key because Workload.id is-// only unique per-node (spec §11) — without it, the same id from two nodes-// would collide. Keying on (nodeId, id, state) means a re-broadcast carrying-// the same triple but updated metadata is treated as a duplicate and dropped,-// not merged — a known and accepted granularity trade-off (spec §4).+// nodeId + Workload.id + state + scheduledOn, removals key on workloadId (see+// keyLifecycle / keyRemove). nodeId is part of the lifecycle key because+// Workload.id is only unique per-node (spec §11) — without it, the same id+// from two nodes would collide. The key still omits the workload's other+// mutable metadata, so a re-broadcast carrying an updated field it does not+// cover — a corrected error string, say — is treated as a duplicate and+// dropped rather than merged. That remains a known granularity trade-off+// (spec §4).type dedupIndex struct {mu sync.Mutexcapacity int@@ -69,14 +72,34 @@ func (d *dedupIndex) seenOrAdd(key string) bool {// keyLifecycle builds the dedup key for a lifecycle event. Workload.id is only// a per-process counter, and both engine proxies count from 1 and reset on-// restart, so the full identity is (originatedFrom, engine, runId, id): nodeId-// disambiguates nodes, engine + the per-process runId nonce disambiguate the-// two local engines and successive proxy runs, and state completes the-// lifecycle-step key. Dropping any component would collapse distinct workloads-// (e.g. a concurrent Ollama + LM Studio job both id "1") and silently discard a-// legitimate peer event.+// restart, so the workload's identity is (originatedFrom, engine, runId, id):+// nodeId disambiguates nodes, and engine plus the per-process runId nonce+// disambiguate the two local engines and successive proxy runs. Dropping any+// component would collapse distinct workloads (e.g. a concurrent Ollama + LM+// Studio job both id "1") and silently discard a legitimate peer event.+//+// seq then completes the key, and it has to be the producer's event sequence+// rather than any property of the workload's current shape. This index is a+// permanent set, so anything derived from shape collides as soon as a workload+// revisits a shape it already had — and a retry does that routinely: queued on+// A, placement cleared between attempts, then queued on A again. Keyed on+// (state, scheduledOn) the third event matched the first and every peer+// dropped it, leaving their brokers holding the interim unplaced record while+// the job actually ran on A, so their schedulers stopped counting it against+// the node doing the work.+//+// state and scheduledOn stay in the key as well. They cost nothing, and they+// keep the key meaningful for a producer that has not stamped a sequence.+//+// This does not weaken what the index is for: a broadcast retry resends an+// identical frame, sequence included, so a true redelivery still dedups — and+// that is what protects a consumer from an out-of-order retry reverting a+// placement. A re-sync heartbeat bypasses this index altogether (see+// isResyncFrame).func keyLifecycle(w *Workload) string {- return "wl\x00" + w.OriginatedFrom + "\x00" + w.Engine + "\x00" + w.RunID + "\x00" + w.ID + "\x00" + string(w.State)+ return "wl\x00" + w.OriginatedFrom + "\x00" + w.Engine + "\x00" + w.RunID + "\x00" + w.ID ++ "\x00" + string(w.State) + "\x00" + w.ScheduledOn ++ "\x00" + strconv.FormatInt(w.Seq, 10)}// keyRemove builds the dedup key for a removal: nodeId + workloadId. As withdiff --git a/services/nvpair-workload-manager/dedup_test.go b/services/nvpair-workload-manager/dedup_test.goindex 557f54d..6c4e8d6 100644--- a/services/nvpair-workload-manager/dedup_test.go+++ b/services/nvpair-workload-manager/dedup_test.go@@ -96,6 +96,75 @@ func TestDedupDistinguishesEngineAndRun(t *testing.T) {}}+// TestDedupDistinguishesPlacement guards the re-point: a failover or a retry+// moves a workload to another node while its state stays "running", so the two+// events differ on scheduledOn and nothing else. With scheduledOn out of the+// key the second was dropped as a duplicate, and every peer went on showing+// the job on the node it was first sent to for the rest of its life. A repeat+// of the same placement must still dedup, which is what keeps a broadcast+// retry from being re-applied.+func TestDedupDistinguishesPlacement(t *testing.T) {+ d := newDedupIndex(8)++ first := &Workload{ID: "1", OriginatedFrom: "host", Engine: "ollama", RunID: "r1", State: StateRunning, ScheduledOn: "node-A"}+ repointed := &Workload{ID: "1", OriginatedFrom: "host", Engine: "ollama", RunID: "r1", State: StateRunning, ScheduledOn: "node-B"}++ if d.seenOrAdd(keyLifecycle(first)) {+ t.Fatal("first placement on node-A should be new")+ }+ if d.seenOrAdd(keyLifecycle(repointed)) {+ t.Fatal("a re-point to node-B differs only in scheduledOn and must not dedup against node-A")+ }+ if !d.seenOrAdd(keyLifecycle(first)) {+ t.Fatal("a resent frame for the node-A placement should still dedup")+ }+}++// TestDedupDistinguishesRepeatedPlacements is the reason the key carries the+// producer's event sequence rather than only the workload's current shape.+//+// This index is a permanent set, so any shape-derived key collides as soon as a+// workload revisits a shape it already had — and the retry loop does that+// routinely: queued on A, placement cleared between attempts, then queued on A+// again. Keyed on shape alone the third event matched the first, so every peer+// dropped it and their brokers kept the interim unplaced record while the job+// was really running on A, which took it out of A's pending load.+//+// The redelivery case still has to dedup, because that is what stops an+// out-of-order broadcast retry from reverting a placement.+func TestDedupDistinguishesRepeatedPlacements(t *testing.T) {+ d := newDedupIndex(8)++ base := func(seq int64, node string) *Workload {+ return &Workload{+ ID: "1", OriginatedFrom: "host", Engine: "ollama", RunID: "r1",+ State: StateQueued, ScheduledOn: node, Seq: seq,+ }+ }++ onA := base(1, "node-A")+ cleared := base(2, "")+ backOnA := base(3, "node-A")++ if d.seenOrAdd(keyLifecycle(onA)) {+ t.Fatal("first placement on node-A should be new")+ }+ if d.seenOrAdd(keyLifecycle(cleared)) {+ t.Fatal("clearing the placement between attempts should be new")+ }+ if d.seenOrAdd(keyLifecycle(backOnA)) {+ t.Fatal("re-dispatching to node-A repeats an earlier shape and must NOT dedup against it")+ }+ // A redelivery of any of those frames carries its original sequence, so it+ // is still recognised as one.+ if !d.seenOrAdd(keyLifecycle(base(1, "node-A"))) {+ t.Fatal("a resent frame for the first placement should still dedup")+ }+ if !d.seenOrAdd(keyLifecycle(base(3, "node-A"))) {+ t.Fatal("a resent frame for the third event should still dedup")+ }+}+// TestDedupRemovalDistinguishesNodes mirrors TestDedupDistinguishesNodes for// the removal path: the same workloadId removed on two nodes must be two// distinct dedup entries.diff --git a/services/nvpair-workload-manager/manager.go b/services/nvpair-workload-manager/manager.goindex d9895e5..84ea634 100644--- a/services/nvpair-workload-manager/manager.go+++ b/services/nvpair-workload-manager/manager.go@@ -372,7 +372,7 @@ func (m *Manager) broadcastFrame(method string, params json.RawMessage) {// times, then it ages out. This is the set a newly-discovered peer is backfilled// with and the heartbeat re-syncs.func (m *Manager) trackActive(key workloadKey, method string, params json.RawMessage, state WorkloadState) {- terminal := state == StateCompleted || state == StateFailed+ terminal := state == StateCompleted || state == StateFailed || state == StateCancelledm.activeMu.Lock()defer m.activeMu.Unlock()// Monotonic: a single inference goes running→terminal and never back, so adiff --git a/services/nvpair-workload-manager/spec.md b/services/nvpair-workload-manager/spec.mdindex c00fb24..194967a 100644--- a/services/nvpair-workload-manager/spec.md+++ b/services/nvpair-workload-manager/spec.md@@ -27,18 +27,18 @@ Tracks inference workloads cluster-wide as they are queued, executed, and retire## 3. Key Use Cases- **Broadcast a local event**: the Broker emits a `workload:*` lifecycle event (`workload:submitted`, `workload:started`, `workload:completed`, `workload:errored`) or `workloads:remove` on `stdin`; the Workload Manager broadcasts it to every discovered peer via `POST /v1/workloads/events` over mTLS.-- **Relay a remote event**: on receiving a peer's `workload:*`, validate and deduplicate (by `(nodeId, Workload.id, state)`), then emit `workloads:upsert` on `stdout` so the Broker updates its catalog. A peer `workloads:remove` (deduplicated by `(nodeId, workloadId)`) is relayed as `workloads:remove` on `stdout`.+- **Relay a remote event**: on receiving a peer's `workload:*`, validate and deduplicate (by `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)`), then emit `workloads:upsert` on `stdout` so the Broker updates its catalog. A peer `workloads:remove` (deduplicated by `(nodeId, workloadId)`) is relayed as `workloads:remove` on `stdout`.- **Discover peers**: the Broker registers this node's `wl` port with the `nvpair-node-scanner` discovery daemon, which carries it on this node's single consolidated record; the Workload Manager subscribes for `wl` nodes and rebuilds the target set from each `discovery:nodes` snapshot the Broker relays, so a node joins the set when it appears in a snapshot and leaves when it is absent from the next one.-- **Edge case — duplicate / re-broadcast events**: retries or re-broadcasts arriving more than once are deduplicated (`(nodeId, Workload.id, state)` for lifecycle, `(nodeId, workloadId)` for removals) so the Broker is updated at most once.+- **Edge case — duplicate / re-broadcast events**: retries or re-broadcasts arriving more than once are deduplicated (`(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)` for lifecycle, `(nodeId, workloadId)` for removals) so the Broker is updated at most once.- **Edge case — peer unreachable**: a target that is down, slow, or partitioned (crash, network split, laptop suspend) does not block others — broadcast runs concurrently with per-peer timeouts and bounded retries.## 4. Open Questions / Risks-- **`initializing` state (open — needs third-party feedback)**: the enum value has no `workload:*` method, so it is never produced or transmitted as specified. Pending the Broker/inference-proxy team's confirmation whether it is a Broker-local pre-queue state only, or needs a new lifecycle method and broadcast path.+- **`initializing` state (closed)**: removed. It had no `workload:*` method and nothing ever produced it, so it was an unreachable member of a closed enum. The proxy has no distinct pre-dispatch moment to represent — admission and the first dispatch are effectively simultaneous — so the value was deleted rather than given a method.- **`EngineType` values (open — needs third-party feedback)**: the valid `engine` set is undefined. Pending the inference-engine team's list, `engine` is treated as an opaque pass-through string (must be present and non-empty, value not validated).- **Risk — late joiners / no replay**: nodes discovered mid-session only get events broadcast after they appear in the target set; their view is incomplete until later events flow (by design).- **Risk — best-effort fan-out**: an unreachable peer misses events; temporary inconsistency until later events arrive or the Broker reconciles by timestamp.- **Risk — snapshot staleness**: the target set is only as current as the last `discovery:nodes` snapshot, so a departed node lingers as a target and a new node appears slowly, bounded by the discovery daemon's own liveness handling rather than by anything this service controls.-- **Risk — dedup granularity**: keying on `(nodeId, Workload.id, state)` means a re-broadcast with the same key but updated metadata (e.g. a corrected `error`) is dropped, not merged.+- **Risk — dedup granularity**: the key is `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)`, so a re-broadcast carrying the same key with updated metadata (e.g. a corrected `error`) is dropped, not merged. `seq` is the producer's event counter and is what makes the key exact, because this index is a permanent set: any key derived only from a workload's current shape collides as soon as the workload revisits a shape it already had, which a retry does routinely (queued on A, placement cleared between attempts, queued on A again). Without it peers dropped that third event, kept the interim unplaced record, and stopped counting an active job against the node running it.## 5. Requirements@@ -46,7 +46,7 @@ Tracks inference workloads cluster-wide as they are queued, executed, and retire- Accept local `workload:*` and `workloads:remove` notifications from the Broker over `stdin` (or a named pipe) and broadcast each to all discovered peers via `POST /v1/workloads/events`.- For each validated, deduplicated inter-node `workload:*`, emit `workloads:upsert` to the Broker (translated, not forwarded unchanged); for each inter-node `workloads:remove`, emit `workloads:remove` to the Broker (not re-broadcast).- Subscribe to the Broker's discovery relay with `discovery:subscribe` filtered to the `wl` service, and rebuild the broadcast target set from every `discovery:nodes` snapshot: take each peer's dialable address and `wl` port from its directory entry, skip entries advertising no `wl` port or no address, and exclude this node's own entry by `hostUuid` rather than by hostname. A snapshot carries the full filtered set and replaces the target set wholesale, so there are no per-node deltas to apply and a peer absent from a snapshot simply stops being a target.-- Deduplicate inbound lifecycle events by `(nodeId, Workload.id, state)` and removals by `(nodeId, workloadId)`, using a configurable bounded LRU index (default ~10,000 entries, sized for session-scoped volume at ~dozen-node scale). `nodeId` is part of the key because `Workload.id` is only unique per node (§11) — keying on `id` alone would collide across nodes and silently drop a legitimate peer's event. This matches the `(nodeId, workloadId)` tuple the Broker uses as the global workload identity (§10).+- Deduplicate inbound lifecycle events by `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)` and removals by `(nodeId, workloadId)`, using a configurable bounded LRU index (default ~10,000 entries, sized for session-scoped volume at ~dozen-node scale). `nodeId` is part of the key because `Workload.id` is only unique per node (§11) — keying on `id` alone would collide across nodes and silently drop a legitimate peer's event. `engine` and `runId` are there for the same reason one level down: `Workload.id` is a per-process counter, both engine proxies count from 1, and the counter resets on restart, so without them a concurrent Ollama and LM Studio job both holding id `"1"` would collapse into one. The client-visible identity the Broker uses as the global key remains the coarser `(nodeId, workloadId)` pair (§10); this key is finer on purpose.- Validate inbound payloads; reject malformed envelopes or unknown `method` values (`400 Bad Request`).**Non-functional**@@ -66,6 +66,8 @@ Tracks inference workloads cluster-wide as they are queued, executed, and retiremodel: stringengine: EngineTypestate: WorkloadState+ runId?: string // Producing process's nonce, minted at proxy startup. Optional/additive (§7.3). Part of the dedup key: `id` is a per-process counter, both engine proxies count from 1, and it resets on restart, so runId is what keeps a reused id from colliding with an older workload. Absent from a producer that does not stamp it, in which case dedup degrades to a coarser key.+ seq?: number // Producer's event counter for this workload, from 1, in emission order. Optional/additive (§7.3). Part of the dedup key, and the component that makes it exact: the index is a permanent set, so any key derived only from a workload's current shape collides when the workload revisits a shape it already had (a retry re-dispatching to a node it already tried). Absent from a producer that does not stamp it, in which case dedup degrades to shape-only and such a repeat is dropped as a duplicate.originatedFrom: string // Owner/origin node of the workload (its identity half of the (originatedFrom, id) global key). Distinct from scheduledOn, the node it was routed to.scheduledOn?: string // Node the workload was routed to / scheduled on (where it actually ran), as opposed to originatedFrom (where it came from). Supplied by the Broker/proxy and passed through opaquely. Optional/additive (§7.3) — absent until a target is chosen (e.g. a still-queued workload:submitted).createdAt: number@@ -76,10 +78,18 @@ Tracks inference workloads cluster-wide as they are queued, executed, and retire}```-`WorkloadState` (enum): `"initializing" | "queued" | "running" | "completed" | "failed"`-- Method → `state`: `workload:submitted` → `queued`, `workload:started` → `running`, `workload:completed` → `completed`, `workload:errored` → `failed`. `initializing` is a pre-queue state with no lifecycle method and is not transmitted (see §4).+`WorkloadState` (enum): `"queued" | "running" | "completed" | "failed" | "cancelled"`+- Method → `state`: `workload:submitted` → `queued`, `workload:started` → `running`, `workload:completed` → `completed`, `workload:errored` → `failed` or `cancelled`.+- `cancelled` is terminal and means the requester stopped waiting — a disconnected client, a dead client whose write deadline tripped, or an in-flight request cancelled by the producer's own shutdown. It rides `workload:errored` rather than having a method of its own, because state is not validated against the method that carried it and consumers read `state` from the payload. Keeping it distinct from `failed` is what lets a consumer's failed bucket mean "an outcome someone might act on" instead of also collecting every time a user pressed stop.-Validation: check every inbound envelope before processing. Inter-node and local interfaces both accept the four `workload:*` methods (with `params.workloadInfo`) and `workloads:remove` (with `params.workloadId`). Reject malformed/unknown-`method` payloads on the inter-node interface (`400`); drop-and-log on the local interface (never broadcast). Deduplicate inter-node lifecycle by `(nodeId, Workload.id, state)` and removals by `(nodeId, workloadId)`. Assume the Broker supplies well-formed notifications with epoch-millisecond timestamps.+Validation: check every inbound envelope before processing. A `Workload` must+carry `id`, `model`, `engine`, `state`, and `originatedFrom`; `runId`, `seq`,+and `scheduledOn` are optional/additive, and the timestamp and nullable fields+are passed through opaquely. Note the asymmetry with the dedup key: `runId` and+`seq` are part of that key but are not required, so a producer that omits them+is accepted and simply gets coarser deduplication rather than a rejection.+Inter-node and local interfaces both accept the four `workload:*` methods (with+`params.workloadInfo`) and `workloads:remove` (with `params.workloadId`). Reject malformed/unknown-`method` payloads on the inter-node interface (`400`); drop-and-log on the local interface (never broadcast). Deduplicate inter-node lifecycle by `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)` and removals by `(nodeId, workloadId)`. Assume the Broker supplies well-formed notifications with epoch-millisecond timestamps.**Outputs** — to the Broker over `stdout` / named pipe, or to peers via HTTP REST. `workloads:upsert` on `stdout` after a remote `workload:*`; `workloads:remove` on `stdout` after a remote inter-node `workloads:remove`. Inter-node broadcast posts the same JSON-RPC notification to `POST /v1/workloads/events` (`200 OK` on success; see §7.2). No response on the local interface (notifications only); `stdout` writes are serialized so frames never interleave; inter-node delivery is best-effort (concurrent, per-peer timeouts, bounded retries).@@ -115,7 +125,10 @@ Example `workloads:upsert` (`stdout`):"model": "llama-3-70b","engine": "trt-llm","state": "running",+ "runId": "9f3c1a7b",+ "seq": 3,"originatedFrom": "node-B",+ "scheduledOn": "node-C","createdAt": 1716998400000,"startedAt": 1716998401000,"completedAt": null,@@ -159,6 +172,8 @@ Example notification:"model": "llama-3-70b","engine": "trt-llm","state": "queued",+ "runId": "9f3c1a7b",+ "seq": 1,"originatedFrom": "node-A","createdAt": 1716998400000,"startedAt": null,@@ -177,7 +192,7 @@ Example notification:- A node broadcasts each local `workload:*` and `workloads:remove` to all discovered peers.- Body: JSON-RPC 2.0 notification — `workload:*` with `params.workloadInfo`, or `workloads:remove` with `params.workloadId` (and optional `params.originatedFrom`).- Responses: `200 OK` (accepted; lifecycle → `workloads:upsert`, removal → `workloads:remove` to the local Broker); `400 Bad Request` (malformed / unknown `method`); `403 Forbidden` (client cert absent or untrusted).-- Idempotency: lifecycle by `(nodeId, Workload.id, state)`; removal by `(nodeId, workloadId)`.+- Idempotency: lifecycle by `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)`; removal by `(nodeId, workloadId)`.Example request (`POST /v1/workloads/events`):```json@@ -239,7 +254,7 @@ Response: `200 OK`.- **Peer unreachable/slow during broadcast** (partition, crash, suspend, GC pause): that peer's Broker misses the event, causing temporary inconsistency. → Concurrent broadcast with per-peer timeouts so one peer can't block others or the local Broker; bounded exponential-backoff retries, drop after max attempts with a metric/alert; peers reconcile via later events.- **Connection exhaustion under burst** (observed in the field): a per-event connection makes every lifecycle event pay a full mTLS handshake, and an unreaped idle connection pins a descriptor for the process lifetime. At inference-burst rates this starves the sender and the receiving listener until handshakes fail outright, and the events lost are disproportionately terminal ones — which the ~two-interval terminal retention cannot recover once the window passes. → Pool connections per peer with a bound on both total and idle connections per host, reap idle connections on both ends (§7.2), and drain response bodies so a connection is actually reusable. Note the residual: fan-out concurrency itself is still per-event, so a heartbeat round with many active workloads issues many simultaneous requests per peer.- **mTLS handshake failure / untrusted or expired cert**: that peer is isolated from broadcasts (`403`). → Reject and log with the presented identity; monitor and alert ahead of cert expiry; keep the trust store hot-reloadable so rotations apply without restart.-- **Duplicate / re-broadcast events**: the Broker could see the same transition repeatedly, corrupting counts/state. → Deduplicate by `(nodeId, Workload.id, state)` (removals by `(nodeId, workloadId)`) before emitting, so the Broker updates at most once.+- **Duplicate / re-broadcast events**: the Broker could see the same transition repeatedly, corrupting counts/state. → Deduplicate by `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)` (removals by `(nodeId, workloadId)`) before emitting, so the Broker updates at most once.- **Malformed payload / unknown `method`**: could crash the parser or propagate garbage. → Validate every envelope; reject (`400`) inter-node or drop-and-log locally; never forward unvalidated payloads.- **Local interface severed** (Broker exited → `stdin` EOF / `stdout` `EPIPE` / `ERROR_BROKEN_PIPE`): an orphaned manager would accept peer events with nowhere to forward them. → Treat EOF/`EPIPE` as the shutdown signal: stop the listener and exit cleanly. No reconnect/buffering — a new Broker spawns a new (stateless) manager. The small load lets the OS pipe buffer absorb serialized `stdout` writes.- **Out-of-order delivery for the same workload (local)**: the Broker could see an inconsistent progression (e.g. `completed` before `started`). → Serialize `stdout` writes; the Broker orders per workload via `(nodeId, workloadId)` (monotonic IDs) and timestamps.diff --git a/services/nvpair-workload-manager/workload.go b/services/nvpair-workload-manager/workload.goindex a64b2ab..36d7c9d 100644--- a/services/nvpair-workload-manager/workload.go+++ b/services/nvpair-workload-manager/workload.go@@ -12,18 +12,22 @@ import (// See versions.json at the repo root for the source of truth.var Version = "dev"-// WorkloadState is the lifecycle state carried in a Workload. The-// "initializing" value exists in the enum but has no lifecycle method and is-// never transmitted (see spec §4); it is accepted on the wire for forward-// compatibility but is not produced by this service.+// WorkloadState is the lifecycle state carried in a Workload. State is not+// validated against the method that carried it (see lifecycleMethods), so+// "cancelled" travels on workload:errored without needing a method of its own.type WorkloadState stringconst (- StateInitializing WorkloadState = "initializing"- StateQueued WorkloadState = "queued"- StateRunning WorkloadState = "running"- StateCompleted WorkloadState = "completed"- StateFailed WorkloadState = "failed"+ StateQueued WorkloadState = "queued"+ StateRunning WorkloadState = "running"+ StateCompleted WorkloadState = "completed"+ StateFailed WorkloadState = "failed"+ // StateCancelled is a job the requester stopped waiting for: a client that+ // disconnected, a dead client whose write deadline tripped, or an in-flight+ // request cancelled by our own shutdown. It is terminal, and distinct from+ // StateFailed so the failed bucket only holds outcomes a user might act on+ // rather than every time somebody pressed stop.+ StateCancelled WorkloadState = "cancelled")// Lifecycle method names on both the local and inter-node interfaces.@@ -74,6 +78,10 @@ type Workload struct {CompletedAt *int64 `json:"completedAt"`Error *string `json:"error"`RequesterID *string `json:"requesterId"`+ // Seq numbers a workload's events in emission order, assigned by the+ // producer. It exists so dedup can tell a new event from a redelivery even+ // when the two carry the same state and placement — see keyLifecycle.+ Seq int64 `json:"seq,omitempty"`}// lifecycleParams / removeParams are the params envelopes for the two kindsdiff --git a/services/tests/workload_identity_interop_test.go b/services/tests/workload_identity_interop_test.goindex c89eebe..864a3e5 100644--- a/services/tests/workload_identity_interop_test.go+++ b/services/tests/workload_identity_interop_test.go@@ -152,7 +152,7 @@ func TestWorkloadCrossEngineIdentityDistinct(t *testing.T) {// TestWorkloadManagerRehydratesActiveWorkloadOnRestart drives the rehydration// path end-to-end. A proxied inference is held open by a blocking fake engine,-// so its workload stays "running" (workload:started is emitted up front). The+// so its workload reaches "running" once the engine emits its first byte. The// supervised workload-manager is then killed; the broker's supervisor respawns// it and replays the still-active local workload. A discovered stub peer must// hear the running workload AGAIN from the restarted manager — which only@@ -162,20 +162,32 @@ func TestWorkloadManagerRehydratesActiveWorkloadOnRestart(t *testing.T) {t.Skip("ollama-proxy default port 11435 already in use; skipping")}- // Fake Ollama that accepts the request then blocks, keeping the proxied- // inference in-flight so the workload never reaches a terminal state. The+ // Fake Ollama that starts a streaming response, flushes one chunk so the+ // proxy commits and the workload becomes "running", then blocks — keeping+ // the proxied inference in-flight so it never reaches a terminal state. The// handler also unblocks if the request is cancelled, so a client-side abort// can't wedge it.+ //+ // The chunk is required: the proxy commits on the first byte of response+ // body, not on the headers, because headers only prove an engine accepted+ // the request. An engine that blocks before producing content leaves the+ // workload "queued", which is the truth but not what this test is about.release := make(chan struct{})var releaseOnce sync.OncestopEngine := func() { releaseOnce.Do(func() { close(release) }) }ollama := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {io.Copy(io.Discard, r.Body)+ w.Header().Set("Content-Type", "application/x-ndjson")+ w.WriteHeader(http.StatusOK)+ _, _ = io.WriteString(w, `{"message":{"role":"assistant","content":""},"done":false}`+"\n")+ if fl, ok := w.(http.Flusher); ok {+ fl.Flush()+ }select {case <-release:case <-r.Context().Done():}- w.WriteHeader(http.StatusOK)+ _, _ = io.WriteString(w, `{"done":true}`+"\n")}))// Defer order matters: stopEngine() must run BEFORE ollama.Close(), because// Close() blocks until the deliberately-held request finishes, which onlydiff --git a/services/versions.json b/services/versions.jsonindex 0a130f7..cdc9bd6 100644--- a/services/versions.json+++ b/services/versions.json@@ -1,16 +1,16 @@{"$comment": "Single source of truth for all version numbers. See VERSIONING.md for bump rules.",- "product": "1.0.0",- "installer": "1.0.0",+ "product": "1.1.0",+ "installer": "1.1.0","components": {- "nvpair-proxy": "1.0.0",+ "nvpair-proxy": "1.1.0","nvpair-node-info": "0.13.3","nvpair-node-scanner": "0.20.3","nvpair-manual-nodes": "0.11.1",- "nvpair-workload-manager": "0.13.3",+ "nvpair-workload-manager": "0.14.0","nvpair-errors": "0.7.4","nvpair-node-settings": "1.0.4",- "nvpair-ui-broker": "1.0.0",+ "nvpair-ui-broker": "1.1.0","nvpair-engine-manager": "0.17.4","nvpair-cluster-manager": "1.1.4","nvpair-job-scheduler": "0.4.2",