Skip to content

Router that virtually distributes inference across connected devices in the home.

Go

Retry jobs when a node stalls or drops out

Noah (Terve) Tervalone0f75ddc841a
diff --git a/desktop/src/shared/constants/workloads.ts b/desktop/src/shared/constants/workloads.ts
index cc5816a..6e1cac7 100644
--- a/desktop/src/shared/constants/workloads.ts
+++ b/desktop/src/shared/constants/workloads.ts
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-export const WorkloadStates = ['initializing', 'queued', 'running', 'completed', 'failed'] as const
+export const WorkloadStates = ['queued', 'running', 'completed', 'failed', 'cancelled'] as const
diff --git a/desktop/src/ui/components/JobsFilter.tsx b/desktop/src/ui/components/JobsFilter.tsx
index 3787576..bba1a96 100644
--- a/desktop/src/ui/components/JobsFilter.tsx
+++ b/desktop/src/ui/components/JobsFilter.tsx
@@ -8,11 +8,7 @@ import { JobsFilterType } from '@/ui/types/types'
interface JobsFilterProps {
setFilter: (type: JobsFilterType, checked: boolean) => void
- values: {
- active: { checked: boolean; count: number }
- completed: { checked: boolean; count: number }
- failed: { checked: boolean; count: number }
- }
+ values: Record<JobsFilterType, { checked: boolean; count: number }>
}
function JobFilterItem({ id, count }: { id: JobsFilterType; count: number }) {
@@ -28,28 +24,19 @@ function JobFilterItem({ id, count }: { id: JobsFilterType; count: number }) {
)
}
+// Dropdown order, listed once so a new bucket needs one entry rather than a
+// parallel edit in three places.
+const FILTER_ORDER: JobsFilterType[] = ['active', 'completed', 'failed', 'cancelled']
+
export default function JobsFilter({ setFilter, values }: JobsFilterProps) {
const dropdownItems: DropdownEntry[] = useMemo(
- () => [
- {
- kind: 'checkbox',
- checked: values.active.checked,
- onCheckedChange: checked => setFilter('active', checked === true),
- children: <JobFilterItem id="active" count={values.active.count} />
- },
- {
- kind: 'checkbox',
- checked: values.completed.checked,
- onCheckedChange: checked => setFilter('completed', checked === true),
- children: <JobFilterItem id="completed" count={values.completed.count} />
- },
- {
+ () =>
+ FILTER_ORDER.map(id => ({
kind: 'checkbox',
- checked: values.failed.checked,
- onCheckedChange: checked => setFilter('failed', checked === true),
- children: <JobFilterItem id="failed" count={values.failed.count} />
- }
- ],
+ checked: values[id].checked,
+ onCheckedChange: checked => setFilter(id, checked === true),
+ children: <JobFilterItem id={id} count={values[id].count} />
+ })),
[setFilter, values]
)
diff --git a/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx b/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx
index 2fa7b47..799de94 100644
--- a/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx
+++ b/desktop/src/ui/components/Workloads/WorkloadItemCard.tsx
@@ -65,7 +65,10 @@ function WorkloadItemCard({ workload }: { workload: Workload }) {
switch (state) {
case 'queued':
value = workload.createdAt
- label = 'Queued at'
+ // Not "Queued at": PAIR runs no queue of its own, and the
+ // elapsed wait is the useful part for a job that has been
+ // accepted but is not generating yet.
+ label = 'Waiting since'
break
case 'running':
value = workload.startedAt ?? 0
@@ -79,6 +82,10 @@ function WorkloadItemCard({ workload }: { workload: Workload }) {
value = workload.completedAt ?? 0
label = 'Failed at'
break
+ case 'cancelled':
+ value = workload.completedAt ?? 0
+ label = 'Cancelled at'
+ break
}
return (
@@ -93,18 +100,16 @@ function WorkloadItemCard({ workload }: { workload: Workload }) {
)
}, [workload.state, workload.createdAt, workload.startedAt, workload.completedAt])
- // const showBadge = useMemo(
- // () =>
- // workload.state === 'initializing' ||
- // workload.state === 'queued' ||
- // workload.state === 'running',
- // [workload.state]
- // )
-
const className = useMemo(() => {
let result = `h-auto dir-ltr max-w-full min-w-0 pair-paper workload-card workload-card-${workload.state}`
- if (workload.state === 'failed' || workload.state === 'completed') {
+ // Terminal states read as history and are dimmed; only live work keeps
+ // full contrast.
+ if (
+ workload.state === 'failed' ||
+ workload.state === 'completed' ||
+ workload.state === 'cancelled'
+ ) {
result += ' card-low'
}
diff --git a/desktop/src/ui/components/Workloads/WorkloadListView.tsx b/desktop/src/ui/components/Workloads/WorkloadListView.tsx
index bfcaa28..2832319 100644
--- a/desktop/src/ui/components/Workloads/WorkloadListView.tsx
+++ b/desktop/src/ui/components/Workloads/WorkloadListView.tsx
@@ -8,13 +8,14 @@ import type { PartialOptions } from 'overlayscrollbars'
import { Flipper, Flipped } from 'react-flip-toolkit'
import {
useActiveWorkloads,
+ useCancelledWorkloads,
useCompletedWorkloads,
useFailedWorkloads
} from '@/ui/stores/workloads.store'
import WorkloadItemCard from './WorkloadItemCard'
import JobsFilter from '@/ui/components/JobsFilter'
import { CONNECTIONS_WIDTH, MAX_HISTORY_ITEMS } from '@/ui/constants/app'
-import type { Workload } from '@/shared/types/workloads'
+import type { Workload, WorkloadState } from '@/shared/types/workloads'
import { workloadKey } from '@/shared/utils/workloads'
import type { JobsFilterType } from '@/ui/types/types'
@@ -30,18 +31,32 @@ interface WorkloadListViewProps {
setFilter: (type: JobsFilterType, checked: boolean) => void
}
+// Which filter bucket a finished job belongs to. Exhaustive over WorkloadState
+// so adding a state is a compile error here rather than a job that silently
+// stops appearing: the history list used to index the filter record by the raw
+// state string, and any state without a matching bucket read as unchecked.
+const HISTORY_BUCKET: Record<WorkloadState, JobsFilterType | null> = {
+ queued: null,
+ running: null,
+ completed: 'completed',
+ failed: 'failed',
+ cancelled: 'cancelled'
+}
+
function WorkloadListView({ filter, setFilter }: WorkloadListViewProps) {
const activeWorkloads = useActiveWorkloads()
const completedWorkloads = useCompletedWorkloads()
const failedWorkloads = useFailedWorkloads()
+ const cancelledWorkloads = useCancelledWorkloads()
const jobValues = useMemo(
() => ({
active: { checked: filter.active, count: activeWorkloads.length },
completed: { checked: filter.completed, count: completedWorkloads.length },
- failed: { checked: filter.failed, count: failedWorkloads.length }
+ failed: { checked: filter.failed, count: failedWorkloads.length },
+ cancelled: { checked: filter.cancelled, count: cancelledWorkloads.length }
}),
- [filter, activeWorkloads, completedWorkloads, failedWorkloads]
+ [filter, activeWorkloads, completedWorkloads, failedWorkloads, cancelledWorkloads]
)
const currentWorkloads = useMemo(() => {
@@ -51,13 +66,16 @@ function WorkloadListView({ filter, setFilter }: WorkloadListViewProps) {
result = activeWorkloads
}
- const rest = [...completedWorkloads, ...failedWorkloads]
- .filter(w => filter[w.state as JobsFilterType])
+ const rest = [...completedWorkloads, ...failedWorkloads, ...cancelledWorkloads]
+ .filter(w => {
+ const bucket = HISTORY_BUCKET[w.state]
+ return bucket !== null && filter[bucket]
+ })
.sort((a, b) => (b.completedAt ?? 0) - (a.completedAt ?? 0))
.slice(0, MAX_HISTORY_ITEMS)
return result.concat(rest)
- }, [filter, completedWorkloads, failedWorkloads, activeWorkloads])
+ }, [filter, completedWorkloads, failedWorkloads, cancelledWorkloads, activeWorkloads])
const flipKey = useMemo(() => {
const filterKey = Object.entries(filter)
diff --git a/desktop/src/ui/constants/app.ts b/desktop/src/ui/constants/app.ts
index 2afc49c..16bb356 100644
--- a/desktop/src/ui/constants/app.ts
+++ b/desktop/src/ui/constants/app.ts
@@ -10,9 +10,9 @@ export const MAX_HISTORY_ITEMS = 30
export const stateOrder: Record<WorkloadState, number> = {
running: 0,
queued: 1,
- initializing: 2,
- completed: 3,
- failed: 4
+ completed: 2,
+ failed: 3,
+ cancelled: 4
}
export const ContentBaseClass = 'grow w-full min-h-0 min-w-0'
diff --git a/desktop/src/ui/stores/workloads.store.ts b/desktop/src/ui/stores/workloads.store.ts
index 9b2799a..7bb9082 100644
--- a/desktop/src/ui/stores/workloads.store.ts
+++ b/desktop/src/ui/stores/workloads.store.ts
@@ -144,8 +144,7 @@ export function useActiveWorkloads(): Workload[] {
useShallow(state => {
const result: Workload[] = []
for (const w of state.workloads.values()) {
- if (w.state === 'running' || w.state === 'queued' || w.state === 'initializing')
- result.push(w)
+ if (w.state === 'running' || w.state === 'queued') result.push(w)
}
return result.sort((a, b) => {
const stateDiff = stateOrder[a.state] - stateOrder[b.state]
@@ -178,16 +177,16 @@ export function useAssignedWorkloads(): Workload[] {
)
}
-// Number of active (running/queued/initializing) workloads scheduled on a node.
-// Returns a primitive so a subscribing node card only re-renders when its own
-// count changes, instead of on every workload event across the cluster.
+// Number of active (running/queued) workloads scheduled on a node. Returns a
+// primitive so a subscribing node card only re-renders when its own count
+// changes, instead of on every workload event across the cluster.
export function useNodeActiveJobCount(nodeId: string): number {
return useWorkloadsStore(state => {
let count = 0
for (const w of state.workloads.values()) {
if (
workloadExecutionNodeId(w) === nodeId &&
- (w.state === 'running' || w.state === 'queued' || w.state === 'initializing')
+ (w.state === 'running' || w.state === 'queued')
)
count++
}
@@ -222,3 +221,20 @@ export function useFailedWorkloads(): Workload[] {
})
)
}
+
+// Jobs the requester stopped waiting for. Kept out of useFailedWorkloads so
+// the failed bucket only holds outcomes a user might act on, rather than also
+// collecting every time somebody pressed stop.
+export function useCancelledWorkloads(): Workload[] {
+ return useWorkloadsStore(
+ useShallow(state => {
+ const result: Workload[] = []
+ for (const w of state.workloads.values()) {
+ if (w.state === 'cancelled') result.push(w)
+ }
+ return result
+ .sort((a, b) => (b.completedAt ?? 0) - (a.completedAt ?? 0))
+ .slice(0, MAX_HISTORY_ITEMS)
+ })
+ )
+}
diff --git a/desktop/src/ui/types/types.ts b/desktop/src/ui/types/types.ts
index fceed83..de24e4e 100644
--- a/desktop/src/ui/types/types.ts
+++ b/desktop/src/ui/types/types.ts
@@ -6,6 +6,6 @@ export interface PerformanceMetric {
value: number
}
-export type JobsFilterType = 'active' | 'completed' | 'failed'
+export type JobsFilterType = 'active' | 'completed' | 'failed' | 'cancelled'
export type BadgeColor = 'yellow' | 'blue' | 'green' | 'gray' | 'red' | 'teal' | 'purple'
diff --git a/desktop/src/ui/utils/colors.ts b/desktop/src/ui/utils/colors.ts
index a220117..ad6b18c 100644
--- a/desktop/src/ui/utils/colors.ts
+++ b/desktop/src/ui/utils/colors.ts
@@ -7,8 +7,6 @@ import { GPU_COLOR_PALETTE, VRAM_COLOR_PALETTE, WORKLOAD_COLOR_MAP } from '@/ui/
export function getWorkloadStateColor(state: WorkloadState): BadgeColor {
switch (state) {
- case 'initializing':
- return 'gray'
case 'queued':
return 'yellow'
case 'running':
@@ -17,6 +15,11 @@ export function getWorkloadStateColor(state: WorkloadState): BadgeColor {
return 'gray'
case 'failed':
return 'red'
+ // A job the requester stopped waiting for is not an error: it reads as
+ // neutral history, so it does not compete with real failures for
+ // attention.
+ case 'cancelled':
+ return 'gray'
default:
return 'gray'
}
@@ -24,8 +27,6 @@ export function getWorkloadStateColor(state: WorkloadState): BadgeColor {
export function getWorkloadColorBar(state: WorkloadState): string {
switch (state) {
- case 'initializing':
- return WORKLOAD_COLOR_MAP.gray
case 'queued':
return WORKLOAD_COLOR_MAP.yellow
case 'running':
@@ -34,6 +35,8 @@ export function getWorkloadColorBar(state: WorkloadState): string {
return WORKLOAD_COLOR_MAP.green
case 'failed':
return WORKLOAD_COLOR_MAP.red
+ case 'cancelled':
+ return WORKLOAD_COLOR_MAP.gray
default:
return WORKLOAD_COLOR_MAP.gray
}
diff --git a/desktop/src/ui/utils/jobs-filter-storage.ts b/desktop/src/ui/utils/jobs-filter-storage.ts
index 1653bb7..704c238 100644
--- a/desktop/src/ui/utils/jobs-filter-storage.ts
+++ b/desktop/src/ui/utils/jobs-filter-storage.ts
@@ -9,10 +9,11 @@ export const JOBS_FILTER_STORAGE_KEY = 'nvpair.jobs.filter'
export const DEFAULT_JOBS_FILTER: Record<JobsFilterType, boolean> = {
active: true,
completed: true,
- failed: true
+ failed: true,
+ cancelled: true
}
-const KEYS: JobsFilterType[] = ['active', 'completed', 'failed']
+const KEYS: JobsFilterType[] = ['active', 'completed', 'failed', 'cancelled']
/** Merge stored booleans with defaults so missing/invalid keys do not break the UI. */
export function parseJobsFilterRecord(raw: unknown): Record<JobsFilterType, boolean> {
diff --git a/services/nvpair-job-scheduler/schedule_test.go b/services/nvpair-job-scheduler/schedule_test.go
index 4bbfbab..6c1a402 100644
--- a/services/nvpair-job-scheduler/schedule_test.go
+++ b/services/nvpair-job-scheduler/schedule_test.go
@@ -269,6 +269,22 @@ func TestRank_TerminalStatesExcluded(t *testing.T) {
assertStrs(t, order, []string{"b", "a"}) // b=0 wins; a would tie only if completed counted
}
+// TestRank_CancelledExcluded pins that isPending is an allow-list, which is
+// what keeps a new terminal state from silently counting as load. "cancelled"
+// is not queued and not running, so it must contribute nothing — a job the
+// requester walked away from is not occupying its node.
+func TestRank_CancelledExcluded(t *testing.T) {
+ m := mgrWith(nopRW{}, []string{"a", "b"},
+ wl("1", "ollama", "running", "x", "a"), // a = 1
+ wl("2", "ollama", "cancelled", "x", "b"), // must not count toward b
+ )
+ order, ranks := m.rank()
+ assertStrs(t, order, []string{"b", "a"})
+ if got := pendingOf(ranks, "b"); got != 0 {
+ t.Fatalf("pending for b = %d, want 0 (cancelled must not count as pending)", got)
+ }
+}
+
// TestRank_NodeWideMixedEngineSynthetic verifies the requested synthetic
// ranking: A has three mixed-engine pending jobs, B has one, and C has none.
// Both engine outputs must therefore receive C,B,A.
diff --git a/services/nvpair-job-scheduler/spec.md b/services/nvpair-job-scheduler/spec.md
index c461e92..9cab523 100644
--- a/services/nvpair-job-scheduler/spec.md
+++ b/services/nvpair-job-scheduler/spec.md
@@ -112,9 +112,10 @@ capacity, model locality, latency, and affinity can grow later behind the same
`scheduler:set-interval`, clamped to a **200 ms** floor (§7.5).
Workload and discovery events do not wait for this timer; emit-only-on-snapshot-
change plus the stable tie-break keep timer recomputes from churning.
-- **Pending-count (decided — node-wide).** *Pending* = `queued` or `running`
- (`completed`/`failed` excluded; `initializing` never transmitted), regardless of
- engine. Counted by **`scheduledOn`** (where work runs), not `originatedFrom`. A
+- **Pending-count (decided — node-wide).** *Pending* = `queued` or `running`,
+ regardless of engine. The test is an allow-list, not a terminal-state
+ exclusion list, so `completed`, `failed`, `cancelled`, and any state added
+ later count as zero load without needing a change here. Counted by **`scheduledOn`** (where work runs), not `originatedFrom`. A
workload with no `scheduledOn` is unplaced and counts toward no node.
- **Identity alignment (decided).** Discovery `hostUuid`, proxy candidate ids, and
workload `scheduledOn` use the same stable UUID namespace. A malformed or legacy
diff --git a/services/nvpair-proxy/README.md b/services/nvpair-proxy/README.md
index 731e6b9..6bcdc50 100644
--- a/services/nvpair-proxy/README.md
+++ b/services/nvpair-proxy/README.md
@@ -201,6 +201,10 @@ the local machine is unaffected.
proxy falls back to auto-select and emits `node/selection-changed`. A
transport error or retryable status, including a model `404` from an
advertised owner with stale inventory, steps to the next eligible owner.
+ Once a round's owners are used up, an inference request re-resolves and keeps
+ trying under a dispatch budget and a wall-clock deadline, and it commits to a
+ node at the first byte of response body rather than at its headers. `spec.md`
+ §5.1–§5.4 is normative for the bounds, the commit point, and the statuses.
### IPC Transport
diff --git a/services/nvpair-proxy/discovery.go b/services/nvpair-proxy/discovery.go
index 87313a4..a31213b 100644
--- a/services/nvpair-proxy/discovery.go
+++ b/services/nvpair-proxy/discovery.go
@@ -102,6 +102,19 @@ func (d *Discovery) Nodes() []Node {
return out
}
+// Has reports whether a node id is still present in either overlay. Used to
+// watch an in-flight request's target: a node that has dropped out of discovery
+// is not coming back to answer the attempt already sent to it.
+func (d *Discovery) Has(id string) bool {
+ d.mu.RLock()
+ defer d.mu.RUnlock()
+ if _, ok := d.subscribedNodes[id]; ok {
+ return true
+ }
+ _, ok := d.manualNodes[id]
+ return ok
+}
+
// SetSubscribed replaces the relay-fed routing overlay with the given set and
// reports what changed versus the previous set (keyed by node ID): nodes newly
// present, nodes whose routable details changed, and nodes that dropped out. The
diff --git a/services/nvpair-proxy/firstbody_test.go b/services/nvpair-proxy/firstbody_test.go
new file mode 100644
index 0000000..294b14f
--- /dev/null
+++ b/services/nvpair-proxy/firstbody_test.go
@@ -0,0 +1,266 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// Commit is the first byte of response body, not the response headers, because
+// headers can mean nothing more than "accepted" (spec §5.2 for the engine
+// measurements behind that). These bodies pin the boundary: what fails over,
+// what commits, what terminates, and what is left alone.
+//
+// firstBodyTimeout stands in at 200ms for its production 120s throughout, so a
+// body that drives a node to its five-dispatch budget costs a second of wall
+// clock rather than ten minutes.
+const testFirstBodyTimeout = 200 * time.Millisecond
+
+// stalledEngine is an upstream that sends 200 and headers immediately and then
+// produces no body until released, which is the shape an engine presents for a
+// streaming request queued behind other work.
+type stalledEngine struct {
+ url string
+ // hits counts dispatches, so a test can show the retry budget was spent
+ // rather than inferring it from timing.
+ hits func() int
+ release func()
+}
+
+func newStalledEngine(t *testing.T) stalledEngine {
+ t.Helper()
+ var n atomic.Int32
+ releaseCh := make(chan struct{})
+ var once sync.Once
+ release := func() { once.Do(func() { close(releaseCh) }) }
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ n.Add(1)
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ if f, ok := w.(http.Flusher); ok {
+ f.Flush() // headers on the wire, body still empty
+ }
+ select {
+ case <-releaseCh:
+ case <-r.Context().Done():
+ }
+ }))
+ // Cleanup is LIFO, so release runs before Close. Close waits for the
+ // handler to return and the handler is parked on release, so the other
+ // order deadlocks the end of the test.
+ t.Cleanup(srv.Close)
+ t.Cleanup(release)
+
+ return stalledEngine{
+ url: srv.URL,
+ hits: func() int { return int(n.Load()) },
+ release: release,
+ }
+}
+
+// TestHandleHTTP_HeadersWithoutContentFailsOver is the regression for the
+// engine that accepts and stalls. The first owner sends headers and no content;
+// the request must move to the next owner rather than binding itself to a node
+// that has not started generating.
+func TestHandleHTTP_HeadersWithoutContentFailsOver(t *testing.T) {
+ setForTest(t, &firstBodyTimeout, testFirstBodyTimeout)
+
+ forEachEngine(t, func(t *testing.T, tc engineCase) {
+ stalled := newStalledEngine(t)
+
+ var servedBody string
+ good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ b, _ := io.ReadAll(r.Body)
+ servedBody = string(b)
+ w.WriteHeader(http.StatusOK)
+ io.WriteString(w, `{"done":true}`)
+ }))
+ defer good.Close()
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "stalled", stalled.url, tc.advertisedModel))
+ disc.AddManual(nodeForModel(t, "good", good.URL, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+ p.soleFacade().SetSelected("stalled") // deterministic: the staller is tried first
+
+ rec := httptest.NewRecorder()
+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200 (should have failed over past the stalled engine)", rec.Code)
+ }
+ if got := decodeJSONBody(t, rec.Body.String()); got["done"] != true {
+ t.Fatalf(`body has done = %v, want true: the response came from the wrong node`, got["done"])
+ }
+ if servedBody != tc.inferenceBody() {
+ t.Errorf("failover node got body %q, want the original request body", servedBody)
+ }
+ })
+}
+
+// TestHandleHTTP_NoContentOnFinalAttemptTerminates covers the exhausted case,
+// which is the one that could hang: a first-content timeout must not commit
+// even with no attempt left, because the peek reader is still blocked on the
+// upstream body (spec §5.2).
+//
+// 504 rather than 502: the node answered, it just never produced anything.
+func TestHandleHTTP_NoContentOnFinalAttemptTerminates(t *testing.T) {
+ setForTest(t, &firstBodyTimeout, testFirstBodyTimeout)
+
+ forEachEngine(t, func(t *testing.T, tc engineCase) {
+ // The only owner, so every attempt in the budget goes to it and the
+ // last one has nowhere to fail over to.
+ stalled := newStalledEngine(t)
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "silent", stalled.url, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+
+ rec := httptest.NewRecorder()
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())
+ }()
+
+ select {
+ case <-done:
+ // Five attempts at the shortened budget is about a second. Any finite
+ // guard catches the regression this test exists for, since a committed
+ // silent upstream blocks forever.
+ case <-time.After(5 * time.Second):
+ t.Fatal("handleHTTP never returned: a silent upstream committed and hung the request")
+ }
+
+ if got := stalled.hits(); got != maxDispatchAttempts {
+ t.Errorf("upstream saw %d dispatches, want the full budget of %d spent on the only owner", got, maxDispatchAttempts)
+ }
+ if rec.Code != http.StatusGatewayTimeout {
+ t.Fatalf("status = %d, want 504 for an upstream that answered but produced no content", rec.Code)
+ }
+ want := "upstream error: " + errFirstBodyTimeout.Error()
+ if got := decodeJSONBody(t, rec.Body.String()); got["error"] != want {
+ t.Fatalf("body error = %v, want %q", got["error"], want)
+ }
+ })
+}
+
+// TestHandleHTTP_FirstBytePreservedOnCommit guards the splice: the byte
+// awaitFirstBody consumes to detect the commit has to be handed back, or every
+// committed response would be served a byte short. Compared bytewise rather
+// than as JSON, since losing the first byte is exactly what would still parse
+// as something.
+func TestHandleHTTP_FirstBytePreservedOnCommit(t *testing.T) {
+ forEachEngine(t, func(t *testing.T, tc engineCase) {
+ const payload = `{"response":"whole body intact","done":true}`
+ good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ io.WriteString(w, payload)
+ }))
+ defer good.Close()
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "good", good.URL, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+
+ rec := httptest.NewRecorder()
+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200", rec.Code)
+ }
+ if got := rec.Body.String(); got != payload {
+ t.Fatalf("body = %q, want %q (the peeked first byte was dropped)", got, payload)
+ }
+ })
+}
+
+// TestHandleHTTP_EmptyBodyCommits: a 200 with no body at all commits rather
+// than burning the budget waiting for content that is never coming (spec §5.2).
+func TestHandleHTTP_EmptyBodyCommits(t *testing.T) {
+ // Long, so a wait would be unmistakable: if the empty body were treated as
+ // a stall, the guard below would trip well before this expires.
+ setForTest(t, &firstBodyTimeout, 30*time.Second)
+
+ forEachEngine(t, func(t *testing.T, tc engineCase) {
+ empty := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer empty.Close()
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "empty", empty.URL, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+
+ done := make(chan int, 1)
+ go func() {
+ rec := httptest.NewRecorder()
+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())
+ done <- rec.Code
+ }()
+
+ select {
+ case code := <-done:
+ if code != http.StatusOK {
+ t.Fatalf("status = %d, want 200", code)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("an empty 200 body blocked on the first-byte wait instead of committing")
+ }
+ })
+}
+
+// TestHandleHTTP_NonInferenceCommitsOnHeaders: the gate is scoped to inference.
+// A control endpoint's headers mean what they say, and some of them (a pull's
+// progress stream) legitimately produce their first byte late, so waiting there
+// would add latency for no benefit.
+func TestHandleHTTP_NonInferenceCommitsOnHeaders(t *testing.T) {
+ setForTest(t, &firstBodyTimeout, testFirstBodyTimeout)
+
+ forEachEngine(t, func(t *testing.T, tc engineCase) {
+ stalled := newStalledEngine(t)
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "stalled", stalled.url, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+
+ rec := httptest.NewRecorder()
+ done := make(chan struct{})
+ go func() {
+ req := httptest.NewRequest(http.MethodGet, tc.nonInferencePath, nil)
+ p.soleFacade().handleHTTP(rec, req)
+ close(done)
+ }()
+
+ // Still blocked on the upstream body well past the first-content
+ // budget, which is the whole assertion: if the gate applied here the
+ // handler would have given up at firstBodyTimeout and returned 504,
+ // and simply waiting for it to finish would pass either way.
+ blocked := 2 * testFirstBodyTimeout
+ time.Sleep(blocked)
+ select {
+ case <-done:
+ t.Fatalf("handler returned after %v with status %d: the first-content gate was applied to a non-inference route",
+ blocked, rec.Code)
+ default:
+ }
+
+ stalled.release()
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("a non-inference request did not complete after the upstream released")
+ }
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200: the request committed on headers and streamed normally", rec.Code)
+ }
+ })
+}
diff --git a/services/nvpair-proxy/proxy.go b/services/nvpair-proxy/proxy.go
index 33a32bd..ff89dbe 100644
--- a/services/nvpair-proxy/proxy.go
+++ b/services/nvpair-proxy/proxy.go
@@ -24,6 +24,7 @@ import (
"strconv"
"strings"
"sync"
+ "sync/atomic"
"time"
"nvpair-shared/applog"
@@ -150,6 +151,7 @@ type RequestEvent struct {
// workload:submitted (the proxy never queues — it forwards immediately) or
// workloads:remove (retirement is a broker concern).
const (
+ workloadSubmittedMethod = "workload:submitted"
workloadStartedMethod = "workload:started"
workloadCompletedMethod = "workload:completed"
workloadErroredMethod = "workload:errored"
@@ -189,6 +191,20 @@ type Workload struct {
CompletedAt *int64 `json:"completedAt"`
Error *string `json:"error"`
RequesterID *string `json:"requesterId"`
+ // Seq counts this workload's events, from 1, in emission order.
+ //
+ // The workload-manager's inter-node dedup is a permanent set, so an event
+ // that repeats an earlier (state, scheduledOn) pair is indistinguishable
+ // from a redelivery and is dropped by every peer. The retry loop produces
+ // exactly that routinely: queued on A, placement cleared between attempts,
+ // then queued on A again. Peers kept the interim unplaced record while the
+ // job was running on A, so their schedulers stopped counting it against the
+ // node actually doing the work.
+ //
+ // Sequencing each event makes the dedup exact without weakening it: a
+ // broadcast retry resends an identical frame, sequence included, so a true
+ // redelivery is still suppressed.
+ Seq int64 `json:"seq"`
}
// workloadParams is the params envelope for a workload:* notification
@@ -503,6 +519,210 @@ func (p *Proxy) Run(ctx context.Context) error {
return err
}
+// Retry bounds for a single inference request.
+//
+// The two are not redundant, because they bound different things. An attempt is
+// consumed by a dispatch, never by a resolution: finding no eligible owner
+// sends nothing, so maxDispatchAttempts governs dispatch failures while
+// jobDeadline governs the wait for an owner to exist at all — a wait that costs
+// no attempts and would otherwise be unbounded.
+//
+// Read only against dispatch failures the deadline looks unreachable, and the
+// arithmetic says so: five attempts at a 120s first-content cap plus 15s of
+// cumulative backoff span 615s, so every reschedule check before the last sees
+// a clock under 600s. It is not dead — it is the only bound on the no-owner
+// wait. Do not remove it as unreachable.
+const maxDispatchAttempts = 5
+
+// jobDeadline is a var (not a const) only so a test can shorten it; production
+// never reassigns it.
+var jobDeadline = 10 * time.Minute
+
+// targetWatchInterval is how often an in-flight attempt rechecks that its
+// target is still in discovery. It bounds detection latency only: the effect is
+// to replace a full first-content budget spent waiting on a node that is
+// already gone with a sub-second abort.
+//
+// A var (not a const) only so a test can shorten it; production never
+// reassigns it.
+var targetWatchInterval = 500 * time.Millisecond
+
+// defaultRetryBackoff is the delay before a retry, indexed by the number of
+// dispatches already made and clamped to the last entry. It also paces the poll
+// while no owner is available.
+var defaultRetryBackoff = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second}
+
+// retryBackoff is the schedule in force. A var (not a const) only so a test can
+// shorten it; production never reassigns it. Naming the default separately lets
+// a test that asserts the schedule itself put the real one back.
+var retryBackoff = defaultRetryBackoff
+
+// backoffFor returns the delay before the next dispatch, jittered and never
+// longer than the time left before the deadline.
+func backoffFor(dispatches int, remaining time.Duration) time.Duration {
+ if len(retryBackoff) == 0 || remaining <= 0 {
+ return 0
+ }
+ i := dispatches - 1
+ if i < 0 {
+ i = 0
+ }
+ if i >= len(retryBackoff) {
+ i = len(retryBackoff) - 1
+ }
+ d := retryBackoff[i]
+ // Jitter by up to ±25% so a burst of requests that all failed against the
+ // same node does not retry in lockstep and re-collide. The clock's
+ // nanosecond low bits are the entropy source: they differ between
+ // concurrent handlers, and `rand` here is crypto/rand, whose interface is
+ // far heavier than spreading a backoff warrants.
+ if spread := int64(d) / 2; spread > 0 {
+ d = time.Duration(int64(d) - spread/2 + time.Now().UnixNano()%(spread+1))
+ }
+ if d > remaining {
+ d = remaining
+ }
+ return d
+}
+
+// waitBeforeRetry sleeps d, reporting false if the request was abandoned while
+// waiting. A client that has gone away should not keep a retry budget alive:
+// nobody is left to receive the answer, so the remaining attempts are better
+// spent on work someone is waiting for.
+func waitBeforeRetry(ctx context.Context, d time.Duration) bool {
+ if ctx.Err() != nil {
+ return false
+ }
+ if d <= 0 {
+ return true
+ }
+ t := time.NewTimer(d)
+ defer t.Stop()
+ select {
+ case <-t.C:
+ return true
+ case <-ctx.Done():
+ return false
+ }
+}
+
+// errFirstBodyTimeout is returned by awaitFirstBody when an upstream sent its
+// headers but produced no content within the budget.
+var errFirstBodyTimeout = stderrors.New("upstream sent no response body within the first-byte budget")
+
+// errTargetGone is the reason an attempt was abandoned because its target left
+// discovery. It replaces the bare "context canceled" the transport reports,
+// which in a log is indistinguishable from the client hanging up.
+var errTargetGone = stderrors.New("target node left discovery mid-request")
+
+// Attempt outcomes for attemptClaim.
+const (
+ attemptPending int32 = iota
+ attemptCommitted
+ attemptAbandoned
+)
+
+// attemptClaim resolves the race between committing an attempt and abandoning
+// it because its target left discovery.
+//
+// Both can become true at once: the watcher's discovery check can pass while
+// the commit path is already running, and between the first body byte arriving
+// and the commit finishing that path does a synchronous notification write and
+// takes two mutexes. A pair of channels cannot express which happened first —
+// closing one says an event occurred, not that it won — so the outcome is one
+// value that each side claims by compare-and-swap.
+//
+// Exactly one of commit and abandon succeeds, and callers must honor the
+// result: a false from commit means this attempt is being cancelled and must
+// not be served, and a false from abandon means the response is already
+// committed and must not be torn down.
+type attemptClaim struct {
+ state atomic.Int32
+}
+
+// commit claims the attempt for the response. Idempotent, so an already
+// committed attempt still reports true.
+func (a *attemptClaim) commit() bool {
+ return a.state.CompareAndSwap(attemptPending, attemptCommitted) ||
+ a.state.Load() == attemptCommitted
+}
+
+// abandon claims the attempt for cancellation, reporting false when the
+// response has already committed.
+//
+// A committed first byte wins on purpose. The byte is direct evidence that this
+// node is serving this request, whereas an absence from discovery can be a
+// transient announcement gap; cancelling then would truncate a working stream.
+func (a *attemptClaim) abandon() bool {
+ return a.state.CompareAndSwap(attemptPending, attemptAbandoned)
+}
+
+// settled reports whether either side has claimed the outcome.
+func (a *attemptClaim) settled() bool {
+ return a.state.Load() != attemptPending
+}
+
+// abandoned reports whether cancellation won.
+func (a *attemptClaim) abandoned() bool {
+ return a.state.Load() == attemptAbandoned
+}
+
+// bodyWithPeek re-attaches an already-read first byte ahead of the rest of an
+// upstream body, while keeping Close bound to the original so the connection
+// is still released.
+type bodyWithPeek struct {
+ io.Reader
+ io.Closer
+}
+
+// awaitFirstBody blocks until the upstream produces the first byte of its
+// response body, and returns that byte spliced back onto the front of the
+// stream.
+//
+// It exists because response headers are not a reliable signal that an engine
+// has started work, and the difference is engine-specific. LM Studio answers a
+// streaming request with 200 and headers ~12ms after accepting it and then
+// holds the connection open, silent, for the entire time the request waits
+// behind other work — measured at 128s behind a single-slot generation. Ollama
+// withholds headers until generation begins. Committing on headers therefore
+// meant that on LM Studio the proxy bound itself to a node that had merely
+// accepted the request: the header timeout could never fire, failover was
+// impossible, and a node dying during the wait took the job with it.
+//
+// The first content byte is the signal both engines agree on, and it is a real
+// one — no SSE keepalive or empty role delta arrives during the wait, so the
+// byte means generation actually started.
+//
+// A read that ends in EOF with no bytes is an empty body, which is a complete
+// (if unusual) response and commits. A genuine read error before any content
+// is retryable, like any other pre-commit upstream failure.
+func awaitFirstBody(body io.ReadCloser, budget time.Duration) (io.ReadCloser, error) {
+ type peeked struct {
+ b []byte
+ err error
+ }
+ // Buffered so the reader goroutine can always finish. On the timeout path
+ // ReverseProxy closes resp.Body, which unblocks the pending Read.
+ ch := make(chan peeked, 1)
+ go func() {
+ var one [1]byte
+ n, err := body.Read(one[:])
+ ch <- peeked{b: one[:n], err: err}
+ }()
+
+ timer := time.NewTimer(budget)
+ defer timer.Stop()
+ select {
+ case p := <-ch:
+ if len(p.b) == 0 && p.err != nil && !stderrors.Is(p.err, io.EOF) {
+ return nil, p.err
+ }
+ return bodyWithPeek{Reader: io.MultiReader(bytes.NewReader(p.b), body), Closer: body}, nil
+ case <-timer.C:
+ return nil, errFirstBodyTimeout
+ }
+}
+
// Timeouts for upstream connections. Logged at startup so they're always
// present in any captured log for post-mortem analysis.
const (
@@ -532,6 +752,18 @@ const (
// deadline against a real socket; production never reassigns it.
var idleClientWriteTimeout = 30 * time.Second
+// firstBodyTimeout bounds how long awaitFirstBody waits for an engine to emit
+// its first content byte. It matches proxyResponseTimeout, the budget the
+// transport applies to headers, because from the caller's point of view the two
+// bound the same thing: how long we wait for a node to start the work. Keeping
+// it generous is deliberate — it also covers a cold model load, which is real
+// work rather than a stall, and which cannot be told apart from a queue wait
+// from outside the engine.
+//
+// It is a var (not a const) only so a test can shorten it; production never
+// reassigns it.
+var firstBodyTimeout = proxyResponseTimeout
+
var modelListClient = &http.Client{
Transport: &http.Transport{
DialContext: (&net.Dialer{
@@ -969,28 +1201,36 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {
wl *Workload
)
- // Emit workload:started up front, the moment we begin forwarding, naming
- // the first candidate we'll try. A burst of concurrent inference requests
- // must surface as job cards immediately; the upstream engine serializes
- // concurrent requests on a single GPU slot, so gating "started" on the
- // upstream response headers (the commit point) left every queued-but-
- // forwarded job invisible until the node dequeued it — only one card at a
- // time (a prior regression). If failover later commits a different
- // node, the commit block re-points scheduledOn; the terminal
- // completed/errored transition is emitted once at the end regardless.
+ // wlSeq numbers this workload's events. Every mutation of wl below happens
+ // either before the watcher goroutine exists or under wlMu, so a plain
+ // counter is enough; see Workload.Seq for why the sequence is needed.
+ var wlSeq int64
+ nextWlSeq := func() int64 { wlSeq++; return wlSeq }
+
+ // Emit workload:submitted the moment the request is admitted, before any
+ // dispatch. A burst of concurrent inference requests must surface as job
+ // cards immediately — the upstream engine serializes work on a single GPU
+ // slot, so a card that waited for the upstream response would leave every
+ // queued job invisible until the node dequeued it, one at a time (a prior
+ // regression). "queued" is what makes that visible without claiming the
+ // engine is generating, which only becomes true at the commit point below.
+ //
+ // scheduledOn is left empty here: nothing has been dispatched yet, and the
+ // scheduler counts pending work by scheduledOn, so naming a node we have
+ // not tried would inflate its load. Each dispatch and each gap between
+ // attempts re-points it.
if isInf && model != "" {
createdMs := start.UnixMilli()
wl = &Workload{
- ID: reqID,
- Model: model,
- Engine: f.profile.Name,
- RunID: p.runID,
- State: "running",
- ScheduledOn: candidates[0].id,
- CreatedAt: createdMs,
- StartedAt: &createdMs,
+ ID: reqID,
+ Model: model,
+ Engine: f.profile.Name,
+ RunID: p.runID,
+ State: "queued",
+ CreatedAt: createdMs,
+ Seq: nextWlSeq(),
}
- p.emitWorkload(workloadStartedMethod, *wl)
+ p.emitWorkload(workloadSubmittedMethod, *wl)
}
// The terminal workload transition (completed/errored) can be reached from
@@ -1017,8 +1257,14 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {
if errMsg != "" {
wl.Error = &errMsg
}
+ wl.Seq = nextWlSeq()
snapshot := *wl
wlMu.Unlock()
+ // Only "completed" gets its own method; every other terminal state,
+ // including "cancelled", rides workload:errored. The workload
+ // manager does not validate method against state and consumers read
+ // the state out of the payload, so a new terminal state needs no new
+ // method on the wire.
method := workloadCompletedMethod
if state != "completed" {
method = workloadErroredMethod
@@ -1043,7 +1289,7 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {
go func() {
select {
case <-reqCtx.Done():
- emitTerminal("failed", "client disconnected before completion")
+ emitTerminal("cancelled", "client disconnected before completion")
case <-finished:
}
}()
@@ -1051,21 +1297,313 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {
// committedSC is the statusCapture of the candidate we committed to
// streaming; its wroteErr tells us after the fact whether the client write
- // failed (dead/half-open client) so we can mark the workload failed.
+ // failed (dead/half-open client) so we can mark the workload failed. It is
+ // assigned at the commit point as well as after the loop, because the
+ // post-loop assignment does not always run — see finalize below.
var committedSC *statusCapture
- // Failover loop: try candidates in order until one returns a
- // usable response or the list is exhausted. We can only retry before the
- // first byte reaches the client; once a response starts streaming we're
- // committed. proxy/request-started fires at that commit point so it names
- // the node that actually serves the request, not one we failed over from;
- // workload:started was already emitted above (and is re-pointed there on a
- // failover). The self-forward guard lives in resolveCandidates.
- for i := range candidates {
- cand := candidates[i]
- last := i == len(candidates)-1
+ // finalize reports this request's outcome, and it is deferred because a
+ // mid-copy error is never returned to us: ReverseProxy converts one into
+ // panic(http.ErrAbortHandler) whenever it detects a real server, and that
+ // panic unwinds straight past the end of this function. Reported inline,
+ // both the terminal workload event and the proxy/request completion were
+ // lost whenever an upstream died mid-stream while the client was healthy,
+ // leaving the job "running" for the life of the broker. That is worse than
+ // a stuck card: the scheduler counts queued and running by scheduledOn, so
+ // the ghost permanently inflated that node's pending count and biased
+ // routing away from a node whose only fault was a crashed engine.
+ //
+ // Client-side failures never had this problem. net/http cancels the
+ // request context synchronously before a write error propagates
+ // (checkConnErrorWriter), so the watcher above has always emitted by the
+ // time the panic arrives. The upstream-died case is the one that had no
+ // reporter at all.
+ //
+ // Registered after the reservation release above, so on unwind this runs
+ // first and the node is still counted as loaded while we report.
+ defer func() {
+ aborted := recover()
+ // An aborted copy is operational detail, not a verdict on the job. A
+ // stream that already committed a 2xx is a completion whether or not
+ // it reached its end, so this never feeds the terminal state below,
+ // and proxyErr is left untouched for that reason.
+ reportErr := proxyErr
+ if aborted != nil && reportErr == "" {
+ reportErr = "upstream stream aborted mid-response"
+ }
+ if committedSC != nil && finalStatus == 0 {
+ // The panic skipped the post-loop assignment, but the committed
+ // capture still carries the status the upstream sent.
+ finalStatus = committedSC.status
+ }
+
+ slog.Debug("proxy request complete",
+ "id", reqID,
+ "node_id", servedNodeID,
+ "method", r.Method,
+ "path", r.URL.Path,
+ "target", servedTarget,
+ "status", finalStatus,
+ "duration_ms", time.Since(start).Milliseconds(),
+ "ttfb_ms", ttfbMs,
+ "err", reportErr,
+ )
+
+ _ = f.notify("proxy/request", RequestEvent{
+ ID: reqID,
+ NodeID: servedNodeID,
+ Method: r.Method,
+ Path: r.URL.Path,
+ Target: servedTarget,
+ Status: finalStatus,
+ Duration: time.Since(start).Milliseconds(),
+ TTFB: ttfbMs,
+ Error: reportErr,
+ })
+
+ // Terminal workload transition pairs with the workload:started emitted
+ // at forward time above. Cancellation, an upstream/transport error, a
+ // failed client write (dead/half-open client), or any non-2xx status is
+ // a failure; a clean 2xx is a completion. The Workload carries the same
+ // id so the broker (and peers) can collapse the start/finish pair.
+ // Routed through emitTerminal so the disconnect watcher and this path
+ // emit exactly once.
+ if wl != nil {
+ switch {
+ case r.Context().Err() != nil:
+ // The request was cancelled before it finished — either the
+ // client disconnected or, on shutdown, we cancelled it to stop
+ // the in-flight inference. A mid-stream cancel never reaches
+ // ErrorHandler (the 200 headers are already sent), so without
+ // this branch it would be misreported as completed. (The
+ // watcher above usually beats us to it; emitTerminal makes that
+ // a no-op.) Cancelled rather than failed: nothing went wrong
+ // here, the requester stopped waiting.
+ emitTerminal("cancelled", "request cancelled before completion")
+ case committedSC != nil && committedSC.wroteErr != nil:
+ // The response committed but a write to (or flush toward) the
+ // client failed — typically the idle deadline tripping on a
+ // dead/half-open client. The same event as the branch above,
+ // differing only in how we noticed: a client that vanished
+ // without a FIN never cancels the context, so the write
+ // deadline is what surfaces it. Classified identically.
+ emitTerminal("cancelled", "client connection lost: "+committedSC.wroteErr.Error())
+ case proxyErr != "" || finalStatus >= http.StatusBadRequest:
+ msg := proxyErr
+ if msg == "" {
+ msg = fmt.Sprintf("upstream returned HTTP %d", finalStatus)
+ }
+ emitTerminal("failed", msg)
+ default:
+ emitTerminal("completed", "")
+ }
+ }
+
+ if aborted != nil {
+ // Preserve ReverseProxy's contract with http.Server, which recovers
+ // ErrAbortHandler silently and closes the connection. Swallowing it
+ // here would leave the client waiting on a response that will never
+ // be finished or closed.
+ panic(aborted)
+ }
+ }()
+
+ // repointWorkload records where the job is currently placed: a node id while
+ // an attempt is in flight, empty between attempts. The scheduler counts
+ // pending work by scheduledOn, so clearing it is what stops a node we have
+ // given up on from still looking busy. The state stays "queued" throughout,
+ // because "running" means the engine is generating and the broker's store
+ // would reject a return to "queued" as a backwards transition.
+ repointWorkload := func(nodeID string) {
+ if wl == nil {
+ return
+ }
+ wlMu.Lock()
+ if terminated || wl.ScheduledOn == nodeID {
+ wlMu.Unlock()
+ return
+ }
+ wl.ScheduledOn = nodeID
+ wl.Seq = nextWlSeq()
+ snapshot := *wl
+ wlMu.Unlock()
+ p.emitWorkload(workloadSubmittedMethod, snapshot)
+ }
+
+ // releaseHeld drops the capacity claim. Called before every backoff: the
+ // reservation map is process-wide and shared with the other engine's
+ // facade, so holding a claim on a node we have stopped using would push
+ // that engine's traffic away from a node that is in fact free. It is the
+ // same reasoning as clearing scheduledOn, applied to the proxy's own
+ // estimate rather than the scheduler's.
+ releaseHeld := func() {
+ resMu.Lock()
+ defer resMu.Unlock()
+ p.releaseReservation(held)
+ held = reservation{}
+ }
+ // takeHeld re-reserves for a fresh round, replacing any claim still held.
+ // reserveCandidate both orders the round and takes the claim, so a retry
+ // that re-resolves has to go back through it.
+ takeHeld := func(cands []candidate) []candidate {
+ resMu.Lock()
+ defer resMu.Unlock()
+ p.releaseReservation(held)
+ var out []candidate
+ out, held = p.reserveCandidate(f, cands)
+ return out
+ }
+
+ // Retry loop. Each iteration dispatches to one candidate; when a round's
+ // candidates are exhausted it backs off and re-resolves, so a node that
+ // recovered or a model that finished pulling becomes eligible mid-retry.
+ //
+ // Bounded by maxDispatchAttempts for dispatch failures and by jobDeadline
+ // for the whole pre-commit life. Both bounds are checked before every
+ // dispatch — the deadline never truncates an attempt already in flight,
+ // because killing one seconds from its first token and then failing the job
+ // for being out of time would spend the entire wait and discard the result.
+ //
+ // Retrying is only possible before the first byte reaches the client; past
+ // the commit point we are bound to that node, and a stream truncated from
+ // there is the caller's to resume. proxy/request-started fires at the commit
+ // so it names the node that actually served. The self-forward guard lives in
+ // resolveCandidates.
+ // The budget is for inference only. Everything not in the engine's route
+ // table is forwarded verbatim, and some of those are state-changing: a
+ // POST /api/pull starts a model download. shouldRetry treats any 5xx as
+ // retryable regardless of route, so without this split a failed pull would
+ // be replayed up to five times — including against the node that just took
+ // the work, since re-resolution can pick it again.
+ //
+ // Non-inference therefore gets one pass over the resolved candidates: no
+ // re-resolution, no backoff, no deadline.
+ maxDispatches := maxDispatchAttempts
+ retryRounds := true
+ if !isInf {
+ maxDispatches = len(candidates)
+ retryRounds = false
+ }
+
+ deadline := start.Add(jobDeadline)
+ dispatches := 0
+ committed := false
+ respondedWithError := false
+ round := candidates
+ next := 0
+
+ for !committed {
+ if dispatches >= maxDispatches {
+ break
+ }
+ if retryRounds && !time.Now().Before(deadline) {
+ break
+ }
+ // Abandon-now: the requester has gone, so there is nobody left to
+ // receive an answer and the remaining attempts are better spent on work
+ // someone is waiting for. The watcher has already emitted the terminal
+ // as cancelled; the guard on the exhaustion response below keeps this
+ // from also inventing a reason for a client that stopped listening.
+ if r.Context().Err() != nil {
+ break
+ }
+ if next >= len(round) {
+ if !retryRounds {
+ // Single pass: the candidate list is the whole budget.
+ break
+ }
+ // Round exhausted, or no owner was available. Either way the job is
+ // on no node right now, so drop both the placement and the capacity
+ // claim before waiting.
+ repointWorkload("")
+ releaseHeld()
+ if !waitBeforeRetry(r.Context(), backoffFor(dispatches, time.Until(deadline))) {
+ break
+ }
+ round = f.resolveCandidates(routingModel)
+ if isInf && model != "" {
+ round = takeHeld(round)
+ }
+ next = 0
+ if len(round) == 0 {
+ // Every owner went away mid-flight. Not a rejection: keep
+ // waiting for one to come back. This consumes no attempt, which
+ // is why jobDeadline is the only bound on it.
+ continue
+ }
+ }
+ cand := round[next]
+ next++
+ dispatches++
+ // lastPermitted means no further dispatch can happen, so this attempt's
+ // failure is the client's answer. It is not "last candidate in the
+ // round" any more: another round may follow.
+ lastPermitted := dispatches >= maxDispatches ||
+ (retryRounds && !time.Now().Before(deadline))
+ last := lastPermitted
+ repointWorkload(cand.id)
+ // The claim follows the node we are about to try, not the one
+ // reserveCandidate happened to pick for the round — an attempt can hold
+ // a node for the whole first-content budget, and for that time it is
+ // the node under load. A same-node move is a no-op.
+ moveHeld(cand.id)
+
+ // Each attempt runs on its own context, a child of the request's, so we
+ // can abandon this dispatch without touching the request itself. The
+ // distinction is load-bearing: the disconnect watcher and the terminal
+ // classification both read r.Context(), the parent, so cancelling a
+ // child cannot be mistaken for the client going away.
+ attemptCtx, cancelAttempt := context.WithCancel(r.Context())
+ // claim decides, once, whether this attempt commits or is abandoned;
+ // see attemptClaim for why a pair of channels could not.
+ claim := &attemptClaim{}
+
+ // Watch the target while the attempt is pre-commit. A node that has
+ // dropped out of discovery is not coming back to answer a request
+ // already sent to it, and without this the attempt sat out the whole
+ // first-content budget before failing over — while the broker had
+ // already stamped the record failed from its own node-loss sweep, so
+ // the UI said failed while the client still waited.
+ //
+ // Polling rather than subscribing keeps this out of Discovery's
+ // contract, and the interval only bounds detection latency. Watching
+ // stops at the commit point, because cancelling after that would kill a
+ // stream that is working.
+ //
+ // Read the interval once, on this goroutine: the watcher must not
+ // depend on a value that could change under it mid-attempt.
+ watchEvery := targetWatchInterval
+ go func() {
+ t := time.NewTicker(watchEvery)
+ defer t.Stop()
+ for {
+ select {
+ case <-attemptCtx.Done():
+ return
+ case <-t.C:
+ // The commit path may have claimed the outcome since the
+ // last tick, in which case this attempt is no longer ours
+ // to cancel.
+ if claim.settled() {
+ return
+ }
+ if f.discovery.Has(cand.id) {
+ continue
+ }
+ if claim.abandon() {
+ cancelAttempt()
+ }
+ return
+ }
+ }
+ }()
+
+ // ReverseProxy derives the outbound request from the one it is given,
+ // so the attempt context has to travel on a copy. The replayed body
+ // goes on the copy for the same reason.
+ attemptReq := r.WithContext(attemptCtx)
if bodyBytes != nil {
- r.Body = io.NopCloser(bytes.NewReader(bodyBytes))
+ attemptReq.Body = io.NopCloser(bytes.NewReader(bodyBytes))
}
retry := false
sc := &statusCapture{ResponseWriter: w, status: http.StatusOK, idle: idleClientWriteTimeout}
@@ -1089,6 +1627,65 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {
retry = true
return retrySignal{}
}
+ // Hold the commit until the engine actually produces content.
+ // Only for inference: a control endpoint's headers mean what
+ // they say, and the model-list fan-out has its own client.
+ // Waiting here costs a non-streaming response nothing, since
+ // its headers and body arrive together, so the request's own
+ // stream flag never has to be consulted — which also avoids
+ // guessing per-dialect defaults (Ollama streams by default,
+ // the OpenAI routes do not).
+ var peeked io.ReadCloser
+ if isInf {
+ body, err := awaitFirstBody(resp.Body, firstBodyTimeout)
+ if err != nil {
+ // This response must not commit, whether or not another
+ // attempt is permitted. awaitFirstBody's reader is still
+ // blocked on resp.Body, so committing would put two
+ // readers on one stream — the peek could swallow a byte
+ // the client never sees — and against a silent upstream
+ // the copy would then block with nothing left to
+ // interrupt it, hanging the request past every bound we
+ // set.
+ //
+ // Returning an error is what releases that reader:
+ // ReverseProxy closes resp.Body and calls ErrorHandler.
+ proxyErr = err.Error()
+ if !last {
+ retry = true
+ slog.Warn("proxy upstream produced no content, failing over",
+ "id", reqID, "node_id", cand.id, "target", cand.url.Host,
+ "path", r.URL.Path, "err", err)
+ return retrySignal{}
+ }
+ slog.Warn("proxy upstream produced no content, retries exhausted",
+ "id", reqID, "node_id", cand.id, "target", cand.url.Host,
+ "method", r.Method, "path", r.URL.Path,
+ "duration_ms", time.Since(start).Milliseconds(), "err", err)
+ return err
+ }
+ peeked = body
+ }
+ // Claim the attempt before any commit side effect. Losing the
+ // claim means the watcher has already cancelled this attempt,
+ // so committing would stream through a context that is about to
+ // die and truncate the response we just started.
+ if !claim.commit() {
+ proxyErr = errTargetGone.Error()
+ if !last {
+ retry = true
+ slog.Warn("proxy target left discovery as the response committed, failing over",
+ "id", reqID, "node_id", cand.id, "target", cand.url.Host,
+ "path", r.URL.Path)
+ return retrySignal{}
+ }
+ return errTargetGone
+ }
+ // The peeked byte goes back only once the commit is ours, so a
+ // lost claim leaves the body untouched for ReverseProxy to close.
+ if peeked != nil {
+ resp.Body = peeked
+ }
// Prefer an engine-declared preflight policy so an exact origin plus
// Allow-Credentials can pass a credentialed browser fetch. Engines
// that publish no policy retain the proxy's permissive 204 fallback.
@@ -1125,21 +1722,31 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {
Path: r.URL.Path,
Target: cand.url.Host,
})
- // workload:started was already emitted up front naming the
- // first candidate. If failover landed us on a different
- // node, re-point scheduledOn so the card — and the terminal
- // completed/errored event, which carries the same wl — name
- // the node that actually served. Guarded by wlMu against the
- // disconnect watcher, and skipped once terminated so a late
- // re-point can't resurrect a workload we've already failed.
- // Move the capacity claim to the node that actually served,
- // for the same reason: the node that refused the request is
- // not doing the work and must not keep counting as loaded.
+ // The claim already follows each dispatch, so this is a
+ // no-op unless something reordered underneath us; it stays
+ // as the belt-and-braces guarantee that the node that
+ // actually served is the one counted as loaded.
moveHeld(cand.id)
+ // Record the commit here, not just after the loop: a copy
+ // error aborts this handler by panic, and finalize needs to
+ // know the response had committed (and on which capture) to
+ // classify it.
+ committedSC = sc
+ // queued -> running: the engine is producing content, so
+ // this is the first moment "running" is true. It also fixes
+ // the placement on the node that actually served, which may
+ // differ from the last node the queued updates named.
+ // Guarded by wlMu against the disconnect watcher, and
+ // skipped once terminated so a late transition cannot
+ // resurrect a workload we have already finalized.
if wl != nil {
wlMu.Lock()
- if !terminated && wl.ScheduledOn != cand.id {
+ if !terminated {
+ startedMs := time.Now().UnixMilli()
+ wl.State = "running"
+ wl.StartedAt = &startedMs
wl.ScheduledOn = cand.id
+ wl.Seq = nextWlSeq()
snapshot := *wl
wlMu.Unlock()
p.emitWorkload(workloadStartedMethod, snapshot)
@@ -1169,7 +1776,8 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {
"path", r.URL.Path, "err", err)
return
}
- // Last candidate failed at the transport: terminal, surface it.
+ // No further dispatch is permitted: terminal, surface it.
+ respondedWithError = true
servedNodeID = cand.id
servedTarget = cand.url.Host
if cors.WritePreflight(ew, r) {
@@ -1177,7 +1785,7 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {
return
}
proxyErr = err.Error()
- slog.Warn("proxy upstream error, candidates exhausted",
+ slog.Warn("proxy upstream error, retries exhausted",
"id", reqID, "node_id", cand.id, "target", cand.url.Host,
"method", r.Method, "path", r.URL.Path,
"duration_ms", time.Since(start).Milliseconds(), "err", err)
@@ -1187,79 +1795,79 @@ func (f *facade) handleHTTP(w http.ResponseWriter, r *http.Request) {
if mErr != nil {
body = []byte(`{"error":"upstream error"}`)
}
+ // A node that answered but never produced content timed out
+ // rather than failed to be reached, and 504 says so.
+ status := http.StatusBadGateway
+ if stderrors.Is(err, errFirstBodyTimeout) {
+ status = http.StatusGatewayTimeout
+ }
cors.Apply(ew.Header())
ew.Header().Set("Content-Type", "application/json")
ew.Header().Set("X-Content-Type-Options", "nosniff")
- ew.WriteHeader(http.StatusBadGateway)
+ ew.WriteHeader(status)
ew.Write(body)
},
}
- proxy.ServeHTTP(sc, r)
+ proxy.ServeHTTP(sc, attemptReq)
+ cancelAttempt()
+ if claim.abandoned() {
+ // Name the real reason rather than the bare "context canceled" the
+ // transport reports, which reads identically to a client hangup.
+ proxyErr = errTargetGone.Error()
+ }
if !retry {
finalStatus = sc.status
committedSC = sc
- break
+ committed = true
}
}
- slog.Debug("proxy request complete",
- "id", reqID,
- "node_id", servedNodeID,
- "method", r.Method,
- "path", r.URL.Path,
- "target", servedTarget,
- "status", finalStatus,
- "duration_ms", time.Since(start).Milliseconds(),
- "ttfb_ms", ttfbMs,
- "err", proxyErr,
- )
-
- _ = f.notify("proxy/request", RequestEvent{
- ID: reqID,
- NodeID: servedNodeID,
- Method: r.Method,
- Path: r.URL.Path,
- Target: servedTarget,
- Status: finalStatus,
- Duration: time.Since(start).Milliseconds(),
- TTFB: ttfbMs,
- Error: proxyErr,
- })
-
- // Terminal workload transition pairs with the workload:started emitted at
- // the commit point above. Cancellation, an upstream/transport error, a
- // failed client write (dead/half-open client), or any non-2xx status is a
- // failure; a clean 2xx is a completion. The Workload carries the same id so
- // the broker (and peers) can collapse the start/finish pair. Routed through
- // emitTerminal so the disconnect watcher and this path emit exactly once.
- if wl != nil {
+ // Nothing committed and no attempt wrote the client's answer, so the loop
+ // ended between attempts: either the budget ran out or, more usually, every
+ // owner went away and none came back before the deadline. The dispatch
+ // paths answer for themselves — a final transport failure writes its own
+ // 502 and a non-retryable status is passed through — so this is the only
+ // outcome left without a response.
+ //
+ // An abandoned request is excluded: writing to a client that has gone
+ // achieves nothing, and attributing its end to a missing node would be a
+ // fabricated reason for something that was the requester's own choice.
+ if !committed && !respondedWithError && r.Context().Err() == nil {
+ // Name what the loop was actually doing when it ran out, which is what
+ // the caller needs to know. The three cases are distinguishable: the
+ // budget is gone; or the deadline passed while an owner was still there
+ // to try, because slow attempts can reach 10 minutes before they reach
+ // five dispatches; or the deadline passed with the candidate set empty,
+ // which is the wait for an owner to come back.
+ reason := "no node advertising the requested model became available before the retry deadline"
switch {
- case r.Context().Err() != nil:
- // The request was cancelled before it finished — either the
- // client disconnected or, on shutdown, we cancelled it to stop
- // the in-flight inference. A mid-stream cancel never reaches
- // ErrorHandler (the 200 headers are already sent), so without this
- // branch it would be misreported as completed. (The watcher above
- // usually beats us to it; emitTerminal makes that a no-op.)
- emitTerminal("failed", "request cancelled before completion")
- case committedSC != nil && committedSC.wroteErr != nil:
- // The response committed but a write to (or flush toward) the
- // client failed — typically the idle deadline tripping on a
- // dead/half-open client. The stream is truncated, so this is a
- // failure, not the completion the 200 status would otherwise
- // suggest.
- emitTerminal("failed", "client connection lost: "+committedSC.wroteErr.Error())
- case proxyErr != "" || finalStatus >= http.StatusBadRequest:
- msg := proxyErr
- if msg == "" {
- msg = fmt.Sprintf("upstream returned HTTP %d", finalStatus)
- }
- emitTerminal("failed", msg)
- default:
- emitTerminal("completed", "")
+ case dispatches >= maxDispatches:
+ reason = "every dispatch attempt failed"
+ case len(round) > 0:
+ reason = "the retry deadline passed while dispatch attempts were still failing"
+ }
+ proxyErr = reason
+ finalStatus = http.StatusServiceUnavailable
+ slog.Warn("proxy request exhausted its retry budget",
+ "id", reqID, "method", r.Method, "path", r.URL.Path,
+ "dispatches", dispatches, "duration_ms", time.Since(start).Milliseconds(),
+ "reason", reason)
+ body, mErr := json.Marshal(map[string]string{"error": reason})
+ if mErr != nil {
+ body = []byte(`{"error":"retry budget exhausted"}`)
}
+ cors.Apply(w.Header())
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("X-Content-Type-Options", "nosniff")
+ // Advisory only: the caller knows its own patience better than we do.
+ w.Header().Set("Retry-After", "1")
+ w.WriteHeader(http.StatusServiceUnavailable)
+ w.Write(body)
}
+
+ // Reporting happens in the deferred finalize above, so it survives the
+ // ErrAbortHandler panic that a mid-copy error raises.
}
// resolveCandidates returns the ordered list of nodes to try for the current
diff --git a/services/nvpair-proxy/retry_test.go b/services/nvpair-proxy/retry_test.go
new file mode 100644
index 0000000..f3634f4
--- /dev/null
+++ b/services/nvpair-proxy/retry_test.go
@@ -0,0 +1,586 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// TestMain shortens the retry backoff for the whole package. Production waits
+// seconds between attempts, and a body that drives a node to its dispatch
+// budget would otherwise pay all of it in wall clock — one all-nodes-down test
+// alone went from under a second to nineteen. The schedule itself is asserted
+// directly in TestBackoffFor, so shortening it here costs no coverage.
+func TestMain(m *testing.M) {
+ retryBackoff = []time.Duration{time.Millisecond}
+ os.Exit(m.Run())
+}
+
+// newCountingServer is an upstream that answers every request the same way and
+// counts how many it saw, which is the quantity the retry tests assert on: not
+// what a node replied, but how many times it was contacted.
+//
+// The count comes back as a func so a caller cannot reset it, and reading it
+// while requests are still in flight is safe.
+func newCountingServer(t *testing.T, status int, body string) (serverURL string, hits func() int) {
+ t.Helper()
+ var n atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ n.Add(1)
+ w.WriteHeader(status)
+ io.WriteString(w, body)
+ }))
+ t.Cleanup(srv.Close)
+ return srv.URL, func() int { return int(n.Load()) }
+}
+
+// waitForCond polls until cond holds, failing the test if it never does.
+//
+// The retry tests need to act at a point in a request's life — "the dispatch
+// has reached the upstream", not "50ms have passed". On a loaded runner those
+// are different statements, and a fixed sleep that loses the race usually makes
+// the test pass while proving nothing.
+func waitForCond(t *testing.T, timeout time.Duration, what string, cond func() bool) {
+ t.Helper()
+ deadline := time.Now().Add(timeout)
+ for time.Now().Before(deadline) {
+ if cond() {
+ return
+ }
+ time.Sleep(time.Millisecond)
+ }
+ t.Fatalf("timed out after %v waiting for %s", timeout, what)
+}
+
+// decodeJSONBody parses a response body as a JSON object so an assertion can
+// name the field it cares about, instead of substring-matching a serialization
+// that a reordered marshal or a changed quote style would break.
+func decodeJSONBody(t *testing.T, body string) map[string]any {
+ t.Helper()
+ var got map[string]any
+ if err := json.Unmarshal([]byte(body), &got); err != nil {
+ t.Fatalf("response body is not a JSON object: %v (body %q)", err, body)
+ }
+ return got
+}
+
+// setForTest overrides a package-level knob for one test and restores it after.
+//
+// The retry bounds are vars precisely so tests can shorten them. Registering
+// the restore with t.Cleanup keeps it on the same line as the override and runs
+// it even when the body ends in t.Fatal, which a hand-written defer pair in
+// each test does not.
+func setForTest[A any](t *testing.T, target *A, value A) {
+ t.Helper()
+ orig := *target
+ t.Cleanup(func() { *target = orig })
+ *target = value
+}
+
+// TestBackoffFor covers the production schedule, which TestMain has replaced
+// for every other body in the package, so it puts the real one back first.
+func TestBackoffFor(t *testing.T) {
+ setForTest(t, &retryBackoff, defaultRetryBackoff)
+
+ // Indexed by dispatches already made, so the wait after the first dispatch
+ // is the first entry. A deadline an hour out keeps the remaining-time clamp
+ // out of it. Jitter is up to ±25%, so assert a band.
+ for _, tc := range []struct {
+ dispatches int
+ want time.Duration
+ }{
+ {dispatches: 1, want: time.Second},
+ {dispatches: 2, want: 2 * time.Second},
+ {dispatches: 3, want: 4 * time.Second},
+ {dispatches: 4, want: 8 * time.Second},
+ {dispatches: 9, want: 8 * time.Second}, // clamped to the last entry
+ } {
+ got := backoffFor(tc.dispatches, time.Hour)
+ lo := tc.want - tc.want/4
+ hi := tc.want + tc.want/4
+ if got < lo || got > hi {
+ t.Errorf("backoffFor(%d) = %v, want within ±25%% of %v", tc.dispatches, got, tc.want)
+ }
+ }
+}
+
+// TestBackoffFor_NeverOutlastsTheDeadline: a wait longer than the time left
+// would guarantee the bounds check after it fails, so the delay would buy
+// latency and no attempt.
+func TestBackoffFor_NeverOutlastsTheDeadline(t *testing.T) {
+ setForTest(t, &retryBackoff, defaultRetryBackoff)
+
+ // Dispatch 3 alone would wait 4s.
+ if got := backoffFor(3, 100*time.Millisecond); got > 100*time.Millisecond {
+ t.Errorf("backoffFor with 100ms left = %v, want no more than 100ms", got)
+ }
+ if got := backoffFor(1, 0); got != 0 {
+ t.Errorf("backoffFor with no time left = %v, want 0", got)
+ }
+}
+
+// Committing an attempt and abandoning it because its target left discovery can
+// become true at once, so the outcome is a single claim rather than a pair of
+// signals (spec §5.3). These cover the claim directly, because the race window
+// in the handler is not reachable deterministically.
+
+func TestAttemptClaim_CommitWinsATie(t *testing.T) {
+ var claim attemptClaim
+
+ if !claim.commit() {
+ t.Fatal("commit on a pending claim should win")
+ }
+ if claim.abandon() {
+ t.Fatal("abandon after commit must lose: a delivered first byte is evidence the node is serving, and cancelling would truncate a working stream")
+ }
+ if claim.abandoned() {
+ t.Fatal("a committed claim must not report as abandoned")
+ }
+}
+
+func TestAttemptClaim_AbandonBlocksALaterCommit(t *testing.T) {
+ var claim attemptClaim
+
+ if !claim.abandon() {
+ t.Fatal("abandon on a pending claim should win")
+ }
+ if claim.commit() {
+ t.Fatal("commit after abandon must lose: the attempt is already being cancelled, so serving it would stream through a dying context")
+ }
+ if !claim.abandoned() {
+ t.Fatal("an abandoned claim must report as abandoned so the caller can name the real reason")
+ }
+}
+
+func TestAttemptClaim_CommitIsIdempotent(t *testing.T) {
+ var claim attemptClaim
+
+ if !claim.commit() || !claim.commit() {
+ t.Fatal("a second commit from the same committed attempt should still report true")
+ }
+}
+
+func TestAttemptClaim_SettledOnlyAfterAClaim(t *testing.T) {
+ var pending, committed, abandoned attemptClaim
+ committed.commit()
+ abandoned.abandon()
+
+ if pending.settled() {
+ t.Error("an unclaimed attempt must read as unsettled, or the watcher would stop watching it")
+ }
+ if !committed.settled() || !abandoned.settled() {
+ t.Error("a claimed attempt must read as settled so the watcher stops")
+ }
+}
+
+// TestAttemptClaim_ExactlyOneWinnerUnderContention is the property the type
+// exists for: with both sides racing, one and only one succeeds.
+func TestAttemptClaim_ExactlyOneWinnerUnderContention(t *testing.T) {
+ for range 500 {
+ var claim attemptClaim
+ var commits, abandons atomic.Int32
+
+ start := make(chan struct{})
+ var wg sync.WaitGroup
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ <-start
+ if claim.commit() {
+ commits.Add(1)
+ }
+ }()
+ go func() {
+ defer wg.Done()
+ <-start
+ if claim.abandon() {
+ abandons.Add(1)
+ }
+ }()
+ close(start)
+ wg.Wait()
+
+ if got := commits.Load() + abandons.Load(); got != 1 {
+ t.Fatalf("winners = %d, want exactly 1 (commit=%d abandon=%d)",
+ got, commits.Load(), abandons.Load())
+ }
+ }
+}
+
+// TestHandleHTTP_RetriesTheOnlyOwner is the case the old loop could not serve.
+// Bounded by the candidate count, a single-owner cluster got exactly one
+// attempt and then a terminal error — which is the common single-desktop
+// shape. The budget is a dispatch count now, so the same node is tried again.
+func TestHandleHTTP_RetriesTheOnlyOwner(t *testing.T) {
+ forEachEngine(t, func(t *testing.T, tc engineCase) {
+ var hits atomic.Int32
+ flaky := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if hits.Add(1) <= 2 {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ io.WriteString(w, `{"error":"loading model"}`)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ io.WriteString(w, `{"done":true}`)
+ }))
+ defer flaky.Close()
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "only", flaky.URL, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+
+ rec := httptest.NewRecorder()
+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200 after retrying the only owner", rec.Code)
+ }
+ if got := hits.Load(); got != 3 {
+ t.Fatalf("upstream saw %d dispatches, want 3 (two failures then a success)", got)
+ }
+ })
+}
+
+// TestHandleHTTP_StopsAtDispatchBudget: retrying is bounded. An owner that
+// always fails is dispatched to exactly maxDispatchAttempts times, and the
+// final attempt's own response is what the client gets — the proxy does not
+// substitute a generated status when the budget is what ran out.
+//
+// The upstream answers 502 rather than 503 so those two outcomes are
+// distinguishable: 503 is what the proxy writes when the loop ends between
+// attempts (spec §5.4), so an upstream 503 here would match either behavior.
+func TestHandleHTTP_StopsAtDispatchBudget(t *testing.T) {
+ forEachEngine(t, func(t *testing.T, tc engineCase) {
+ const upstreamBody = `{"error":"still loading"}`
+ brokenURL, hits := newCountingServer(t, http.StatusBadGateway, upstreamBody)
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "broken", brokenURL, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+
+ rec := httptest.NewRecorder()
+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())
+
+ if got := hits(); got != maxDispatchAttempts {
+ t.Fatalf("upstream saw %d dispatches, want exactly %d", got, maxDispatchAttempts)
+ }
+ if rec.Code != http.StatusBadGateway {
+ t.Fatalf("status = %d, want the final attempt's own 502 (503 would mean the proxy answered for it)", rec.Code)
+ }
+ if got := rec.Body.String(); got != upstreamBody {
+ t.Fatalf("body = %q, want the upstream's %q passed through unchanged", got, upstreamBody)
+ }
+ })
+}
+
+// TestHandleHTTP_NonInferenceIsNotRetriedBeyondItsCandidates keeps the retry
+// budget off state-changing control-plane routes (spec §5.1). A 5xx is
+// retryable regardless of route, so without the bound a failed POST /api/pull
+// would be replayed into repeated model downloads. One owner means one
+// dispatch.
+func TestHandleHTTP_NonInferenceIsNotRetriedBeyondItsCandidates(t *testing.T) {
+ forEachEngine(t, func(t *testing.T, tc engineCase) {
+ failingURL, hits := newCountingServer(t, http.StatusInternalServerError, `{"error":"pull failed"}`)
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "only", failingURL, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, tc.nonInferencePath, strings.NewReader(`{"name":"m"}`))
+ p.soleFacade().handleHTTP(rec, req)
+
+ if got := hits(); got != 1 {
+ t.Fatalf("upstream saw %d dispatches for a non-inference POST, want exactly 1 (the retry budget must not replay a state-changing route)", got)
+ }
+ if rec.Code != http.StatusInternalServerError {
+ t.Fatalf("status = %d, want the upstream 500 passed through", rec.Code)
+ }
+ })
+}
+
+// TestHandleHTTP_ReservationReleasedBetweenAttempts is the shared-scheduler
+// half of the retry policy, and the reason it is not just a loop.
+//
+// The reservation map is process-wide: both engine facades estimate load from
+// the same counts. A retry that kept its claim while sitting in a backoff would
+// leave a node looking busy when nothing is running on it, and would push the
+// OTHER engine's traffic away from a node that is in fact free. So the claim is
+// dropped before every wait and re-taken on the next dispatch, which is the
+// same reasoning that clears scheduledOn — applied to the proxy's own estimate
+// rather than the scheduler's.
+//
+// Two failing owners, so the claim has to move as well as clear. A leak on the
+// first node would make it look permanently loaded, which is observable twice:
+// the count it is left holding, and the skew in how the five dispatches were
+// split between two nodes that are equally broken.
+func TestHandleHTTP_ReservationReleasedBetweenAttempts(t *testing.T) {
+ tc := anyCase(t)
+
+ firstURL, firstHits := newCountingServer(t, http.StatusServiceUnavailable, "")
+ secondURL, secondHits := newCountingServer(t, http.StatusServiceUnavailable, "")
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "broken-a", firstURL, tc.advertisedModel))
+ disc.AddManual(nodeForModel(t, "broken-b", secondURL, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+ // A snapshot so both nodes are scheduler-listed and therefore reservable.
+ p.SetPriority([]string{"broken-a", "broken-b"})
+
+ rec := httptest.NewRecorder()
+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())
+
+ if got := firstHits() + secondHits(); got != maxDispatchAttempts {
+ t.Fatalf("dispatches = %d (broken-a %d, broken-b %d), want %d in total",
+ got, firstHits(), secondHits(), maxDispatchAttempts)
+ }
+ if firstHits() == 0 || secondHits() == 0 {
+ t.Fatalf("dispatches = broken-a %d, broken-b %d, want both owners tried: a claim stuck on one node steers every later attempt away from it",
+ firstHits(), secondHits())
+ }
+
+ p.priorityMu.Lock()
+ leftover := map[string]int{
+ "broken-a": p.priorityReservations["broken-a"],
+ "broken-b": p.priorityReservations["broken-b"],
+ }
+ p.priorityMu.Unlock()
+ for id, n := range leftover {
+ if n != 0 {
+ t.Errorf("reservations left on %s after the request ended = %d, want 0: a retry leaked its claim, so the node looks loaded to both engines forever", id, n)
+ }
+ }
+}
+
+// TestHandleHTTP_WaitsForAnOwnerWithoutSpendingAttempts pins the split between
+// the two bounds. When every owner disappears mid-flight the request waits for
+// one to come back, and that wait consumes no dispatch attempt — so the only
+// thing that ends it is the deadline. Read against dispatch failures alone the
+// deadline looks unreachable, which is exactly why this path needs a test: it
+// is the reason the deadline is not dead configuration.
+func TestHandleHTTP_WaitsForAnOwnerWithoutSpendingAttempts(t *testing.T) {
+ setForTest(t, &jobDeadline, 1500*time.Millisecond)
+ setForTest(t, &retryBackoff, []time.Duration{150 * time.Millisecond})
+
+ tc := anyCase(t)
+
+ // Present at admission so the request is accepted, but its server is
+ // already closed, so the one dispatch it gets fails at the transport.
+ dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+ deadURL := dead.URL
+ dead.Close()
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "gone", deadURL, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+
+ rec := httptest.NewRecorder()
+ start := time.Now()
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())
+ }()
+
+ // Drop the owner once that first dispatch has failed, so every later
+ // re-resolution finds nothing at all.
+ time.Sleep(75 * time.Millisecond)
+ disc.RemoveManual("gone")
+
+ select {
+ case <-done:
+ case <-time.After(10 * time.Second):
+ t.Fatal("handleHTTP never returned")
+ }
+ elapsed := time.Since(start)
+
+ if rec.Code != http.StatusServiceUnavailable {
+ t.Fatalf("status = %d, want 503 once the deadline passed with no owner", rec.Code)
+ }
+ // The distinguishing evidence: the budget was never spent, so the reason is
+ // the missing owner rather than exhausted dispatches.
+ const want = "no node advertising the requested model became available before the retry deadline"
+ if got := decodeJSONBody(t, rec.Body.String()); got["error"] != want {
+ t.Fatalf("body error = %v, want %q (attempts must not have been consumed by empty resolutions)", got["error"], want)
+ }
+ // It waited rather than giving up at once, which is the point of the wait.
+ if elapsed < jobDeadline/2 {
+ t.Fatalf("returned after %v, want it to keep waiting for an owner until close to the %v deadline", elapsed, jobDeadline)
+ }
+}
+
+// TestHandleHTTP_DeadlineWithAnOwnerPresentSaysSo is the other way to run out
+// of time, and the one whose reason is easy to get wrong.
+//
+// A slow attempt can spend 240s — the header cap and the first-content cap back
+// to back — so a job can reach the 10-minute deadline before it reaches five
+// dispatches. The owner is still right there, so reporting that no node became
+// available would blame a missing node for attempts that were made and failed.
+func TestHandleHTTP_DeadlineWithAnOwnerPresentSaysSo(t *testing.T) {
+ // A deadline the first slow attempt outlives, so the loop breaks on time
+ // rather than on the dispatch budget, with the owner still advertising.
+ setForTest(t, &firstBodyTimeout, 300*time.Millisecond)
+ setForTest(t, &jobDeadline, 150*time.Millisecond)
+
+ tc := anyCase(t)
+ stalled := newStalledEngine(t)
+
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "slow", stalled.url, tc.advertisedModel))
+ p := testProxy(tc.profile, disc, tc.profile.FacadePort)
+
+ rec := httptest.NewRecorder()
+ p.soleFacade().handleHTTP(rec, tc.inferenceRequest())
+
+ if got := stalled.hits(); got == 0 || got >= maxDispatchAttempts {
+ t.Fatalf("upstream saw %d dispatches, want between 1 and %d: the deadline, not the budget, must be what ended this",
+ got, maxDispatchAttempts-1)
+ }
+ if rec.Code != http.StatusServiceUnavailable {
+ t.Fatalf("status = %d, want 503", rec.Code)
+ }
+ const want = "the retry deadline passed while dispatch attempts were still failing"
+ if got := decodeJSONBody(t, rec.Body.String()); got["error"] != want {
+ t.Fatalf("body error = %v, want %q: an owner was advertising throughout, so a missing-node reason would be fabricated",
+ got["error"], want)
+ }
+}
+
+// TestHandleHTTP_TargetLeavingDiscoveryAbortsAttempt covers the in-flight
+// abort, and with it the distinction the whole mechanism rests on.
+//
+// Without it, a node that vanished mid-request was noticed only when the
+// first-content budget expired: the attempt sat for the full 120s while the
+// broker's own node-loss sweep had already stamped the record failed, so the UI
+// reported a failure the client was still waiting on. Here the budget is left
+// long on purpose, so finishing quickly is only possible via the discovery
+// watcher.
+//
+// The second assertion is the subtle half. Aborting an attempt cancels a
+// context, and so does a client hanging up; if the two were confused, either a
+// retarget would be reported as cancelled — losing a job that went on to
+// succeed — or a real disconnect would keep burning attempts. The attempt
+// context is a child, and both the disconnect watcher and the terminal
+// classification read the parent, which is what keeps them apart.
+func TestHandleHTTP_TargetLeavingDiscoveryAbortsAttempt(t *testing.T) {
+ // Left long on purpose: if the watcher does not end the attempt, nothing
+ // else will inside the test's patience.
+ setForTest(t, &firstBodyTimeout, 30*time.Second)
+ setForTest(t, &targetWatchInterval, 25*time.Millisecond)
+
+ tc := anyCase(t)
+
+ stalled := newStalledEngine(t)
+
+ good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ io.WriteString(w, `{"done":true}`)
+ }))
+ defer good.Close()
+
+ rec := &recRW{}
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "vanishing", stalled.url, tc.advertisedModel))
+ disc.AddManual(nodeForModel(t, "survivor", good.URL, tc.advertisedModel))
+ p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)
+ p.soleFacade().SetSelected("vanishing") // deterministic: the doomed node first
+
+ resp := httptest.NewRecorder()
+ start := time.Now()
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ p.soleFacade().handleHTTP(resp, tc.inferenceRequest())
+ }()
+
+ // Take the node away only once the attempt has actually reached the stalled
+ // engine. Removing it first would leave the survivor as the only owner at
+ // resolution time, and the request would succeed through it without the
+ // watcher ever being involved — passing while proving nothing.
+ waitForCond(t, 5*time.Second, "the first dispatch to reach the stalled engine",
+ func() bool { return stalled.hits() > 0 })
+ disc.RemoveManual("vanishing")
+
+ select {
+ case <-done:
+ case <-time.After(20 * time.Second):
+ t.Fatal("handleHTTP never returned: the target leaving discovery did not abort the attempt")
+ }
+ elapsed := time.Since(start)
+
+ if elapsed > 10*time.Second {
+ t.Fatalf("took %v: the attempt waited out the first-content budget instead of aborting on node loss", elapsed)
+ }
+ if resp.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200 from the surviving node", resp.Code)
+ }
+ if !strings.Contains(resp.Body.String(), `"done":true`) {
+ t.Fatalf("body came from the wrong node: %q", resp.Body.String())
+ }
+ if !rec.has(`"state":"completed"`) {
+ t.Fatal("the job must complete: aborting an attempt is not the client disconnecting")
+ }
+ if rec.has(`"state":"cancelled"`) {
+ t.Fatal("a retarget was misreported as cancelled: the attempt cancel was mistaken for the client leaving")
+ }
+}
+
+// TestHandleHTTP_AbandonedRequestStopsRetrying: a client that has gone away
+// should not keep a retry budget alive. Nobody is left to receive the answer,
+// so the remaining attempts belong to work someone is waiting for.
+func TestHandleHTTP_AbandonedRequestStopsRetrying(t *testing.T) {
+ // Long enough that the cancel below lands inside the wait rather than
+ // between two dispatches.
+ setForTest(t, &retryBackoff, []time.Duration{300 * time.Millisecond})
+
+ tc := anyCase(t)
+
+ brokenURL, hits := newCountingServer(t, http.StatusServiceUnavailable, "")
+
+ rec := &recRW{}
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "broken", brokenURL, tc.advertisedModel))
+ p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ req := tc.inferenceRequest().WithContext(ctx)
+
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ p.soleFacade().handleHTTP(httptest.NewRecorder(), req)
+ }()
+
+ // Abandon the request once the first dispatch has been made, which puts the
+ // cancel inside the backoff before the second. Waiting on the count rather
+ // than the clock is what makes the assertion below mean something: with no
+ // dispatch yet observed there would be nothing for a retry to exceed.
+ waitForCond(t, 5*time.Second, "the first dispatch to reach the upstream",
+ func() bool { return hits() > 0 })
+ afterFirst := hits()
+ cancel()
+
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("handleHTTP did not return after the client abandoned the request")
+ }
+
+ if got := hits(); got > afterFirst {
+ t.Fatalf("upstream saw %d dispatches after the client left (was %d): an abandoned request must stop retrying", got, afterFirst)
+ }
+ if !rec.has(`"state":"cancelled"`) {
+ t.Fatal("an abandoned request must terminate as cancelled")
+ }
+}
diff --git a/services/nvpair-proxy/spec.md b/services/nvpair-proxy/spec.md
index 69d486f..92bce19 100644
--- a/services/nvpair-proxy/spec.md
+++ b/services/nvpair-proxy/spec.md
@@ -17,10 +17,62 @@ engine's own dialect, filtering to nodes that advertise the requested model,
choosing among them, and failing over when one refuses.
It is a data plane, not a policy component. Ranking belongs to
-`nvpair-job-scheduler`, membership and trust to `nvpair-cluster-manager`,
-lifecycle to `nvpair-engine-manager`. What this process owns is the choice of
-*which advertised owner receives this request*, and the local accounting that
-makes that choice good under bursts.
+`nvpair-job-scheduler`, and cluster membership and trust to
+`nvpair-cluster-manager`. `nvpair-engine-manager` owns an engine's *install and
+process* lifecycle; the per-request *workload* lifecycle events in §5.5 are this
+process's own. What it owns beyond those events is the choice of *which
+advertised owner receives this request*, and the local accounting that makes
+that choice good under bursts.
+
+### 1.1 What a caller is promised
+
+The proxy aims to get every inference request **started** on some node that can
+serve it, and spends a bounded amount of effort doing so. Stated precisely:
+
+> While budget remains (§5.1), a failure that happens *before* the engine begins
+> producing output is retried rather than returned. Once output begins, the
+> request is committed to that node and is the caller's to finish with.
+
+Failures absorbed that way include a retryable status, a transport error, an
+engine that answers its headers and then produces nothing, and a target that
+leaves discovery mid-request. Waiting for an owner to *exist* costs no attempt
+at all, so a node blinking out does not spend the budget.
+
+Four things bound that aim, and a caller should design for them.
+
+**Effort is bounded, and which bound ran out changes what the caller gets.**
+Absorption holds only while budget remains. Past that there are two shapes, both
+detailed in §5.4:
+
+- **The dispatch budget ran out.** The final permitted attempt is never
+ retried, so its own failure is the answer: the upstream's status and body
+ passed through unchanged, or a `502` if it failed at the transport and a
+ `504` if it answered and produced nothing.
+- **The deadline passed between attempts, or no owner came back.** Nothing is
+ in flight to answer, so the proxy generates a `503` with `Retry-After` that
+ names which of those it was.
+
+Either way this is a persistent, bounded attempt to start — not an assurance
+that every submission eventually runs.
+
+**"Started" is the guarantee, not "finished".** Commitment ends at the first
+byte of response body (§5.2), so `completed` means the engine began generating,
+not that a whole answer arrived.
+
+**A truncated stream still reports `completed`**, for two reasons that compound.
+To the caller it is the same situation either way — it holds partial output and
+is the only party that can decide what to do with it, which is what the
+retrying harness above this layer is for. And the proxy genuinely cannot tell a
+truncated stream from a complete one: a streaming response carries no
+`Content-Length`, and the end-of-turn marker is engine-specific, so recognising
+it would mean parsing response bodies, which §9 forbids.
+
+**A non-streaming request is the weakest case.** Its headers arrive only when
+the response is complete, so "queued inside the engine" and "generating right
+now" look identical from outside. The attempt cap therefore has to cover whole
+generation, and when it expires the proxy cannot tell whether it abandoned a
+stalled request or a working one — so retrying may discard real progress. That
+is a genuine limitation, not a safety margin (§5.6).
## 2. Scope
@@ -54,8 +106,10 @@ single-valued flag carries only one plan.
Between scheduler snapshots a facade takes short-lived **reservations** for work
it has dispatched but that the scheduler has not yet observed. Those live in the
-process. With a process per engine each held half the picture, so simultaneous
-Ollama and LM Studio bursts could both pick the same node believing it idle.
+process, and every facade shares them. If each engine had its own proxy
+process, neither could see the other's reservations, so simultaneous bursts of
+Ollama and LM Studio requests could both select the same node, each incorrectly
+believing it was idle.
Sharing the map is the reason the engines share a process.
### 3.2 What sharing costs
@@ -72,8 +126,10 @@ Inside the process the boundaries are finer, and deliberately so:
| Panic handling a request | That request only; logged with stack, answered as an internal error |
| Process crash or exit | Every facade |
-The first three used to be process-fatal, which was acceptable when the process
-was one engine. They are contained now because they are not.
+The first three failures are contained within their listed boundaries. Only a
+process crash or exit takes down every facade. Containment matters here in a way
+it would not for a single-engine process: one engine's bind race must not take
+another engine's working listener with it.
## 4. Addressing
@@ -116,22 +172,340 @@ For a model-bearing inference request:
4. Forward, failing over through the remaining candidates on a retryable status
or transport error. An upstream model `404` counts as retryable, because
positive inventory can be stale.
+5. When a round's candidates are exhausted, back off and re-resolve (§5.1),
+ until the dispatch budget or the deadline runs out.
An ineligible manual selection cannot override the capability gate, and failover
never broadens to an excluded node.
+### 5.1 Retry bounds
+
+| Bound | Value | Governs |
+| --- | --- | --- |
+| `maxDispatchAttempts` | 5 | how many dispatches may fail |
+| `jobDeadline` | 10 minutes from request creation | how long new attempts may keep being started, including while waiting for an owner to exist |
+| `firstBodyTimeout` | `proxyResponseTimeout` (120s) | one attempt's wait for first content, measured from the arrival of headers |
+
+The retry budget applies only to inference requests. Routes not classified in
+the engine's route table are still forwarded verbatim, and some perform
+state-changing operations; for example, `POST /api/pull` starts a model
+download. Since any `5xx` is retryable regardless of route, re-resolving and
+redispatching such a request could start the operation several times, which is
+both non-obvious and wasteful. For a non-inference request the proxy therefore
+resolves candidates once and tries each at most once. It does not re-resolve
+candidates, back off between rounds, or apply the inference deadline.
+
+**An attempt is consumed by a dispatch, never by a resolution.** Finding no
+eligible owner sends nothing, so a job whose only owner briefly drops out does
+not burn its budget while the node comes back.
+
+**Elapsed time per attempt is the sum of three budgets, not one.** They run in
+sequence: the dial (`proxyDialTimeout`, 10s), then headers
+(`proxyResponseTimeout`, 120s, enforced by the transport), then first content
+(`firstBodyTimeout`, 120s, which starts only once headers arrive). One attempt
+can therefore occupy around 240s — the two 120s caps back to back, since a slow
+dial is a LAN rarity — and nothing caps their sum.
+
+Both bounds are checked before every dispatch and **never truncate an attempt
+already in flight**. Killing one seconds from its first token and then failing
+the job for being out of time would spend the entire wait and discard the
+result. `jobDeadline` is therefore a "no new attempts after" line rather than a
+ceiling on elapsed time, and the worst case is the deadline plus one whole
+attempt: a last dispatch starting just under 600s and running its full 240s
+puts time to first content near fourteen minutes.
+
+Which bound ends a job depends on how its attempts fail, and all three cases
+are real:
+
+- **Fast failures** — a refused connection, a prompt `503` — cost about a
+ second, so all five attempts fit well inside the deadline and the dispatch
+ budget is what ends the job. This is the common case.
+- **Slow failures** — an engine that accepts and then withholds everything —
+ cost close to a whole attempt each, so the deadline arrives first. At 240s
+ per attempt the fourth dispatch would start past 600s, so the job ends after
+ three attempts with budget unspent.
+- **No owner at all** costs no attempt, so the deadline is the only thing that
+ can end the wait.
+
+The last case is why the deadline cannot be dropped as redundant with the
+attempt count, and the middle one is why five attempts is a ceiling rather than
+a promise. **Do not remove either bound as unreachable.**
+
+#### Waiting for an owner
+
+There is no memory of nodes seen in the past: every resolution runs against the
+live discovery snapshot, so "eligible owner" always means "advertising this
+model right now". What differs is the position in the request's life.
+
+- **No owner at admission** — nothing has been dispatched, so there is nothing
+ to wait for being interrupted. Return `502` immediately (§5.4). A model
+ nobody advertises will not become servable by waiting.
+- **No owner after at least one dispatch** — an owner existed a moment ago and
+ went away, so the proxy waits for one to reappear, re-resolving on the backoff
+ schedule. This costs no attempt, so `jobDeadline` is the only thing that ends
+ it: the deadline *is* the grace period for a node coming back.
+
+The asymmetry is deliberate. Waiting is justified by evidence that a capable
+owner existed within this request's lifetime, and nothing weaker.
+
+Candidates are re-resolved every round, so a node that recovered or a model that
+finished pulling becomes eligible mid-retry. The same node may be retried — with
+one owner, all five attempts go to it.
+
+A requester that has gone away ends the retry immediately: nobody is left to
+receive the answer, so the remaining attempts belong to work someone is waiting
+for. Its end is reported as `cancelled` and is never attributed to a missing
+node.
+
+### 5.2 The commit point
+
+**Commit is the first byte of response body, not the response headers.**
+
+Headers are not a valid commit signal: they may show only that the engine
+accepted the request. Treating them as commitment would disable the header
+timeout and prevent failover while the request remains queued inside the
+engine. The first content byte is the first engine-independent evidence that
+generation has begun.
+
+The difference is engine-specific and large. Measured against LM Studio 0.4.x
+with a model loaded at `--parallel 1`, probing while a long generation held the
+single slot:
+
+| Request | Headers | First body byte |
+| --- | --- | --- |
+| streaming | 12ms, on accept | 35.5s, when the slot freed |
+| non-streaming | 38.9s, withheld until complete | with the headers |
+
+Nothing arrives during that wait — no SSE keepalive, no empty role delta — so a
+byte is real content rather than a heartbeat.
+
+**Only the existence of the byte matters, never its value.** The proxy does not
+look at it: inspecting content would make the gate engine-specific, and it
+would mean reading response data §9 forbids. Whatever the byte is, it is
+spliced back onto the front of the stream so the client receives the response
+intact.
+
+`EOF` with no bytes counts as a commit. This is defensive rather than expected
+— an inference route answering `200` with an empty body is an engine
+misbehaving — but treating it as a stall would spend the whole budget waiting
+for content that is never coming and then fail a request the engine considered
+answered.
+
+A first-content timeout **never** commits, including on the final permitted
+attempt, where it is terminal with `504`. Committing there would leave the peek
+reader blocked on the same body, racing two readers on one stream, and against
+a silent upstream the copy would block with nothing left to interrupt it.
+
+The gate is scoped to inference, and costs a non-streaming response nothing,
+since its headers and body arrive together. That is why the request's own
+`stream` flag never has to be consulted and no per-dialect default has to be
+guessed, and it is what makes the boundary engine-independent — correct whether
+an engine withholds headers or sends them on accept.
+
+**After commit there is no retry.** A stream truncated from there is the
+caller's to finish with (§1.1).
+
+#### Known gap: a silent upstream after commit
+
+There is no upstream read or idle deadline. If a committed engine keeps the
+connection open and stops producing, nothing in this process ends the request:
+the client write deadline needs a write to trip, and a silent upstream produces
+nothing to write. A *crashed* engine is handled — the copy fails and §5.4's
+deferred reporter emits the terminal — but a *frozen* one holds the request
+until the client gives up.
+
+Closing it needs a read deadline refreshed on every chunk, and a value chosen
+so a slow-but-healthy generation is not cut off mid-answer. That judgement is
+not made here.
+
+### 5.3 Per-attempt cancellation
+
+A watcher cancels an attempt promptly when its target leaves discovery.
+Otherwise the attempt could stay blocked until the first-content timeout while
+the broker's node-loss sweep had already marked the workload failed, so the UI
+would report failure while the client was still waiting.
+
+Each attempt runs on a context derived from the request's, and that child
+relationship is load-bearing: the disconnect watcher and the outcome
+classification both read the *parent*, so cancelling a child cannot be mistaken
+for the requester leaving. Confusing the two would either report a job that went
+on to succeed as `cancelled`, or let a real disconnect keep burning attempts.
+
+**Commit and cancellation are mutually exclusive, decided by one atomic
+claim.** Both can become true at once — the watcher's discovery check can pass
+while the commit path is already running, and the work between the arrival of
+the first byte and the end of the commit includes a synchronous notification
+write and two mutex acquisitions. A pair of signals cannot express which
+happened first, so the attempt has a single state that each side claims by
+compare-and-swap; the loser becomes a no-op.
+
+**A committed first byte wins that claim.** A delivered byte is direct evidence
+that this node is serving this request, whereas an absence from discovery can be
+a transient announcement gap. If the watcher claims first the attempt is
+abandoned without committing, so the request retries rather than being served a
+stream the proxy then truncates itself.
+
+### 5.4 Status handling and outcome classification
+
+Whether a status is retried depends on *when* it arrives, not only on its class.
+Before commit the proxy may try elsewhere; at commit the status becomes the
+client's answer.
+
+| Upstream status | Pre-commit, budget remaining | Workload state if it is the answer |
+| --- | --- | --- |
+| `2xx` | commits (§5.2) | `completed` |
+| `3xx` | not retried; not expected from an engine, and forwarded verbatim if one sends it | `failed` |
+| `4xx` except `404`/`408`/`429` | not retried — the same request fails identically everywhere, so retrying only spends budget | `failed` |
+| `404` | retried on an inference route only: positive model inventory can be stale | `failed` |
+| `408`, `429` | retried | `failed` |
+| `5xx` | retried | `failed` |
+
+Anything not retried commits and becomes the client's answer immediately, with
+no further dispatch, whatever budget is left.
+
+These are the statuses the proxy itself returns:
+
+| Condition | Status |
+| --- | --- |
+| No owner advertises the model, nothing dispatched | `502` |
+| Final permitted attempt returned a status | that status and body, unchanged |
+| Final permitted attempt failed at the transport | `502` |
+| Final permitted attempt answered but produced no content | `504` |
+| The loop ended between attempts, nothing in flight | `503` with `Retry-After` |
+
+The final permitted attempt is never retried, so it answers for itself. That is
+why exhausting the dispatch budget does not produce a `503`: the fifth attempt
+is the answer. The `503` is for a loop that ended with no attempt to answer,
+which means time ran out.
+
+Its body names which, because the two call for different responses from the
+caller:
+
+- **the deadline passed while attempts were still failing** — an owner was
+ present throughout, and slow attempts reached 10 minutes before they reached
+ five dispatches (§5.1);
+- **no owner became available** — the candidate set was empty and the wait for
+ a node to come back timed out.
+
+A third reason, **every dispatch attempt failed**, guards the case where the
+loop ends on the count with nothing having answered. The paths above make that
+unreachable today; it is cheap to keep and wrong to rely on.
+
+The resulting workload state:
+
+| Outcome | State |
+| --- | --- |
+| committed 2xx, streamed to the end | `completed` |
+| committed 2xx, truncated by the node dying | `completed` |
+| committed non-2xx | `failed` |
+| client disconnected, or dead-client write deadline tripped | `cancelled` |
+| cancelled by our own shutdown | `cancelled` |
+| budget or deadline exhausted pre-commit | `failed` |
+
+`cancelled` distinguishes requester-initiated termination from an actionable
+engine, node, or routing failure. Client disconnects and shutdown cancellation
+therefore do not appear in the failed bucket.
+
+**Reporting is deferred.** A mid-copy error is never returned to the handler:
+`httputil.ReverseProxy` converts it into `panic(http.ErrAbortHandler)` when
+running under a real server. That panic bypasses ordinary code following the
+proxy call, so terminal workload and request reporting must run in a `defer`.
+The deferred reporter records the abort as operational detail, emits the
+terminal event, and re-panics so `http.Server` still closes the connection.
+Reservation release is also deferred so it survives the same unwind.
+
+That panic is not the failure mode §3.2 describes, despite the shared word.
+`http.ErrAbortHandler` is a sentinel `net/http` defines to mean "abandon this
+response without logging a stack", and `http.Server` recovers it silently by
+contract; re-panicking is how the connection gets closed, and swallowing it
+would leave the client waiting on a response that will never be finished.
+§3.2's entry covers an *unexpected* panic — a bug — which is logged with its
+stack and answered as an internal error.
+
+### 5.5 Workload lifecycle
+
+| Event | When | State | `scheduledOn` |
+| --- | --- | --- | --- |
+| `workload:submitted` | admitted | `queued` | empty |
+| `workload:submitted` | each dispatch | `queued` | the target |
+| `workload:submitted` | between attempts | `queued` | empty |
+| `workload:started` | commit | `running` | the node that served |
+| `workload:completed` / `workload:errored` | terminal | per §5.4 | unchanged |
+
+`submitted` covers the whole pre-commit life; `started` means the engine is
+producing content. That split is forced as well as honest: the broker's store
+merges by state rank and rejects a lower one, so a job that claimed `running` at
+dispatch time could not return to `queued` for its next attempt and the retry
+would be invisible. Keeping the loop inside one state, with `scheduledOn`
+carrying the detail, is the only shape that store can represent.
+
+Every event carries `seq`, the producer's event counter from 1. The
+workload-manager's inter-node dedup is a permanent set, so a retry returning to
+a placement it already used — `queued` on A, cleared, `queued` on A again, which
+this loop produces routinely — would otherwise be indistinguishable from a
+redelivery and dropped by every peer.
+
+A job admitted but not yet dispatched has no execution node. Consumers must
+treat an absent `scheduledOn` as "not placed" rather than assuming a node.
+
+### 5.6 Coverage gap: non-streaming requests
+
+A non-streaming request is the case the retry policy serves worst, and the
+weakness is the opposite of too little retrying.
+
+Its headers arrive only when the response is complete, so from outside the
+engine "queued behind other work" and "generating right now" are
+indistinguishable. The 120s header cap therefore has to cover whole generation
+— the first-content wait adds nothing here, since the body arrives with the
+headers — and when it expires the proxy retries without being able to tell
+whether it just abandoned a stalled request or a working one. A generation
+legitimately slower than that cap is restarted elsewhere, discarding real
+progress, and the restart faces the same cap.
+
+Nothing in the current engine surface fixes this: there is no progress signal
+before the response is complete. A caller that cares should prefer a streaming
+request, where the first content byte gives the proxy the evidence it lacks
+here.
+
+### 5.7 Accepted behavior
+
+**An abandoned attempt may still run inside the engine.** Neither engine PAIR
+supports today exposes a way to withdraw a request it has accepted, and both
+queue internally, so a retried job can have more than one copy generating while
+the abandoned copy keeps its node busy.
+
+Whether closing the connection cancels anything upstream is **unverified** —
+plausible, engine-specific, and never measured. The behavior above is stated as
+the pessimistic case for that reason.
+
+This is accepted rather than overlooked: those internal queues are invisible and
+un-retargetable, which is the problem the retry policy works around rather than
+a mechanism it can build on. Do not reach for engine environment bounds or
+occupancy probes to "fix" it — LM Studio exposes no such signal at all, and an
+adopted engine keeps its own environment regardless. An engine that did offer a
+real cancellation API would change this calculus, and the claim should be
+revisited per engine rather than assumed to hold forever.
+
## 6. Reservations
A reservation is one in-flight dispatch this process has made since the last
scheduler snapshot. Estimated load for a node is
`pending + gpuPressure + reservations`, the first two from the scheduler.
-- **Taken** when a candidate is chosen, unless an eligible manual pin applies.
-- **Moved** to the node a failover actually lands on, because the node that
- refused the request is not doing the work.
+- **Taken** when a candidate is chosen, unless an eligible manual pin applies,
+ and re-taken through the same path on each retry round (§5.1).
+- **Moved** to the node an attempt actually targets, because the node that
+ refused the request is not doing the work. An attempt can hold a node for the
+ whole first-content budget, so the claim follows each dispatch rather than
+ waiting for the commit.
- **Released** when the request ends, so a node stops counting as loaded as soon
as it stops working. Without release a trickle of short requests makes an idle
node read as busy for the rest of the snapshot interval.
+- **Released before every retry backoff**, for the same reason applied to a
+ waiting job: between attempts nothing is running, and because the map is
+ process-wide a stale claim would push the *other* engine's traffic away from a
+ node that is in fact free. This mirrors clearing `scheduledOn`, one being the
+ proxy's own estimate and the other the scheduler's.
Each is stamped with the snapshot **generation** it was counted against. A
snapshot supersedes every reservation taken before it — its pending counts
diff --git a/services/nvpair-proxy/workload_test.go b/services/nvpair-proxy/workload_test.go
index 07acdd8..a0dbe71 100644
--- a/services/nvpair-proxy/workload_test.go
+++ b/services/nvpair-proxy/workload_test.go
@@ -37,19 +37,24 @@ func (r *recRW) has(s string) bool {
return strings.Contains(string(r.b), s)
}
-// TestHandleHTTP_WorkloadStartedAtForward is a regression test: a job's
-// workload:started must be emitted when the request is FORWARDED, not when the
-// upstream first responds. Ollama serializes inference on a single GPU, so a
-// burst of parallel submissions is forwarded concurrently but streams back one
-// at a time; emitting at first byte made the queued submissions invisible
-// (they appeared one card at a time). Here the upstream accepts the connection
-// but never sends a byte until released, so a first-byte emission could never
-// fire — yet workload:started must already be on the wire.
+// TestHandleHTTP_WorkloadVisibleBeforeFirstByte is a regression test: a job
+// must be on the wire as soon as it is admitted, not when the upstream first
+// responds. An engine serializes inference on a single GPU slot, so a burst of
+// parallel submissions is dispatched concurrently but streams back one at a
+// time; emitting only at first byte made the waiting submissions invisible and
+// they appeared one card at a time. Here the upstream accepts the connection
+// but never sends a byte until released, so a first-byte emission could not
+// have fired — yet the job must already be visible.
//
-// proxy/request-started intentionally stays at the commit point (so it names
-// the node that actually served after any failover); only workload:started
-// moves to forward time, so this test asserts only that.
-func TestHandleHTTP_WorkloadStartedAtForward(t *testing.T) {
+// The event is workload:submitted, carrying state "queued". "running" is
+// reserved for the commit point, when the engine is actually producing content:
+// a job waiting its turn inside the engine is not running, and the broker's
+// store rejects a backwards transition, so a job that claimed "running" at
+// dispatch time could never return to "queued" for a retry.
+//
+// proxy/request-started also stays at the commit point, so it names the node
+// that actually served after any failover.
+func TestHandleHTTP_WorkloadVisibleBeforeFirstByte(t *testing.T) {
forEachEngine(t, func(t *testing.T, tc engineCase) {
received := make(chan struct{}, 1)
release := make(chan struct{})
@@ -92,13 +97,28 @@ func TestHandleHTTP_WorkloadStartedAtForward(t *testing.T) {
}
// The upstream has the request but has sent no response byte, so a
- // first-byte emission could not have fired. workload:started must
- // still be on the wire — that is the forward-time emission.
- if !waitFor(t, rec, "workload:started") {
- t.Fatal("workload:started not emitted at forward time")
+ // first-byte emission could not have fired. The job must still be
+ // visible, as queued.
+ if !waitFor(t, rec, "workload:submitted") {
+ t.Fatal("workload:submitted not emitted when the request was admitted")
+ }
+ if !rec.has(`"state":"queued"`) {
+ t.Fatal("an admitted job must be queued until the engine produces content")
+ }
+ if rec.has("workload:started") {
+ t.Fatal("workload:started emitted before the engine produced any content")
}
doRelease()
<-done
+
+ // Released: the engine produced content, so the job is now running and
+ // then completes. queued -> running -> completed, never backwards.
+ if !waitFor(t, rec, "workload:started") {
+ t.Fatal("workload:started not emitted once the engine produced content")
+ }
+ if !rec.has(`"state":"running"`) {
+ t.Fatal("the commit point must transition the job to running")
+ }
})
}
diff --git a/services/nvpair-proxy/zombie_test.go b/services/nvpair-proxy/zombie_test.go
index 372d8f3..306bfff 100644
--- a/services/nvpair-proxy/zombie_test.go
+++ b/services/nvpair-proxy/zombie_test.go
@@ -64,14 +64,14 @@ func (c *clientGoneWriter) Write(b []byte) (int, error) {
func (c *clientGoneWriter) Flush() {}
-// TestHandleHTTP_ClientWriteError_MarksFailed is the zombie-job regression: a
+// TestHandleHTTP_ClientWriteError_MarksCancelled is the zombie-job regression: a
// streaming inference response that has committed (200 headers sent) but whose
// body write to the client fails — the signature of a killed / half-open client
-// whose write deadline tripped — must terminate the workload as FAILED, not be
-// silently reported completed. Before the fix the terminal event fired only in
-// the post-handler switch, whose default arm (no ctx cancel, no upstream error,
-// 2xx status) marked it completed, so the truncated stream looked successful.
-func TestHandleHTTP_ClientWriteError_MarksFailed(t *testing.T) {
+// whose write deadline tripped — must terminate the workload as `cancelled`,
+// not be silently reported completed. A default arm that sees no ctx cancel, no
+// upstream error, and a 2xx status would call this a success and leave a
+// truncated stream looking clean.
+func TestHandleHTTP_ClientWriteError_MarksCancelled(t *testing.T) {
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
@@ -106,7 +106,10 @@ func TestHandleHTTP_ClientWriteError_MarksFailed(t *testing.T) {
t.Fatalf("workload:errored emitted %d times, want exactly 1", got)
}
if rec.has("workload:completed") {
- t.Fatal("workload:completed emitted for a request whose client write failed (should be failed)")
+ t.Fatal("workload:completed emitted for a request whose client write failed")
+ }
+ if !rec.has(`"state":"cancelled"`) {
+ t.Fatal("a client that vanished mid-stream must terminate as cancelled, not failed")
}
}
@@ -409,7 +412,10 @@ func TestHandleHTTP_RealSocketWriteDeadline(t *testing.T) {
t.Fatal("workload never terminated after the client stopped reading (zombie: write deadline did not trip / no terminal emitted)")
}
if rec.has("workload:completed") {
- t.Fatal("workload:completed emitted for a client that stopped reading (should be failed)")
+ t.Fatal("workload:completed emitted for a client that stopped reading")
+ }
+ if !rec.has(`"state":"cancelled"`) {
+ t.Fatal("a client that stopped reading must terminate as cancelled, not failed")
}
}
@@ -507,6 +513,117 @@ func TestHandleHTTP_RealSocketFlushDeadline(t *testing.T) {
t.Fatal("workload never terminated after the client stopped reading (flush path not deadline-aware)")
}
if rec.has("workload:completed") {
- t.Fatal("workload:completed emitted for a stalled client (should be failed)")
+ t.Fatal("workload:completed emitted for a stalled client")
+ }
+ if !rec.has(`"state":"cancelled"`) {
+ t.Fatal("a stalled client must terminate as cancelled, not failed")
+ }
+}
+
+// TestHandleHTTP_UpstreamDiesMidStream_EmitsTerminal is the other half of the
+// zombie-job story, and the half no in-process test can express. Here the
+// CLIENT stays healthy — it reads everything the proxy sends — and the
+// UPSTREAM dies mid-body instead.
+//
+// That combination defeats both existing reporters. Nothing cancels the
+// request context, because no write to the client ever fails, so the
+// disconnect watcher never fires. And the copy error is never returned to us:
+// ReverseProxy converts a mid-copy failure into panic(http.ErrAbortHandler)
+// whenever it detects a real server, and that panic unwinds straight past the
+// end of handleHTTP. A terminal emitted inline is therefore lost outright and
+// the workload stays "running" for the life of the broker. The cost is not
+// just a stuck card: the scheduler counts queued and running by scheduledOn,
+// so the ghost permanently inflates that node's pending count and biases
+// routing away from a node whose only sin was a crashed engine.
+//
+// A real http.Server is essential. shouldPanicOnCopyError keys off
+// ServerContextKey, so calling handleHTTP directly — as the in-process tests
+// above do — suppresses the panic and cannot reach this path at all.
+//
+// The response committed a 2xx before truncating, so the terminal is a
+// COMPLETION: a cut-off stream is a success the caller resumes from whatever
+// it received, not a cluster failure.
+func TestHandleHTTP_UpstreamDiesMidStream_EmitsTerminal(t *testing.T) {
+ // A raw listener rather than httptest, so the abrupt close is exact: a
+ // chunked response that stops without its terminating zero-length chunk,
+ // which is what the proxy sees when an engine is killed mid-generation.
+ // Accepting in a loop keeps the address probe in targetURL from consuming
+ // the one connection the request needs.
+ upLn, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen upstream: %v", err)
+ }
+ defer upLn.Close()
+ go func() {
+ for {
+ conn, err := upLn.Accept()
+ if err != nil {
+ return
+ }
+ go func(c net.Conn) {
+ br := bufio.NewReader(c)
+ for {
+ line, err := br.ReadString('\n')
+ if err != nil {
+ c.Close()
+ return
+ }
+ if line == "\r\n" {
+ break
+ }
+ }
+ chunk := `{"response":"partial tokens","done":false}`
+ io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nTransfer-Encoding: chunked\r\n\r\n")
+ fmt.Fprintf(c, "%x\r\n%s\r\n", len(chunk), chunk)
+ c.Close()
+ }(conn)
+ }
+ }()
+
+ tc := anyCase(t)
+ rec := &recRW{}
+ disc := NewDiscovery()
+ disc.AddManual(nodeForModel(t, "node-a", "http://"+upLn.Addr().String(), tc.advertisedModel))
+ p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)
+
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("listen proxy: %v", err)
+ }
+ srv := &http.Server{Handler: http.HandlerFunc(p.soleFacade().handleHTTP)}
+ go func() { _ = srv.Serve(ln) }()
+ defer srv.Close()
+
+ conn, err := net.Dial("tcp", ln.Addr().String())
+ if err != nil {
+ t.Fatalf("dial proxy: %v", err)
+ }
+ defer conn.Close()
+
+ body := tc.inferenceBody()
+ reqText := fmt.Sprintf("POST %s HTTP/1.1\r\n", tc.inferencePath) +
+ "Host: localhost\r\n" +
+ "Content-Type: application/json\r\n" +
+ fmt.Sprintf("Content-Length: %d\r\n", len(body)) +
+ "\r\n" + body
+ if _, err := conn.Write([]byte(reqText)); err != nil {
+ t.Fatalf("write request: %v", err)
+ }
+
+ // Keep the client healthy by draining until the proxy hangs up. A client
+ // that stopped reading would trip the write deadline and cancel the
+ // context, which is the case the tests above cover and would mask this one.
+ go func() { _, _ = io.Copy(io.Discard, conn) }()
+
+ deadline := time.Now().Add(5 * time.Second)
+ terminals := func() int { return rec.count("workload:completed") + rec.count("workload:errored") }
+ for time.Now().Before(deadline) && terminals() == 0 {
+ time.Sleep(10 * time.Millisecond)
+ }
+ if got := terminals(); got != 1 {
+ t.Fatalf("terminal workload events = %d, want exactly 1 (zombie: the ErrAbortHandler panic skipped the terminal)", got)
+ }
+ if !rec.has("workload:completed") {
+ t.Fatal("a truncated stream that had already committed 2xx must terminate as completed, not failed")
}
}
diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go
index cf43461..3dcb6f7 100644
--- a/services/nvpair-ui-broker/broker.go
+++ b/services/nvpair-ui-broker/broker.go
@@ -1011,7 +1011,12 @@ func stateReplayMethod(state string) string {
return "workload:started"
case "completed":
return "workload:completed"
- case "failed":
+ // "cancelled" rides workload:errored, the same frame the proxy emits it on:
+ // the manager does not validate method against state, and consumers read
+ // the state out of the payload. Every state the store can hold has to
+ // appear here, because an omitted one is silently left out of the replay
+ // and drops off the manager's re-sync set across a supervised restart.
+ case "failed", "cancelled":
return "workload:errored"
default:
return ""
diff --git a/services/nvpair-ui-broker/workloadstore/store.go b/services/nvpair-ui-broker/workloadstore/store.go
index 154d56b..4af3c49 100644
--- a/services/nvpair-ui-broker/workloadstore/store.go
+++ b/services/nvpair-ui-broker/workloadstore/store.go
@@ -548,21 +548,26 @@ func recordFrom(in Incoming, now time.Time) Record {
}
}
-// isTerminal reports whether a state ends a workload's life.
+// isTerminal reports whether a state ends a workload's life. A state missing
+// from here is treated as live forever: it would keep counting toward the
+// scheduler's pending total for its node and stay exposed to the staleness
+// sweeps, so every terminal state must be listed.
func isTerminal(state string) bool {
- return state == "completed" || state == "failed"
+ return state == "completed" || state == "failed" || state == "cancelled"
}
// rank orders lifecycle states so state only moves forward within a
-// generation: queued < running < {completed, failed}. "initializing" and any
-// unknown value rank below queued (they're never transmitted; see spec §4).
+// generation: queued < running < {completed, failed, cancelled}. An unknown
+// value ranks below queued, which means a state missing from here is not
+// merely mis-sorted — applyLocked rejects every event carrying it, so the
+// transition is dropped and never reaches the scheduler or any client.
func rank(state string) int {
switch state {
case "queued":
return 1
case "running":
return 2
- case "completed", "failed":
+ case "completed", "failed", "cancelled":
return 3
default:
return 0
diff --git a/services/nvpair-ui-broker/workloadstore/store_test.go b/services/nvpair-ui-broker/workloadstore/store_test.go
index 5ee5bb3..d515e88 100644
--- a/services/nvpair-ui-broker/workloadstore/store_test.go
+++ b/services/nvpair-ui-broker/workloadstore/store_test.go
@@ -118,6 +118,38 @@ func TestApplyEqualRankReemit(t *testing.T) {
}
}
+// TestApplyCancelledIsTerminal covers the two gates a new terminal state has
+// to pass, both of which fail silently and in opposite directions. Omitted from
+// rank() it sorts below queued, so applyLocked rejects the event outright and
+// the transition never reaches the scheduler or any client. Omitted from
+// isTerminal() it is accepted but treated as live forever, so it keeps counting
+// toward its node's pending total and stays exposed to the staleness sweeps.
+func TestApplyCancelledIsTerminal(t *testing.T) {
+ s := New()
+ if !s.Apply(mkIn("1", "a", "running", "node1", 100)) {
+ t.Fatal("initial running should be accepted")
+ }
+ // Ranks above running, so the transition is accepted rather than dropped.
+ if !s.Apply(mkIn("1", "a", "cancelled", "node1", 100)) {
+ t.Fatal("cancelled after running should be accepted (missing from rank(): sorts below queued and is rejected)")
+ }
+ // Terminal, so a late running cannot resurrect it.
+ if s.Apply(mkIn("1", "a", "running", "node1", 100)) {
+ t.Fatal("running after cancelled should be rejected as backwards")
+ }
+ // Terminal, so it drops out of the active set rather than counting as load.
+ r, ok := s.Get("a", "1")
+ if !ok {
+ t.Fatal("record should still be stored")
+ }
+ if !r.Terminal {
+ t.Fatal("cancelled must be terminal (missing from isTerminal(): counts as pending forever and stays exposed to the staleness sweeps)")
+ }
+ if got := len(s.ActiveSnapshot()); got != 0 {
+ t.Fatalf("active snapshot has %d entries, want 0 for a cancelled workload", got)
+ }
+}
+
// TestApplyCrossNodeIsolation: the same numeric id from two origins is two
// distinct workloads and must never merge against each other.
func TestApplyCrossNodeIsolation(t *testing.T) {
diff --git a/services/nvpair-workload-manager/README.md b/services/nvpair-workload-manager/README.md
index 474d938..3a76cfd 100644
--- a/services/nvpair-workload-manager/README.md
+++ b/services/nvpair-workload-manager/README.md
@@ -81,13 +81,14 @@ Defined in [`workload.go`](workload.go):
| --- | --- |
| `id` | Stable workload identifier |
| `model`, `engine` | What was requested and by which engine |
-| `runId` | Optional grouping key |
-| `state` | `initializing`, `queued`, `running`, `completed`, or `failed` |
+| `runId` | Producing process's nonce, minted at proxy startup. Part of the dedup key: `id` is a per-process counter that both engine proxies start at 1 and that resets on restart, so `runId` keeps a reused id from colliding with an older workload |
+| `state` | `queued`, `running`, `completed`, `failed`, or `cancelled` |
| `originatedFrom` | Node the request entered the cluster on |
| `scheduledOn` | Node it was routed to; absent until a target is chosen |
| `createdAt`, `startedAt`, `completedAt` | Epoch milliseconds; the last two are nullable |
| `error` | Normalized failure text, nullable |
| `requesterId` | Optional client attribution, nullable |
+| `seq` | Producer's event counter, from 1. Part of the dedup key, so a workload that returns to a placement it already had is not mistaken for a redelivery |
Optional and nullable fields use pointers so a peer's payload round-trips without
inventing zero values.
diff --git a/services/nvpair-workload-manager/dedup.go b/services/nvpair-workload-manager/dedup.go
index c265a62..4d44dd5 100644
--- a/services/nvpair-workload-manager/dedup.go
+++ b/services/nvpair-workload-manager/dedup.go
@@ -5,6 +5,7 @@ package main
import (
"container/list"
+ "strconv"
"sync"
)
@@ -19,12 +20,14 @@ const defaultDedupCapacity = 10000
// request.
//
// Keys are opaque strings built by the caller: lifecycle events key on
-// nodeId + Workload.id + state, removals key on workloadId (see keyLifecycle
-// / keyRemove). nodeId is part of the lifecycle key because Workload.id is
-// only unique per-node (spec §11) — without it, the same id from two nodes
-// would collide. Keying on (nodeId, id, state) means a re-broadcast carrying
-// the same triple but updated metadata is treated as a duplicate and dropped,
-// not merged — a known and accepted granularity trade-off (spec §4).
+// nodeId + Workload.id + state + scheduledOn, removals key on workloadId (see
+// keyLifecycle / keyRemove). nodeId is part of the lifecycle key because
+// Workload.id is only unique per-node (spec §11) — without it, the same id
+// from two nodes would collide. The key still omits the workload's other
+// mutable metadata, so a re-broadcast carrying an updated field it does not
+// cover — a corrected error string, say — is treated as a duplicate and
+// dropped rather than merged. That remains a known granularity trade-off
+// (spec §4).
type dedupIndex struct {
mu sync.Mutex
capacity int
@@ -69,14 +72,34 @@ func (d *dedupIndex) seenOrAdd(key string) bool {
// keyLifecycle builds the dedup key for a lifecycle event. Workload.id is only
// a per-process counter, and both engine proxies count from 1 and reset on
-// restart, so the full identity is (originatedFrom, engine, runId, id): nodeId
-// disambiguates nodes, engine + the per-process runId nonce disambiguate the
-// two local engines and successive proxy runs, and state completes the
-// lifecycle-step key. Dropping any component would collapse distinct workloads
-// (e.g. a concurrent Ollama + LM Studio job both id "1") and silently discard a
-// legitimate peer event.
+// restart, so the workload's identity is (originatedFrom, engine, runId, id):
+// nodeId disambiguates nodes, and engine plus the per-process runId nonce
+// disambiguate the two local engines and successive proxy runs. Dropping any
+// component would collapse distinct workloads (e.g. a concurrent Ollama + LM
+// Studio job both id "1") and silently discard a legitimate peer event.
+//
+// seq then completes the key, and it has to be the producer's event sequence
+// rather than any property of the workload's current shape. This index is a
+// permanent set, so anything derived from shape collides as soon as a workload
+// revisits a shape it already had — and a retry does that routinely: queued on
+// A, placement cleared between attempts, then queued on A again. Keyed on
+// (state, scheduledOn) the third event matched the first and every peer
+// dropped it, leaving their brokers holding the interim unplaced record while
+// the job actually ran on A, so their schedulers stopped counting it against
+// the node doing the work.
+//
+// state and scheduledOn stay in the key as well. They cost nothing, and they
+// keep the key meaningful for a producer that has not stamped a sequence.
+//
+// This does not weaken what the index is for: a broadcast retry resends an
+// identical frame, sequence included, so a true redelivery still dedups — and
+// that is what protects a consumer from an out-of-order retry reverting a
+// placement. A re-sync heartbeat bypasses this index altogether (see
+// isResyncFrame).
func keyLifecycle(w *Workload) string {
- return "wl\x00" + w.OriginatedFrom + "\x00" + w.Engine + "\x00" + w.RunID + "\x00" + w.ID + "\x00" + string(w.State)
+ return "wl\x00" + w.OriginatedFrom + "\x00" + w.Engine + "\x00" + w.RunID + "\x00" + w.ID +
+ "\x00" + string(w.State) + "\x00" + w.ScheduledOn +
+ "\x00" + strconv.FormatInt(w.Seq, 10)
}
// keyRemove builds the dedup key for a removal: nodeId + workloadId. As with
diff --git a/services/nvpair-workload-manager/dedup_test.go b/services/nvpair-workload-manager/dedup_test.go
index 557f54d..6c4e8d6 100644
--- a/services/nvpair-workload-manager/dedup_test.go
+++ b/services/nvpair-workload-manager/dedup_test.go
@@ -96,6 +96,75 @@ func TestDedupDistinguishesEngineAndRun(t *testing.T) {
}
}
+// TestDedupDistinguishesPlacement guards the re-point: a failover or a retry
+// moves a workload to another node while its state stays "running", so the two
+// events differ on scheduledOn and nothing else. With scheduledOn out of the
+// key the second was dropped as a duplicate, and every peer went on showing
+// the job on the node it was first sent to for the rest of its life. A repeat
+// of the same placement must still dedup, which is what keeps a broadcast
+// retry from being re-applied.
+func TestDedupDistinguishesPlacement(t *testing.T) {
+ d := newDedupIndex(8)
+
+ first := &Workload{ID: "1", OriginatedFrom: "host", Engine: "ollama", RunID: "r1", State: StateRunning, ScheduledOn: "node-A"}
+ repointed := &Workload{ID: "1", OriginatedFrom: "host", Engine: "ollama", RunID: "r1", State: StateRunning, ScheduledOn: "node-B"}
+
+ if d.seenOrAdd(keyLifecycle(first)) {
+ t.Fatal("first placement on node-A should be new")
+ }
+ if d.seenOrAdd(keyLifecycle(repointed)) {
+ t.Fatal("a re-point to node-B differs only in scheduledOn and must not dedup against node-A")
+ }
+ if !d.seenOrAdd(keyLifecycle(first)) {
+ t.Fatal("a resent frame for the node-A placement should still dedup")
+ }
+}
+
+// TestDedupDistinguishesRepeatedPlacements is the reason the key carries the
+// producer's event sequence rather than only the workload's current shape.
+//
+// This index is a permanent set, so any shape-derived key collides as soon as a
+// workload revisits a shape it already had — and the retry loop does that
+// routinely: queued on A, placement cleared between attempts, then queued on A
+// again. Keyed on shape alone the third event matched the first, so every peer
+// dropped it and their brokers kept the interim unplaced record while the job
+// was really running on A, which took it out of A's pending load.
+//
+// The redelivery case still has to dedup, because that is what stops an
+// out-of-order broadcast retry from reverting a placement.
+func TestDedupDistinguishesRepeatedPlacements(t *testing.T) {
+ d := newDedupIndex(8)
+
+ base := func(seq int64, node string) *Workload {
+ return &Workload{
+ ID: "1", OriginatedFrom: "host", Engine: "ollama", RunID: "r1",
+ State: StateQueued, ScheduledOn: node, Seq: seq,
+ }
+ }
+
+ onA := base(1, "node-A")
+ cleared := base(2, "")
+ backOnA := base(3, "node-A")
+
+ if d.seenOrAdd(keyLifecycle(onA)) {
+ t.Fatal("first placement on node-A should be new")
+ }
+ if d.seenOrAdd(keyLifecycle(cleared)) {
+ t.Fatal("clearing the placement between attempts should be new")
+ }
+ if d.seenOrAdd(keyLifecycle(backOnA)) {
+ t.Fatal("re-dispatching to node-A repeats an earlier shape and must NOT dedup against it")
+ }
+ // A redelivery of any of those frames carries its original sequence, so it
+ // is still recognised as one.
+ if !d.seenOrAdd(keyLifecycle(base(1, "node-A"))) {
+ t.Fatal("a resent frame for the first placement should still dedup")
+ }
+ if !d.seenOrAdd(keyLifecycle(base(3, "node-A"))) {
+ t.Fatal("a resent frame for the third event should still dedup")
+ }
+}
+
// TestDedupRemovalDistinguishesNodes mirrors TestDedupDistinguishesNodes for
// the removal path: the same workloadId removed on two nodes must be two
// distinct dedup entries.
diff --git a/services/nvpair-workload-manager/manager.go b/services/nvpair-workload-manager/manager.go
index d9895e5..84ea634 100644
--- a/services/nvpair-workload-manager/manager.go
+++ b/services/nvpair-workload-manager/manager.go
@@ -372,7 +372,7 @@ func (m *Manager) broadcastFrame(method string, params json.RawMessage) {
// times, then it ages out. This is the set a newly-discovered peer is backfilled
// with and the heartbeat re-syncs.
func (m *Manager) trackActive(key workloadKey, method string, params json.RawMessage, state WorkloadState) {
- terminal := state == StateCompleted || state == StateFailed
+ terminal := state == StateCompleted || state == StateFailed || state == StateCancelled
m.activeMu.Lock()
defer m.activeMu.Unlock()
// Monotonic: a single inference goes running→terminal and never back, so a
diff --git a/services/nvpair-workload-manager/spec.md b/services/nvpair-workload-manager/spec.md
index c00fb24..194967a 100644
--- a/services/nvpair-workload-manager/spec.md
+++ b/services/nvpair-workload-manager/spec.md
@@ -27,18 +27,18 @@ Tracks inference workloads cluster-wide as they are queued, executed, and retire
## 3. Key Use Cases
- **Broadcast a local event**: the Broker emits a `workload:*` lifecycle event (`workload:submitted`, `workload:started`, `workload:completed`, `workload:errored`) or `workloads:remove` on `stdin`; the Workload Manager broadcasts it to every discovered peer via `POST /v1/workloads/events` over mTLS.
-- **Relay a remote event**: on receiving a peer's `workload:*`, validate and deduplicate (by `(nodeId, Workload.id, state)`), then emit `workloads:upsert` on `stdout` so the Broker updates its catalog. A peer `workloads:remove` (deduplicated by `(nodeId, workloadId)`) is relayed as `workloads:remove` on `stdout`.
+- **Relay a remote event**: on receiving a peer's `workload:*`, validate and deduplicate (by `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)`), then emit `workloads:upsert` on `stdout` so the Broker updates its catalog. A peer `workloads:remove` (deduplicated by `(nodeId, workloadId)`) is relayed as `workloads:remove` on `stdout`.
- **Discover peers**: the Broker registers this node's `wl` port with the `nvpair-node-scanner` discovery daemon, which carries it on this node's single consolidated record; the Workload Manager subscribes for `wl` nodes and rebuilds the target set from each `discovery:nodes` snapshot the Broker relays, so a node joins the set when it appears in a snapshot and leaves when it is absent from the next one.
-- **Edge case — duplicate / re-broadcast events**: retries or re-broadcasts arriving more than once are deduplicated (`(nodeId, Workload.id, state)` for lifecycle, `(nodeId, workloadId)` for removals) so the Broker is updated at most once.
+- **Edge case — duplicate / re-broadcast events**: retries or re-broadcasts arriving more than once are deduplicated (`(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)` for lifecycle, `(nodeId, workloadId)` for removals) so the Broker is updated at most once.
- **Edge case — peer unreachable**: a target that is down, slow, or partitioned (crash, network split, laptop suspend) does not block others — broadcast runs concurrently with per-peer timeouts and bounded retries.
## 4. Open Questions / Risks
-- **`initializing` state (open — needs third-party feedback)**: the enum value has no `workload:*` method, so it is never produced or transmitted as specified. Pending the Broker/inference-proxy team's confirmation whether it is a Broker-local pre-queue state only, or needs a new lifecycle method and broadcast path.
+- **`initializing` state (closed)**: removed. It had no `workload:*` method and nothing ever produced it, so it was an unreachable member of a closed enum. The proxy has no distinct pre-dispatch moment to represent — admission and the first dispatch are effectively simultaneous — so the value was deleted rather than given a method.
- **`EngineType` values (open — needs third-party feedback)**: the valid `engine` set is undefined. Pending the inference-engine team's list, `engine` is treated as an opaque pass-through string (must be present and non-empty, value not validated).
- **Risk — late joiners / no replay**: nodes discovered mid-session only get events broadcast after they appear in the target set; their view is incomplete until later events flow (by design).
- **Risk — best-effort fan-out**: an unreachable peer misses events; temporary inconsistency until later events arrive or the Broker reconciles by timestamp.
- **Risk — snapshot staleness**: the target set is only as current as the last `discovery:nodes` snapshot, so a departed node lingers as a target and a new node appears slowly, bounded by the discovery daemon's own liveness handling rather than by anything this service controls.
-- **Risk — dedup granularity**: keying on `(nodeId, Workload.id, state)` means a re-broadcast with the same key but updated metadata (e.g. a corrected `error`) is dropped, not merged.
+- **Risk — dedup granularity**: the key is `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)`, so a re-broadcast carrying the same key with updated metadata (e.g. a corrected `error`) is dropped, not merged. `seq` is the producer's event counter and is what makes the key exact, because this index is a permanent set: any key derived only from a workload's current shape collides as soon as the workload revisits a shape it already had, which a retry does routinely (queued on A, placement cleared between attempts, queued on A again). Without it peers dropped that third event, kept the interim unplaced record, and stopped counting an active job against the node running it.
## 5. Requirements
@@ -46,7 +46,7 @@ Tracks inference workloads cluster-wide as they are queued, executed, and retire
- Accept local `workload:*` and `workloads:remove` notifications from the Broker over `stdin` (or a named pipe) and broadcast each to all discovered peers via `POST /v1/workloads/events`.
- For each validated, deduplicated inter-node `workload:*`, emit `workloads:upsert` to the Broker (translated, not forwarded unchanged); for each inter-node `workloads:remove`, emit `workloads:remove` to the Broker (not re-broadcast).
- Subscribe to the Broker's discovery relay with `discovery:subscribe` filtered to the `wl` service, and rebuild the broadcast target set from every `discovery:nodes` snapshot: take each peer's dialable address and `wl` port from its directory entry, skip entries advertising no `wl` port or no address, and exclude this node's own entry by `hostUuid` rather than by hostname. A snapshot carries the full filtered set and replaces the target set wholesale, so there are no per-node deltas to apply and a peer absent from a snapshot simply stops being a target.
-- Deduplicate inbound lifecycle events by `(nodeId, Workload.id, state)` and removals by `(nodeId, workloadId)`, using a configurable bounded LRU index (default ~10,000 entries, sized for session-scoped volume at ~dozen-node scale). `nodeId` is part of the key because `Workload.id` is only unique per node (§11) — keying on `id` alone would collide across nodes and silently drop a legitimate peer's event. This matches the `(nodeId, workloadId)` tuple the Broker uses as the global workload identity (§10).
+- Deduplicate inbound lifecycle events by `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)` and removals by `(nodeId, workloadId)`, using a configurable bounded LRU index (default ~10,000 entries, sized for session-scoped volume at ~dozen-node scale). `nodeId` is part of the key because `Workload.id` is only unique per node (§11) — keying on `id` alone would collide across nodes and silently drop a legitimate peer's event. `engine` and `runId` are there for the same reason one level down: `Workload.id` is a per-process counter, both engine proxies count from 1, and the counter resets on restart, so without them a concurrent Ollama and LM Studio job both holding id `"1"` would collapse into one. The client-visible identity the Broker uses as the global key remains the coarser `(nodeId, workloadId)` pair (§10); this key is finer on purpose.
- Validate inbound payloads; reject malformed envelopes or unknown `method` values (`400 Bad Request`).
**Non-functional**
@@ -66,6 +66,8 @@ Tracks inference workloads cluster-wide as they are queued, executed, and retire
model: string
engine: EngineType
state: WorkloadState
+ runId?: string // Producing process's nonce, minted at proxy startup. Optional/additive (§7.3). Part of the dedup key: `id` is a per-process counter, both engine proxies count from 1, and it resets on restart, so runId is what keeps a reused id from colliding with an older workload. Absent from a producer that does not stamp it, in which case dedup degrades to a coarser key.
+ seq?: number // Producer's event counter for this workload, from 1, in emission order. Optional/additive (§7.3). Part of the dedup key, and the component that makes it exact: the index is a permanent set, so any key derived only from a workload's current shape collides when the workload revisits a shape it already had (a retry re-dispatching to a node it already tried). Absent from a producer that does not stamp it, in which case dedup degrades to shape-only and such a repeat is dropped as a duplicate.
originatedFrom: string // Owner/origin node of the workload (its identity half of the (originatedFrom, id) global key). Distinct from scheduledOn, the node it was routed to.
scheduledOn?: string // Node the workload was routed to / scheduled on (where it actually ran), as opposed to originatedFrom (where it came from). Supplied by the Broker/proxy and passed through opaquely. Optional/additive (§7.3) — absent until a target is chosen (e.g. a still-queued workload:submitted).
createdAt: number
@@ -76,10 +78,18 @@ Tracks inference workloads cluster-wide as they are queued, executed, and retire
}
```
-`WorkloadState` (enum): `"initializing" | "queued" | "running" | "completed" | "failed"`
-- Method → `state`: `workload:submitted` → `queued`, `workload:started` → `running`, `workload:completed` → `completed`, `workload:errored` → `failed`. `initializing` is a pre-queue state with no lifecycle method and is not transmitted (see §4).
+`WorkloadState` (enum): `"queued" | "running" | "completed" | "failed" | "cancelled"`
+- Method → `state`: `workload:submitted` → `queued`, `workload:started` → `running`, `workload:completed` → `completed`, `workload:errored` → `failed` or `cancelled`.
+- `cancelled` is terminal and means the requester stopped waiting — a disconnected client, a dead client whose write deadline tripped, or an in-flight request cancelled by the producer's own shutdown. It rides `workload:errored` rather than having a method of its own, because state is not validated against the method that carried it and consumers read `state` from the payload. Keeping it distinct from `failed` is what lets a consumer's failed bucket mean "an outcome someone might act on" instead of also collecting every time a user pressed stop.
-Validation: check every inbound envelope before processing. Inter-node and local interfaces both accept the four `workload:*` methods (with `params.workloadInfo`) and `workloads:remove` (with `params.workloadId`). Reject malformed/unknown-`method` payloads on the inter-node interface (`400`); drop-and-log on the local interface (never broadcast). Deduplicate inter-node lifecycle by `(nodeId, Workload.id, state)` and removals by `(nodeId, workloadId)`. Assume the Broker supplies well-formed notifications with epoch-millisecond timestamps.
+Validation: check every inbound envelope before processing. A `Workload` must
+carry `id`, `model`, `engine`, `state`, and `originatedFrom`; `runId`, `seq`,
+and `scheduledOn` are optional/additive, and the timestamp and nullable fields
+are passed through opaquely. Note the asymmetry with the dedup key: `runId` and
+`seq` are part of that key but are not required, so a producer that omits them
+is accepted and simply gets coarser deduplication rather than a rejection.
+Inter-node and local interfaces both accept the four `workload:*` methods (with
+`params.workloadInfo`) and `workloads:remove` (with `params.workloadId`). Reject malformed/unknown-`method` payloads on the inter-node interface (`400`); drop-and-log on the local interface (never broadcast). Deduplicate inter-node lifecycle by `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)` and removals by `(nodeId, workloadId)`. Assume the Broker supplies well-formed notifications with epoch-millisecond timestamps.
**Outputs** — to the Broker over `stdout` / named pipe, or to peers via HTTP REST. `workloads:upsert` on `stdout` after a remote `workload:*`; `workloads:remove` on `stdout` after a remote inter-node `workloads:remove`. Inter-node broadcast posts the same JSON-RPC notification to `POST /v1/workloads/events` (`200 OK` on success; see §7.2). No response on the local interface (notifications only); `stdout` writes are serialized so frames never interleave; inter-node delivery is best-effort (concurrent, per-peer timeouts, bounded retries).
@@ -115,7 +125,10 @@ Example `workloads:upsert` (`stdout`):
"model": "llama-3-70b",
"engine": "trt-llm",
"state": "running",
+ "runId": "9f3c1a7b",
+ "seq": 3,
"originatedFrom": "node-B",
+ "scheduledOn": "node-C",
"createdAt": 1716998400000,
"startedAt": 1716998401000,
"completedAt": null,
@@ -159,6 +172,8 @@ Example notification:
"model": "llama-3-70b",
"engine": "trt-llm",
"state": "queued",
+ "runId": "9f3c1a7b",
+ "seq": 1,
"originatedFrom": "node-A",
"createdAt": 1716998400000,
"startedAt": null,
@@ -177,7 +192,7 @@ Example notification:
- A node broadcasts each local `workload:*` and `workloads:remove` to all discovered peers.
- Body: JSON-RPC 2.0 notification — `workload:*` with `params.workloadInfo`, or `workloads:remove` with `params.workloadId` (and optional `params.originatedFrom`).
- Responses: `200 OK` (accepted; lifecycle → `workloads:upsert`, removal → `workloads:remove` to the local Broker); `400 Bad Request` (malformed / unknown `method`); `403 Forbidden` (client cert absent or untrusted).
-- Idempotency: lifecycle by `(nodeId, Workload.id, state)`; removal by `(nodeId, workloadId)`.
+- Idempotency: lifecycle by `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)`; removal by `(nodeId, workloadId)`.
Example request (`POST /v1/workloads/events`):
```json
@@ -239,7 +254,7 @@ Response: `200 OK`.
- **Peer unreachable/slow during broadcast** (partition, crash, suspend, GC pause): that peer's Broker misses the event, causing temporary inconsistency. → Concurrent broadcast with per-peer timeouts so one peer can't block others or the local Broker; bounded exponential-backoff retries, drop after max attempts with a metric/alert; peers reconcile via later events.
- **Connection exhaustion under burst** (observed in the field): a per-event connection makes every lifecycle event pay a full mTLS handshake, and an unreaped idle connection pins a descriptor for the process lifetime. At inference-burst rates this starves the sender and the receiving listener until handshakes fail outright, and the events lost are disproportionately terminal ones — which the ~two-interval terminal retention cannot recover once the window passes. → Pool connections per peer with a bound on both total and idle connections per host, reap idle connections on both ends (§7.2), and drain response bodies so a connection is actually reusable. Note the residual: fan-out concurrency itself is still per-event, so a heartbeat round with many active workloads issues many simultaneous requests per peer.
- **mTLS handshake failure / untrusted or expired cert**: that peer is isolated from broadcasts (`403`). → Reject and log with the presented identity; monitor and alert ahead of cert expiry; keep the trust store hot-reloadable so rotations apply without restart.
-- **Duplicate / re-broadcast events**: the Broker could see the same transition repeatedly, corrupting counts/state. → Deduplicate by `(nodeId, Workload.id, state)` (removals by `(nodeId, workloadId)`) before emitting, so the Broker updates at most once.
+- **Duplicate / re-broadcast events**: the Broker could see the same transition repeatedly, corrupting counts/state. → Deduplicate by `(nodeId, engine, runId, Workload.id, state, scheduledOn, seq)` (removals by `(nodeId, workloadId)`) before emitting, so the Broker updates at most once.
- **Malformed payload / unknown `method`**: could crash the parser or propagate garbage. → Validate every envelope; reject (`400`) inter-node or drop-and-log locally; never forward unvalidated payloads.
- **Local interface severed** (Broker exited → `stdin` EOF / `stdout` `EPIPE` / `ERROR_BROKEN_PIPE`): an orphaned manager would accept peer events with nowhere to forward them. → Treat EOF/`EPIPE` as the shutdown signal: stop the listener and exit cleanly. No reconnect/buffering — a new Broker spawns a new (stateless) manager. The small load lets the OS pipe buffer absorb serialized `stdout` writes.
- **Out-of-order delivery for the same workload (local)**: the Broker could see an inconsistent progression (e.g. `completed` before `started`). → Serialize `stdout` writes; the Broker orders per workload via `(nodeId, workloadId)` (monotonic IDs) and timestamps.
diff --git a/services/nvpair-workload-manager/workload.go b/services/nvpair-workload-manager/workload.go
index a64b2ab..36d7c9d 100644
--- a/services/nvpair-workload-manager/workload.go
+++ b/services/nvpair-workload-manager/workload.go
@@ -12,18 +12,22 @@ import (
// See versions.json at the repo root for the source of truth.
var Version = "dev"
-// WorkloadState is the lifecycle state carried in a Workload. The
-// "initializing" value exists in the enum but has no lifecycle method and is
-// never transmitted (see spec §4); it is accepted on the wire for forward
-// compatibility but is not produced by this service.
+// WorkloadState is the lifecycle state carried in a Workload. State is not
+// validated against the method that carried it (see lifecycleMethods), so
+// "cancelled" travels on workload:errored without needing a method of its own.
type WorkloadState string
const (
- StateInitializing WorkloadState = "initializing"
- StateQueued WorkloadState = "queued"
- StateRunning WorkloadState = "running"
- StateCompleted WorkloadState = "completed"
- StateFailed WorkloadState = "failed"
+ StateQueued WorkloadState = "queued"
+ StateRunning WorkloadState = "running"
+ StateCompleted WorkloadState = "completed"
+ StateFailed WorkloadState = "failed"
+ // StateCancelled is a job the requester stopped waiting for: a client that
+ // disconnected, a dead client whose write deadline tripped, or an in-flight
+ // request cancelled by our own shutdown. It is terminal, and distinct from
+ // StateFailed so the failed bucket only holds outcomes a user might act on
+ // rather than every time somebody pressed stop.
+ StateCancelled WorkloadState = "cancelled"
)
// Lifecycle method names on both the local and inter-node interfaces.
@@ -74,6 +78,10 @@ type Workload struct {
CompletedAt *int64 `json:"completedAt"`
Error *string `json:"error"`
RequesterID *string `json:"requesterId"`
+ // Seq numbers a workload's events in emission order, assigned by the
+ // producer. It exists so dedup can tell a new event from a redelivery even
+ // when the two carry the same state and placement — see keyLifecycle.
+ Seq int64 `json:"seq,omitempty"`
}
// lifecycleParams / removeParams are the params envelopes for the two kinds
diff --git a/services/tests/workload_identity_interop_test.go b/services/tests/workload_identity_interop_test.go
index c89eebe..864a3e5 100644
--- a/services/tests/workload_identity_interop_test.go
+++ b/services/tests/workload_identity_interop_test.go
@@ -152,7 +152,7 @@ func TestWorkloadCrossEngineIdentityDistinct(t *testing.T) {
// TestWorkloadManagerRehydratesActiveWorkloadOnRestart drives the rehydration
// path end-to-end. A proxied inference is held open by a blocking fake engine,
-// so its workload stays "running" (workload:started is emitted up front). The
+// so its workload reaches "running" once the engine emits its first byte. The
// supervised workload-manager is then killed; the broker's supervisor respawns
// it and replays the still-active local workload. A discovered stub peer must
// hear the running workload AGAIN from the restarted manager — which only
@@ -162,20 +162,32 @@ func TestWorkloadManagerRehydratesActiveWorkloadOnRestart(t *testing.T) {
t.Skip("ollama-proxy default port 11435 already in use; skipping")
}
- // Fake Ollama that accepts the request then blocks, keeping the proxied
- // inference in-flight so the workload never reaches a terminal state. The
+ // Fake Ollama that starts a streaming response, flushes one chunk so the
+ // proxy commits and the workload becomes "running", then blocks — keeping
+ // the proxied inference in-flight so it never reaches a terminal state. The
// handler also unblocks if the request is cancelled, so a client-side abort
// can't wedge it.
+ //
+ // The chunk is required: the proxy commits on the first byte of response
+ // body, not on the headers, because headers only prove an engine accepted
+ // the request. An engine that blocks before producing content leaves the
+ // workload "queued", which is the truth but not what this test is about.
release := make(chan struct{})
var releaseOnce sync.Once
stopEngine := func() { releaseOnce.Do(func() { close(release) }) }
ollama := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
io.Copy(io.Discard, r.Body)
+ w.Header().Set("Content-Type", "application/x-ndjson")
+ w.WriteHeader(http.StatusOK)
+ _, _ = io.WriteString(w, `{"message":{"role":"assistant","content":""},"done":false}`+"\n")
+ if fl, ok := w.(http.Flusher); ok {
+ fl.Flush()
+ }
select {
case <-release:
case <-r.Context().Done():
}
- w.WriteHeader(http.StatusOK)
+ _, _ = io.WriteString(w, `{"done":true}`+"\n")
}))
// Defer order matters: stopEngine() must run BEFORE ollama.Close(), because
// Close() blocks until the deliberately-held request finishes, which only
diff --git a/services/versions.json b/services/versions.json
index 0a130f7..cdc9bd6 100644
--- a/services/versions.json
+++ b/services/versions.json
@@ -1,16 +1,16 @@
{
"$comment": "Single source of truth for all version numbers. See VERSIONING.md for bump rules.",
- "product": "1.0.0",
- "installer": "1.0.0",
+ "product": "1.1.0",
+ "installer": "1.1.0",
"components": {
- "nvpair-proxy": "1.0.0",
+ "nvpair-proxy": "1.1.0",
"nvpair-node-info": "0.13.3",
"nvpair-node-scanner": "0.20.3",
"nvpair-manual-nodes": "0.11.1",
- "nvpair-workload-manager": "0.13.3",
+ "nvpair-workload-manager": "0.14.0",
"nvpair-errors": "0.7.4",
"nvpair-node-settings": "1.0.4",
- "nvpair-ui-broker": "1.0.0",
+ "nvpair-ui-broker": "1.1.0",
"nvpair-engine-manager": "0.17.4",
"nvpair-cluster-manager": "1.1.4",
"nvpair-job-scheduler": "0.4.2",