Skip to content

Router that virtually distributes inference across connected devices in the home.

Go
22.9 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package main

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"
	"testing"
	"time"
)

// This file runs on one arbitrary engine (anyCase), not both. Its subject is
// statusCapture's write and flush deadlines and the terminalOnce guard, which
// never consult the profile; the inference request is only the vehicle that
// reaches the streaming path. Two of these drive real sockets against
// multi-second deadlines, so a second pass would cost real time for no branch.

// count reports how many newline-delimited frames the codec wrote that mention
// s. Method names appear once per emitted notification, so counting the
// substring counts emissions.
func (r *recRW) count(s string) int {
	r.mu.Lock()
	defer r.mu.Unlock()
	return strings.Count(string(r.b), s)
}

// clientGoneWriter is an http.ResponseWriter whose body Write fails after the
// status line is sent, standing in for a client that vanished mid-stream: the
// idle write deadline trips (statusCapture.Write) and the underlying
// connection write returns an error. It records the status and supports Flush
// so the reverse proxy streams through it. This is the shape of the zombie-job
// bug — the response has committed (200), so without the wroteErr check the
// terminal would be misreported as completed.
type clientGoneWriter struct {
	header http.Header
	status int
	err    error
	wrote  bool
}

func (c *clientGoneWriter) Header() http.Header {
	if c.header == nil {
		c.header = make(http.Header)
	}
	return c.header
}

func (c *clientGoneWriter) WriteHeader(code int) { c.status = code }

func (c *clientGoneWriter) Write(b []byte) (int, error) {
	c.wrote = true
	return 0, c.err
}

func (c *clientGoneWriter) Flush() {}

// TestHandleHTTP_ClientWriteError_MarksCancelled is the zombie-job regression: a
// streaming inference response that has committed (200 headers sent) but whose
// body write to the client fails — the signature of a killed / half-open client
// whose write deadline tripped — must terminate the workload as `cancelled`,
// not be silently reported completed. A default arm that sees no ctx cancel, no
// upstream error, and a 2xx status would call this a success and leave a
// truncated stream looking clean.
func TestHandleHTTP_ClientWriteError_MarksCancelled(t *testing.T) {
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		io.WriteString(w, `{"model":"llama","response":"partial tokens before the client vanished","done":false}`)
	}))
	defer upstream.Close()

	tc := anyCase(t)
	rec := &recRW{}
	disc := NewDiscovery()
	disc.AddManual(nodeForModel(t, "node-a", upstream.URL, tc.advertisedModel))
	p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)

	cw := &clientGoneWriter{err: errors.New("write tcp: connection reset by peer")}

	done := make(chan struct{})
	go func() {
		defer close(done)
		p.soleFacade().handleHTTP(cw, tc.inferenceRequest())
	}()

	select {
	case <-done:
	case <-time.After(5 * time.Second):
		t.Fatal("handleHTTP did not return after the client write failed (zombie: handler blocked)")
	}

	if !cw.wrote {
		t.Fatal("reverse proxy never attempted a body write to the client; test did not exercise the streaming path")
	}
	if got := rec.count("workload:errored"); got != 1 {
		t.Fatalf("workload:errored emitted %d times, want exactly 1", got)
	}
	if rec.has("workload:completed") {
		t.Fatal("workload:completed emitted for a request whose client write failed")
	}
	if !rec.has(`"state":"cancelled"`) {
		t.Fatal("a client that vanished mid-stream must terminate as cancelled, not failed")
	}
}

// TestHandleHTTP_ClientDisconnect_TerminalOnce covers the disconnect watcher: a
// request whose context is cancelled mid-flight (client disconnect / shutdown)
// must emit exactly one terminal (errored) — the watcher and the post-handler
// path are guarded by terminalOnce so they can't double-emit — and handleHTTP
// must return promptly rather than hang.
func TestHandleHTTP_ClientDisconnect_TerminalOnce(t *testing.T) {
	received := make(chan struct{}, 1)
	release := make(chan struct{})
	var releaseOnce sync.Once
	doRelease := func() { releaseOnce.Do(func() { close(release) }) }

	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		if f, ok := w.(http.Flusher); ok {
			io.WriteString(w, `{"response":"first chunk","done":false}`+"\n")
			f.Flush()
		}
		select {
		case received <- struct{}{}:
		default:
		}
		<-release // block mid-stream until the test releases us
	}))
	// LIFO: unblock the upstream handler before tearing the server down.
	defer upstream.Close()
	defer doRelease()

	tc := anyCase(t)
	rec := &recRW{}
	disc := NewDiscovery()
	disc.AddManual(nodeForModel(t, "node-a", upstream.URL, tc.advertisedModel))
	p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)

	ctx, cancel := context.WithCancel(context.Background())
	req := tc.inferenceRequest().WithContext(ctx)

	done := make(chan struct{})
	go func() {
		defer close(done)
		p.soleFacade().handleHTTP(httptest.NewRecorder(), req)
	}()

	// Wait until the upstream is actively streaming, then simulate the client
	// going away.
	select {
	case <-received:
	case <-time.After(5 * time.Second):
		t.Fatal("upstream never started streaming")
	}
	cancel()

	select {
	case <-done:
	case <-time.After(5 * time.Second):
		t.Fatal("handleHTTP did not return after client disconnect (zombie: handler blocked)")
	}

	if got := rec.count("workload:errored"); got != 1 {
		t.Fatalf("workload:errored emitted %d times, want exactly 1 (terminalOnce guard)", got)
	}
	if rec.has("workload:completed") {
		t.Fatal("workload:completed emitted for a cancelled request")
	}
}

// deadlineRW records SetWriteDeadline calls and can force a Write error, so the
// statusCapture write-deadline mechanics can be tested without a real socket.
type deadlineRW struct {
	*httptest.ResponseRecorder
	deadlines []time.Time
	writeErr  error
	flushErr  error
	flushed   int
}

func (d *deadlineRW) SetWriteDeadline(t time.Time) error {
	d.deadlines = append(d.deadlines, t)
	return nil
}

func (d *deadlineRW) Write(b []byte) (int, error) {
	if d.writeErr != nil {
		return 0, d.writeErr
	}
	return d.ResponseRecorder.Write(b)
}

// FlushError lets a test drive statusCapture.FlushError against a controllable
// flush outcome (recorded so the deadline arm/clear can be asserted).
func (d *deadlineRW) FlushError() error {
	d.flushed++
	return d.flushErr
}

// TestStatusCapture_WriteDeadline verifies statusCapture arms a write deadline
// around each streamed write and clears it after a successful one (so idle gaps
// between tokens never penalize a live-but-slow generation), and that the first
// write error is retained for the caller to classify the workload failed.
func TestStatusCapture_WriteDeadline(t *testing.T) {
	t.Run("armed then cleared on success", func(t *testing.T) {
		d := &deadlineRW{ResponseRecorder: httptest.NewRecorder()}
		sc := &statusCapture{ResponseWriter: d, status: http.StatusOK, idle: 50 * time.Millisecond}
		if _, err := sc.Write([]byte("tokens")); err != nil {
			t.Fatalf("Write returned error: %v", err)
		}
		if len(d.deadlines) != 2 {
			t.Fatalf("SetWriteDeadline called %d times, want 2 (arm + clear)", len(d.deadlines))
		}
		if d.deadlines[0].IsZero() {
			t.Fatal("first SetWriteDeadline should arm a future deadline, got zero")
		}
		if !d.deadlines[1].IsZero() {
			t.Fatal("second SetWriteDeadline should clear the deadline (zero time)")
		}
		if sc.wroteErr != nil {
			t.Fatalf("wroteErr set after a successful write: %v", sc.wroteErr)
		}
	})

	t.Run("write error retained, deadline not cleared", func(t *testing.T) {
		boom := errors.New("i/o timeout")
		d := &deadlineRW{ResponseRecorder: httptest.NewRecorder(), writeErr: boom}
		sc := &statusCapture{ResponseWriter: d, status: http.StatusOK, idle: 50 * time.Millisecond}
		if _, err := sc.Write([]byte("tokens")); !errors.Is(err, boom) {
			t.Fatalf("Write err = %v, want %v", err, boom)
		}
		if !errors.Is(sc.wroteErr, boom) {
			t.Fatalf("wroteErr = %v, want %v", sc.wroteErr, boom)
		}
		if len(d.deadlines) != 1 {
			t.Fatalf("SetWriteDeadline called %d times, want 1 (arm only; not cleared on error)", len(d.deadlines))
		}
	})

	t.Run("no deadline when idle is zero", func(t *testing.T) {
		d := &deadlineRW{ResponseRecorder: httptest.NewRecorder()}
		sc := &statusCapture{ResponseWriter: d, status: http.StatusOK}
		if _, err := sc.Write([]byte("tokens")); err != nil {
			t.Fatalf("Write returned error: %v", err)
		}
		if len(d.deadlines) != 0 {
			t.Fatalf("SetWriteDeadline called %d times with idle=0, want 0", len(d.deadlines))
		}
	})
}

// TestStatusCapture_FlushDeadline verifies the flush path is deadline-aware:
// statusCapture.FlushError arms the idle deadline around the underlying flush,
// clears it after a successful flush, and retains a real flush error (but not an
// unsupported-flush) so a stalled client's blocked flush is classified failed
// rather than hanging forever.
func TestStatusCapture_FlushDeadline(t *testing.T) {
	t.Run("armed then cleared on success", func(t *testing.T) {
		d := &deadlineRW{ResponseRecorder: httptest.NewRecorder()}
		sc := &statusCapture{ResponseWriter: d, status: http.StatusOK, idle: 50 * time.Millisecond}
		if err := sc.FlushError(); err != nil {
			t.Fatalf("FlushError returned error: %v", err)
		}
		if d.flushed != 1 {
			t.Fatalf("underlying flushed %d times, want 1", d.flushed)
		}
		if len(d.deadlines) != 2 {
			t.Fatalf("SetWriteDeadline called %d times, want 2 (arm + clear)", len(d.deadlines))
		}
		if d.deadlines[0].IsZero() {
			t.Fatal("flush should arm a future deadline, got zero")
		}
		if !d.deadlines[1].IsZero() {
			t.Fatal("flush should clear the deadline on success (zero time)")
		}
		if sc.wroteErr != nil {
			t.Fatalf("wroteErr set after a successful flush: %v", sc.wroteErr)
		}
	})

	t.Run("flush error retained, deadline not cleared", func(t *testing.T) {
		boom := errors.New("i/o timeout")
		d := &deadlineRW{ResponseRecorder: httptest.NewRecorder(), flushErr: boom}
		sc := &statusCapture{ResponseWriter: d, status: http.StatusOK, idle: 50 * time.Millisecond}
		if err := sc.FlushError(); !errors.Is(err, boom) {
			t.Fatalf("FlushError = %v, want %v", err, boom)
		}
		if !errors.Is(sc.wroteErr, boom) {
			t.Fatalf("wroteErr = %v, want %v", sc.wroteErr, boom)
		}
		if len(d.deadlines) != 1 {
			t.Fatalf("SetWriteDeadline called %d times, want 1 (arm only; not cleared on error)", len(d.deadlines))
		}
	})

	t.Run("unsupported flush is not a client failure", func(t *testing.T) {
		d := &deadlineRW{ResponseRecorder: httptest.NewRecorder(), flushErr: http.ErrNotSupported}
		sc := &statusCapture{ResponseWriter: d, status: http.StatusOK, idle: 50 * time.Millisecond}
		_ = sc.FlushError()
		if sc.wroteErr != nil {
			t.Fatalf("ErrNotSupported must not be retained as wroteErr, got %v", sc.wroteErr)
		}
	})
}

// TestHandleHTTP_RealSocketWriteDeadline is the end-to-end, OS-level proof of
// the zombie-job fix. It drives handleHTTP over a REAL TCP socket with a client
// that reads the response headers and then stops reading — the shape of a
// killed / half-open client whose receive window closes without a FIN/RST. The
// upstream streams without end, so the proxy's kernel send buffer to the client
// fills and its next write blocks. Before the fix that write blocks ~forever
// (no terminal event; the zombie job), so r.Context() never fires and the
// handler never returns. With the fix, statusCapture arms a real
// SetWriteDeadline that the Go runtime's netpoller enforces on every platform
// (IOCP on Windows, epoll/kqueue elsewhere) regardless of the peer's TCP state,
// so the stuck write fails and the workload terminates as failed. This is the
// piece the in-process tests stub out — here the deadline is genuinely enforced
// by the OS/runtime.
func TestHandleHTTP_RealSocketWriteDeadline(t *testing.T) {
	// Shorten the idle write deadline so a stuck write trips quickly; restore
	// the production default for any test that runs after this one.
	orig := idleClientWriteTimeout
	idleClientWriteTimeout = 300 * time.Millisecond
	defer func() { idleClientWriteTimeout = orig }()

	// Upstream streams 64 KiB chunks endlessly. Once the proxy stops reading
	// from it (because the proxy is itself blocked writing to the stalled
	// client), the upstream's own writes block too — no busy loop — and it
	// unwinds when the proxy tears the connection down (write error or context
	// cancel).
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		flusher, _ := w.(http.Flusher)
		chunk := bytes.Repeat([]byte("x"), 64*1024)
		for {
			if _, err := w.Write(chunk); err != nil {
				return
			}
			if flusher != nil {
				flusher.Flush()
			}
			select {
			case <-r.Context().Done():
				return
			default:
			}
		}
	}))
	defer upstream.Close()

	tc := anyCase(t)
	rec := &recRW{}
	disc := NewDiscovery()
	disc.AddManual(nodeForModel(t, "node-a", upstream.URL, tc.advertisedModel))
	p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)

	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	srv := &http.Server{Handler: http.HandlerFunc(p.soleFacade().handleHTTP)}
	go func() { _ = srv.Serve(ln) }()
	defer srv.Close()

	conn, err := net.Dial("tcp", ln.Addr().String())
	if err != nil {
		t.Fatalf("dial proxy: %v", err)
	}
	defer conn.Close()

	body := tc.inferenceBody()
	reqText := fmt.Sprintf("POST %s HTTP/1.1\r\n", tc.inferencePath) +
		"Host: localhost\r\n" +
		"Content-Type: application/json\r\n" +
		fmt.Sprintf("Content-Length: %d\r\n", len(body)) +
		"\r\n" + body
	if _, err := conn.Write([]byte(reqText)); err != nil {
		t.Fatalf("write request: %v", err)
	}

	// Read the status line only — enough to confirm the response committed and
	// started streaming — then STOP reading so the proxy's send buffer backs
	// up. A read deadline guards against a hang if the proxy never responds.
	_ = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
	statusLine, err := bufio.NewReader(conn).ReadString('\n')
	if err != nil {
		t.Fatalf("read status line: %v", err)
	}
	if !strings.Contains(statusLine, "200") {
		t.Fatalf("unexpected status line: %q", statusLine)
	}

	// The stuck write must trip the deadline and terminate the workload as
	// failed within a few multiples of the deadline — never completed.
	deadline := time.Now().Add(5 * time.Second)
	for time.Now().Before(deadline) && !rec.has("workload:errored") {
		time.Sleep(10 * time.Millisecond)
	}
	if !rec.has("workload:errored") {
		t.Fatal("workload never terminated after the client stopped reading (zombie: write deadline did not trip / no terminal emitted)")
	}
	if rec.has("workload:completed") {
		t.Fatal("workload:completed emitted for a client that stopped reading")
	}
	if !rec.has(`"state":"cancelled"`) {
		t.Fatal("a client that stopped reading must terminate as cancelled, not failed")
	}
}

// TestHandleHTTP_RealSocketFlushDeadline is the flush-path counterpart to the
// write-deadline test. A streaming response is flushed after every chunk
// (ReverseProxy uses immediate flushing for chunked/streaming upstreams), so a
// small chunk is buffered by a successful Write (no network I/O) and the actual
// network write happens in a *separate* Flush. If only Write is deadline-aware,
// a stalled client makes that Flush block unbounded and the handler never
// returns — a zombie the 64 KiB Write-blocking test does not catch. This drives
// real, paced small flushed chunks over a real socket and asserts the flush
// deadline terminates the workload as failed.
func TestHandleHTTP_RealSocketFlushDeadline(t *testing.T) {
	orig := idleClientWriteTimeout
	idleClientWriteTimeout = 300 * time.Millisecond
	defer func() { idleClientWriteTimeout = orig }()

	// Small (<2 KiB) chunks each followed by Flush: every chunk buffers on
	// Write without touching the socket, then the network write happens in
	// Flush. ReverseProxy flushes after each write for a chunked response, so
	// the block lands in Flush, never in Write.
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/x-ndjson")
		w.WriteHeader(http.StatusOK)
		flusher, ok := w.(http.Flusher)
		if !ok {
			t.Errorf("upstream ResponseWriter is not a Flusher")
			return
		}
		chunk := bytes.Repeat([]byte("x"), 1500)
		for {
			if _, err := w.Write(chunk); err != nil {
				return
			}
			flusher.Flush()
			// Pace so the reverse proxy's 32 KiB copy read returns one small
			// chunk per iteration rather than coalescing many into a >2 KiB
			// write (which would block inside Write, not Flush).
			time.Sleep(2 * time.Millisecond)
			select {
			case <-r.Context().Done():
				return
			default:
			}
		}
	}))
	defer upstream.Close()

	tc := anyCase(t)
	rec := &recRW{}
	disc := NewDiscovery()
	disc.AddManual(nodeForModel(t, "node-a", upstream.URL, tc.advertisedModel))
	p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)

	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	srv := &http.Server{Handler: http.HandlerFunc(p.soleFacade().handleHTTP)}
	go func() { _ = srv.Serve(ln) }()
	defer srv.Close()

	conn, err := net.Dial("tcp", ln.Addr().String())
	if err != nil {
		t.Fatalf("dial proxy: %v", err)
	}
	defer conn.Close()

	body := tc.inferenceBody()
	reqText := fmt.Sprintf("POST %s HTTP/1.1\r\n", tc.inferencePath) +
		"Host: localhost\r\n" +
		"Content-Type: application/json\r\n" +
		fmt.Sprintf("Content-Length: %d\r\n", len(body)) +
		"\r\n" + body
	if _, err := conn.Write([]byte(reqText)); err != nil {
		t.Fatalf("write request: %v", err)
	}

	// Read the status line only, then stop reading so the proxy's send buffer
	// backs up and a Flush blocks.
	_ = conn.SetReadDeadline(time.Now().Add(5 * time.Second))
	statusLine, err := bufio.NewReader(conn).ReadString('\n')
	if err != nil {
		t.Fatalf("read status line: %v", err)
	}
	if !strings.Contains(statusLine, "200") {
		t.Fatalf("unexpected status line: %q", statusLine)
	}

	deadline := time.Now().Add(5 * time.Second)
	for time.Now().Before(deadline) && !rec.has("workload:errored") {
		time.Sleep(10 * time.Millisecond)
	}
	if !rec.has("workload:errored") {
		t.Fatal("workload never terminated after the client stopped reading (flush path not deadline-aware)")
	}
	if rec.has("workload:completed") {
		t.Fatal("workload:completed emitted for a stalled client")
	}
	if !rec.has(`"state":"cancelled"`) {
		t.Fatal("a stalled client must terminate as cancelled, not failed")
	}
}

// TestHandleHTTP_UpstreamDiesMidStream_EmitsTerminal is the other half of the
// zombie-job story, and the half no in-process test can express. Here the
// CLIENT stays healthy — it reads everything the proxy sends — and the
// UPSTREAM dies mid-body instead.
//
// That combination defeats both existing reporters. Nothing cancels the
// request context, because no write to the client ever fails, so the
// disconnect watcher never fires. And the copy error is never returned to us:
// ReverseProxy converts a mid-copy failure into panic(http.ErrAbortHandler)
// whenever it detects a real server, and that panic unwinds straight past the
// end of handleHTTP. A terminal emitted inline is therefore lost outright and
// the workload stays "running" for the life of the broker. The cost is not
// just a stuck card: the scheduler counts queued and running by scheduledOn,
// so the ghost permanently inflates that node's pending count and biases
// routing away from a node whose only sin was a crashed engine.
//
// A real http.Server is essential. shouldPanicOnCopyError keys off
// ServerContextKey, so calling handleHTTP directly — as the in-process tests
// above do — suppresses the panic and cannot reach this path at all.
//
// The response committed a 2xx before truncating, so the terminal is a
// COMPLETION: a cut-off stream is a success the caller resumes from whatever
// it received, not a cluster failure.
func TestHandleHTTP_UpstreamDiesMidStream_EmitsTerminal(t *testing.T) {
	// A raw listener rather than httptest, so the abrupt close is exact: a
	// chunked response that stops without its terminating zero-length chunk,
	// which is what the proxy sees when an engine is killed mid-generation.
	// Accepting in a loop keeps the address probe in targetURL from consuming
	// the one connection the request needs.
	upLn, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen upstream: %v", err)
	}
	defer upLn.Close()
	go func() {
		for {
			conn, err := upLn.Accept()
			if err != nil {
				return
			}
			go func(c net.Conn) {
				br := bufio.NewReader(c)
				for {
					line, err := br.ReadString('\n')
					if err != nil {
						c.Close()
						return
					}
					if line == "\r\n" {
						break
					}
				}
				chunk := `{"response":"partial tokens","done":false}`
				io.WriteString(c, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nTransfer-Encoding: chunked\r\n\r\n")
				fmt.Fprintf(c, "%x\r\n%s\r\n", len(chunk), chunk)
				c.Close()
			}(conn)
		}
	}()

	tc := anyCase(t)
	rec := &recRW{}
	disc := NewDiscovery()
	disc.AddManual(nodeForModel(t, "node-a", "http://"+upLn.Addr().String(), tc.advertisedModel))
	p := newTestProxy(tc.profile, NewCodec(rec), disc, tc.profile.FacadePort)

	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen proxy: %v", err)
	}
	srv := &http.Server{Handler: http.HandlerFunc(p.soleFacade().handleHTTP)}
	go func() { _ = srv.Serve(ln) }()
	defer srv.Close()

	conn, err := net.Dial("tcp", ln.Addr().String())
	if err != nil {
		t.Fatalf("dial proxy: %v", err)
	}
	defer conn.Close()

	body := tc.inferenceBody()
	reqText := fmt.Sprintf("POST %s HTTP/1.1\r\n", tc.inferencePath) +
		"Host: localhost\r\n" +
		"Content-Type: application/json\r\n" +
		fmt.Sprintf("Content-Length: %d\r\n", len(body)) +
		"\r\n" + body
	if _, err := conn.Write([]byte(reqText)); err != nil {
		t.Fatalf("write request: %v", err)
	}

	// Keep the client healthy by draining until the proxy hangs up. A client
	// that stopped reading would trip the write deadline and cancel the
	// context, which is the case the tests above cover and would mask this one.
	go func() { _, _ = io.Copy(io.Discard, conn) }()

	deadline := time.Now().Add(5 * time.Second)
	terminals := func() int { return rec.count("workload:completed") + rec.count("workload:errored") }
	for time.Now().Before(deadline) && terminals() == 0 {
		time.Sleep(10 * time.Millisecond)
	}
	if got := terminals(); got != 1 {
		t.Fatalf("terminal workload events = %d, want exactly 1 (zombie: the ErrAbortHandler panic skipped the terminal)", got)
	}
	if !rec.has("workload:completed") {
		t.Fatal("a truncated stream that had already committed 2xx must terminate as completed, not failed")
	}
}