Multiplayer agent harness for work.
fix: keep session titles on model failure (#1139)
Joshua France51bf455ea414
diff --git a/plugins/web-ui/test/document-title.test.ts b/plugins/web-ui/test/document-title.test.tsindex 54e92b279..f1a8b481a 100644--- a/plugins/web-ui/test/document-title.test.ts+++ b/plugins/web-ui/test/document-title.test.ts@@ -80,15 +80,16 @@ test("document title follows session switches, split-pane focus, and sign-out",try {const { appState, signOut, syncDocumentTitle } = await vite.ssrLoadModule("/src/shell.ts");const { mainConversation } = await vite.ssrLoadModule("/src/conversations.ts");- const { sessionsState } = await vite.ssrLoadModule("/src/sessions.ts");- const { activateCanvas, splitInterceptsOpen } = await vite.ssrLoadModule("/src/split.ts");+ const { refreshSessions, sessionsState } = await vite.ssrLoadModule("/src/sessions.ts");+ const { activateCanvas } = await vite.ssrLoadModule("/src/split.ts");const oldSession = { id: "old", threadRef: "web:old", scopeId: "personal:tester", title: "Old title" };const newSession = { id: "new", threadRef: "web:new", scopeId: "personal:tester", title: "New title" };sessionsState.list = [oldSession, newSession];appState.me = { user: "tester", org: "test" };appState.currentView = "chats";appState.mainEl = document.createElement("main");- document.body.append(appState.mainEl);+ appState.listEl = document.createElement("aside");+ document.body.append(appState.listEl, appState.mainEl);mainConversation().state.sessionId = "old";mainConversation().state.threadRef = "web:old";syncDocumentTitle();@@ -100,8 +101,39 @@ test("document title follows session switches, split-pane focus, and sign-out",activateCanvas({ sessionId: "old", threadRef: "web:old" }, { sessionId: "new", threadRef: "web:new" }, "right");assert.equal(document.title, `New title · ${PRODUCT_TITLE}`);- assert.equal(splitInterceptsOpen(oldSession), true);- assert.equal(document.title, `Old title · ${PRODUCT_TITLE}`);+ await new Promise((resolve) => setTimeout(resolve, 0));++ const paneTitles = () =>+ Array.from(document.querySelectorAll(".split-pane-title-text"), (node) => node.textContent?.trim());+ const refreshTitles = async (oldTitle: string, newTitle: string) => {+ globalThis.fetch = async () =>+ Response.json({+ sessions: [+ { ...oldSession, title: oldTitle },+ { ...newSession, title: newTitle },+ ],+ });+ assert.equal(await refreshSessions({ silent: true }), true);+ };+ const focusPane = (index: number) => {+ document+ .querySelectorAll(".dv-tab")+ .item(index)+ .dispatchEvent(new dom.window.MouseEvent("pointerdown", { bubbles: true }));+ };+ assert.deepEqual(paneTitles(), ["Old title", "New title"]);+ await refreshTitles("", "New title");+ assert.deepEqual(paneTitles(), ["Web chat", "New title"]);+ assert.equal(document.title, `New title · ${PRODUCT_TITLE}`);+ await refreshTitles("Fallback for overloaded title model", "New title");+ assert.deepEqual(paneTitles(), ["Fallback for overloaded title model", "New title"]);+ focusPane(0);+ assert.equal(document.title, `Fallback for overloaded title model · ${PRODUCT_TITLE}`);+ await refreshTitles("Fallback for overloaded title model", "Fallback for OAuth callback");+ assert.deepEqual(paneTitles(), ["Fallback for overloaded title model", "Fallback for OAuth callback"]);+ assert.equal(document.title, `Fallback for overloaded title model · ${PRODUCT_TITLE}`);+ focusPane(1);+ assert.equal(document.title, `Fallback for OAuth callback · ${PRODUCT_TITLE}`);globalThis.fetch = async () => new Response(null, { status: 204 });await signOut();diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.tsindex 683579cfa..9ed742c2f 100644--- a/src/core/orchestrator.ts+++ b/src/core/orchestrator.ts@@ -150,7 +150,7 @@ import {} from "../harness/replay.ts";import { errMessage, swallow, swallowAs } from "../util/errors.ts";import { isObj } from "../util/objects.ts";-import { jsonbSafeStringify } from "../util/text.ts";+import { headSlice, jsonbSafeStringify } from "../util/text.ts";import { NonRetryableTurnError, turnFailureMessage, type TurnFailurePayload } from "./turn-error.ts";import { personKey, samePerson } from "../directory/person.ts";import { sleep } from "../util/async.ts";@@ -180,6 +180,7 @@ import {stripAckPrefix,stripTurnBoilerplate,turnPostKeys,+ visibleTitleEntryText,} from "./orchestrator/turn-helpers.ts";import {currentTimeBlock,@@ -311,20 +312,23 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {const classifySecurityData = createSecurityClassifier(deps);+ function fallbackSessionTitle(text: string): string | undefined {+ const clean = stripTurnBoilerplate(text).replace(/\s+/g, " ").trim();+ if (!clean) return undefined;+ return clean.length > 60 ? `${headSlice(clean, 59).trimEnd()}…` : clean;+ }+async function generateAndStoreTitle(sessionId: string,scopeId: ScopeId,transcript: string,principalId?: string,+ fallbackText?: string,): Promise<string | undefined> {- if (!deps.harness.models.generateTitle || !transcript.trim()) return undefined;+ if (!transcript.trim()) return undefined;+ let title: string | undefined;try {- const title = await deps.harness.models.generateTitle(transcript);- if (title) {- if (principalId) await deps.sessions.updateParticipantView(sessionId, principalId, { title });- else await deps.sessions.updateTitle(sessionId, title);- }- return title;+ title = await deps.harness.models.generateTitle?.(transcript);} catch (e) {deps.errors?.record({category: "session_title",@@ -333,8 +337,13 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {scopeLabel: scopeId,sessionId,});- return undefined;}+ title ??= fallbackText ? fallbackSessionTitle(fallbackText) : undefined;+ if (title) {+ if (principalId) await deps.sessions.updateParticipantView(sessionId, principalId, { title });+ else await deps.sessions.updateTitle(sessionId, title);+ }+ return title;}function recordSessionBusy(busy: {@@ -490,11 +499,13 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {}if (entries.length === 0) return null;const transcript = renderTitleTranscript(entries);+ const fallbackEntry = entries.find((entry) => entry.type === "user" && !isOverheardEntry(entry));const title = await generateAndStoreTitle(session.id,session.scopeId,transcript,participantIds?.length ? principalId : undefined,+ fallbackEntry ? visibleTitleEntryText(fallbackEntry) : undefined,);return { title: title ?? (participantIds ? null : (session.title ?? null)) };},@@ -2641,10 +2652,15 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {...(input.displayText?.trim() ? { display: input.displayText } : {}),}: undefined;- const earlyTitleGen: Promise<string | undefined> | undefined =- humanTurn && !session.title && !syntheticPrompt && input.text.trim()- ? generateAndStoreTitle(session.id, scopeId, `User:\n${stripTurnBoilerplate(input.text)}`)- : undefined;+ const titleText = input.displayText?.trim() || input.text;+ const fallbackTitle = !session.title && !syntheticPrompt ? fallbackSessionTitle(titleText) : undefined;+ const fallbackTitleWrite = fallbackTitle ? deps.sessions.updateTitle(session.id, fallbackTitle) : undefined;+ if (fallbackTitleWrite && deps.harness.models.generateTitle) {+ void fallbackTitleWrite+ .then(() => generateAndStoreTitle(session.id, scopeId, `User:\n${stripTurnBoilerplate(titleText)}`))+ .finally(() => deps.errors?.flush())+ .catch(swallowAs("orchestrator: session title", undefined));+ }const requestedTurnWallClockMs =typeof input.turnWallClockMs === "number" && input.turnWallClockMs > 0 ? input.turnWallClockMs : undefined;const configuredTurnWallClockSec = await deps.config?.getTurnWallClockSecDurable(resolution.orgScopeId);@@ -3455,8 +3471,8 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {}}}- if (!pausing && turnCompleted && !session.title && !(earlyTitleGen && (await earlyTitleGen))) {- await generateAndStoreTitle(session.id, scopeId, `User:\n${input.text}\n\nAssistant:\n${result.reply}`);+ if (!pausing && turnCompleted && !session.title && !fallbackTitleWrite) {+ await generateAndStoreTitle(session.id, scopeId, `User:\n${titleText}\n\nAssistant:\n${result.reply}`);}} finally {await reclaimBox();@@ -3556,6 +3572,8 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {};}+ await fallbackTitleWrite;+if (input.background && finalResult.status !== "pending_approval") {tailOwnsCleanup = true;await catchUpMessageRevisions();diff --git a/src/core/orchestrator/turn-helpers.ts b/src/core/orchestrator/turn-helpers.tsindex 0b6a00dc4..f5f9435bd 100644--- a/src/core/orchestrator/turn-helpers.ts+++ b/src/core/orchestrator/turn-helpers.ts@@ -158,12 +158,18 @@ export function stripTurnBoilerplate(text: string): string {return kept || text.trim();}+export function visibleTitleEntryText(entry: SessionEntry): string | undefined {+ const payload = entry.payload as { text?: unknown; display?: unknown } | null;+ const value = entry.type === "user" && typeof payload?.display === "string" ? payload.display : payload?.text;+ return typeof value === "string" ? value.trim() || undefined : undefined;+}+export function renderTitleTranscript(entries: SessionEntry[]): string {const lines: string[] = [];for (const e of entries) {if (e.type !== "user" && e.type !== "assistant") continue;if (isOverheardEntry(e)) continue;- const raw = (e.payload as { text?: string } | null)?.text?.trim();+ const raw = visibleTitleEntryText(e);if (!raw) continue;const text = e.type === "user" ? stripTurnBoilerplate(raw) : raw;if (!text) continue;diff --git a/src/harness/mock-harness.ts b/src/harness/mock-harness.tsindex 3f30c08b0..d38e92840 100644--- a/src/harness/mock-harness.ts+++ b/src/harness/mock-harness.ts@@ -959,6 +959,9 @@ export function createMockHarness(): Harness {},generateTitle(transcript: string): Promise<string | undefined> {+ if (transcript.includes("Simulate title provider exception"))+ return Promise.reject(new Error("title model overloaded"));+ if (transcript.includes("Simulate four-way title outage")) return Promise.resolve(undefined);const line = transcript.split("\n").map((l) => l.trim())diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.tsindex ef56a0fc9..85bd62546 100644--- a/src/harness/pi-harness.ts+++ b/src/harness/pi-harness.ts@@ -2492,22 +2492,18 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {async generateTitle(transcript: string): Promise<string | undefined> {if (!transcript.trim()) return undefined;- try {- const model = getRequiredModel(titleModelId());- const providerKeys = await resolveProviderKeys();- if (!keyForModel(providerKeys, model)) return undefined;- const out = await oneShot(- "pi-title",- model,- providerKeys,- TITLE_GENERATION_PROMPT,- titleUserPrompt(transcript),- { modelGateway },- );- return sanitizeTitle(out);- } catch {- return undefined;- }+ const model = getRequiredModel(titleModelId());+ const providerKeys = await resolveProviderKeys();+ if (!keyForModel(providerKeys, model)) return undefined;+ const out = await oneShot(+ "pi-title",+ model,+ providerKeys,+ TITLE_GENERATION_PROMPT,+ titleUserPrompt(transcript),+ { modelGateway },+ );+ return sanitizeTitle(out);},async summarizeApproval(command: string, reason: string, purpose?: string): Promise<string | undefined> {diff --git a/test/finalize-on-reply-ready.test.ts b/test/finalize-on-reply-ready.test.tsindex f04c55184..1fbac1cb4 100644--- a/test/finalize-on-reply-ready.test.ts+++ b/test/finalize-on-reply-ready.test.ts@@ -160,6 +160,50 @@ test("background: the run finishes as soon as the reply is ready; backup/teardowassert.equal(g.teardownFinished, 1, "the detached tail completed once unblocked");});+test("background: title persistence finishes before the run while teardown stays detached", async () => {+ const g = gatedSandbox();+ const base = createMockHarness();+ const harness = { ...base, models: { ...base.models, generateTitle: () => new Promise<string>(() => {}) } };+ const { orch, sessions } = buildOrchestrator(g.sandbox, undefined, harness);+ const result = await orch.handleTurn({+ ...dm("dm:U1:title-background", "!run echo hi"),+ displayText: "Simulate four-way title outage from the visible message",+ runId: "title-background",+ background: true,+ });++ assert.equal(result.status, "ok");+ assert.equal(+ (await sessions.get(result.sessionId!))?.title,+ "Simulate four-way title outage from the visible message",+ );+ assert.equal(g.teardownFinished, 0);+ for (let i = 0; i < 200 && g.teardownStarted === 0; i++) await tick();+ g.release();+});++test("spine-routed titles use visible text instead of the internal wake envelope", async () => {+ const { orch, sessions } = buildOrchestrator(gatedSandbox().sandbox);+ const result = await orch.handleTurn({+ ...dm(+ "dm:U1:title-spine",+ '<wake reason="addressed">Simulate four-way title outage from the internal envelope</wake>',+ ),+ displayText: "Simulate four-way title outage from the visible message",+ runId: "title-spine",+ background: true,+ });++ assert.equal(+ (await sessions.get(result.sessionId!))?.title,+ "Simulate four-way title outage from the visible message",+ );+ assert.equal(+ (await orch.regenerateTitle(result.sessionId!, actor.id))?.title,+ "Simulate four-way title outage from the visible message",+ );+});+test("background: a turn that never used its eagerly provisioned box returns before teardown and marks the home unchanged", async () => {const g = gatedSandbox();const { orch } = buildOrchestrator(g.sandbox, undefined, createMockHarness(), true);diff --git a/test/pi-harness-oneshot.test.ts b/test/pi-harness-oneshot.test.tsindex 2b7be989e..cd3a87afe 100644--- a/test/pi-harness-oneshot.test.ts+++ b/test/pi-harness-oneshot.test.ts@@ -197,6 +197,37 @@ test("Pi title generation returns no title without an auxiliary-model credentialassert.equal(await harness.models.generateTitle!("User:\nPrioritize the public qm issues"), undefined);});+test("Pi title generation surfaces provider failures to its caller", async (t) => {+ const server = createServer((_request, response) => {+ response.writeHead(401, { "content-type": "application/json" });+ response.end(JSON.stringify({ error: { message: "title model rejected request" } }));+ });+ await new Promise<void>((resolve, reject) => {+ server.once("error", reject);+ server.listen(0, "127.0.0.1", resolve);+ });+ t.after(+ () =>+ new Promise<void>((resolve, reject) => {+ server.close((error) => (error ? reject(error) : resolve()));+ }),+ );+ const address = server.address();+ assert(address && typeof address !== "string");+ const harness = createPiHarness({+ defaultModelId: "claude-opus-4-8",+ titleModelId: "claude-haiku-4-5",+ modelGateway: {+ url: `http://127.0.0.1:${address.port}`,+ apiKey: "gateway-key",+ apiKeyHeader: "api-key",+ models: { "claude-haiku-4-5": "title-model" },+ },+ });++ await assert.rejects(harness.models.generateTitle!("User:\nInvestigate the deploy"));+});+test("piHarnessConfigOptions omits the optional fields when the config leaves them unset", () => {const opts = piHarnessConfigOptions(testConfig());for (const key of ["defaultModelId", "detectModelId", "titleModelId", "apiKey"] as const) {diff --git a/test/projects.test.ts b/test/projects.test.tsindex 5ea6d6699..eb2ed0cc4 100644--- a/test/projects.test.ts+++ b/test/projects.test.ts@@ -405,7 +405,7 @@ test("Project routes use ordinary group sessions with the durable roster as authgrantedBy: "owner",});assert.equal((await deploy.reachDeployment(deployment.id, "member")).status, "ok");- assert.equal((await turn("owner", "web:owner:first", "after joining")).status, "ok");+ assert.equal((await turn("owner", "web:owner:first", "Simulate four-way title outage after joining")).status, "ok");assert.ok((await built.app.listSessions("member")).some((session) => session.id === first.id));const latestRequest = (await built.sessions.listLlmRequests(first.id)).at(-1)!;assert.match(JSON.stringify(latestRequest.promptEnvelope), /secret-before-join/);@@ -428,12 +428,24 @@ test("Project routes use ordinary group sessions with the durable roster as authconst globalTitle = (await built.sessions.get(first.id))?.title ?? null;const regenerated = await built.app.regenerateTitle(first.id, "member");- assert.ok(regenerated?.title);+ assert.equal(regenerated?.title, "secret-before-join");assert.equal((await built.sessions.get(first.id))?.title ?? null, globalTitle);assert.equal(- (await built.sessions.listByParticipant("member")).find((session) => session.id === first.id)?.title,+ (await built.app.listSessions("member")).find((session) => session.id === first.id)?.title,regenerated.title,);+ assert.equal((await built.app.listSessions("owner")).find((session) => session.id === first.id)?.title, globalTitle);+ assert.equal(await built.app.regenerateTitle(first.id, "outsider"), null);+ assert.ok(await built.app.updateSession(first.id, "owner", { title: "Owner-only project title" }));+ assert.equal((await built.sessions.get(first.id))?.title ?? null, globalTitle);+ assert.equal(+ (await built.app.listSessions("member")).find((session) => session.id === first.id)?.title,+ regenerated.title,+ );+ assert.equal(+ (await built.app.listSessions("owner")).find((session) => session.id === first.id)?.title,+ "Owner-only project title",+ );const unchangedAdd = await built.app.addProjectMember(project.id, "owner", "member");assert.equal(unchangedAdd.status, "ok");assert.equal(unchangedAdd.status === "ok" && unchangedAdd.changed, false);diff --git a/test/session-title.test.ts b/test/session-title.test.tsindex 09cbf8fdd..6e0af38e9 100644--- a/test/session-title.test.ts+++ b/test/session-title.test.ts@@ -29,6 +29,65 @@ test("names a conversation from its first completed turn (auto-title)", async ()assert.equal(got?.session.title, "Chat: How do I roll back");});+test("concurrent completed turns keep durable titles when title generation is unavailable", async () => {+ const { app } = freshApp();+ const turns = await Promise.all(+ Array.from({ length: 4 }, (_, i) =>+ app.turn(dm(`Simulate four-way title outage ${i + 1}`, `web:U1:title-load-${i + 1}`)),+ ),+ );++ for (const [i, turn] of turns.entries()) {+ assert.equal(turn.status, "ok");+ assert.equal((await app.getSession(turn.sessionId!))?.session.title, `Simulate four-way title outage ${i + 1}`);+ }+});++test("a title provider exception is recorded before the completed turn gets its fallback title", async () => {+ const { app, errors } = freshApp();+ const turn = await app.turn(dm("Simulate title provider exception", "web:U1:title-error"));++ assert.equal(turn.status, "ok");+ assert.equal((await app.getSession(turn.sessionId!))?.session.title, "Simulate title provider exception");+ const failures = (await errors.list({ sessionId: turn.sessionId! })).filter(+ (error) => error.category === "session_title" && error.code === "generation_failed",+ );+ assert.equal(failures.length, 1);+ assert.match(failures[0]!.message, /title model overloaded/);+});++test("the durable fallback strips turn boilerplate and stays within the generated title limit", async () => {+ const { app } = freshApp();+ const turn = await app.turn(+ dm(+ "[browser context that must not become the title]\n\nSimulate four-way title outage with deliberately long distinguishing words for truncation",+ "web:U1:title-fallback-shape",+ ),+ );+ const title = (await app.getSession(turn.sessionId!))?.session.title;++ assert.equal(turn.status, "ok");+ assert.equal(title?.length, 60);+ assert.match(title!, /^Simulate four-way title outage/);+ assert.match(title!, /…$/);+ assert.doesNotMatch(title!, /browser context/);+});++test("the shared fallback covers approval pauses and manual regeneration", async () => {+ const { app } = freshApp();+ const turn = await app.turn(dm("!paused-approval Simulate four-way title outage deploy", "web:U1:title-paused"));++ assert.ok(turn.pendingApprovals?.length);+ assert.equal(+ (await app.getSession(turn.sessionId!))?.session.title,+ "!paused-approval Simulate four-way title outage deploy",+ );+ assert.equal(+ (await app.regenerateTitle(turn.sessionId!, "U1"))?.title,+ "!paused-approval Simulate four-way title outage deploy",+ );+});+test("the title is generated ONCE — a later turn does not rewrite it", async () => {const { app } = freshApp();const r1 = await app.turn(dm("First topic about pricing tiers", "web:U1:t2"));