Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

fix: keep session titles on model failure (#1139)

Joshua France51bf455ea414
diff --git a/plugins/web-ui/test/document-title.test.ts b/plugins/web-ui/test/document-title.test.ts
index 54e92b279..f1a8b481a 100644
--- a/plugins/web-ui/test/document-title.test.ts
+++ b/plugins/web-ui/test/document-title.test.ts
@@ -80,15 +80,16 @@ test("document title follows session switches, split-pane focus, and sign-out",
try {
const { appState, signOut, syncDocumentTitle } = await vite.ssrLoadModule("/src/shell.ts");
const { mainConversation } = await vite.ssrLoadModule("/src/conversations.ts");
- const { sessionsState } = await vite.ssrLoadModule("/src/sessions.ts");
- const { activateCanvas, splitInterceptsOpen } = await vite.ssrLoadModule("/src/split.ts");
+ const { refreshSessions, sessionsState } = await vite.ssrLoadModule("/src/sessions.ts");
+ const { activateCanvas } = await vite.ssrLoadModule("/src/split.ts");
const oldSession = { id: "old", threadRef: "web:old", scopeId: "personal:tester", title: "Old title" };
const newSession = { id: "new", threadRef: "web:new", scopeId: "personal:tester", title: "New title" };
sessionsState.list = [oldSession, newSession];
appState.me = { user: "tester", org: "test" };
appState.currentView = "chats";
appState.mainEl = document.createElement("main");
- document.body.append(appState.mainEl);
+ appState.listEl = document.createElement("aside");
+ document.body.append(appState.listEl, appState.mainEl);
mainConversation().state.sessionId = "old";
mainConversation().state.threadRef = "web:old";
syncDocumentTitle();
@@ -100,8 +101,39 @@ test("document title follows session switches, split-pane focus, and sign-out",
activateCanvas({ sessionId: "old", threadRef: "web:old" }, { sessionId: "new", threadRef: "web:new" }, "right");
assert.equal(document.title, `New title · ${PRODUCT_TITLE}`);
- assert.equal(splitInterceptsOpen(oldSession), true);
- assert.equal(document.title, `Old title · ${PRODUCT_TITLE}`);
+ await new Promise((resolve) => setTimeout(resolve, 0));
+
+ const paneTitles = () =>
+ Array.from(document.querySelectorAll(".split-pane-title-text"), (node) => node.textContent?.trim());
+ const refreshTitles = async (oldTitle: string, newTitle: string) => {
+ globalThis.fetch = async () =>
+ Response.json({
+ sessions: [
+ { ...oldSession, title: oldTitle },
+ { ...newSession, title: newTitle },
+ ],
+ });
+ assert.equal(await refreshSessions({ silent: true }), true);
+ };
+ const focusPane = (index: number) => {
+ document
+ .querySelectorAll(".dv-tab")
+ .item(index)
+ .dispatchEvent(new dom.window.MouseEvent("pointerdown", { bubbles: true }));
+ };
+ assert.deepEqual(paneTitles(), ["Old title", "New title"]);
+ await refreshTitles("", "New title");
+ assert.deepEqual(paneTitles(), ["Web chat", "New title"]);
+ assert.equal(document.title, `New title · ${PRODUCT_TITLE}`);
+ await refreshTitles("Fallback for overloaded title model", "New title");
+ assert.deepEqual(paneTitles(), ["Fallback for overloaded title model", "New title"]);
+ focusPane(0);
+ assert.equal(document.title, `Fallback for overloaded title model · ${PRODUCT_TITLE}`);
+ await refreshTitles("Fallback for overloaded title model", "Fallback for OAuth callback");
+ assert.deepEqual(paneTitles(), ["Fallback for overloaded title model", "Fallback for OAuth callback"]);
+ assert.equal(document.title, `Fallback for overloaded title model · ${PRODUCT_TITLE}`);
+ focusPane(1);
+ assert.equal(document.title, `Fallback for OAuth callback · ${PRODUCT_TITLE}`);
globalThis.fetch = async () => new Response(null, { status: 204 });
await signOut();
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts
index 683579cfa..9ed742c2f 100644
--- a/src/core/orchestrator.ts
+++ b/src/core/orchestrator.ts
@@ -150,7 +150,7 @@ import {
} from "../harness/replay.ts";
import { errMessage, swallow, swallowAs } from "../util/errors.ts";
import { isObj } from "../util/objects.ts";
-import { jsonbSafeStringify } from "../util/text.ts";
+import { headSlice, jsonbSafeStringify } from "../util/text.ts";
import { NonRetryableTurnError, turnFailureMessage, type TurnFailurePayload } from "./turn-error.ts";
import { personKey, samePerson } from "../directory/person.ts";
import { sleep } from "../util/async.ts";
@@ -180,6 +180,7 @@ import {
stripAckPrefix,
stripTurnBoilerplate,
turnPostKeys,
+ visibleTitleEntryText,
} from "./orchestrator/turn-helpers.ts";
import {
currentTimeBlock,
@@ -311,20 +312,23 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
const classifySecurityData = createSecurityClassifier(deps);
+ function fallbackSessionTitle(text: string): string | undefined {
+ const clean = stripTurnBoilerplate(text).replace(/\s+/g, " ").trim();
+ if (!clean) return undefined;
+ return clean.length > 60 ? `${headSlice(clean, 59).trimEnd()}…` : clean;
+ }
+
async function generateAndStoreTitle(
sessionId: string,
scopeId: ScopeId,
transcript: string,
principalId?: string,
+ fallbackText?: string,
): Promise<string | undefined> {
- if (!deps.harness.models.generateTitle || !transcript.trim()) return undefined;
+ if (!transcript.trim()) return undefined;
+ let title: string | undefined;
try {
- const title = await deps.harness.models.generateTitle(transcript);
- if (title) {
- if (principalId) await deps.sessions.updateParticipantView(sessionId, principalId, { title });
- else await deps.sessions.updateTitle(sessionId, title);
- }
- return title;
+ title = await deps.harness.models.generateTitle?.(transcript);
} catch (e) {
deps.errors?.record({
category: "session_title",
@@ -333,8 +337,13 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
scopeLabel: scopeId,
sessionId,
});
- return undefined;
}
+ title ??= fallbackText ? fallbackSessionTitle(fallbackText) : undefined;
+ if (title) {
+ if (principalId) await deps.sessions.updateParticipantView(sessionId, principalId, { title });
+ else await deps.sessions.updateTitle(sessionId, title);
+ }
+ return title;
}
function recordSessionBusy(busy: {
@@ -490,11 +499,13 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
}
if (entries.length === 0) return null;
const transcript = renderTitleTranscript(entries);
+ const fallbackEntry = entries.find((entry) => entry.type === "user" && !isOverheardEntry(entry));
const title = await generateAndStoreTitle(
session.id,
session.scopeId,
transcript,
participantIds?.length ? principalId : undefined,
+ fallbackEntry ? visibleTitleEntryText(fallbackEntry) : undefined,
);
return { title: title ?? (participantIds ? null : (session.title ?? null)) };
},
@@ -2641,10 +2652,15 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
...(input.displayText?.trim() ? { display: input.displayText } : {}),
}
: undefined;
- const earlyTitleGen: Promise<string | undefined> | undefined =
- humanTurn && !session.title && !syntheticPrompt && input.text.trim()
- ? generateAndStoreTitle(session.id, scopeId, `User:\n${stripTurnBoilerplate(input.text)}`)
- : undefined;
+ const titleText = input.displayText?.trim() || input.text;
+ const fallbackTitle = !session.title && !syntheticPrompt ? fallbackSessionTitle(titleText) : undefined;
+ const fallbackTitleWrite = fallbackTitle ? deps.sessions.updateTitle(session.id, fallbackTitle) : undefined;
+ if (fallbackTitleWrite && deps.harness.models.generateTitle) {
+ void fallbackTitleWrite
+ .then(() => generateAndStoreTitle(session.id, scopeId, `User:\n${stripTurnBoilerplate(titleText)}`))
+ .finally(() => deps.errors?.flush())
+ .catch(swallowAs("orchestrator: session title", undefined));
+ }
const requestedTurnWallClockMs =
typeof input.turnWallClockMs === "number" && input.turnWallClockMs > 0 ? input.turnWallClockMs : undefined;
const configuredTurnWallClockSec = await deps.config?.getTurnWallClockSecDurable(resolution.orgScopeId);
@@ -3455,8 +3471,8 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
}
}
}
- if (!pausing && turnCompleted && !session.title && !(earlyTitleGen && (await earlyTitleGen))) {
- await generateAndStoreTitle(session.id, scopeId, `User:\n${input.text}\n\nAssistant:\n${result.reply}`);
+ if (!pausing && turnCompleted && !session.title && !fallbackTitleWrite) {
+ await generateAndStoreTitle(session.id, scopeId, `User:\n${titleText}\n\nAssistant:\n${result.reply}`);
}
} finally {
await reclaimBox();
@@ -3556,6 +3572,8 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
};
}
+ await fallbackTitleWrite;
+
if (input.background && finalResult.status !== "pending_approval") {
tailOwnsCleanup = true;
await catchUpMessageRevisions();
diff --git a/src/core/orchestrator/turn-helpers.ts b/src/core/orchestrator/turn-helpers.ts
index 0b6a00dc4..f5f9435bd 100644
--- a/src/core/orchestrator/turn-helpers.ts
+++ b/src/core/orchestrator/turn-helpers.ts
@@ -158,12 +158,18 @@ export function stripTurnBoilerplate(text: string): string {
return kept || text.trim();
}
+export function visibleTitleEntryText(entry: SessionEntry): string | undefined {
+ const payload = entry.payload as { text?: unknown; display?: unknown } | null;
+ const value = entry.type === "user" && typeof payload?.display === "string" ? payload.display : payload?.text;
+ return typeof value === "string" ? value.trim() || undefined : undefined;
+}
+
export function renderTitleTranscript(entries: SessionEntry[]): string {
const lines: string[] = [];
for (const e of entries) {
if (e.type !== "user" && e.type !== "assistant") continue;
if (isOverheardEntry(e)) continue;
- const raw = (e.payload as { text?: string } | null)?.text?.trim();
+ const raw = visibleTitleEntryText(e);
if (!raw) continue;
const text = e.type === "user" ? stripTurnBoilerplate(raw) : raw;
if (!text) continue;
diff --git a/src/harness/mock-harness.ts b/src/harness/mock-harness.ts
index 3f30c08b0..d38e92840 100644
--- a/src/harness/mock-harness.ts
+++ b/src/harness/mock-harness.ts
@@ -959,6 +959,9 @@ export function createMockHarness(): Harness {
},
generateTitle(transcript: string): Promise<string | undefined> {
+ if (transcript.includes("Simulate title provider exception"))
+ return Promise.reject(new Error("title model overloaded"));
+ if (transcript.includes("Simulate four-way title outage")) return Promise.resolve(undefined);
const line = transcript
.split("\n")
.map((l) => l.trim())
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index ef56a0fc9..85bd62546 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -2492,22 +2492,18 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
async generateTitle(transcript: string): Promise<string | undefined> {
if (!transcript.trim()) return undefined;
- try {
- const model = getRequiredModel(titleModelId());
- const providerKeys = await resolveProviderKeys();
- if (!keyForModel(providerKeys, model)) return undefined;
- const out = await oneShot(
- "pi-title",
- model,
- providerKeys,
- TITLE_GENERATION_PROMPT,
- titleUserPrompt(transcript),
- { modelGateway },
- );
- return sanitizeTitle(out);
- } catch {
- return undefined;
- }
+ const model = getRequiredModel(titleModelId());
+ const providerKeys = await resolveProviderKeys();
+ if (!keyForModel(providerKeys, model)) return undefined;
+ const out = await oneShot(
+ "pi-title",
+ model,
+ providerKeys,
+ TITLE_GENERATION_PROMPT,
+ titleUserPrompt(transcript),
+ { modelGateway },
+ );
+ return sanitizeTitle(out);
},
async summarizeApproval(command: string, reason: string, purpose?: string): Promise<string | undefined> {
diff --git a/test/finalize-on-reply-ready.test.ts b/test/finalize-on-reply-ready.test.ts
index f04c55184..1fbac1cb4 100644
--- a/test/finalize-on-reply-ready.test.ts
+++ b/test/finalize-on-reply-ready.test.ts
@@ -160,6 +160,50 @@ test("background: the run finishes as soon as the reply is ready; backup/teardow
assert.equal(g.teardownFinished, 1, "the detached tail completed once unblocked");
});
+test("background: title persistence finishes before the run while teardown stays detached", async () => {
+ const g = gatedSandbox();
+ const base = createMockHarness();
+ const harness = { ...base, models: { ...base.models, generateTitle: () => new Promise<string>(() => {}) } };
+ const { orch, sessions } = buildOrchestrator(g.sandbox, undefined, harness);
+ const result = await orch.handleTurn({
+ ...dm("dm:U1:title-background", "!run echo hi"),
+ displayText: "Simulate four-way title outage from the visible message",
+ runId: "title-background",
+ background: true,
+ });
+
+ assert.equal(result.status, "ok");
+ assert.equal(
+ (await sessions.get(result.sessionId!))?.title,
+ "Simulate four-way title outage from the visible message",
+ );
+ assert.equal(g.teardownFinished, 0);
+ for (let i = 0; i < 200 && g.teardownStarted === 0; i++) await tick();
+ g.release();
+});
+
+test("spine-routed titles use visible text instead of the internal wake envelope", async () => {
+ const { orch, sessions } = buildOrchestrator(gatedSandbox().sandbox);
+ const result = await orch.handleTurn({
+ ...dm(
+ "dm:U1:title-spine",
+ '<wake reason="addressed">Simulate four-way title outage from the internal envelope</wake>',
+ ),
+ displayText: "Simulate four-way title outage from the visible message",
+ runId: "title-spine",
+ background: true,
+ });
+
+ assert.equal(
+ (await sessions.get(result.sessionId!))?.title,
+ "Simulate four-way title outage from the visible message",
+ );
+ assert.equal(
+ (await orch.regenerateTitle(result.sessionId!, actor.id))?.title,
+ "Simulate four-way title outage from the visible message",
+ );
+});
+
test("background: a turn that never used its eagerly provisioned box returns before teardown and marks the home unchanged", async () => {
const g = gatedSandbox();
const { orch } = buildOrchestrator(g.sandbox, undefined, createMockHarness(), true);
diff --git a/test/pi-harness-oneshot.test.ts b/test/pi-harness-oneshot.test.ts
index 2b7be989e..cd3a87afe 100644
--- a/test/pi-harness-oneshot.test.ts
+++ b/test/pi-harness-oneshot.test.ts
@@ -197,6 +197,37 @@ test("Pi title generation returns no title without an auxiliary-model credential
assert.equal(await harness.models.generateTitle!("User:\nPrioritize the public qm issues"), undefined);
});
+test("Pi title generation surfaces provider failures to its caller", async (t) => {
+ const server = createServer((_request, response) => {
+ response.writeHead(401, { "content-type": "application/json" });
+ response.end(JSON.stringify({ error: { message: "title model rejected request" } }));
+ });
+ await new Promise<void>((resolve, reject) => {
+ server.once("error", reject);
+ server.listen(0, "127.0.0.1", resolve);
+ });
+ t.after(
+ () =>
+ new Promise<void>((resolve, reject) => {
+ server.close((error) => (error ? reject(error) : resolve()));
+ }),
+ );
+ const address = server.address();
+ assert(address && typeof address !== "string");
+ const harness = createPiHarness({
+ defaultModelId: "claude-opus-4-8",
+ titleModelId: "claude-haiku-4-5",
+ modelGateway: {
+ url: `http://127.0.0.1:${address.port}`,
+ apiKey: "gateway-key",
+ apiKeyHeader: "api-key",
+ models: { "claude-haiku-4-5": "title-model" },
+ },
+ });
+
+ await assert.rejects(harness.models.generateTitle!("User:\nInvestigate the deploy"));
+});
+
test("piHarnessConfigOptions omits the optional fields when the config leaves them unset", () => {
const opts = piHarnessConfigOptions(testConfig());
for (const key of ["defaultModelId", "detectModelId", "titleModelId", "apiKey"] as const) {
diff --git a/test/projects.test.ts b/test/projects.test.ts
index 5ea6d6699..eb2ed0cc4 100644
--- a/test/projects.test.ts
+++ b/test/projects.test.ts
@@ -405,7 +405,7 @@ test("Project routes use ordinary group sessions with the durable roster as auth
grantedBy: "owner",
});
assert.equal((await deploy.reachDeployment(deployment.id, "member")).status, "ok");
- assert.equal((await turn("owner", "web:owner:first", "after joining")).status, "ok");
+ assert.equal((await turn("owner", "web:owner:first", "Simulate four-way title outage after joining")).status, "ok");
assert.ok((await built.app.listSessions("member")).some((session) => session.id === first.id));
const latestRequest = (await built.sessions.listLlmRequests(first.id)).at(-1)!;
assert.match(JSON.stringify(latestRequest.promptEnvelope), /secret-before-join/);
@@ -428,12 +428,24 @@ test("Project routes use ordinary group sessions with the durable roster as auth
const globalTitle = (await built.sessions.get(first.id))?.title ?? null;
const regenerated = await built.app.regenerateTitle(first.id, "member");
- assert.ok(regenerated?.title);
+ assert.equal(regenerated?.title, "secret-before-join");
assert.equal((await built.sessions.get(first.id))?.title ?? null, globalTitle);
assert.equal(
- (await built.sessions.listByParticipant("member")).find((session) => session.id === first.id)?.title,
+ (await built.app.listSessions("member")).find((session) => session.id === first.id)?.title,
regenerated.title,
);
+ assert.equal((await built.app.listSessions("owner")).find((session) => session.id === first.id)?.title, globalTitle);
+ assert.equal(await built.app.regenerateTitle(first.id, "outsider"), null);
+ assert.ok(await built.app.updateSession(first.id, "owner", { title: "Owner-only project title" }));
+ assert.equal((await built.sessions.get(first.id))?.title ?? null, globalTitle);
+ assert.equal(
+ (await built.app.listSessions("member")).find((session) => session.id === first.id)?.title,
+ regenerated.title,
+ );
+ assert.equal(
+ (await built.app.listSessions("owner")).find((session) => session.id === first.id)?.title,
+ "Owner-only project title",
+ );
const unchangedAdd = await built.app.addProjectMember(project.id, "owner", "member");
assert.equal(unchangedAdd.status, "ok");
assert.equal(unchangedAdd.status === "ok" && unchangedAdd.changed, false);
diff --git a/test/session-title.test.ts b/test/session-title.test.ts
index 09cbf8fdd..6e0af38e9 100644
--- a/test/session-title.test.ts
+++ b/test/session-title.test.ts
@@ -29,6 +29,65 @@ test("names a conversation from its first completed turn (auto-title)", async ()
assert.equal(got?.session.title, "Chat: How do I roll back");
});
+test("concurrent completed turns keep durable titles when title generation is unavailable", async () => {
+ const { app } = freshApp();
+ const turns = await Promise.all(
+ Array.from({ length: 4 }, (_, i) =>
+ app.turn(dm(`Simulate four-way title outage ${i + 1}`, `web:U1:title-load-${i + 1}`)),
+ ),
+ );
+
+ for (const [i, turn] of turns.entries()) {
+ assert.equal(turn.status, "ok");
+ assert.equal((await app.getSession(turn.sessionId!))?.session.title, `Simulate four-way title outage ${i + 1}`);
+ }
+});
+
+test("a title provider exception is recorded before the completed turn gets its fallback title", async () => {
+ const { app, errors } = freshApp();
+ const turn = await app.turn(dm("Simulate title provider exception", "web:U1:title-error"));
+
+ assert.equal(turn.status, "ok");
+ assert.equal((await app.getSession(turn.sessionId!))?.session.title, "Simulate title provider exception");
+ const failures = (await errors.list({ sessionId: turn.sessionId! })).filter(
+ (error) => error.category === "session_title" && error.code === "generation_failed",
+ );
+ assert.equal(failures.length, 1);
+ assert.match(failures[0]!.message, /title model overloaded/);
+});
+
+test("the durable fallback strips turn boilerplate and stays within the generated title limit", async () => {
+ const { app } = freshApp();
+ const turn = await app.turn(
+ dm(
+ "[browser context that must not become the title]\n\nSimulate four-way title outage with deliberately long distinguishing words for truncation",
+ "web:U1:title-fallback-shape",
+ ),
+ );
+ const title = (await app.getSession(turn.sessionId!))?.session.title;
+
+ assert.equal(turn.status, "ok");
+ assert.equal(title?.length, 60);
+ assert.match(title!, /^Simulate four-way title outage/);
+ assert.match(title!, /…$/);
+ assert.doesNotMatch(title!, /browser context/);
+});
+
+test("the shared fallback covers approval pauses and manual regeneration", async () => {
+ const { app } = freshApp();
+ const turn = await app.turn(dm("!paused-approval Simulate four-way title outage deploy", "web:U1:title-paused"));
+
+ assert.ok(turn.pendingApprovals?.length);
+ assert.equal(
+ (await app.getSession(turn.sessionId!))?.session.title,
+ "!paused-approval Simulate four-way title outage deploy",
+ );
+ assert.equal(
+ (await app.regenerateTitle(turn.sessionId!, "U1"))?.title,
+ "!paused-approval Simulate four-way title outage deploy",
+ );
+});
+
test("the title is generated ONCE — a later turn does not rewrite it", async () => {
const { app } = freshApp();
const r1 = await app.turn(dm("First topic about pricing tiers", "web:U1:t2"));