The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
test(evals): qualify native question resume paths (#15616)
Dotta428b617fc91e
diff --git a/doc/evals.md b/doc/evals.mdindex 06037daea6..61e0d385cf 100644--- a/doc/evals.md+++ b/doc/evals.md@@ -391,6 +391,11 @@ The 26 native `first-task` cells exercise onboarding before native selectionbecomes the UI default. Live results and semantic answer reviews must accompanyany qualification claim; catalog presence alone is not a pass.+The explicit-only [native question/resume qualification](../tests/runner-e2e/README.md#task-continuation)+separates a completed two-answer user journey from semantic-tool documentation+qualification. It verifies the exact provider-pause or semantic-response-wake+binding for each answer and preserves prior grades when the definition changes.+## Lifecycle behavior baselineThe credential-free [lifecycle baseline](../tests/lifecycle-baseline/README.md)diff --git a/server/src/__tests__/idle-admission.test.ts b/server/src/__tests__/idle-admission.test.tsindex 81f268a1ff..d9b7cd269f 100644--- a/server/src/__tests__/idle-admission.test.ts+++ b/server/src/__tests__/idle-admission.test.ts@@ -49,6 +49,36 @@ describe("idle admission", () => {expect(idleWorkSnapshot().active).toBe(0);});+ it("tracks real Vite middleware without confusing Connect mount paths with Express routes", async () => {+ const { createServer } = await import("vite");+ const vite = await createServer({+ configFile: false,+ appType: "custom",+ server: { middlewareMode: true, hmr: false, watch: null },+ optimizeDeps: { noDiscovery: true, include: [] },+ });+ const server = app(), done = deferred();+ vite.middlewares.use("/vite-work", async (_req, res) => {+ res.statusCode = 202;+ res.end();+ await done.promise;+ });+ server.use(vite.middlewares);+ try {+ expect(() => trackIdleRequestHandlers(server)).not.toThrow();+ await request(server).get("/vite-work").expect(202);+ expect(idleWorkSnapshot().active).toBe(1);+ startTaskDrain({ purpose: "idle", ttlMs: 60_000 });+ await request(server).get("/vite-work").expect(503);+ done.resolve();+ await new Promise(resolve => setImmediate(resolve));+ expect(idleWorkSnapshot().active).toBe(0);+ } finally {+ done.resolve();+ await vite.close();+ }+ });+it("does not mistake a client disconnect for completed accepted work", async () => {const server = app();const entered = deferred(), done = deferred(), closed = deferred();diff --git a/server/src/middleware/idle-admission.ts b/server/src/middleware/idle-admission.tsindex c9de5f03ec..9540d704dd 100644--- a/server/src/middleware/idle-admission.ts+++ b/server/src/middleware/idle-admission.ts@@ -53,14 +53,16 @@ export const idleAdmissionMiddleware: RequestHandler = (req, res, next) => {// preserving router objects and error-handler arity. Waiting only for finish// loses async work after res.json(), including the disconnected-client case.// Keep this adapter isolated and exercise it against real Express in tests.-type Layer = { handle: Function & { stack?: Layer[] }; route?: { stack: Layer[] } };+type Layer = { handle: Function & { stack?: Layer[] }; route?: { stack: Layer[] } | string };export function trackIdleRequestHandlers(app: Application): void {const seen = new Set<Layer>();const visit = (stack: Layer[], route = false) => {for (const layer of stack) {if (seen.has(layer)) continue;seen.add(layer);- if (layer.route) { visit(layer.route.stack, true); continue; }+ // Connect/Vite layers use a mount-path string for route; only Express+ // Route objects contain a nested handler stack.+ if (layer.route && typeof layer.route !== "string") { visit(layer.route.stack, true); continue; }if (layer.handle.stack) { visit(layer.handle.stack); continue; }const original = layer.handle;if (original === idleAdmissionMiddleware) continue;diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.mdindex fd015d0d62..d24e660c24 100644--- a/tests/runner-e2e/README.md+++ b/tests/runner-e2e/README.md@@ -1271,6 +1271,30 @@ contract tests separately verify that the advertised tool carries the documentatfor fresh and resumed native executions. This tests the current documentationplacement; it is not a statistical comparison with the former prompt arrangement.+The separate explicit-only `question-resume` suite uses the same neutral two-question+request on native Codex and ACPX Claude. It qualifies the user journey through+either the provider's built-in question or Paperclip's semantic question tool.+It does not replace or regrade `question-tool-documentation`.++The oracle classifies each wait from its exact task/agent/run identity. Provider+questions require the server-generated request key and retain the paused run.+Semantic questions require an applied `request_human_input` receipt and one new+response wake bound to that interaction and source run. Only a verified provider+choice may carry the adapter's optional Other field. A second substantive question+is still rejected; the subsequent reference question must be text-only. Both real+board answers must remain bound to unchanged forms, and one saved document must+include both answers. Existing lifecycle, ownership, premature-output, terminal+state and no-extra-task checks apply. Three checkpoints are required even when+both answers resume one run; missing native inputs or unexplained runs fail.++Select `question-resume.runner-acpx-claude.local.question-answer-resume` for the+bounded Claude qualification. The suite is excluded from `--all`; it allows one+attempt, one to three actual runs, a ten-minute cell deadline and verified+1,000-cent company/agent hard stops. The historical failure where the optional+Other field stopped the driver before any answer remains a FAIL. A new behavior+pass does not establish semantic-tool selection, documentation placement, default+hiring behavior or general reliability.+Continuation screenshots wait for the correct task heading and fully revealedconversation before capture. A loading screen or wrong task fails capture.Browser-only regressions exercise delayed rendering without provider calls:diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.tsindex 6b386d6e18..6e049a4841 100644--- a/tests/runner-e2e/catalog.test.ts+++ b/tests/runner-e2e/catalog.test.ts@@ -146,10 +146,10 @@ describe("runner E2E catalog", () => {expect(localIntegrityTasks).toHaveLength(2);expect(openRouterBreadthTasks).toHaveLength(3);expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([- 63, 12, 6, 8, 2, 2, 2, 30, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116,+ 63, 12, 6, 8, 2, 2, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116,]);- expect(validateRunnerCatalog()).toHaveLength(698);- expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(698);+ expect(validateRunnerCatalog()).toHaveLength(700);+ expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(700);expect(runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),).toHaveLength(48);diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.tsindex 0d2b996f93..bae3099052 100644--- a/tests/runner-e2e/catalog.ts+++ b/tests/runner-e2e/catalog.ts@@ -12,7 +12,7 @@ import { apiResponseReadingTask } from "./api-response-reading.js";import { taskTitleTasks, taskTitleDefinitionDigest, TASK_TITLE_BUDGET_CENTS } from "./task-titles.js";import { blockerTasks, blockerProfile } from "./blocker-cases.js";import { accountingTasks } from "./accounting-cases.js";-import { continuationTasks } from "./continuation-cases.js";+import { continuationTasks, questionResumeTask } from "./continuation-cases.js";import { contextIntegrityTasks, paperclipDocumentTask } from "./context-integrity-cases.js";import { productionDefaultHireProfile, stockHarnessSourceDigest, stockHarnessSkillSources } from "./stock-harness.js";import { lifecycleLiveTasks, lifecycleLiveDefinitionDigest } from "./lifecycle-live-cases.js";@@ -1262,6 +1262,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [excludedExecutionIds: ["neutral", "challenge"].map(variant => `lifecycle-baseline.runner-codex.local.lifecycle-repair-${variant}`),definitionMetadata: { version: 5, clarificationGrading: "wrapped-information-fields-list", narrativeDigest: lifecycleLiveDefinitionDigest, grading: "durable-state-and-attributed-narrative", scheduling: "explicit-only" },},+ {+ id: "question-resume", label: "Native question and answer continuation", manualOnly: true,+ description: "Two real UI answers, exact provider-pause or semantic-wake identity, and one saved document.",+ groups: ["native", "local"], environments: [localEnvironment],+ profiles: runnerProfiles.filter(p => ["runner-codex", "runner-acpx-claude"].includes(p.id)).map(productionStoryProfile),+ tasks: [questionResumeTask], expectedMatrixSize: 2,+ definitionMetadata: { version: 1, grading: "bound-question-path-and-two-durable-answers", instructions: "unchanged-production",+ scheduling: "explicit-only", automaticRetryPolicy: "single_attempt", companyAndAgentBudgetCents: 1_000,+ providerRuns: "1-3; each semantic answer starts one exactly bound wake; provider answers retain their paused run" },+ },{id: "continuation", label: "Task continuation",description: "Human direction, approval boundaries, untrusted evidence, and completed actions across turns.",@@ -1668,6 +1678,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {...accountingTasks,...lifecycleLiveTasks,...continuationTasks,+ questionResumeTask,...everydayTasks,...taskTitleTasks,...runnerTasks,diff --git a/tests/runner-e2e/continuation-cases.ts b/tests/runner-e2e/continuation-cases.tsindex 65810ad4fb..781d44e394 100644--- a/tests/runner-e2e/continuation-cases.ts+++ b/tests/runner-e2e/continuation-cases.ts@@ -10,9 +10,9 @@ export const CONTINUATION_CASES = ["question-tool-documentation","provider-question-bridge",] as const;-export type ContinuationCase = (typeof CONTINUATION_CASES)[number];+export type ContinuationCase = (typeof CONTINUATION_CASES)[number] | "question-answer-resume";export function continuationScenario(id: string, nonce: string) {- if (!CONTINUATION_CASES.includes(id as ContinuationCase))+ if (id !== "question-answer-resume" && !CONTINUATION_CASES.includes(id as (typeof CONTINUATION_CASES)[number]))throw new Error(`Unknown continuation case: ${id}`);const marker = `AMBER${nonce.replace(/[^a-z0-9]/gi, "")}`;const old = `COBALT${nonce.replace(/[^a-z0-9]/gi, "")}`;@@ -21,9 +21,11 @@ export function continuationScenario(id: string, nonce: string) {const childTitle = `Follow-up ${marker}`;const document ="Save the final note as a task document. No other deliverables or child tasks are needed.";+ const questionPrompt = "Help me write a one-sentence welcome note for a club meetup. First let me choose Morning or Afternoon using clickable choices. After I choose, ask me for a reference to include using an open text field. Ask only one question at a time and wait for my answers. Then save the note as a task document, including the selected time and my reference exactly as supplied, and finish. Do not create any other tasks or deliverables.";const prompts: Record<ContinuationCase, string> = {+ "question-answer-resume": questionPrompt,"provider-question-bridge": `Use your built-in AskUserQuestion tool (not Paperclip's request_human_input) to ask which reference to include, with two choices: ${marker} and ${old}. Wait for my real answer, then save a one-sentence welcome note including only my selected reference as a task document and finish. No other tasks or deliverables are needed.`,- "question-tool-documentation": `Help me write a one-sentence welcome note for a club meetup. First let me choose Morning or Afternoon using clickable choices. After I choose, ask me for a reference to include using an open text field. Ask only one question at a time and wait for my answers. Then save the note as a task document, including the selected time and my reference exactly as supplied, and finish. Do not create any other tasks or deliverables.`,+ "question-tool-documentation": questionPrompt,"answer-updates-scope": `I need a one-sentence welcome note containing ${old}. Before writing it, ask me one open-ended structured question about any changes I want. Then apply my answer and finish. ${document}`,"clarification-not-approval": `I need a one-sentence welcome note. First ask me one open-ended structured question for the word to include. After my answer, propose your approach and wait for my explicit approval before writing the note. ${document}`,"revision-preserves-approval": `Propose an approach for a one-sentence welcome note containing ${old}, and save that approach as the plan document. Wait for my explicit approval before writing the note. ${document}`,@@ -85,3 +87,12 @@ export function continuationScreenshotFile(? "final-state.png": `question-continuation-${phase}.png`;}++/** Separate behavior contract; leave the semantic documentation case unchanged. */+export const questionResumeTask: RunnerTaskFixture = {+ ...continuationTasks.find(t => t.id === "question-tool-documentation")!,+ id: "question-answer-resume", label: "Two answered questions across native paths",+ expectedRunCount: 3, minimumExpectedRunCount: 1,+ buildTitle: nonce => `Question resume ${nonce}`,+ buildPrompt: nonce => continuationScenario("question-answer-resume", nonce).prompt,+};diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.tsindex 6381d03b5c..8c0ad9ea22 100644--- a/tests/runner-e2e/continuation-flow.ts+++ b/tests/runner-e2e/continuation-flow.ts@@ -1,3 +1,4 @@+import { validResumeQuestionForm } from "./question-resume-scoring.js";import { gradeLifecycleBaseline, type LifecycleCheckpoint } from "./lifecycle-baseline.js";import { lifecycleLiveCase, lifecycleLiveContinuation, gradeLifecycleNarrative } from "./lifecycle-live-cases.js";import { prepareLegacyContinuationSkill, prepareContinuationBudget } from "./continuation-fixtures.js";@@ -106,7 +107,7 @@ export async function runContinuationFlow(input: {return ready;},reject: (state) =>- state.runs.length > 12+ state.runs.length > (scenario.id === "question-answer-resume" ? 3 : 12)? "Bounded continuation run count exceeded": state.runs.some((r) =>["failed", "timed_out", "cancelled"].includes(r.status),@@ -185,7 +186,9 @@ export async function runContinuationFlow(input: {);expect(questions, "one real question must be shown").toHaveLength(1);const set = chatQuestionPresentation(questions[0].payload);- if (scenario.id === "provider-question-bridge") {+ if (scenario.id === "question-answer-resume") {+ expect(validResumeQuestionForm(checkpoints.at(-1)!, questions[0], !!choice), "a verified native question path and the requested form").toBe(true);+ } else if (scenario.id === "provider-question-bridge") {expect(isSingleClaudeQuestion(set.questions), "one choice question with only the optional provider Other field").toBe(true);} else expect(set.questions, "ask only the requested next question").toHaveLength(1);const before = new Set(runs.map((r) => r.id));@@ -267,7 +270,7 @@ export async function runContinuationFlow(input: {await input.restart();await open();}- if (scenario.id === "question-tool-documentation") {+ if (["question-tool-documentation", "question-answer-resume"].includes(scenario.id)) {await answer("Afternoon");await snapshot("answered");assertWaiting();diff --git a/tests/runner-e2e/continuation-scoring.ts b/tests/runner-e2e/continuation-scoring.tsindex ada89b0916..42e79f8ed0 100644--- a/tests/runner-e2e/continuation-scoring.ts+++ b/tests/runner-e2e/continuation-scoring.ts@@ -1,3 +1,4 @@+import { gradeQuestionResume } from "./question-resume-scoring.js";import { createHash } from "node:crypto";import { gradeQuestionDocumentation } from "./question-documentation-scoring.js";import type { ContinuationCase } from "./continuation-cases.js";@@ -49,7 +50,7 @@ export function gradeContinuation(input: {const before = input.checkpoints.filter((c) => c.phase !== "final");check("recorded-continuation",- before.length > 0 && !!final && final.runs.length >= (input.id === "provider-question-bridge" ? 1 : 2),+ before.length > 0 && !!final && final.runs.length >= (["provider-question-bridge", "question-answer-resume"].includes(input.id) ? 1 : 2),"Initial and final turns must both be recorded.",);for (const c of before) {@@ -135,5 +136,6 @@ export function gradeContinuation(input: {"A real provider-native card must be answered and resume the same run to completion.");}if (input.id === "question-tool-documentation") checks.push(...gradeQuestionDocumentation(input.checkpoints, input.marker));+ if (input.id === "question-answer-resume") checks.push(...gradeQuestionResume(input.checkpoints, input.marker));return checks;}diff --git a/tests/runner-e2e/question-resume-scoring.ts b/tests/runner-e2e/question-resume-scoring.tsnew file mode 100644index 0000000000..4f494bdae0--- /dev/null+++ b/tests/runner-e2e/question-resume-scoring.ts@@ -0,0 +1,131 @@+import { isDeepStrictEqual } from "node:util";+import type { ContinuationCheckpoint } from "./continuation-scoring.js";+import { isSingleClaudeQuestion } from "./runtime-question-readiness.js";++type Row = Record<string, any>;+export type QuestionPath = "provider" | "semantic";+const nonempty = (v: unknown): v is string => typeof v === "string" && v.trim().length > 0;+const questions = (c?: ContinuationCheckpoint): Row[] =>+ (c?.interactions ?? []).filter((i: any) => i.kind === "ask_user_questions") as Row[];+export const resumeQuestionFields = (i?: Row): Row[] =>+ Array.isArray(i?.payload?.questionSet?.questions) ? i.payload.questionSet.questions : [];++/** Classify by server-owned identities and the applied semantic receipt. A+ * missing receipt is unknown, never inferred from form shape or run count. */+export function questionPath(checkpoint: ContinuationCheckpoint | undefined, i?: Row): QuestionPath | null {+ if (!checkpoint || !i || !nonempty(i.id) || !nonempty(i.sourceRunId) || !nonempty(i.idempotencyKey)) return null;+ const matches = checkpoint.runs.filter(r => r.id === i.sourceRunId) as Row[];+ const run = matches.length === 1 ? matches[0] : undefined;+ if (!run || run.runtimeMode !== "native" || !nonempty(run.companyId) ||+ run.companyId !== (checkpoint.issue as Row).companyId ||+ i.companyId !== run.companyId || i.issueId !== checkpoint.issue.id || run.issueId !== i.issueId ||+ !nonempty(checkpoint.issue.assigneeAgentId) || i.createdByAgentId !== checkpoint.issue.assigneeAgentId ||+ run.agentId !== checkpoint.issue.assigneeAgentId) return null;+ const request = i.payload?.runtimeRequestId;+ if (request !== undefined) {+ return nonempty(request) && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$/.test(request) &&+ i.idempotencyKey === `paperclip-runner-question:${run.id}:${request}` &&+ i.continuationPolicy === "none" && run.status === "running" ? "provider" : null;+ }+ const receipt = run.resultJson?.semanticToolReceipts?.[i.idempotencyKey];+ const created = receipt?.result?.interaction;+ return run.status === "succeeded" && i.continuationPolicy === "wake_assignee" &&+ receipt?.operationId === "request_human_input" && receipt.result?.disposition === "applied" &&+ created?.id === i.id && created?.sourceRunId === run.id && created?.issueId === i.issueId &&+ created?.companyId === i.companyId && created?.kind === "ask_user_questions" &&+ isDeepStrictEqual(created?.payload?.questionSet, i.payload?.questionSet) ? "semantic" : null;+}++export function validResumeQuestionForm(checkpoint: ContinuationCheckpoint, i: Row, choice: boolean): boolean {+ const path = questionPath(checkpoint, i);+ const fields = resumeQuestionFields(i);+ if (!path || fields.some(f => !nonempty(f.id)) || new Set(fields.map(f => f.id)).size !== fields.length) return false;+ if (!choice) return fields.length === 1 && fields[0].answerMode === "text" &&+ !fields[0].options?.length && !fields[0].customAnswer;+ return fields[0]?.answerMode === "single_select" &&+ (fields.length === 1 || (path === "provider" && isSingleClaudeQuestion(fields)));+}++function sameQuestion(a: Row, b?: Row) {+ return !!b && ["id", "kind", "sourceRunId", "issueId", "companyId", "createdByAgentId", "idempotencyKey", "continuationPolicy"]+ .every(k => a[k] === b[k]) && isDeepStrictEqual(a.payload, b.payload);+}+function answeredByUser(i?: Row) {+ return i?.status === "answered" && nonempty(i.resolvedByUserId) && !i.resolvedByAgentId;+}+function resumedRun(before: ContinuationCheckpoint | undefined, after: ContinuationCheckpoint | undefined, i?: Row) {+ if (!before || !after || !i) return false;+ const path = questionPath(before, i);+ const prior = new Set(before.runs.map(r => r.id));+ const next = after.runs.filter(r => !prior.has(r.id)) as Row[];+ if (prior.size !== before.runs.length || new Set(after.runs.map(r => r.id)).size !== after.runs.length ||+ before.runs.some(r => !after.runs.some(n => n.id === r.id))) return false;+ if (path === "provider") return next.length === 0 &&+ after.runs.some(r => r.id === i.sourceRunId && ["running", "succeeded"].includes(r.status));+ if (path !== "semantic" || next.length !== 1) return false;+ const context = next[0].contextSnapshot;+ return context?.issueId === i.issueId && context?.sourceRunId === i.sourceRunId &&+ context?.interactionId === i.id && context?.interactionKind === "ask_user_questions" &&+ context?.interactionStatus === "answered";+}++/** Behavioral qualification supports either valid native path. It does not+ * replace the separate documentation-placement/semantic-tool qualification. */+export function gradeQuestionResume(checkpoints: ContinuationCheckpoint[], marker: string) {+ const initial = checkpoints.find(c => c.phase === "initial");+ const middle = checkpoints.find(c => c.phase === "answered");+ const final = checkpoints.find(c => c.phase === "final");+ const first = questions(initial)[0];+ const second = questions(middle).find(i => i.status === "pending");+ const choice = resumeQuestionFields(first)[0];+ const text = resumeQuestionFields(second)[0];+ const options: Row[] = choice?.options ?? [];+ const afternoon = options.find(o => /^afternoon\b/i.test(String(o.label).trim()));+ const checks = [{+ id: "question-resume.forms",+ passed: !!initial && !!middle && questions(initial).length === 1 && first?.status === "pending" &&+ validResumeQuestionForm(initial, first, true) && options.length >= 2 &&+ options.every(o => nonempty(o.id)) && new Set(options.map(o => o.id)).size === options.length &&+ options.some(o => /^morning\b/i.test(String(o.label).trim())) && !!afternoon &&+ questions(middle).length === 2 && questions(middle).filter(i => i.status === "pending").length === 1 &&+ nonempty(second?.id) && second?.id !== first?.id && validResumeQuestionForm(middle, second, false),+ detail: "One choice plus only a verified provider Other companion, then a distinct text-only question.",+ }];+ for (const [before, after, i, field, answer] of [+ [initial, middle, first, choice, "choice"], [middle, final, second, text, "text"],+ ] as const) {+ const committed = questions(after).filter(q => q.id === i?.id);+ const retained = questions(final).filter(q => q.id === i?.id);+ const response: Row[] = committed[0]?.result?.answers ?? [];+ const primary = response.filter(a => a.questionId === field?.id);+ const optionalIds = resumeQuestionFields(i).slice(1).map(f => f.id);+ const validAnswer = primary.length === 1 && (answer === "choice"+ ? primary[0].optionIds?.length === 1 && primary[0].optionIds[0] === afternoon?.id && !primary[0].otherText+ : !primary[0].optionIds?.length && primary[0].otherText === `Use ${marker}.`);+ checks.push({+ id: `question-resume.${answer}.answer`,+ passed: !!i && committed.length === 1 && retained.length === 1 &&+ sameQuestion(i, committed[0]) && sameQuestion(i, retained[0]) && answeredByUser(committed[0]) &&+ answeredByUser(retained[0]) && isDeepStrictEqual(committed[0].result, retained[0].result) &&+ validAnswer && response.every(a => a.questionId === field?.id ||+ (optionalIds.includes(a.questionId) && !a.optionIds?.length && !a.otherText)),+ detail: "The real board answer must remain bound to the unchanged question, with no fabricated or substituted response.",+ }, {+ id: `question-resume.${answer}.path`,+ passed: resumedRun(before, after, i),+ detail: `Observed path: ${questionPath(before, i) ?? "unknown"}; provider answers resume the same run; semantic answers require one exactly bound response wake.`,+ });+ }+ checks.push({+ id: "question-resume.complete-evidence",+ passed: checkpoints.length === 3 && !!initial && !!middle && !!final && initial.runs.length === 1 &&+ questions(final).length === 2 && final.runs.length >= 1 && final.runs.length <= 3 &&+ final.runs.every((r: Row) => r.companyId === (final.issue as Row).companyId &&+ r.issueId === final.issue.id && r.agentId === final.issue.assigneeAgentId &&+ typeof r.runnerProfileJson?.nativeExecutionInput?.task?.prompt === "string") &&+ final.documents.length === 1 && !!final.documents[0].latestRevisionId &&+ final.documents[0].body.includes(`Use ${marker}.`) && /\bafternoon\b/i.test(final.documents[0].body),+ detail: "Retain all three checkpoints, every native input and exactly one saved document containing the choice and the complete submitted reference.",+ });+ return checks.map(c => ({ ...c, passed: Boolean(c.passed) }));+}diff --git a/tests/runner-e2e/question-resume.test.ts b/tests/runner-e2e/question-resume.test.tsnew file mode 100644index 0000000000..525bce0c19--- /dev/null+++ b/tests/runner-e2e/question-resume.test.ts@@ -0,0 +1,161 @@+import { describe, expect, it } from "vitest";+import { gradeQuestionResume, questionPath, validResumeQuestionForm, type QuestionPath } from "./question-resume-scoring.js";+import { gradeContinuation } from "./continuation-scoring.js";+import { gradeLifecycleBaseline } from "./lifecycle-baseline.js";+import { continuationScenario } from "./continuation-cases.js";+import { runnerMatrix } from "./catalog.js";++type Row = Record<string, any>;+function recording(paths: QuestionPath[] = ["provider", "semantic"]) {+ const runs: Row[] = [];+ const cards: Row[] = [];+ const points: any[] = [];+ const newRun = (previous?: Row) => ({ id: `run-${runs.length + 1}`, status: "running", runtimeMode: "native",+ companyId: "company", issueId: "task", agentId: "agent", resultJson: { semanticToolReceipts: {} },+ runnerProfileJson: { nativeExecutionInput: { task: { prompt: "actual native task input" } } },+ contextSnapshot: previous ? { issueId: "task", sourceRunId: previous.sourceRunId,+ interactionId: previous.id, interactionKind: "ask_user_questions", interactionStatus: "answered" } : {},+ });+ runs.push(newRun());+ const snap = (phase: string, documents: Row[] = []) => {+ const active = runs.find(r => r.status === "running");+ points.push(structuredClone({ phase, issue: { id: "task", companyId: "company", assigneeAgentId: "agent",+ status: phase === "final" ? "done" : active ? "in_progress" : "in_review" },+ lifecycle: { executionRunId: active?.id ?? null, scheduledRetry: null, activeRecoveryAction: null, monitorNextCheckAt: null },+ runs, interactions: cards, children: [], attachments: [], comments: [], documents }));+ };+ for (let n = 0; n < 2; n++) {+ const run = runs.at(-1)!;+ const provider = paths[n] === "provider";+ const request = `request-${n}`;+ const idempotencyKey = provider ? `paperclip-runner-question:${run.id}:${request}` : `question-${n}`;+ const field = n === 0 ? { id: "field-1-question_0-hash", answerMode: "single_select", options: [+ { id: "am", label: "Morning" }, { id: "pm", label: "Afternoon" },+ ] } : { id: "reference", answerMode: "text" };+ const questions: Row[] = [field];+ if (provider && n === 0) questions.push({ id: "field-2-question_0_custom-hash", answerMode: "text", header: "Other", required: false });+ const card: Row = { id: `card-${n}`, kind: "ask_user_questions", status: "pending",+ companyId: "company", issueId: "task", createdByAgentId: "agent", sourceRunId: run.id, idempotencyKey,+ continuationPolicy: provider ? "none" : "wake_assignee",+ payload: { ...(provider ? { runtimeRequestId: request } : {}), questionSet: { questions } } };+ cards.push(card);+ if (!provider) {+ run.status = "succeeded";+ run.resultJson.semanticToolReceipts[idempotencyKey] = { operationId: "request_human_input",+ result: { disposition: "applied", interaction: structuredClone(card) } };+ }+ snap(n === 0 ? "initial" : "answered");+ card.status = "answered"; card.resolvedByUserId = "local-board";+ card.result = { answers: [n === 0 ? { questionId: field.id, optionIds: ["pm"] } :+ { questionId: field.id, optionIds: [], otherText: "Use AMBERnonce." }] };+ if (!provider) runs.push(newRun(card));+ }+ runs.at(-1)!.status = "succeeded";+ snap("final", [{ key: "welcome-note", latestRevisionId: "revision", body: "Welcome to the afternoon meetup. Use AMBERnonce." }]);+ return points;+}+const failed = (r: any[]) => gradeQuestionResume(r, "AMBERnonce").filter(c => !c.passed).map(c => c.id);+const question = (r: any[], n = 0) => r[n].interactions[n];++describe("native question/resume path qualification", () => {+ it("keeps the new journey explicit-only and the original documentation cells intact", () => {+ const cells = runnerMatrix.filter(c => c.suite.id === "question-resume");+ expect(cells.map(c => c.profile.id).sort()).toEqual(["runner-acpx-claude", "runner-codex"]);+ for (const c of cells) {+ expect(c.suite.manualOnly).toBe(true);+ expect(c.task).toMatchObject({ expectedRunCount: 3, minimumExpectedRunCount: 1, automaticRetryPolicy: "single_attempt" });+ expect(c.task.buildPrompt("nonce")).toBe(continuationScenario("question-tool-documentation", "nonce").prompt);+ }+ expect(runnerMatrix.filter(c => c.suite.id === "continuation")).toHaveLength(23);+ });+ it.each([+ ["provider", "provider"], ["provider", "semantic"], ["semantic", "provider"], ["semantic", "semantic"],+ ] as QuestionPath[][])("accepts exactly bound %s then %s journeys", (a, b) => {+ const r = recording([a,b]);+ expect(failed(r)).toEqual([]);+ expect(gradeContinuation({ ...continuationScenario("question-answer-resume", "nonce"), runtimeMode: "native", checkpoints: r }).filter(c => !c.passed)).toEqual([]);+ expect(gradeLifecycleBaseline(r).filter(c => !c.passed)).toEqual([]);+ });+ it.each([+ "Welcome to the afternoon meetup, AMBERnonce.",+ "Welcome to the afternoon meetup. Use AMBERnonce",+ ])("rejects a shortened supplied reference in the saved document: %s", body => {+ const r = recording();+ r[2].documents[0].body = body;+ expect(failed(r)).toContain("question-resume.complete-evidence");+ });+ it("does not admit an optional Other field on the semantic path", () => {+ const r = recording(["semantic", "semantic"]);+ const i = question(r);+ i.payload.questionSet.questions.push({ id: "field-2-question_0_custom-hash", answerMode: "text", header: "Other", required: false });+ expect(validResumeQuestionForm(r[0], i, true)).toBe(false);+ });+ it.each(["sourceRunId", "issueId", "companyId", "createdByAgentId", "idempotencyKey"])("rejects a provider card with mismatched %s", key => {+ const r = recording(); question(r)[key] = "unrelated";+ expect(failed(r)).toContain("question-resume.choice.path");+ });+ it("rejects a consistent foreign-company card/run pair on this task", () => {+ const r = recording(); question(r).companyId = "foreign"; r[0].runs[0].companyId = "foreign";+ expect(questionPath(r[0], question(r))).toBeNull();+ });+ it.each(["missing-request", "wrong-request", "terminal-run", "wrong-policy", "legacy-run"])("rejects invalid provider binding: %s", kind => {+ const r = recording(); const i = question(r);+ if (kind === "missing-request") delete i.payload.runtimeRequestId;+ if (kind === "wrong-request") i.payload.runtimeRequestId = "unrelated";+ if (kind === "terminal-run") r[0].runs[0].status = "succeeded";+ if (kind === "wrong-policy") i.continuationPolicy = "wake_assignee";+ if (kind === "legacy-run") r[0].runs[0].runtimeMode = "legacy";+ expect(questionPath(r[0], i)).toBeNull();+ });+ it.each(["missing", "wrong-operation", "rejected", "wrong-interaction", "wrong-source", "wrong-form"])("rejects missing or mismatched semantic receipt: %s", kind => {+ const r = recording(["semantic", "semantic"]); const i = question(r);+ const receipt = r[0].runs[0].resultJson.semanticToolReceipts[i.idempotencyKey];+ if (kind === "missing") delete r[0].runs[0].resultJson.semanticToolReceipts[i.idempotencyKey];+ if (kind === "wrong-operation") receipt.operationId = "write_document";+ if (kind === "rejected") receipt.result.disposition = "rejected";+ if (kind === "wrong-interaction") receipt.result.interaction.id = "unrelated";+ if (kind === "wrong-source") receipt.result.interaction.sourceRunId = "unrelated";+ if (kind === "wrong-form") receipt.result.interaction.payload.questionSet.questions = [];+ expect(questionPath(r[0], i)).toBeNull();+ });+ it.each(["required-other", "extra-question", "text-with-options", "two-early-cards"])("rejects the wrong form: %s", kind => {+ const r = recording(); const fields = question(r).payload.questionSet.questions;+ if (kind === "required-other") fields[1].required = true;+ if (kind === "extra-question") fields.push({ id: "extra", answerMode: "text" });+ if (kind === "text-with-options") question(r,1).payload.questionSet.questions[0].options = [{ id: "other" }];+ if (kind === "two-early-cards") r[0].interactions.push(structuredClone(question(r,1)));+ expect(failed(r)).toContain("question-resume.forms");+ });+ it.each(["agent-answer", "wrong-choice", "wrong-text", "duplicate-answer", "replaced-card", "changed-form", "changed-answer", "unanswered"])("rejects corrupt answer evidence: %s", kind => {+ const r = recording(); const first = r[1].interactions[0]; const last = r[2].interactions[1];+ if (kind === "agent-answer") { first.resolvedByUserId = null; first.resolvedByAgentId = "agent"; }+ if (kind === "wrong-choice") first.result.answers[0].optionIds = ["am"];+ if (kind === "wrong-text") last.result.answers[0].otherText = "wrong";+ if (kind === "duplicate-answer") first.result.answers.push(structuredClone(first.result.answers[0]));+ if (kind === "replaced-card") last.id = "replacement";+ if (kind === "changed-form") last.payload.questionSet.questions[0].prompt = "Changed question";+ if (kind === "changed-answer") r[2].interactions[0].result.answers[0].optionIds = ["am"];+ if (kind === "unanswered") last.status = "pending";+ expect(failed(r).some(id => id.endsWith(".answer"))).toBe(true);+ });+ it("rejects replacing a paused provider run with a fresh run", () => {+ const r = recording(); r[1].runs.push({ ...structuredClone(r[1].runs[0]), id: "replacement" });+ expect(failed(r)).toContain("question-resume.choice.path");+ });+ it.each(["interactionId", "sourceRunId", "issueId", "interactionStatus"])("rejects the wrong semantic response wake %s", key => {+ const r = recording(); r[2].runs[1].contextSnapshot[key] = "unrelated";+ expect(failed(r)).toContain("question-resume.text.path");+ });+ it.each(["lost-run", "duplicate-run", "extra-run", "missing-input", "missing-final", "missing-middle", "missing-output", "wrong-output"])("rejects incomplete continuation evidence: %s", kind => {+ const r = recording();+ if (kind === "lost-run") r[2].runs.shift();+ if (kind === "duplicate-run") r[2].runs.push(structuredClone(r[2].runs[0]));+ if (kind === "extra-run") r[2].runs.push({ ...structuredClone(r[2].runs[0]), id: "extra" });+ if (kind === "missing-input") delete r[2].runs[0].runnerProfileJson;+ if (kind === "missing-output") r[2].documents = [];+ if (kind === "wrong-output") r[2].documents[0].body = "Welcome AMBERnonce.";+ if (kind === "missing-final") r.pop();+ if (kind === "missing-middle") r.splice(1,1);+ expect(failed(r).length).toBeGreaterThan(0);+ });+});