Skip to content

The open-source app everyone uses to manage agents at work

test(evals): qualify native question resume paths (#15616)

Dotta428b617fc91e
diff --git a/doc/evals.md b/doc/evals.md
index 06037daea6..61e0d385cf 100644
--- a/doc/evals.md
+++ b/doc/evals.md
@@ -391,6 +391,11 @@ The 26 native `first-task` cells exercise onboarding before native selection
becomes the UI default. Live results and semantic answer reviews must accompany
any qualification claim; catalog presence alone is not a pass.
+The explicit-only [native question/resume qualification](../tests/runner-e2e/README.md#task-continuation)
+separates a completed two-answer user journey from semantic-tool documentation
+qualification. It verifies the exact provider-pause or semantic-response-wake
+binding for each answer and preserves prior grades when the definition changes.
+
## Lifecycle behavior baseline
The credential-free [lifecycle baseline](../tests/lifecycle-baseline/README.md)
diff --git a/server/src/__tests__/idle-admission.test.ts b/server/src/__tests__/idle-admission.test.ts
index 81f268a1ff..d9b7cd269f 100644
--- a/server/src/__tests__/idle-admission.test.ts
+++ b/server/src/__tests__/idle-admission.test.ts
@@ -49,6 +49,36 @@ describe("idle admission", () => {
expect(idleWorkSnapshot().active).toBe(0);
});
+ it("tracks real Vite middleware without confusing Connect mount paths with Express routes", async () => {
+ const { createServer } = await import("vite");
+ const vite = await createServer({
+ configFile: false,
+ appType: "custom",
+ server: { middlewareMode: true, hmr: false, watch: null },
+ optimizeDeps: { noDiscovery: true, include: [] },
+ });
+ const server = app(), done = deferred();
+ vite.middlewares.use("/vite-work", async (_req, res) => {
+ res.statusCode = 202;
+ res.end();
+ await done.promise;
+ });
+ server.use(vite.middlewares);
+ try {
+ expect(() => trackIdleRequestHandlers(server)).not.toThrow();
+ await request(server).get("/vite-work").expect(202);
+ expect(idleWorkSnapshot().active).toBe(1);
+ startTaskDrain({ purpose: "idle", ttlMs: 60_000 });
+ await request(server).get("/vite-work").expect(503);
+ done.resolve();
+ await new Promise(resolve => setImmediate(resolve));
+ expect(idleWorkSnapshot().active).toBe(0);
+ } finally {
+ done.resolve();
+ await vite.close();
+ }
+ });
+
it("does not mistake a client disconnect for completed accepted work", async () => {
const server = app();
const entered = deferred(), done = deferred(), closed = deferred();
diff --git a/server/src/middleware/idle-admission.ts b/server/src/middleware/idle-admission.ts
index c9de5f03ec..9540d704dd 100644
--- a/server/src/middleware/idle-admission.ts
+++ b/server/src/middleware/idle-admission.ts
@@ -53,14 +53,16 @@ export const idleAdmissionMiddleware: RequestHandler = (req, res, next) => {
// preserving router objects and error-handler arity. Waiting only for finish
// loses async work after res.json(), including the disconnected-client case.
// Keep this adapter isolated and exercise it against real Express in tests.
-type Layer = { handle: Function & { stack?: Layer[] }; route?: { stack: Layer[] } };
+type Layer = { handle: Function & { stack?: Layer[] }; route?: { stack: Layer[] } | string };
export function trackIdleRequestHandlers(app: Application): void {
const seen = new Set<Layer>();
const visit = (stack: Layer[], route = false) => {
for (const layer of stack) {
if (seen.has(layer)) continue;
seen.add(layer);
- if (layer.route) { visit(layer.route.stack, true); continue; }
+ // Connect/Vite layers use a mount-path string for route; only Express
+ // Route objects contain a nested handler stack.
+ if (layer.route && typeof layer.route !== "string") { visit(layer.route.stack, true); continue; }
if (layer.handle.stack) { visit(layer.handle.stack); continue; }
const original = layer.handle;
if (original === idleAdmissionMiddleware) continue;
diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md
index fd015d0d62..d24e660c24 100644
--- a/tests/runner-e2e/README.md
+++ b/tests/runner-e2e/README.md
@@ -1271,6 +1271,30 @@ contract tests separately verify that the advertised tool carries the documentat
for fresh and resumed native executions. This tests the current documentation
placement; it is not a statistical comparison with the former prompt arrangement.
+The separate explicit-only `question-resume` suite uses the same neutral two-question
+request on native Codex and ACPX Claude. It qualifies the user journey through
+either the provider's built-in question or Paperclip's semantic question tool.
+It does not replace or regrade `question-tool-documentation`.
+
+The oracle classifies each wait from its exact task/agent/run identity. Provider
+questions require the server-generated request key and retain the paused run.
+Semantic questions require an applied `request_human_input` receipt and one new
+response wake bound to that interaction and source run. Only a verified provider
+choice may carry the adapter's optional Other field. A second substantive question
+is still rejected; the subsequent reference question must be text-only. Both real
+board answers must remain bound to unchanged forms, and one saved document must
+include both answers. Existing lifecycle, ownership, premature-output, terminal
+state and no-extra-task checks apply. Three checkpoints are required even when
+both answers resume one run; missing native inputs or unexplained runs fail.
+
+Select `question-resume.runner-acpx-claude.local.question-answer-resume` for the
+bounded Claude qualification. The suite is excluded from `--all`; it allows one
+attempt, one to three actual runs, a ten-minute cell deadline and verified
+1,000-cent company/agent hard stops. The historical failure where the optional
+Other field stopped the driver before any answer remains a FAIL. A new behavior
+pass does not establish semantic-tool selection, documentation placement, default
+hiring behavior or general reliability.
+
Continuation screenshots wait for the correct task heading and fully revealed
conversation before capture. A loading screen or wrong task fails capture.
Browser-only regressions exercise delayed rendering without provider calls:
diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts
index 6b386d6e18..6e049a4841 100644
--- a/tests/runner-e2e/catalog.test.ts
+++ b/tests/runner-e2e/catalog.test.ts
@@ -146,10 +146,10 @@ describe("runner E2E catalog", () => {
expect(localIntegrityTasks).toHaveLength(2);
expect(openRouterBreadthTasks).toHaveLength(3);
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
- 63, 12, 6, 8, 2, 2, 2, 30, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116,
+ 63, 12, 6, 8, 2, 2, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116,
]);
- expect(validateRunnerCatalog()).toHaveLength(698);
- expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(698);
+ expect(validateRunnerCatalog()).toHaveLength(700);
+ expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(700);
expect(
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
).toHaveLength(48);
diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts
index 0d2b996f93..bae3099052 100644
--- a/tests/runner-e2e/catalog.ts
+++ b/tests/runner-e2e/catalog.ts
@@ -12,7 +12,7 @@ import { apiResponseReadingTask } from "./api-response-reading.js";
import { taskTitleTasks, taskTitleDefinitionDigest, TASK_TITLE_BUDGET_CENTS } from "./task-titles.js";
import { blockerTasks, blockerProfile } from "./blocker-cases.js";
import { accountingTasks } from "./accounting-cases.js";
-import { continuationTasks } from "./continuation-cases.js";
+import { continuationTasks, questionResumeTask } from "./continuation-cases.js";
import { contextIntegrityTasks, paperclipDocumentTask } from "./context-integrity-cases.js";
import { productionDefaultHireProfile, stockHarnessSourceDigest, stockHarnessSkillSources } from "./stock-harness.js";
import { lifecycleLiveTasks, lifecycleLiveDefinitionDigest } from "./lifecycle-live-cases.js";
@@ -1262,6 +1262,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
excludedExecutionIds: ["neutral", "challenge"].map(variant => `lifecycle-baseline.runner-codex.local.lifecycle-repair-${variant}`),
definitionMetadata: { version: 5, clarificationGrading: "wrapped-information-fields-list", narrativeDigest: lifecycleLiveDefinitionDigest, grading: "durable-state-and-attributed-narrative", scheduling: "explicit-only" },
},
+ {
+ id: "question-resume", label: "Native question and answer continuation", manualOnly: true,
+ description: "Two real UI answers, exact provider-pause or semantic-wake identity, and one saved document.",
+ groups: ["native", "local"], environments: [localEnvironment],
+ profiles: runnerProfiles.filter(p => ["runner-codex", "runner-acpx-claude"].includes(p.id)).map(productionStoryProfile),
+ tasks: [questionResumeTask], expectedMatrixSize: 2,
+ definitionMetadata: { version: 1, grading: "bound-question-path-and-two-durable-answers", instructions: "unchanged-production",
+ scheduling: "explicit-only", automaticRetryPolicy: "single_attempt", companyAndAgentBudgetCents: 1_000,
+ providerRuns: "1-3; each semantic answer starts one exactly bound wake; provider answers retain their paused run" },
+ },
{
id: "continuation", label: "Task continuation",
description: "Human direction, approval boundaries, untrusted evidence, and completed actions across turns.",
@@ -1668,6 +1678,7 @@ export function validateRunnerCatalog(): MatrixExecution[] {
...accountingTasks,
...lifecycleLiveTasks,
...continuationTasks,
+ questionResumeTask,
...everydayTasks,
...taskTitleTasks,
...runnerTasks,
diff --git a/tests/runner-e2e/continuation-cases.ts b/tests/runner-e2e/continuation-cases.ts
index 65810ad4fb..781d44e394 100644
--- a/tests/runner-e2e/continuation-cases.ts
+++ b/tests/runner-e2e/continuation-cases.ts
@@ -10,9 +10,9 @@ export const CONTINUATION_CASES = [
"question-tool-documentation",
"provider-question-bridge",
] as const;
-export type ContinuationCase = (typeof CONTINUATION_CASES)[number];
+export type ContinuationCase = (typeof CONTINUATION_CASES)[number] | "question-answer-resume";
export function continuationScenario(id: string, nonce: string) {
- if (!CONTINUATION_CASES.includes(id as ContinuationCase))
+ if (id !== "question-answer-resume" && !CONTINUATION_CASES.includes(id as (typeof CONTINUATION_CASES)[number]))
throw new Error(`Unknown continuation case: ${id}`);
const marker = `AMBER${nonce.replace(/[^a-z0-9]/gi, "")}`;
const old = `COBALT${nonce.replace(/[^a-z0-9]/gi, "")}`;
@@ -21,9 +21,11 @@ export function continuationScenario(id: string, nonce: string) {
const childTitle = `Follow-up ${marker}`;
const document =
"Save the final note as a task document. No other deliverables or child tasks are needed.";
+ const questionPrompt = "Help me write a one-sentence welcome note for a club meetup. First let me choose Morning or Afternoon using clickable choices. After I choose, ask me for a reference to include using an open text field. Ask only one question at a time and wait for my answers. Then save the note as a task document, including the selected time and my reference exactly as supplied, and finish. Do not create any other tasks or deliverables.";
const prompts: Record<ContinuationCase, string> = {
+ "question-answer-resume": questionPrompt,
"provider-question-bridge": `Use your built-in AskUserQuestion tool (not Paperclip's request_human_input) to ask which reference to include, with two choices: ${marker} and ${old}. Wait for my real answer, then save a one-sentence welcome note including only my selected reference as a task document and finish. No other tasks or deliverables are needed.`,
- "question-tool-documentation": `Help me write a one-sentence welcome note for a club meetup. First let me choose Morning or Afternoon using clickable choices. After I choose, ask me for a reference to include using an open text field. Ask only one question at a time and wait for my answers. Then save the note as a task document, including the selected time and my reference exactly as supplied, and finish. Do not create any other tasks or deliverables.`,
+ "question-tool-documentation": questionPrompt,
"answer-updates-scope": `I need a one-sentence welcome note containing ${old}. Before writing it, ask me one open-ended structured question about any changes I want. Then apply my answer and finish. ${document}`,
"clarification-not-approval": `I need a one-sentence welcome note. First ask me one open-ended structured question for the word to include. After my answer, propose your approach and wait for my explicit approval before writing the note. ${document}`,
"revision-preserves-approval": `Propose an approach for a one-sentence welcome note containing ${old}, and save that approach as the plan document. Wait for my explicit approval before writing the note. ${document}`,
@@ -85,3 +87,12 @@ export function continuationScreenshotFile(
? "final-state.png"
: `question-continuation-${phase}.png`;
}
+
+/** Separate behavior contract; leave the semantic documentation case unchanged. */
+export const questionResumeTask: RunnerTaskFixture = {
+ ...continuationTasks.find(t => t.id === "question-tool-documentation")!,
+ id: "question-answer-resume", label: "Two answered questions across native paths",
+ expectedRunCount: 3, minimumExpectedRunCount: 1,
+ buildTitle: nonce => `Question resume ${nonce}`,
+ buildPrompt: nonce => continuationScenario("question-answer-resume", nonce).prompt,
+};
diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts
index 6381d03b5c..8c0ad9ea22 100644
--- a/tests/runner-e2e/continuation-flow.ts
+++ b/tests/runner-e2e/continuation-flow.ts
@@ -1,3 +1,4 @@
+import { validResumeQuestionForm } from "./question-resume-scoring.js";
import { gradeLifecycleBaseline, type LifecycleCheckpoint } from "./lifecycle-baseline.js";
import { lifecycleLiveCase, lifecycleLiveContinuation, gradeLifecycleNarrative } from "./lifecycle-live-cases.js";
import { prepareLegacyContinuationSkill, prepareContinuationBudget } from "./continuation-fixtures.js";
@@ -106,7 +107,7 @@ export async function runContinuationFlow(input: {
return ready;
},
reject: (state) =>
- state.runs.length > 12
+ state.runs.length > (scenario.id === "question-answer-resume" ? 3 : 12)
? "Bounded continuation run count exceeded"
: state.runs.some((r) =>
["failed", "timed_out", "cancelled"].includes(r.status),
@@ -185,7 +186,9 @@ export async function runContinuationFlow(input: {
);
expect(questions, "one real question must be shown").toHaveLength(1);
const set = chatQuestionPresentation(questions[0].payload);
- if (scenario.id === "provider-question-bridge") {
+ if (scenario.id === "question-answer-resume") {
+ expect(validResumeQuestionForm(checkpoints.at(-1)!, questions[0], !!choice), "a verified native question path and the requested form").toBe(true);
+ } else if (scenario.id === "provider-question-bridge") {
expect(isSingleClaudeQuestion(set.questions), "one choice question with only the optional provider Other field").toBe(true);
} else expect(set.questions, "ask only the requested next question").toHaveLength(1);
const before = new Set(runs.map((r) => r.id));
@@ -267,7 +270,7 @@ export async function runContinuationFlow(input: {
await input.restart();
await open();
}
- if (scenario.id === "question-tool-documentation") {
+ if (["question-tool-documentation", "question-answer-resume"].includes(scenario.id)) {
await answer("Afternoon");
await snapshot("answered");
assertWaiting();
diff --git a/tests/runner-e2e/continuation-scoring.ts b/tests/runner-e2e/continuation-scoring.ts
index ada89b0916..42e79f8ed0 100644
--- a/tests/runner-e2e/continuation-scoring.ts
+++ b/tests/runner-e2e/continuation-scoring.ts
@@ -1,3 +1,4 @@
+import { gradeQuestionResume } from "./question-resume-scoring.js";
import { createHash } from "node:crypto";
import { gradeQuestionDocumentation } from "./question-documentation-scoring.js";
import type { ContinuationCase } from "./continuation-cases.js";
@@ -49,7 +50,7 @@ export function gradeContinuation(input: {
const before = input.checkpoints.filter((c) => c.phase !== "final");
check(
"recorded-continuation",
- before.length > 0 && !!final && final.runs.length >= (input.id === "provider-question-bridge" ? 1 : 2),
+ before.length > 0 && !!final && final.runs.length >= (["provider-question-bridge", "question-answer-resume"].includes(input.id) ? 1 : 2),
"Initial and final turns must both be recorded.",
);
for (const c of before) {
@@ -135,5 +136,6 @@ export function gradeContinuation(input: {
"A real provider-native card must be answered and resume the same run to completion.");
}
if (input.id === "question-tool-documentation") checks.push(...gradeQuestionDocumentation(input.checkpoints, input.marker));
+ if (input.id === "question-answer-resume") checks.push(...gradeQuestionResume(input.checkpoints, input.marker));
return checks;
}
diff --git a/tests/runner-e2e/question-resume-scoring.ts b/tests/runner-e2e/question-resume-scoring.ts
new file mode 100644
index 0000000000..4f494bdae0
--- /dev/null
+++ b/tests/runner-e2e/question-resume-scoring.ts
@@ -0,0 +1,131 @@
+import { isDeepStrictEqual } from "node:util";
+import type { ContinuationCheckpoint } from "./continuation-scoring.js";
+import { isSingleClaudeQuestion } from "./runtime-question-readiness.js";
+
+type Row = Record<string, any>;
+export type QuestionPath = "provider" | "semantic";
+const nonempty = (v: unknown): v is string => typeof v === "string" && v.trim().length > 0;
+const questions = (c?: ContinuationCheckpoint): Row[] =>
+ (c?.interactions ?? []).filter((i: any) => i.kind === "ask_user_questions") as Row[];
+export const resumeQuestionFields = (i?: Row): Row[] =>
+ Array.isArray(i?.payload?.questionSet?.questions) ? i.payload.questionSet.questions : [];
+
+/** Classify by server-owned identities and the applied semantic receipt. A
+ * missing receipt is unknown, never inferred from form shape or run count. */
+export function questionPath(checkpoint: ContinuationCheckpoint | undefined, i?: Row): QuestionPath | null {
+ if (!checkpoint || !i || !nonempty(i.id) || !nonempty(i.sourceRunId) || !nonempty(i.idempotencyKey)) return null;
+ const matches = checkpoint.runs.filter(r => r.id === i.sourceRunId) as Row[];
+ const run = matches.length === 1 ? matches[0] : undefined;
+ if (!run || run.runtimeMode !== "native" || !nonempty(run.companyId) ||
+ run.companyId !== (checkpoint.issue as Row).companyId ||
+ i.companyId !== run.companyId || i.issueId !== checkpoint.issue.id || run.issueId !== i.issueId ||
+ !nonempty(checkpoint.issue.assigneeAgentId) || i.createdByAgentId !== checkpoint.issue.assigneeAgentId ||
+ run.agentId !== checkpoint.issue.assigneeAgentId) return null;
+ const request = i.payload?.runtimeRequestId;
+ if (request !== undefined) {
+ return nonempty(request) && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$/.test(request) &&
+ i.idempotencyKey === `paperclip-runner-question:${run.id}:${request}` &&
+ i.continuationPolicy === "none" && run.status === "running" ? "provider" : null;
+ }
+ const receipt = run.resultJson?.semanticToolReceipts?.[i.idempotencyKey];
+ const created = receipt?.result?.interaction;
+ return run.status === "succeeded" && i.continuationPolicy === "wake_assignee" &&
+ receipt?.operationId === "request_human_input" && receipt.result?.disposition === "applied" &&
+ created?.id === i.id && created?.sourceRunId === run.id && created?.issueId === i.issueId &&
+ created?.companyId === i.companyId && created?.kind === "ask_user_questions" &&
+ isDeepStrictEqual(created?.payload?.questionSet, i.payload?.questionSet) ? "semantic" : null;
+}
+
+export function validResumeQuestionForm(checkpoint: ContinuationCheckpoint, i: Row, choice: boolean): boolean {
+ const path = questionPath(checkpoint, i);
+ const fields = resumeQuestionFields(i);
+ if (!path || fields.some(f => !nonempty(f.id)) || new Set(fields.map(f => f.id)).size !== fields.length) return false;
+ if (!choice) return fields.length === 1 && fields[0].answerMode === "text" &&
+ !fields[0].options?.length && !fields[0].customAnswer;
+ return fields[0]?.answerMode === "single_select" &&
+ (fields.length === 1 || (path === "provider" && isSingleClaudeQuestion(fields)));
+}
+
+function sameQuestion(a: Row, b?: Row) {
+ return !!b && ["id", "kind", "sourceRunId", "issueId", "companyId", "createdByAgentId", "idempotencyKey", "continuationPolicy"]
+ .every(k => a[k] === b[k]) && isDeepStrictEqual(a.payload, b.payload);
+}
+function answeredByUser(i?: Row) {
+ return i?.status === "answered" && nonempty(i.resolvedByUserId) && !i.resolvedByAgentId;
+}
+function resumedRun(before: ContinuationCheckpoint | undefined, after: ContinuationCheckpoint | undefined, i?: Row) {
+ if (!before || !after || !i) return false;
+ const path = questionPath(before, i);
+ const prior = new Set(before.runs.map(r => r.id));
+ const next = after.runs.filter(r => !prior.has(r.id)) as Row[];
+ if (prior.size !== before.runs.length || new Set(after.runs.map(r => r.id)).size !== after.runs.length ||
+ before.runs.some(r => !after.runs.some(n => n.id === r.id))) return false;
+ if (path === "provider") return next.length === 0 &&
+ after.runs.some(r => r.id === i.sourceRunId && ["running", "succeeded"].includes(r.status));
+ if (path !== "semantic" || next.length !== 1) return false;
+ const context = next[0].contextSnapshot;
+ return context?.issueId === i.issueId && context?.sourceRunId === i.sourceRunId &&
+ context?.interactionId === i.id && context?.interactionKind === "ask_user_questions" &&
+ context?.interactionStatus === "answered";
+}
+
+/** Behavioral qualification supports either valid native path. It does not
+ * replace the separate documentation-placement/semantic-tool qualification. */
+export function gradeQuestionResume(checkpoints: ContinuationCheckpoint[], marker: string) {
+ const initial = checkpoints.find(c => c.phase === "initial");
+ const middle = checkpoints.find(c => c.phase === "answered");
+ const final = checkpoints.find(c => c.phase === "final");
+ const first = questions(initial)[0];
+ const second = questions(middle).find(i => i.status === "pending");
+ const choice = resumeQuestionFields(first)[0];
+ const text = resumeQuestionFields(second)[0];
+ const options: Row[] = choice?.options ?? [];
+ const afternoon = options.find(o => /^afternoon\b/i.test(String(o.label).trim()));
+ const checks = [{
+ id: "question-resume.forms",
+ passed: !!initial && !!middle && questions(initial).length === 1 && first?.status === "pending" &&
+ validResumeQuestionForm(initial, first, true) && options.length >= 2 &&
+ options.every(o => nonempty(o.id)) && new Set(options.map(o => o.id)).size === options.length &&
+ options.some(o => /^morning\b/i.test(String(o.label).trim())) && !!afternoon &&
+ questions(middle).length === 2 && questions(middle).filter(i => i.status === "pending").length === 1 &&
+ nonempty(second?.id) && second?.id !== first?.id && validResumeQuestionForm(middle, second, false),
+ detail: "One choice plus only a verified provider Other companion, then a distinct text-only question.",
+ }];
+ for (const [before, after, i, field, answer] of [
+ [initial, middle, first, choice, "choice"], [middle, final, second, text, "text"],
+ ] as const) {
+ const committed = questions(after).filter(q => q.id === i?.id);
+ const retained = questions(final).filter(q => q.id === i?.id);
+ const response: Row[] = committed[0]?.result?.answers ?? [];
+ const primary = response.filter(a => a.questionId === field?.id);
+ const optionalIds = resumeQuestionFields(i).slice(1).map(f => f.id);
+ const validAnswer = primary.length === 1 && (answer === "choice"
+ ? primary[0].optionIds?.length === 1 && primary[0].optionIds[0] === afternoon?.id && !primary[0].otherText
+ : !primary[0].optionIds?.length && primary[0].otherText === `Use ${marker}.`);
+ checks.push({
+ id: `question-resume.${answer}.answer`,
+ passed: !!i && committed.length === 1 && retained.length === 1 &&
+ sameQuestion(i, committed[0]) && sameQuestion(i, retained[0]) && answeredByUser(committed[0]) &&
+ answeredByUser(retained[0]) && isDeepStrictEqual(committed[0].result, retained[0].result) &&
+ validAnswer && response.every(a => a.questionId === field?.id ||
+ (optionalIds.includes(a.questionId) && !a.optionIds?.length && !a.otherText)),
+ detail: "The real board answer must remain bound to the unchanged question, with no fabricated or substituted response.",
+ }, {
+ id: `question-resume.${answer}.path`,
+ passed: resumedRun(before, after, i),
+ detail: `Observed path: ${questionPath(before, i) ?? "unknown"}; provider answers resume the same run; semantic answers require one exactly bound response wake.`,
+ });
+ }
+ checks.push({
+ id: "question-resume.complete-evidence",
+ passed: checkpoints.length === 3 && !!initial && !!middle && !!final && initial.runs.length === 1 &&
+ questions(final).length === 2 && final.runs.length >= 1 && final.runs.length <= 3 &&
+ final.runs.every((r: Row) => r.companyId === (final.issue as Row).companyId &&
+ r.issueId === final.issue.id && r.agentId === final.issue.assigneeAgentId &&
+ typeof r.runnerProfileJson?.nativeExecutionInput?.task?.prompt === "string") &&
+ final.documents.length === 1 && !!final.documents[0].latestRevisionId &&
+ final.documents[0].body.includes(`Use ${marker}.`) && /\bafternoon\b/i.test(final.documents[0].body),
+ detail: "Retain all three checkpoints, every native input and exactly one saved document containing the choice and the complete submitted reference.",
+ });
+ return checks.map(c => ({ ...c, passed: Boolean(c.passed) }));
+}
diff --git a/tests/runner-e2e/question-resume.test.ts b/tests/runner-e2e/question-resume.test.ts
new file mode 100644
index 0000000000..525bce0c19
--- /dev/null
+++ b/tests/runner-e2e/question-resume.test.ts
@@ -0,0 +1,161 @@
+import { describe, expect, it } from "vitest";
+import { gradeQuestionResume, questionPath, validResumeQuestionForm, type QuestionPath } from "./question-resume-scoring.js";
+import { gradeContinuation } from "./continuation-scoring.js";
+import { gradeLifecycleBaseline } from "./lifecycle-baseline.js";
+import { continuationScenario } from "./continuation-cases.js";
+import { runnerMatrix } from "./catalog.js";
+
+type Row = Record<string, any>;
+function recording(paths: QuestionPath[] = ["provider", "semantic"]) {
+ const runs: Row[] = [];
+ const cards: Row[] = [];
+ const points: any[] = [];
+ const newRun = (previous?: Row) => ({ id: `run-${runs.length + 1}`, status: "running", runtimeMode: "native",
+ companyId: "company", issueId: "task", agentId: "agent", resultJson: { semanticToolReceipts: {} },
+ runnerProfileJson: { nativeExecutionInput: { task: { prompt: "actual native task input" } } },
+ contextSnapshot: previous ? { issueId: "task", sourceRunId: previous.sourceRunId,
+ interactionId: previous.id, interactionKind: "ask_user_questions", interactionStatus: "answered" } : {},
+ });
+ runs.push(newRun());
+ const snap = (phase: string, documents: Row[] = []) => {
+ const active = runs.find(r => r.status === "running");
+ points.push(structuredClone({ phase, issue: { id: "task", companyId: "company", assigneeAgentId: "agent",
+ status: phase === "final" ? "done" : active ? "in_progress" : "in_review" },
+ lifecycle: { executionRunId: active?.id ?? null, scheduledRetry: null, activeRecoveryAction: null, monitorNextCheckAt: null },
+ runs, interactions: cards, children: [], attachments: [], comments: [], documents }));
+ };
+ for (let n = 0; n < 2; n++) {
+ const run = runs.at(-1)!;
+ const provider = paths[n] === "provider";
+ const request = `request-${n}`;
+ const idempotencyKey = provider ? `paperclip-runner-question:${run.id}:${request}` : `question-${n}`;
+ const field = n === 0 ? { id: "field-1-question_0-hash", answerMode: "single_select", options: [
+ { id: "am", label: "Morning" }, { id: "pm", label: "Afternoon" },
+ ] } : { id: "reference", answerMode: "text" };
+ const questions: Row[] = [field];
+ if (provider && n === 0) questions.push({ id: "field-2-question_0_custom-hash", answerMode: "text", header: "Other", required: false });
+ const card: Row = { id: `card-${n}`, kind: "ask_user_questions", status: "pending",
+ companyId: "company", issueId: "task", createdByAgentId: "agent", sourceRunId: run.id, idempotencyKey,
+ continuationPolicy: provider ? "none" : "wake_assignee",
+ payload: { ...(provider ? { runtimeRequestId: request } : {}), questionSet: { questions } } };
+ cards.push(card);
+ if (!provider) {
+ run.status = "succeeded";
+ run.resultJson.semanticToolReceipts[idempotencyKey] = { operationId: "request_human_input",
+ result: { disposition: "applied", interaction: structuredClone(card) } };
+ }
+ snap(n === 0 ? "initial" : "answered");
+ card.status = "answered"; card.resolvedByUserId = "local-board";
+ card.result = { answers: [n === 0 ? { questionId: field.id, optionIds: ["pm"] } :
+ { questionId: field.id, optionIds: [], otherText: "Use AMBERnonce." }] };
+ if (!provider) runs.push(newRun(card));
+ }
+ runs.at(-1)!.status = "succeeded";
+ snap("final", [{ key: "welcome-note", latestRevisionId: "revision", body: "Welcome to the afternoon meetup. Use AMBERnonce." }]);
+ return points;
+}
+const failed = (r: any[]) => gradeQuestionResume(r, "AMBERnonce").filter(c => !c.passed).map(c => c.id);
+const question = (r: any[], n = 0) => r[n].interactions[n];
+
+describe("native question/resume path qualification", () => {
+ it("keeps the new journey explicit-only and the original documentation cells intact", () => {
+ const cells = runnerMatrix.filter(c => c.suite.id === "question-resume");
+ expect(cells.map(c => c.profile.id).sort()).toEqual(["runner-acpx-claude", "runner-codex"]);
+ for (const c of cells) {
+ expect(c.suite.manualOnly).toBe(true);
+ expect(c.task).toMatchObject({ expectedRunCount: 3, minimumExpectedRunCount: 1, automaticRetryPolicy: "single_attempt" });
+ expect(c.task.buildPrompt("nonce")).toBe(continuationScenario("question-tool-documentation", "nonce").prompt);
+ }
+ expect(runnerMatrix.filter(c => c.suite.id === "continuation")).toHaveLength(23);
+ });
+ it.each([
+ ["provider", "provider"], ["provider", "semantic"], ["semantic", "provider"], ["semantic", "semantic"],
+ ] as QuestionPath[][])("accepts exactly bound %s then %s journeys", (a, b) => {
+ const r = recording([a,b]);
+ expect(failed(r)).toEqual([]);
+ expect(gradeContinuation({ ...continuationScenario("question-answer-resume", "nonce"), runtimeMode: "native", checkpoints: r }).filter(c => !c.passed)).toEqual([]);
+ expect(gradeLifecycleBaseline(r).filter(c => !c.passed)).toEqual([]);
+ });
+ it.each([
+ "Welcome to the afternoon meetup, AMBERnonce.",
+ "Welcome to the afternoon meetup. Use AMBERnonce",
+ ])("rejects a shortened supplied reference in the saved document: %s", body => {
+ const r = recording();
+ r[2].documents[0].body = body;
+ expect(failed(r)).toContain("question-resume.complete-evidence");
+ });
+ it("does not admit an optional Other field on the semantic path", () => {
+ const r = recording(["semantic", "semantic"]);
+ const i = question(r);
+ i.payload.questionSet.questions.push({ id: "field-2-question_0_custom-hash", answerMode: "text", header: "Other", required: false });
+ expect(validResumeQuestionForm(r[0], i, true)).toBe(false);
+ });
+ it.each(["sourceRunId", "issueId", "companyId", "createdByAgentId", "idempotencyKey"])("rejects a provider card with mismatched %s", key => {
+ const r = recording(); question(r)[key] = "unrelated";
+ expect(failed(r)).toContain("question-resume.choice.path");
+ });
+ it("rejects a consistent foreign-company card/run pair on this task", () => {
+ const r = recording(); question(r).companyId = "foreign"; r[0].runs[0].companyId = "foreign";
+ expect(questionPath(r[0], question(r))).toBeNull();
+ });
+ it.each(["missing-request", "wrong-request", "terminal-run", "wrong-policy", "legacy-run"])("rejects invalid provider binding: %s", kind => {
+ const r = recording(); const i = question(r);
+ if (kind === "missing-request") delete i.payload.runtimeRequestId;
+ if (kind === "wrong-request") i.payload.runtimeRequestId = "unrelated";
+ if (kind === "terminal-run") r[0].runs[0].status = "succeeded";
+ if (kind === "wrong-policy") i.continuationPolicy = "wake_assignee";
+ if (kind === "legacy-run") r[0].runs[0].runtimeMode = "legacy";
+ expect(questionPath(r[0], i)).toBeNull();
+ });
+ it.each(["missing", "wrong-operation", "rejected", "wrong-interaction", "wrong-source", "wrong-form"])("rejects missing or mismatched semantic receipt: %s", kind => {
+ const r = recording(["semantic", "semantic"]); const i = question(r);
+ const receipt = r[0].runs[0].resultJson.semanticToolReceipts[i.idempotencyKey];
+ if (kind === "missing") delete r[0].runs[0].resultJson.semanticToolReceipts[i.idempotencyKey];
+ if (kind === "wrong-operation") receipt.operationId = "write_document";
+ if (kind === "rejected") receipt.result.disposition = "rejected";
+ if (kind === "wrong-interaction") receipt.result.interaction.id = "unrelated";
+ if (kind === "wrong-source") receipt.result.interaction.sourceRunId = "unrelated";
+ if (kind === "wrong-form") receipt.result.interaction.payload.questionSet.questions = [];
+ expect(questionPath(r[0], i)).toBeNull();
+ });
+ it.each(["required-other", "extra-question", "text-with-options", "two-early-cards"])("rejects the wrong form: %s", kind => {
+ const r = recording(); const fields = question(r).payload.questionSet.questions;
+ if (kind === "required-other") fields[1].required = true;
+ if (kind === "extra-question") fields.push({ id: "extra", answerMode: "text" });
+ if (kind === "text-with-options") question(r,1).payload.questionSet.questions[0].options = [{ id: "other" }];
+ if (kind === "two-early-cards") r[0].interactions.push(structuredClone(question(r,1)));
+ expect(failed(r)).toContain("question-resume.forms");
+ });
+ it.each(["agent-answer", "wrong-choice", "wrong-text", "duplicate-answer", "replaced-card", "changed-form", "changed-answer", "unanswered"])("rejects corrupt answer evidence: %s", kind => {
+ const r = recording(); const first = r[1].interactions[0]; const last = r[2].interactions[1];
+ if (kind === "agent-answer") { first.resolvedByUserId = null; first.resolvedByAgentId = "agent"; }
+ if (kind === "wrong-choice") first.result.answers[0].optionIds = ["am"];
+ if (kind === "wrong-text") last.result.answers[0].otherText = "wrong";
+ if (kind === "duplicate-answer") first.result.answers.push(structuredClone(first.result.answers[0]));
+ if (kind === "replaced-card") last.id = "replacement";
+ if (kind === "changed-form") last.payload.questionSet.questions[0].prompt = "Changed question";
+ if (kind === "changed-answer") r[2].interactions[0].result.answers[0].optionIds = ["am"];
+ if (kind === "unanswered") last.status = "pending";
+ expect(failed(r).some(id => id.endsWith(".answer"))).toBe(true);
+ });
+ it("rejects replacing a paused provider run with a fresh run", () => {
+ const r = recording(); r[1].runs.push({ ...structuredClone(r[1].runs[0]), id: "replacement" });
+ expect(failed(r)).toContain("question-resume.choice.path");
+ });
+ it.each(["interactionId", "sourceRunId", "issueId", "interactionStatus"])("rejects the wrong semantic response wake %s", key => {
+ const r = recording(); r[2].runs[1].contextSnapshot[key] = "unrelated";
+ expect(failed(r)).toContain("question-resume.text.path");
+ });
+ it.each(["lost-run", "duplicate-run", "extra-run", "missing-input", "missing-final", "missing-middle", "missing-output", "wrong-output"])("rejects incomplete continuation evidence: %s", kind => {
+ const r = recording();
+ if (kind === "lost-run") r[2].runs.shift();
+ if (kind === "duplicate-run") r[2].runs.push(structuredClone(r[2].runs[0]));
+ if (kind === "extra-run") r[2].runs.push({ ...structuredClone(r[2].runs[0]), id: "extra" });
+ if (kind === "missing-input") delete r[2].runs[0].runnerProfileJson;
+ if (kind === "missing-output") r[2].documents = [];
+ if (kind === "wrong-output") r[2].documents[0].body = "Welcome AMBERnonce.";
+ if (kind === "missing-final") r.pop();
+ if (kind === "missing-middle") r.splice(1,1);
+ expect(failed(r).length).toBeGreaterThan(0);
+ });
+});