Skip to content

The open-source app everyone uses to manage agents at work

test(runner): add question adapter conformance (#12409)

Dottab93ad538b63c
diff --git a/packages/paperclip-runner/protocol/fixtures/questions/acpx.json b/packages/paperclip-runner/protocol/fixtures/questions/acpx.json
new file mode 100644
index 0000000000..d6ef05edc0
--- /dev/null
+++ b/packages/paperclip-runner/protocol/fixtures/questions/acpx.json
@@ -0,0 +1,75 @@
+{
+ "schema": "paperclip.question_adapter_fixture.v1",
+ "adapter": "acpx",
+ "nativeRequest": {
+ "method": "elicitation/create",
+ "params": {
+ "mode": "form",
+ "message": "Deployment input",
+ "requestedSchema": {
+ "type": "object",
+ "title": "Deployment input",
+ "required": ["environment"],
+ "properties": {
+ "environment": {
+ "type": "string",
+ "title": "Environment",
+ "description": "Where should we deploy?",
+ "oneOf": [
+ {
+ "const": "staging",
+ "title": "Staging",
+ "description": "Deploy to staging first."
+ },
+ {
+ "const": "production",
+ "title": "Production",
+ "description": "Deploy directly to production."
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "canonicalQuestionSet": {
+ "schema": "paperclip.question_set.v1",
+ "title": "Deployment input",
+ "submitLabel": "Submit answers",
+ "questions": [
+ {
+ "id": "field-1-environment-ba5285161ba6",
+ "header": "Environment",
+ "prompt": "Where should we deploy?",
+ "required": true,
+ "answerMode": "single_select",
+ "options": [
+ {
+ "id": "option-1",
+ "label": "Staging",
+ "description": "Deploy to staging first."
+ },
+ {
+ "id": "option-2",
+ "label": "Production",
+ "description": "Deploy directly to production."
+ }
+ ]
+ }
+ ]
+ },
+ "canonicalResponse": {
+ "schema": "paperclip.question_response.v1",
+ "answers": {
+ "field-1-environment-ba5285161ba6": {
+ "selectedOptionIds": ["option-1"]
+ }
+ }
+ },
+ "nativeResponse": {
+ "action": "accept",
+ "content": {
+ "environment": "staging"
+ }
+ }
+}
diff --git a/packages/paperclip-runner/protocol/fixtures/questions/codex.json b/packages/paperclip-runner/protocol/fixtures/questions/codex.json
index d1ec06850f..4f90d1b513 100644
--- a/packages/paperclip-runner/protocol/fixtures/questions/codex.json
+++ b/packages/paperclip-runner/protocol/fixtures/questions/codex.json
@@ -9,6 +9,7 @@
"id": "environment",
"header": "Environment",
"question": "Where should we deploy?",
+ "required": true,
"options": [
{ "label": "Staging", "description": "Deploy to staging first." },
{ "label": "Production", "description": "Deploy directly to production." }
diff --git a/packages/paperclip-runner/protocol/manifest.json b/packages/paperclip-runner/protocol/manifest.json
index 591bbd88ac..c7a5b5f0c3 100644
--- a/packages/paperclip-runner/protocol/manifest.json
+++ b/packages/paperclip-runner/protocol/manifest.json
@@ -175,9 +175,15 @@
"expectation": "accept",
"compatibilityCase": "canonical"
},
+ {
+ "path": "fixtures/questions/acpx.json",
+ "sha256": "6f1b65d5168b39f4bc821e778901bf077b1df10495f4bc1217b515eca4a339e4",
+ "expectation": "accept",
+ "compatibilityCase": "acpx-structured-input"
+ },
{
"path": "fixtures/questions/codex.json",
- "sha256": "ad673b6f9d70b74b53656e622d15fad1a66430f39d62b9ef71479d194a198bd6",
+ "sha256": "a02a862b008d8327b4665bbde8ae3b73ad1f8dca7891072b0839b0e84b5920fa",
"expectation": "accept",
"compatibilityCase": "codex-structured-input"
},
diff --git a/packages/paperclip-runner/scripts/generate-protocol-manifest.mjs b/packages/paperclip-runner/scripts/generate-protocol-manifest.mjs
index eda018031a..a9964449d1 100644
--- a/packages/paperclip-runner/scripts/generate-protocol-manifest.mjs
+++ b/packages/paperclip-runner/scripts/generate-protocol-manifest.mjs
@@ -5,8 +5,10 @@ import { fileURLToPath } from "node:url";
import {
SUPPORTED_FIXTURE_VERSION,
SUPPORTED_PROTOCOL_VERSION,
+ assertAcpxQuestionFixture,
assertCodexQuestionFixture,
assertConformanceFixturePair,
+ assertQuestionAdapterFixture,
assertReplayFixtureCompatibility,
assertSchemaInstance,
compileProtocolValidators,
@@ -64,14 +66,19 @@ export async function buildProtocolManifest() {
compatibilityCase = "additive-optional-fields";
}
}
- } else if (relativePath === "fixtures/questions/codex.json") {
- assertCodexQuestionFixture(value);
+ } else if (relativePath.startsWith("fixtures/questions/")) {
assertSchemaInstance(
validators.questionAdapterFixture,
value,
relativePath,
);
- compatibilityCase = "codex-structured-input";
+ assertQuestionAdapterFixture(value);
+ if (relativePath === "fixtures/questions/codex.json") {
+ assertCodexQuestionFixture(value);
+ } else if (relativePath === "fixtures/questions/acpx.json") {
+ assertAcpxQuestionFixture(value);
+ }
+ compatibilityCase = `${value.adapter}-structured-input`;
} else if (relativePath === "fixtures/conformance-minimal-run.json") {
assertSchemaInstance(validators.conformanceFixture, value, relativePath);
compatibilityCase = "cross-language-input";
diff --git a/packages/paperclip-runner/scripts/protocol-contract.mjs b/packages/paperclip-runner/scripts/protocol-contract.mjs
index a800538915..f58ef848d1 100644
--- a/packages/paperclip-runner/scripts/protocol-contract.mjs
+++ b/packages/paperclip-runner/scripts/protocol-contract.mjs
@@ -1,3 +1,4 @@
+import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import { readdir, readFile } from "node:fs/promises";
import { relative, resolve, sep } from "node:path";
@@ -180,48 +181,521 @@ export function assertReplayFixtureCompatibility(fixture) {
return fixture;
}
-export function assertCodexQuestionFixture(fixture) {
+export function assertQuestionAdapterFixture(fixture) {
requireSchema(fixture, "paperclip.question_adapter_fixture.v1", "question fixture");
- if (fixture.adapter !== "codex") throw contractError("unsupported_provider", String(fixture.adapter));
requireSchema(fixture.canonicalQuestionSet, "paperclip.question_set.v1", "canonicalQuestionSet");
requireSchema(fixture.canonicalResponse, "paperclip.question_response.v1", "canonicalResponse");
- if (fixture.nativeRequest?.method !== "item/tool/requestUserInput") {
- throw contractError("invalid_codex_question_fixture", "native request method");
- }
const questions = fixture.canonicalQuestionSet.questions;
if (!Array.isArray(questions) || questions.length === 0) {
- throw contractError("invalid_codex_question_fixture", "questions must be non-empty");
+ throw contractError("invalid_question_adapter_fixture", "questions must be non-empty");
}
- const questionIds = new Set();
+ const questionsById = new Map();
const optionIdsByQuestion = new Map();
for (const question of questions) {
- if (typeof question.id !== "string" || question.id.length === 0 || questionIds.has(question.id)) {
- throw contractError("invalid_codex_question_fixture", "question IDs must be unique");
+ if (typeof question.id !== "string" || question.id.length === 0 || questionsById.has(question.id)) {
+ throw contractError("invalid_question_adapter_fixture", "question IDs must be unique");
}
- questionIds.add(question.id);
+ questionsById.set(question.id, question);
const optionIds = new Set();
for (const option of question.options ?? []) {
if (typeof option.id !== "string" || option.id.length === 0 || optionIds.has(option.id)) {
- throw contractError("invalid_codex_question_fixture", `option IDs for ${question.id} must be unique`);
+ throw contractError("invalid_question_adapter_fixture", `option IDs for ${question.id} must be unique`);
}
optionIds.add(option.id);
}
optionIdsByQuestion.set(question.id, optionIds);
+ const validation = question.textValidation;
+ if (
+ validation?.minLength !== undefined
+ && validation.maxLength !== undefined
+ && validation.minLength > validation.maxLength
+ ) {
+ throw contractError("invalid_question_adapter_fixture", `text validation for ${question.id} has minLength greater than maxLength`);
+ }
+ if (
+ validation?.minimum !== undefined
+ && validation.maximum !== undefined
+ && validation.minimum > validation.maximum
+ ) {
+ throw contractError("invalid_question_adapter_fixture", `text validation for ${question.id} has minimum greater than maximum`);
+ }
+ if (validation?.pattern !== undefined) {
+ try {
+ new RegExp(validation.pattern);
+ } catch {
+ throw contractError("invalid_question_adapter_fixture", `text validation pattern for ${question.id} is invalid`);
+ }
+ }
}
- for (const [answerId, answer] of Object.entries(fixture.canonicalResponse.answers ?? {})) {
- if (!questionIds.has(answerId)) {
- throw contractError("invalid_codex_question_fixture", `answer has unknown question ID ${answerId}`);
+ const answers = fixture.canonicalResponse.answers ?? {};
+ for (const [answerId, answer] of Object.entries(answers)) {
+ if (!questionsById.has(answerId)) {
+ throw contractError("invalid_question_adapter_fixture", `answer has unknown question ID ${answerId}`);
+ }
+ if (answer === null || typeof answer !== "object" || Array.isArray(answer)) {
+ throw contractError("invalid_question_adapter_fixture", `answer for ${answerId} must be an object`);
+ }
+ const selectedOptionIds = answer.selectedOptionIds ?? [];
+ if (!Array.isArray(selectedOptionIds) || selectedOptionIds.some((id) => typeof id !== "string")) {
+ throw contractError("invalid_question_adapter_fixture", `answer for ${answerId} has invalid option IDs`);
+ }
+ if (new Set(selectedOptionIds).size !== selectedOptionIds.length) {
+ throw contractError("invalid_question_adapter_fixture", `answer for ${answerId} repeats an option ID`);
}
- for (const optionId of answer.selectedOptionIds ?? []) {
+ for (const optionId of selectedOptionIds) {
if (!optionIdsByQuestion.get(answerId)?.has(optionId)) {
- throw contractError("invalid_codex_question_fixture", `answer has unknown option ID ${optionId}`);
+ throw contractError("invalid_question_adapter_fixture", `answer has unknown option ID ${optionId}`);
}
}
}
+ for (const question of questions) {
+ const answer = answers[question.id];
+ if (answer === undefined) {
+ if (question.required) {
+ throw contractError("invalid_question_adapter_fixture", `answer for required question ${question.id} is missing`);
+ }
+ continue;
+ }
+ const selectedOptionIds = answer.selectedOptionIds ?? [];
+ const text = answer.text;
+ const customText = answer.customText;
+ if (question.answerMode === "text") {
+ if (selectedOptionIds.length > 0 || customText !== undefined) {
+ throw contractError("invalid_question_adapter_fixture", `text answer for ${question.id} carries select-only fields`);
+ }
+ } else {
+ if (text !== undefined) {
+ throw contractError("invalid_question_adapter_fixture", `select answer for ${question.id} carries text`);
+ }
+ if (question.answerMode === "single_select" && selectedOptionIds.length > 1) {
+ throw contractError("invalid_question_adapter_fixture", `single-select answer for ${question.id} chooses more than one option`);
+ }
+ if (customText !== undefined && question.customAnswer?.enabled !== true) {
+ throw contractError("invalid_question_adapter_fixture", `custom answer for ${question.id} is not enabled`);
+ }
+ if (
+ question.answerMode === "single_select"
+ && typeof customText === "string"
+ && customText.trim().length > 0
+ && selectedOptionIds.length > 0
+ ) {
+ throw contractError("invalid_question_adapter_fixture", `single-select answer for ${question.id} mixes option and custom values`);
+ }
+ }
+ const hasValue = fixtureAnswerHasValue(answer);
+ if (question.required && !hasValue) {
+ throw contractError("invalid_question_adapter_fixture", `answer for required question ${question.id} is empty`);
+ }
+ const boundedText = question.answerMode === "text" ? text : customText;
+ if (boundedText !== undefined) {
+ const validation = question.textValidation;
+ if (validation?.minLength !== undefined && boundedText.length < validation.minLength) {
+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} must contain at least ${validation.minLength} characters`);
+ }
+ if (validation?.maxLength !== undefined && boundedText.length > validation.maxLength) {
+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} must contain at most ${validation.maxLength} characters`);
+ }
+ if (validation?.pattern !== undefined) {
+ let pattern;
+ try {
+ pattern = new RegExp(validation.pattern);
+ } catch {
+ throw contractError("invalid_question_adapter_fixture", `text validation pattern for ${question.id} is invalid`);
+ }
+ if (!testFixturePatternWithDeadline(pattern.source, boundedText, question.id)) {
+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} does not match the required format`);
+ }
+ }
+ if (validation?.inputType === "number" || validation?.inputType === "integer") {
+ const numeric = Number(boundedText);
+ if (!Number.isFinite(numeric) || (validation.inputType === "integer" && !Number.isInteger(numeric))) {
+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} must be a valid ${validation.inputType}`);
+ }
+ if (validation.minimum !== undefined && numeric < validation.minimum) {
+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} must be at least ${validation.minimum}`);
+ }
+ if (validation.maximum !== undefined && numeric > validation.maximum) {
+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} must be at most ${validation.maximum}`);
+ }
+ }
+ }
+ }
+ return fixture;
+}
+
+export function assertCodexQuestionFixture(fixture) {
+ assertQuestionAdapterFixture(fixture);
+ if (fixture.adapter !== "codex") throw contractError("unsupported_provider", String(fixture.adapter));
+ if (fixture.nativeRequest?.method !== "item/tool/requestUserInput") {
+ throw contractError("invalid_codex_question_fixture", "native request method");
+ }
+ return fixture;
+}
+
+export function assertAcpxQuestionFixture(fixture) {
+ assertQuestionAdapterFixture(fixture);
+ if (fixture.adapter !== "acpx") {
+ throw contractError("unsupported_provider", String(fixture.adapter));
+ }
+ const request = fixture.nativeRequest;
+ if (!isPlainRecord(request) || request.method !== "elicitation/create") {
+ throw contractError("invalid_acpx_question_fixture", "native request method");
+ }
+ const params = request.params;
+ const requestedSchema = isPlainRecord(params) ? params.requestedSchema : null;
+ if (
+ !isPlainRecord(params)
+ || params.mode !== "form"
+ || !isPlainRecord(requestedSchema)
+ || !isPlainRecord(requestedSchema.properties)
+ ) {
+ throw contractError("invalid_acpx_question_fixture", "native form request");
+ }
+ const properties = Object.entries(requestedSchema.properties);
+ if (properties.length === 0 || properties.length > 64) {
+ throw contractError("invalid_acpx_question_fixture", "native form property count");
+ }
+ const required = normalizedAcpxRequired(requestedSchema);
+ for (const [name, property] of properties) {
+ assertAcpxProperty(name, property);
+ if (
+ isPlainRecord(property)
+ && property.type === "string"
+ && acpxEnumValues(property, "oneOf").length === 0
+ && optionalFixtureText(property.pattern) !== undefined
+ ) {
+ throw contractError(
+ "invalid_acpx_question_fixture",
+ `native property ${name} uses an unsupported pattern`,
+ );
+ }
+ }
+
+ const response = fixture.nativeResponse;
+ if (
+ !isPlainRecord(response)
+ || response.action !== "accept"
+ || !isPlainRecord(response.content)
+ ) {
+ throw contractError("invalid_acpx_question_fixture", "native accept response");
+ }
+ for (const [name] of properties) {
+ if (required.has(name) && !Object.hasOwn(response.content, name)) {
+ throw contractError("invalid_acpx_question_fixture", `native response omits required property ${name}`);
+ }
+ }
+ for (const [name, value] of Object.entries(response.content)) {
+ const property = requestedSchema.properties[name];
+ if (!isPlainRecord(property)) {
+ throw contractError("invalid_acpx_question_fixture", `native response has unknown property ${name}`);
+ }
+ assertAcpxPropertyValue(name, property, value);
+ }
+ const projected = projectAcpxFixture(params, fixture.canonicalResponse);
+ if (canonicalFixtureJson(projected.questionSet) !== canonicalFixtureJson(fixture.canonicalQuestionSet)) {
+ throw contractError("invalid_acpx_question_fixture", "native and canonical question sets differ");
+ }
+ if (canonicalFixtureJson(projected.nativeResponse) !== canonicalFixtureJson(fixture.nativeResponse)) {
+ throw contractError("invalid_acpx_question_fixture", "canonical and native responses differ");
+ }
return fixture;
}
+function projectAcpxFixture(params, canonicalResponse) {
+ const schema = params.requestedSchema;
+ const required = normalizedAcpxRequired(schema);
+ const bindings = Object.entries(schema.properties).map(([name, property], index) =>
+ projectAcpxProperty(name, property, index, required.has(name))
+ );
+ const title = optionalFixtureText(schema.title) ?? "Additional information needed";
+ const descriptions = [
+ optionalFixtureText(params.message),
+ optionalFixtureText(schema.description),
+ ].filter((value, index, all) => value !== undefined && value !== title && all.indexOf(value) === index);
+ const questionSet = {
+ schema: "paperclip.question_set.v1",
+ title,
+ ...(descriptions.length > 0 ? { description: descriptions.join("\n\n") } : {}),
+ submitLabel: "Submit answers",
+ questions: bindings.map((binding) => binding.question),
+ };
+ const content = {};
+ for (const binding of bindings) {
+ const answer = canonicalResponse.answers?.[binding.question.id];
+ // The production parser accepts explicit empty optional answers and omits
+ // them from its normalized response. Mirror that before projecting ACP
+ // content so the fixture gate certifies the same wire behavior.
+ if (!answer || !fixtureAnswerHasValue(answer)) continue;
+ if (binding.type === "string" && binding.question.answerMode === "text") {
+ if (answer.text !== undefined) defineAcpxResponseProperty(content, binding.name, answer.text);
+ } else if (binding.type === "number" || binding.type === "integer") {
+ if (answer.text !== undefined) {
+ defineAcpxResponseProperty(content, binding.name, Number(answer.text));
+ }
+ } else if (binding.type === "boolean") {
+ const selected = answer.selectedOptionIds?.[0];
+ if (selected !== undefined) {
+ defineAcpxResponseProperty(
+ content,
+ binding.name,
+ binding.optionValues.get(selected) === "true",
+ );
+ }
+ } else {
+ const selected = (answer.selectedOptionIds ?? []).map((id) => binding.optionValues.get(id));
+ if (binding.type === "array") {
+ defineAcpxResponseProperty(content, binding.name, selected);
+ } else if (selected[0] !== undefined) {
+ defineAcpxResponseProperty(content, binding.name, selected[0]);
+ }
+ }
+ }
+ return { questionSet, nativeResponse: { action: "accept", content } };
+}
+
+function fixtureAnswerHasValue(answer) {
+ return (
+ (typeof answer.text === "string" && answer.text.trim().length > 0)
+ || (typeof answer.customText === "string" && answer.customText.trim().length > 0)
+ || (Array.isArray(answer.selectedOptionIds) && answer.selectedOptionIds.length > 0)
+ );
+}
+
+function defineAcpxResponseProperty(content, name, value) {
+ Object.defineProperty(content, name, {
+ value,
+ enumerable: true,
+ configurable: true,
+ writable: true,
+ });
+}
+
+function normalizedAcpxRequired(schema) {
+ const propertyNames = new Set(
+ isPlainRecord(schema.properties) ? Object.keys(schema.properties) : [],
+ );
+ return new Set(
+ Array.isArray(schema.required)
+ ? schema.required
+ .filter(
+ (value) => typeof value === "string" && propertyNames.has(value),
+ )
+ : []
+ );
+}
+
+function projectAcpxProperty(name, property, index, required) {
+ const id = stableAcpxFieldId(name, index);
+ const header = optionalFixtureText(property.title) ?? name;
+ const base = {
+ id,
+ header,
+ prompt: optionalFixtureText(property.description) ?? header,
+ required,
+ };
+ if (property.type === "string") {
+ const options = acpxNativeOptions(property, "oneOf");
+ if (options.length > 0) return projectedAcpxOptions(name, property.type, base, options, "single_select");
+ const pattern = optionalFixtureText(property.pattern);
+ return {
+ name,
+ type: property.type,
+ optionValues: new Map(),
+ question: {
+ ...base,
+ answerMode: "text",
+ textValidation: {
+ inputType: "text",
+ ...(finiteNonNegativeFixtureInteger(property.minLength) !== undefined
+ ? { minLength: property.minLength }
+ : {}),
+ ...(finiteNonNegativeFixtureInteger(property.maxLength) !== undefined
+ ? { maxLength: property.maxLength }
+ : {}),
+ ...(pattern !== undefined ? { pattern } : {}),
+ },
+ },
+ };
+ }
+ if (property.type === "number" || property.type === "integer") {
+ return {
+ name,
+ type: property.type,
+ optionValues: new Map(),
+ question: {
+ ...base,
+ answerMode: "text",
+ textValidation: {
+ inputType: property.type,
+ ...(Number.isFinite(property.minimum) ? { minimum: property.minimum } : {}),
+ ...(Number.isFinite(property.maximum) ? { maximum: property.maximum } : {}),
+ },
+ },
+ };
+ }
+ if (property.type === "boolean") {
+ return projectedAcpxOptions(name, property.type, base, [
+ { value: "true", label: "Yes" },
+ { value: "false", label: "No" },
+ ], "single_select");
+ }
+ return projectedAcpxOptions(
+ name,
+ property.type,
+ base,
+ acpxNativeOptions(property.items, "anyOf"),
+ "multi_select",
+ );
+}
+
+function testFixturePatternWithDeadline(pattern, value, questionId) {
+ const outcome = spawnSync(
+ process.execPath,
+ [
+ "--input-type=module",
+ "--eval",
+ "const chunks=[];for await(const chunk of process.stdin)chunks.push(chunk);const {pattern,value}=JSON.parse(Buffer.concat(chunks).toString());process.stdout.write(new RegExp(pattern).test(value)?'1':'0');",
+ ],
+ {
+ input: JSON.stringify({ pattern, value }),
+ encoding: "utf8",
+ timeout: 1_000,
+ maxBuffer: 1_024,
+ windowsHide: true,
+ },
+ );
+ if (outcome.error || outcome.signal || outcome.status !== 0) {
+ throw contractError(
+ "invalid_question_adapter_fixture",
+ `text validation pattern for ${questionId} could not be evaluated safely`,
+ );
+ }
+ return outcome.stdout === "1";
+}
+
+function projectedAcpxOptions(name, type, base, nativeOptions, answerMode) {
+ const optionValues = new Map();
+ const options = nativeOptions.map((option, index) => {
+ const id = `option-${index + 1}`;
+ optionValues.set(id, option.value);
+ return {
+ id,
+ label: option.label,
+ ...(option.description !== undefined ? { description: option.description } : {}),
+ };
+ });
+ return { name, type, optionValues, question: { ...base, answerMode, options } };
+}
+
+function acpxNativeOptions(property, preferredTitledKey) {
+ const alternateTitledKey = preferredTitledKey === "anyOf" ? "oneOf" : "anyOf";
+ const titled = property[preferredTitledKey]
+ ?? property[alternateTitledKey];
+ if (Array.isArray(titled)) {
+ return titled.map((entry) => ({
+ value: entry.const,
+ label: optionalFixtureText(entry.title) ?? entry.const,
+ ...(optionalFixtureText(entry.description) !== undefined
+ ? { description: entry.description }
+ : {}),
+ }));
+ }
+ return (Array.isArray(property.enum) ? property.enum : [])
+ .map((value) => ({ value, label: value }));
+}
+
+function stableAcpxFieldId(value, index) {
+ const readable = value
+ .trim()
+ .replace(/[^A-Za-z0-9._:-]+/g, "-")
+ .replace(/^-+|-+$/g, "")
+ .slice(0, 96);
+ return `field-${index + 1}-${readable || "value"}-${sha256(value).slice(0, 12)}`;
+}
+
+function optionalFixtureText(value) {
+ return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
+function finiteNonNegativeFixtureInteger(value) {
+ return Number.isSafeInteger(value) && value >= 0 ? value : undefined;
+}
+
+function canonicalFixtureJson(value) {
+ if (Array.isArray(value)) return `[${value.map(canonicalFixtureJson).join(",")}]`;
+ if (isPlainRecord(value)) {
+ return `{${Object.keys(value).sort().map((key) =>
+ `${JSON.stringify(key)}:${canonicalFixtureJson(value[key])}`
+ ).join(",")}}`;
+ }
+ return JSON.stringify(value);
+}
+
+function assertAcpxProperty(name, value) {
+ if (!isPlainRecord(value)) {
+ throw contractError("invalid_acpx_question_fixture", `native property ${name}`);
+ }
+ if (!["string", "number", "integer", "boolean", "array"].includes(value.type)) {
+ throw contractError("invalid_acpx_question_fixture", `unsupported native property ${name}`);
+ }
+ if (value.type === "array") {
+ if (!isPlainRecord(value.items) || acpxEnumValues(value.items, "anyOf").length === 0) {
+ throw contractError("invalid_acpx_question_fixture", `native array property ${name} requires options`);
+ }
+ } else if (value.type === "string") {
+ acpxEnumValues(value, "oneOf");
+ }
+}
+
+function assertAcpxPropertyValue(name, property, value) {
+ const typeMatches = property.type === "string"
+ ? typeof value === "string"
+ : property.type === "number"
+ ? typeof value === "number" && Number.isFinite(value)
+ : property.type === "integer"
+ ? Number.isInteger(value)
+ : property.type === "boolean"
+ ? typeof value === "boolean"
+ : Array.isArray(value) && value.every((item) => typeof item === "string");
+ if (!typeMatches) {
+ throw contractError("invalid_acpx_question_fixture", `native response type for ${name}`);
+ }
+ const enumValues = property.type === "array"
+ ? acpxEnumValues(property.items, "anyOf")
+ : property.type === "string"
+ ? acpxEnumValues(property, "oneOf")
+ : [];
+ const responseValues = Array.isArray(value) ? value : [value];
+ if (enumValues.length > 0 && responseValues.some((item) => !enumValues.includes(item))) {
+ throw contractError("invalid_acpx_question_fixture", `native response option for ${name}`);
+ }
+}
+
+function acpxEnumValues(property, preferredTitledKey) {
+ if (!isPlainRecord(property)) return [];
+ const alternateTitledKey = preferredTitledKey === "anyOf" ? "oneOf" : "anyOf";
+ const titled = property[preferredTitledKey]
+ ?? property[alternateTitledKey];
+ const values = Array.isArray(titled)
+ ? titled.map((entry) => isPlainRecord(entry) ? entry.const : undefined)
+ : Array.isArray(property.enum)
+ ? property.enum
+ : [];
+ if (
+ !Array.isArray(values)
+ || values.length > 128
+ || values.some((value) => typeof value !== "string" || value.length === 0)
+ ) {
+ throw contractError("invalid_acpx_question_fixture", "native options must be non-empty bounded strings");
+ }
+ return values;
+}
+
+function isPlainRecord(value) {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
export function assertConformanceFixturePair(fixture, output) {
if (fixture?.schemaVersion !== "paperclip.runner.conformance.fixture.v1") {
throw contractError("unsupported_required_schema", `conformance fixture requires ${String(fixture?.schemaVersion)}`);
diff --git a/packages/paperclip-runner/src/contracts/question-adapter-conformance.test.ts b/packages/paperclip-runner/src/contracts/question-adapter-conformance.test.ts
new file mode 100644
index 0000000000..c43f4c2c46
--- /dev/null
+++ b/packages/paperclip-runner/src/contracts/question-adapter-conformance.test.ts
@@ -0,0 +1,136 @@
+import { readFile } from "node:fs/promises";
+
+import type { AcpElicitationRequest } from "acpx/runtime";
+import { describe, expect, it } from "vitest";
+
+import { normalizeAcpFormElicitation } from "../drivers/acpx/acp-question-adapter.js";
+import {
+ createCodexQuestionResponseContext,
+ normalizeCodexQuestionSet,
+ runtimeRequestResponse,
+} from "../drivers/codex/codex-question-adapter.js";
+import {
+ parsePaperclipQuestionResponse,
+ parsePaperclipQuestionSet,
+ type PaperclipQuestionSet,
+} from "./question-set.js";
+import type { HarnessRuntimeRequest } from "./harness-driver.js";
+
+interface QuestionAdapterFixture {
+ schema: "paperclip.question_adapter_fixture.v1";
+ adapter: "codex" | "acpx";
+ nativeRequest: Record<string, unknown>;
+ canonicalQuestionSet: unknown;
+ canonicalResponse: unknown;
+ nativeResponse: unknown;
+}
+
+async function fixture(
+ adapter: QuestionAdapterFixture["adapter"],
+): Promise<QuestionAdapterFixture> {
+ const source = await readFile(
+ new URL(
+ `../../protocol/fixtures/questions/${adapter}.json`,
+ import.meta.url,
+ ),
+ "utf8",
+ );
+ return JSON.parse(source) as QuestionAdapterFixture;
+}
+
+describe("question adapter conformance fixtures", () => {
+ it("normalizes equivalent Codex and ACPX forms to one canonical question set", async () => {
+ const [codex, acpx] = await Promise.all([
+ fixture("codex"),
+ fixture("acpx"),
+ ]);
+ const codexQuestionSet = normalizeCodexQuestionSet(
+ String(codex.nativeRequest.method),
+ codex.nativeRequest.params as Record<string, unknown>,
+ createCodexQuestionResponseContext(),
+ );
+ const acpxQuestionSet = normalizeAcpFormElicitation(
+ acpx.nativeRequest.params as AcpElicitationRequest,
+ )?.questionSet;
+ const codexExpected = parsePaperclipQuestionSet(codex.canonicalQuestionSet);
+ const acpxExpected = parsePaperclipQuestionSet(acpx.canonicalQuestionSet);
+
+ expect(codexQuestionSet).toEqual(codexExpected);
+ expect(acpxQuestionSet).toEqual(acpxExpected);
+ expect(questionPresentation(acpxExpected)).toEqual(
+ questionPresentation(codexExpected),
+ );
+ expect(
+ parsePaperclipQuestionResponse(codexExpected, codex.canonicalResponse),
+ ).toEqual(codex.canonicalResponse);
+ expect(
+ parsePaperclipQuestionResponse(acpxExpected, acpx.canonicalResponse),
+ ).toEqual(acpx.canonicalResponse);
+ });
+
+ it("converts one canonical response back to each provider shape", async () => {
+ const [codex, acpx] = await Promise.all([
+ fixture("codex"),
+ fixture("acpx"),
+ ]);
+ const responseContext = createCodexQuestionResponseContext();
+ const codexQuestionSet = normalizeCodexQuestionSet(
+ String(codex.nativeRequest.method),
+ codex.nativeRequest.params as Record<string, unknown>,
+ responseContext,
+ );
+ if (codexQuestionSet === null) {
+ throw new Error("Codex conformance fixture did not contain a question set");
+ }
+ const codexRequest: HarnessRuntimeRequest = {
+ requestId: "fixture-request",
+ requestKind: "user_input",
+ method: String(codex.nativeRequest.method),
+ turnId: "fixture-turn",
+ itemId: "fixture-item",
+ status: "pending",
+ prompt: "Deployment input",
+ details: {},
+ input: codexQuestionSet,
+ origin: {
+ adapter: "codex",
+ method: String(codex.nativeRequest.method),
+ },
+ };
+ const codexResponse = parsePaperclipQuestionResponse(
+ codexQuestionSet,
+ codex.canonicalResponse,
+ );
+ const normalizedAcpx = normalizeAcpFormElicitation(
+ acpx.nativeRequest.params as AcpElicitationRequest,
+ );
+ const acpxResponse = parsePaperclipQuestionResponse(
+ normalizedAcpx!.questionSet,
+ acpx.canonicalResponse,
+ );
+
+ expect(
+ runtimeRequestResponse(codexRequest, {
+ action: "submit",
+ response: codexResponse,
+ }, responseContext),
+ ).toEqual(codex.nativeResponse);
+ expect(normalizedAcpx?.accept(acpxResponse)).toEqual(acpx.nativeResponse);
+ });
+});
+
+function questionPresentation(questionSet: PaperclipQuestionSet): unknown {
+ return {
+ ...questionSet,
+ questions: questionSet.questions.map(
+ ({ id: _questionId, options, ...question }) => ({
+ ...question,
+ ...(options
+ ? {
+ options: options.map(({ id: _optionId, ...option }) => option),
+ }
+ : {}),
+ }),
+ ),
+ };
+}
diff --git a/packages/paperclip-runner/test/protocol-contract.test.mjs b/packages/paperclip-runner/test/protocol-contract.test.mjs
index fb63206d6a..45c3ce4e95 100644
--- a/packages/paperclip-runner/test/protocol-contract.test.mjs
+++ b/packages/paperclip-runner/test/protocol-contract.test.mjs
@@ -1,4 +1,5 @@
import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import test from "node:test";
@@ -6,8 +7,10 @@ import { fileURLToPath } from "node:url";
import { buildProtocolManifest } from "../scripts/generate-protocol-manifest.mjs";
import {
+ assertAcpxQuestionFixture,
assertCodexQuestionFixture,
assertConformanceFixturePair,
+ assertQuestionAdapterFixture,
assertReplayFixtureCompatibility,
assertSchemaInstance,
compileProtocolValidators,
@@ -112,6 +115,495 @@ test("the Codex question fixture uses stable provider-neutral IDs", async () =>
assert.deepEqual(Object.keys(value.canonicalResponse.answers), ["environment"]);
});
+test("the ACPX question fixture enforces its provider-native contract", async () => {
+ const canonical = await fixture("questions/acpx.json");
+ assert.doesNotThrow(() => assertAcpxQuestionFixture(canonical));
+
+ const implicitObjectSchema = structuredClone(canonical);
+ delete implicitObjectSchema.nativeRequest.params.requestedSchema.type;
+ assert.doesNotThrow(() => assertAcpxQuestionFixture(implicitObjectSchema));
+
+ const unionObjectSchema = structuredClone(canonical);
+ unionObjectSchema.nativeRequest.params.requestedSchema.type = ["object", "null"];
+ assert.doesNotThrow(() => assertAcpxQuestionFixture(unionObjectSchema));
+
+ const malformedRequest = structuredClone(canonical);
+ malformedRequest.nativeRequest.params.requestedSchema.properties.environment.type = "object";
+ assert.throws(
+ () => assertAcpxQuestionFixture(malformedRequest),
+ /unsupported native property environment/,
+ );
+
+ const malformedResponse = structuredClone(canonical);
+ malformedResponse.nativeResponse.content.environment = "unknown";
+ assert.throws(
+ () => assertAcpxQuestionFixture(malformedResponse),
+ /native response option for environment/,
+ );
+
+ const patternBearingQuestion = structuredClone(canonical);
+ const patternBearingProperty = patternBearingQuestion.nativeRequest.params
+ .requestedSchema.properties.environment;
+ delete patternBearingProperty.oneOf;
+ patternBearingProperty.pattern = "^staging$";
+ patternBearingQuestion.canonicalQuestionSet.questions[0] = {
+ id: "field-1-environment-ba5285161ba6",
+ header: "Environment",
+ prompt: "Where should we deploy?",
+ required: true,
+ answerMode: "text",
+ textValidation: { inputType: "text", pattern: "^staging$" },
+ };
+ patternBearingQuestion.canonicalResponse.answers = {
+ "field-1-environment-ba5285161ba6": { text: "staging" },
+ };
+ assert.throws(
+ () => assertAcpxQuestionFixture(patternBearingQuestion),
+ /native property environment uses an unsupported pattern/,
+ );
+
+ const optionPattern = structuredClone(canonical);
+ optionPattern.nativeRequest.params.requestedSchema.properties.environment
+ .pattern = "^staging$";
+ assert.doesNotThrow(() => assertAcpxQuestionFixture(optionPattern));
+
+ const emptyOptionalAnswer = structuredClone(canonical);
+ emptyOptionalAnswer.nativeRequest.params.requestedSchema.required = [];
+ emptyOptionalAnswer.nativeResponse.content = {};
+ emptyOptionalAnswer.canonicalQuestionSet.questions[0] = {
+ ...emptyOptionalAnswer.canonicalQuestionSet.questions[0],
+ required: false,
+ };
+ emptyOptionalAnswer.canonicalResponse.answers = {
+ [emptyOptionalAnswer.canonicalQuestionSet.questions[0].id]: {},
+ };
+ assert.doesNotThrow(() => assertAcpxQuestionFixture(emptyOptionalAnswer));
+
+ const emptyOption = structuredClone(canonical);
+ emptyOption.nativeRequest.params.requestedSchema.properties.environment
+ .oneOf[0].const = "";
+ assert.throws(
+ () => assertAcpxQuestionFixture(emptyOption),
+ /native options must be non-empty bounded strings/,
+ );
+
+ const missingResponse = structuredClone(canonical);
+ missingResponse.nativeResponse.content = {};
+ assert.throws(
+ () => assertAcpxQuestionFixture(missingResponse),
+ /native response omits required property environment/,
+ );
+
+ const divergentQuestion = structuredClone(canonical);
+ divergentQuestion.nativeRequest.params.requestedSchema.required = [];
+ assert.throws(
+ () => assertAcpxQuestionFixture(divergentQuestion),
+ /native and canonical question sets differ/,
+ );
+
+ const normalizedRequired = structuredClone(canonical);
+ normalizedRequired.nativeRequest.params.requestedSchema.required = [
+ "environment",
+ "environment",
+ 42,
+ "not-a-property",
+ ];
+ assert.doesNotThrow(() => assertAcpxQuestionFixture(normalizedRequired));
+
+ const manyUnknownRequired = structuredClone(canonical);
+ manyUnknownRequired.nativeRequest.params.requestedSchema.required = [
+ ...Array.from({ length: 80 }, (_, index) => `unknown-${index}`),
+ "environment",
+ ];
+ assert.doesNotThrow(() => assertAcpxQuestionFixture(manyUnknownRequired));
+
+ const divergentResponse = structuredClone(canonical);
+ divergentResponse.nativeResponse.content.environment = "production";
+ assert.throws(
+ () => assertAcpxQuestionFixture(divergentResponse),
+ /canonical and native responses differ/,
+ );
+
+ const arrayWithBothOptionForms = structuredClone(canonical);
+ const environment = arrayWithBothOptionForms.nativeRequest.params
+ .requestedSchema.properties.environment;
+ environment.type = "array";
+ environment.items = {
+ anyOf: environment.oneOf,
+ oneOf: [{ const: "wrong", title: "Wrong precedence" }],
+ };
+ delete environment.oneOf;
+ arrayWithBothOptionForms.canonicalQuestionSet.questions[0].answerMode = "multi_select";
+ arrayWithBothOptionForms.nativeResponse.content.environment = ["staging"];
+ assert.doesNotThrow(() => assertAcpxQuestionFixture(arrayWithBothOptionForms));
+
+ const malformedPreferredUnion = structuredClone(arrayWithBothOptionForms);
+ malformedPreferredUnion.nativeRequest.params.requestedSchema.properties
+ .environment.items.anyOf = { const: "staging" };
+ assert.throws(
+ () => assertAcpxQuestionFixture(malformedPreferredUnion),
+ /native array property environment requires options/,
+ );
+
+ const enumFallback = structuredClone(malformedPreferredUnion);
+ enumFallback.nativeRequest.params.requestedSchema.properties.environment.items = {
+ anyOf: { const: "ignored-malformed-preferred-union" },
+ oneOf: [{ const: "wrong", title: "Wrong alternate union" }],
+ enum: ["staging", "production"],
+ };
+ enumFallback.canonicalQuestionSet.questions[0].options = [
+ { id: "option-1", label: "staging" },
+ { id: "option-2", label: "production" },
+ ];
+ assert.doesNotThrow(() => assertAcpxQuestionFixture(enumFallback));
+
+ const specialPropertyName = "__proto__";
+ const specialProperty = structuredClone(
+ canonical.nativeRequest.params.requestedSchema.properties.environment,
+ );
+ const specialPropertyFixture = structuredClone(canonical);
+ const specialProperties = specialPropertyFixture.nativeRequest.params
+ .requestedSchema.properties;
+ delete specialProperties.environment;
+ Object.defineProperty(specialProperties, specialPropertyName, {
+ value: specialProperty,
+ enumerable: true,
+ configurable: true,
+ writable: true,
+ });
+ specialPropertyFixture.nativeRequest.params.requestedSchema.required = [
+ specialPropertyName,
+ ];
+ const specialQuestionId = `field-1-__proto__-${createHash("sha256")
+ .update(specialPropertyName)
+ .digest("hex")
+ .slice(0, 12)}`;
+ specialPropertyFixture.canonicalQuestionSet.questions[0].id = specialQuestionId;
+ specialPropertyFixture.canonicalResponse.answers = {
+ [specialQuestionId]: { selectedOptionIds: ["option-1"] },
+ };
+ specialPropertyFixture.nativeResponse.content = {};
+ Object.defineProperty(
+ specialPropertyFixture.nativeResponse.content,
+ specialPropertyName,
+ {
+ value: "staging",
+ enumerable: true,
+ configurable: true,
+ writable: true,
+ },
+ );
+ assert.doesNotThrow(() => assertAcpxQuestionFixture(specialPropertyFixture));
+});
+
+test("the ACPX question fixture ignores scalar enums the adapter does not project", async () => {
+ const base = await fixture("questions/acpx.json");
+ const id = "field-1-environment-ba5285161ba6";
+ const cases = [
+ {
+ type: "boolean",
+ property: { type: "boolean", title: "Confirm", enum: [true] },
+ question: {
+ id,
+ header: "Confirm",
+ prompt: "Confirm",
+ required: true,
+ answerMode: "single_select",
+ options: [
+ { id: "option-1", label: "Yes" },
+ { id: "option-2", label: "No" },
+ ],
+ },
+ canonicalAnswer: { selectedOptionIds: ["option-2"] },
+ nativeAnswer: false,
+ },
+ {
+ type: "number",
+ property: { type: "number", title: "Threshold", enum: [1.5] },
+ question: {
+ id,
+ header: "Threshold",
+ prompt: "Threshold",
+ required: true,
+ answerMode: "text",
+ textValidation: { inputType: "number" },
+ },
+ canonicalAnswer: { text: "2.5" },
+ nativeAnswer: 2.5,
+ },
+ {
+ type: "integer",
+ property: { type: "integer", title: "Replicas", enum: [1] },
+ question: {
+ id,
+ header: "Replicas",
+ prompt: "Replicas",
+ required: true,
+ answerMode: "text",
+ textValidation: { inputType: "integer" },
+ },
+ canonicalAnswer: { text: "2" },
+ nativeAnswer: 2,
+ },
+ ];
+
+ for (const testCase of cases) {
+ const value = structuredClone(base);
+ value.nativeRequest.params.requestedSchema.properties.environment = testCase.property;
+ value.canonicalQuestionSet.questions[0] = testCase.question;
+ value.canonicalResponse.answers[id] = testCase.canonicalAnswer;
+ value.nativeResponse.content.environment = testCase.nativeAnswer;
+ assert.doesNotThrow(
+ () => assertAcpxQuestionFixture(value),
+ `${testCase.type} enum must not constrain the adapter's projected scalar answer`,
+ );
+ }
+
+ const invalidNativeType = structuredClone(base);
+ invalidNativeType.nativeRequest.params.requestedSchema.properties.environment =
+ cases[0].property;
+ invalidNativeType.canonicalQuestionSet.questions[0] = cases[0].question;
+ invalidNativeType.canonicalResponse.answers[id] = cases[0].canonicalAnswer;
+ invalidNativeType.nativeResponse.content.environment = "false";
+ assert.throws(
+ () => assertAcpxQuestionFixture(invalidNativeType),
+ /native response type for environment/,
+ );
+});
+
+test("every question adapter fixture satisfies its declared schema", async () => {
+ const schemas = await loadSchemaCatalog(resolve(protocolRoot, "schemas"));
+ const validators = compileProtocolValidators(schemas);
+ for (const adapter of ["codex", "acpx"]) {
+ const value = await fixture(`questions/${adapter}.json`);
+ assert.doesNotThrow(() =>
+ assertSchemaInstance(
+ validators.questionAdapterFixture,
+ value,
+ `${adapter}-question-fixture`,
+ ),
+ );
+ assert.doesNotThrow(() => assertQuestionAdapterFixture(value));
+ }
+
+ const malformed = structuredClone(await fixture("questions/acpx.json"));
+ delete malformed.canonicalQuestionSet.schema;
+ assert.throws(
+ () =>
+ assertSchemaInstance(
+ validators.questionAdapterFixture,
+ malformed,
+ "malformed-acpx-question-fixture",
+ ),
+ /schema_validation_failed/,
+ );
+
+ const unknownQuestion = structuredClone(await fixture("questions/acpx.json"));
+ unknownQuestion.canonicalResponse.answers = {
+ "unknown-question": { selectedOptionIds: ["option-1"] },
+ };
+ assert.throws(
+ () => assertQuestionAdapterFixture(unknownQuestion),
+ /answer has unknown question ID unknown-question/,
+ );
+
+ const unknownOption = structuredClone(await fixture("questions/acpx.json"));
+ const [questionId] = Object.keys(unknownOption.canonicalResponse.answers);
+ unknownOption.canonicalResponse.answers[questionId].selectedOptionIds = [
+ "unknown-option",
+ ];
+ assert.throws(
+ () => assertQuestionAdapterFixture(unknownOption),
+ /answer has unknown option ID unknown-option/,
+ );
+
+ const canonical = await fixture("questions/acpx.json");
+ const [requiredQuestion] = canonical.canonicalQuestionSet.questions;
+ const requiredQuestionId = requiredQuestion.id;
+ const malformedAnswers = [
+ {
+ label: "missing required answer",
+ answer: undefined,
+ pattern: /required question .* is missing/,
+ },
+ {
+ label: "empty required answer",
+ answer: {},
+ pattern: /required question .* is empty/,
+ },
+ {
+ label: "multiple single-select values",
+ answer: { selectedOptionIds: requiredQuestion.options.map((option) => option.id) },
+ pattern: /single-select answer .* chooses more than one option/,
+ },
+ {
+ label: "text on a select question",
+ answer: { text: "staging" },
+ pattern: /select answer .* carries text/,
+ },
+ {
+ label: "disabled custom answer",
+ answer: { customText: "canary" },
+ pattern: /custom answer .* is not enabled/,
+ },
+ ];
+ for (const malformedAnswer of malformedAnswers) {
+ const malformedResponse = structuredClone(canonical);
+ malformedResponse.canonicalResponse.answers = malformedAnswer.answer === undefined
+ ? {}
+ : { [requiredQuestionId]: malformedAnswer.answer };
+ assert.throws(
+ () => assertQuestionAdapterFixture(malformedResponse),
+ malformedAnswer.pattern,
+ malformedAnswer.label,
+ );
+ }
+
+ const emptyOptionalAnswer = structuredClone(canonical);
+ emptyOptionalAnswer.canonicalQuestionSet.questions[0] = {
+ ...requiredQuestion,
+ required: false,
+ };
+ emptyOptionalAnswer.canonicalResponse.answers = {
+ [requiredQuestionId]: {},
+ };
+ assert.doesNotThrow(() => assertQuestionAdapterFixture(emptyOptionalAnswer));
+
+ const textModeMismatch = structuredClone(canonical);
+ textModeMismatch.canonicalQuestionSet.questions[0] = {
+ ...requiredQuestion,
+ answerMode: "text",
+ options: [],
+ };
+ textModeMismatch.canonicalResponse.answers[requiredQuestionId] = {
+ customText: "select-only custom value",
+ };
+ assert.throws(
+ () => assertQuestionAdapterFixture(textModeMismatch),
+ /text answer .* carries select-only fields/,
+ );
+
+ const invalidTextAnswers = [
+ {
+ label: "minimum text length",
+ validation: { minLength: 4 },
+ text: "abc",
+ pattern: /must contain at least 4 characters/,
+ },
+ {
+ label: "maximum text length",
+ validation: { maxLength: 2 },
+ text: "abc",
+ pattern: /must contain at most 2 characters/,
+ },
+ {
+ label: "text pattern",
+ validation: { pattern: "^z+$" },
+ text: "abc",
+ pattern: /does not match the required format/,
+ },
+ {
+ label: "numeric input",
+ validation: { inputType: "number" },
+ text: "not-a-number",
+ pattern: /must be a valid number/,
+ },
+ {
+ label: "integer input",
+ validation: { inputType: "integer" },
+ text: "1.5",
+ pattern: /must be a valid integer/,
+ },
+ {
+ label: "numeric minimum",
+ validation: { inputType: "number", minimum: 2 },
+ text: "1",
+ pattern: /must be at least 2/,
+ },
+ {
+ label: "numeric maximum",
+ validation: { inputType: "number", maximum: 2 },
+ text: "3",
+ pattern: /must be at most 2/,
+ },
+ ];
+ for (const invalid of invalidTextAnswers) {
+ const malformedResponse = structuredClone(canonical);
+ malformedResponse.canonicalQuestionSet.questions[0] = {
+ ...requiredQuestion,
+ answerMode: "text",
+ options: [],
+ textValidation: invalid.validation,
+ };
+ malformedResponse.canonicalResponse.answers[requiredQuestionId] = {
+ text: invalid.text,
+ };
+ assert.throws(
+ () => assertQuestionAdapterFixture(malformedResponse),
+ invalid.pattern,
+ invalid.label,
+ );
+ }
+
+ const invalidCustomText = structuredClone(canonical);
+ invalidCustomText.canonicalQuestionSet.questions[0] = {
+ ...requiredQuestion,
+ customAnswer: { enabled: true },
+ textValidation: { minLength: 2, inputType: "text" },
+ };
+ invalidCustomText.canonicalResponse.answers[requiredQuestionId] = {
+ customText: "x",
+ };
+ assert.throws(
+ () => assertQuestionAdapterFixture(invalidCustomText),
+ /must contain at least 2 characters/,
+ "select custom text validation",
+ );
+
+ for (const invalid of [
+ {
+ label: "contradictory optional text lengths",
+ validation: { minLength: 3, maxLength: 2 },
+ pattern: /minLength greater than maxLength/,
+ },
+ {
+ label: "contradictory optional numeric bounds",
+ validation: { inputType: "number", minimum: 3, maximum: 2 },
+ pattern: /minimum greater than maximum/,
+ },
+ ]) {
+ const malformedQuestion = structuredClone(canonical);
+ malformedQuestion.canonicalQuestionSet.questions[0] = {
+ ...requiredQuestion,
+ required: false,
+ answerMode: "text",
+ options: [],
+ textValidation: invalid.validation,
+ };
+ malformedQuestion.canonicalResponse.answers = {};
+ assert.throws(
+ () => assertQuestionAdapterFixture(malformedQuestion),
+ invalid.pattern,
+ invalid.label,
+ );
+ }
+
+ const pathologicalPattern = structuredClone(canonical);
+ pathologicalPattern.canonicalQuestionSet.questions[0] = {
+ ...requiredQuestion,
+ answerMode: "text",
+ options: [],
+ textValidation: { pattern: "(a+)+$" },
+ };
+ pathologicalPattern.canonicalResponse.answers[requiredQuestionId] = {
+ text: `${"a".repeat(100_000)}!`,
+ };
+ assert.throws(
+ () => assertQuestionAdapterFixture(pathologicalPattern),
+ /could not be evaluated safely/,
+ );
+});
+
test("the cross-language conformance input and output have one stable identity", async () => {
const input = await fixture("conformance-minimal-run.json");
const output = await fixture("conformance-expected-output.json");