The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
test(runner): add question adapter conformance (#12409)
Dottab93ad538b63c
diff --git a/packages/paperclip-runner/protocol/fixtures/questions/acpx.json b/packages/paperclip-runner/protocol/fixtures/questions/acpx.jsonnew file mode 100644index 0000000000..d6ef05edc0--- /dev/null+++ b/packages/paperclip-runner/protocol/fixtures/questions/acpx.json@@ -0,0 +1,75 @@+{+ "schema": "paperclip.question_adapter_fixture.v1",+ "adapter": "acpx",+ "nativeRequest": {+ "method": "elicitation/create",+ "params": {+ "mode": "form",+ "message": "Deployment input",+ "requestedSchema": {+ "type": "object",+ "title": "Deployment input",+ "required": ["environment"],+ "properties": {+ "environment": {+ "type": "string",+ "title": "Environment",+ "description": "Where should we deploy?",+ "oneOf": [+ {+ "const": "staging",+ "title": "Staging",+ "description": "Deploy to staging first."+ },+ {+ "const": "production",+ "title": "Production",+ "description": "Deploy directly to production."+ }+ ]+ }+ }+ }+ }+ },+ "canonicalQuestionSet": {+ "schema": "paperclip.question_set.v1",+ "title": "Deployment input",+ "submitLabel": "Submit answers",+ "questions": [+ {+ "id": "field-1-environment-ba5285161ba6",+ "header": "Environment",+ "prompt": "Where should we deploy?",+ "required": true,+ "answerMode": "single_select",+ "options": [+ {+ "id": "option-1",+ "label": "Staging",+ "description": "Deploy to staging first."+ },+ {+ "id": "option-2",+ "label": "Production",+ "description": "Deploy directly to production."+ }+ ]+ }+ ]+ },+ "canonicalResponse": {+ "schema": "paperclip.question_response.v1",+ "answers": {+ "field-1-environment-ba5285161ba6": {+ "selectedOptionIds": ["option-1"]+ }+ }+ },+ "nativeResponse": {+ "action": "accept",+ "content": {+ "environment": "staging"+ }+ }+}diff --git a/packages/paperclip-runner/protocol/fixtures/questions/codex.json b/packages/paperclip-runner/protocol/fixtures/questions/codex.jsonindex d1ec06850f..4f90d1b513 100644--- a/packages/paperclip-runner/protocol/fixtures/questions/codex.json+++ b/packages/paperclip-runner/protocol/fixtures/questions/codex.json@@ -9,6 +9,7 @@"id": "environment","header": "Environment","question": "Where should we deploy?",+ "required": true,"options": [{ "label": "Staging", "description": "Deploy to staging first." },{ "label": "Production", "description": "Deploy directly to production." }diff --git a/packages/paperclip-runner/protocol/manifest.json b/packages/paperclip-runner/protocol/manifest.jsonindex 591bbd88ac..c7a5b5f0c3 100644--- a/packages/paperclip-runner/protocol/manifest.json+++ b/packages/paperclip-runner/protocol/manifest.json@@ -175,9 +175,15 @@"expectation": "accept","compatibilityCase": "canonical"},+ {+ "path": "fixtures/questions/acpx.json",+ "sha256": "6f1b65d5168b39f4bc821e778901bf077b1df10495f4bc1217b515eca4a339e4",+ "expectation": "accept",+ "compatibilityCase": "acpx-structured-input"+ },{"path": "fixtures/questions/codex.json",- "sha256": "ad673b6f9d70b74b53656e622d15fad1a66430f39d62b9ef71479d194a198bd6",+ "sha256": "a02a862b008d8327b4665bbde8ae3b73ad1f8dca7891072b0839b0e84b5920fa","expectation": "accept","compatibilityCase": "codex-structured-input"},diff --git a/packages/paperclip-runner/scripts/generate-protocol-manifest.mjs b/packages/paperclip-runner/scripts/generate-protocol-manifest.mjsindex eda018031a..a9964449d1 100644--- a/packages/paperclip-runner/scripts/generate-protocol-manifest.mjs+++ b/packages/paperclip-runner/scripts/generate-protocol-manifest.mjs@@ -5,8 +5,10 @@ import { fileURLToPath } from "node:url";import {SUPPORTED_FIXTURE_VERSION,SUPPORTED_PROTOCOL_VERSION,+ assertAcpxQuestionFixture,assertCodexQuestionFixture,assertConformanceFixturePair,+ assertQuestionAdapterFixture,assertReplayFixtureCompatibility,assertSchemaInstance,compileProtocolValidators,@@ -64,14 +66,19 @@ export async function buildProtocolManifest() {compatibilityCase = "additive-optional-fields";}}- } else if (relativePath === "fixtures/questions/codex.json") {- assertCodexQuestionFixture(value);+ } else if (relativePath.startsWith("fixtures/questions/")) {assertSchemaInstance(validators.questionAdapterFixture,value,relativePath,);- compatibilityCase = "codex-structured-input";+ assertQuestionAdapterFixture(value);+ if (relativePath === "fixtures/questions/codex.json") {+ assertCodexQuestionFixture(value);+ } else if (relativePath === "fixtures/questions/acpx.json") {+ assertAcpxQuestionFixture(value);+ }+ compatibilityCase = `${value.adapter}-structured-input`;} else if (relativePath === "fixtures/conformance-minimal-run.json") {assertSchemaInstance(validators.conformanceFixture, value, relativePath);compatibilityCase = "cross-language-input";diff --git a/packages/paperclip-runner/scripts/protocol-contract.mjs b/packages/paperclip-runner/scripts/protocol-contract.mjsindex a800538915..f58ef848d1 100644--- a/packages/paperclip-runner/scripts/protocol-contract.mjs+++ b/packages/paperclip-runner/scripts/protocol-contract.mjs@@ -1,3 +1,4 @@+import { spawnSync } from "node:child_process";import { createHash } from "node:crypto";import { readdir, readFile } from "node:fs/promises";import { relative, resolve, sep } from "node:path";@@ -180,48 +181,521 @@ export function assertReplayFixtureCompatibility(fixture) {return fixture;}-export function assertCodexQuestionFixture(fixture) {+export function assertQuestionAdapterFixture(fixture) {requireSchema(fixture, "paperclip.question_adapter_fixture.v1", "question fixture");- if (fixture.adapter !== "codex") throw contractError("unsupported_provider", String(fixture.adapter));requireSchema(fixture.canonicalQuestionSet, "paperclip.question_set.v1", "canonicalQuestionSet");requireSchema(fixture.canonicalResponse, "paperclip.question_response.v1", "canonicalResponse");- if (fixture.nativeRequest?.method !== "item/tool/requestUserInput") {- throw contractError("invalid_codex_question_fixture", "native request method");- }const questions = fixture.canonicalQuestionSet.questions;if (!Array.isArray(questions) || questions.length === 0) {- throw contractError("invalid_codex_question_fixture", "questions must be non-empty");+ throw contractError("invalid_question_adapter_fixture", "questions must be non-empty");}- const questionIds = new Set();+ const questionsById = new Map();const optionIdsByQuestion = new Map();for (const question of questions) {- if (typeof question.id !== "string" || question.id.length === 0 || questionIds.has(question.id)) {- throw contractError("invalid_codex_question_fixture", "question IDs must be unique");+ if (typeof question.id !== "string" || question.id.length === 0 || questionsById.has(question.id)) {+ throw contractError("invalid_question_adapter_fixture", "question IDs must be unique");}- questionIds.add(question.id);+ questionsById.set(question.id, question);const optionIds = new Set();for (const option of question.options ?? []) {if (typeof option.id !== "string" || option.id.length === 0 || optionIds.has(option.id)) {- throw contractError("invalid_codex_question_fixture", `option IDs for ${question.id} must be unique`);+ throw contractError("invalid_question_adapter_fixture", `option IDs for ${question.id} must be unique`);}optionIds.add(option.id);}optionIdsByQuestion.set(question.id, optionIds);+ const validation = question.textValidation;+ if (+ validation?.minLength !== undefined+ && validation.maxLength !== undefined+ && validation.minLength > validation.maxLength+ ) {+ throw contractError("invalid_question_adapter_fixture", `text validation for ${question.id} has minLength greater than maxLength`);+ }+ if (+ validation?.minimum !== undefined+ && validation.maximum !== undefined+ && validation.minimum > validation.maximum+ ) {+ throw contractError("invalid_question_adapter_fixture", `text validation for ${question.id} has minimum greater than maximum`);+ }+ if (validation?.pattern !== undefined) {+ try {+ new RegExp(validation.pattern);+ } catch {+ throw contractError("invalid_question_adapter_fixture", `text validation pattern for ${question.id} is invalid`);+ }+ }}- for (const [answerId, answer] of Object.entries(fixture.canonicalResponse.answers ?? {})) {- if (!questionIds.has(answerId)) {- throw contractError("invalid_codex_question_fixture", `answer has unknown question ID ${answerId}`);+ const answers = fixture.canonicalResponse.answers ?? {};+ for (const [answerId, answer] of Object.entries(answers)) {+ if (!questionsById.has(answerId)) {+ throw contractError("invalid_question_adapter_fixture", `answer has unknown question ID ${answerId}`);+ }+ if (answer === null || typeof answer !== "object" || Array.isArray(answer)) {+ throw contractError("invalid_question_adapter_fixture", `answer for ${answerId} must be an object`);+ }+ const selectedOptionIds = answer.selectedOptionIds ?? [];+ if (!Array.isArray(selectedOptionIds) || selectedOptionIds.some((id) => typeof id !== "string")) {+ throw contractError("invalid_question_adapter_fixture", `answer for ${answerId} has invalid option IDs`);+ }+ if (new Set(selectedOptionIds).size !== selectedOptionIds.length) {+ throw contractError("invalid_question_adapter_fixture", `answer for ${answerId} repeats an option ID`);}- for (const optionId of answer.selectedOptionIds ?? []) {+ for (const optionId of selectedOptionIds) {if (!optionIdsByQuestion.get(answerId)?.has(optionId)) {- throw contractError("invalid_codex_question_fixture", `answer has unknown option ID ${optionId}`);+ throw contractError("invalid_question_adapter_fixture", `answer has unknown option ID ${optionId}`);}}}+ for (const question of questions) {+ const answer = answers[question.id];+ if (answer === undefined) {+ if (question.required) {+ throw contractError("invalid_question_adapter_fixture", `answer for required question ${question.id} is missing`);+ }+ continue;+ }+ const selectedOptionIds = answer.selectedOptionIds ?? [];+ const text = answer.text;+ const customText = answer.customText;+ if (question.answerMode === "text") {+ if (selectedOptionIds.length > 0 || customText !== undefined) {+ throw contractError("invalid_question_adapter_fixture", `text answer for ${question.id} carries select-only fields`);+ }+ } else {+ if (text !== undefined) {+ throw contractError("invalid_question_adapter_fixture", `select answer for ${question.id} carries text`);+ }+ if (question.answerMode === "single_select" && selectedOptionIds.length > 1) {+ throw contractError("invalid_question_adapter_fixture", `single-select answer for ${question.id} chooses more than one option`);+ }+ if (customText !== undefined && question.customAnswer?.enabled !== true) {+ throw contractError("invalid_question_adapter_fixture", `custom answer for ${question.id} is not enabled`);+ }+ if (+ question.answerMode === "single_select"+ && typeof customText === "string"+ && customText.trim().length > 0+ && selectedOptionIds.length > 0+ ) {+ throw contractError("invalid_question_adapter_fixture", `single-select answer for ${question.id} mixes option and custom values`);+ }+ }+ const hasValue = fixtureAnswerHasValue(answer);+ if (question.required && !hasValue) {+ throw contractError("invalid_question_adapter_fixture", `answer for required question ${question.id} is empty`);+ }+ const boundedText = question.answerMode === "text" ? text : customText;+ if (boundedText !== undefined) {+ const validation = question.textValidation;+ if (validation?.minLength !== undefined && boundedText.length < validation.minLength) {+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} must contain at least ${validation.minLength} characters`);+ }+ if (validation?.maxLength !== undefined && boundedText.length > validation.maxLength) {+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} must contain at most ${validation.maxLength} characters`);+ }+ if (validation?.pattern !== undefined) {+ let pattern;+ try {+ pattern = new RegExp(validation.pattern);+ } catch {+ throw contractError("invalid_question_adapter_fixture", `text validation pattern for ${question.id} is invalid`);+ }+ if (!testFixturePatternWithDeadline(pattern.source, boundedText, question.id)) {+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} does not match the required format`);+ }+ }+ if (validation?.inputType === "number" || validation?.inputType === "integer") {+ const numeric = Number(boundedText);+ if (!Number.isFinite(numeric) || (validation.inputType === "integer" && !Number.isInteger(numeric))) {+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} must be a valid ${validation.inputType}`);+ }+ if (validation.minimum !== undefined && numeric < validation.minimum) {+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} must be at least ${validation.minimum}`);+ }+ if (validation.maximum !== undefined && numeric > validation.maximum) {+ throw contractError("invalid_question_adapter_fixture", `answer for ${question.id} must be at most ${validation.maximum}`);+ }+ }+ }+ }+ return fixture;+}++export function assertCodexQuestionFixture(fixture) {+ assertQuestionAdapterFixture(fixture);+ if (fixture.adapter !== "codex") throw contractError("unsupported_provider", String(fixture.adapter));+ if (fixture.nativeRequest?.method !== "item/tool/requestUserInput") {+ throw contractError("invalid_codex_question_fixture", "native request method");+ }+ return fixture;+}++export function assertAcpxQuestionFixture(fixture) {+ assertQuestionAdapterFixture(fixture);+ if (fixture.adapter !== "acpx") {+ throw contractError("unsupported_provider", String(fixture.adapter));+ }+ const request = fixture.nativeRequest;+ if (!isPlainRecord(request) || request.method !== "elicitation/create") {+ throw contractError("invalid_acpx_question_fixture", "native request method");+ }+ const params = request.params;+ const requestedSchema = isPlainRecord(params) ? params.requestedSchema : null;+ if (+ !isPlainRecord(params)+ || params.mode !== "form"+ || !isPlainRecord(requestedSchema)+ || !isPlainRecord(requestedSchema.properties)+ ) {+ throw contractError("invalid_acpx_question_fixture", "native form request");+ }+ const properties = Object.entries(requestedSchema.properties);+ if (properties.length === 0 || properties.length > 64) {+ throw contractError("invalid_acpx_question_fixture", "native form property count");+ }+ const required = normalizedAcpxRequired(requestedSchema);+ for (const [name, property] of properties) {+ assertAcpxProperty(name, property);+ if (+ isPlainRecord(property)+ && property.type === "string"+ && acpxEnumValues(property, "oneOf").length === 0+ && optionalFixtureText(property.pattern) !== undefined+ ) {+ throw contractError(+ "invalid_acpx_question_fixture",+ `native property ${name} uses an unsupported pattern`,+ );+ }+ }++ const response = fixture.nativeResponse;+ if (+ !isPlainRecord(response)+ || response.action !== "accept"+ || !isPlainRecord(response.content)+ ) {+ throw contractError("invalid_acpx_question_fixture", "native accept response");+ }+ for (const [name] of properties) {+ if (required.has(name) && !Object.hasOwn(response.content, name)) {+ throw contractError("invalid_acpx_question_fixture", `native response omits required property ${name}`);+ }+ }+ for (const [name, value] of Object.entries(response.content)) {+ const property = requestedSchema.properties[name];+ if (!isPlainRecord(property)) {+ throw contractError("invalid_acpx_question_fixture", `native response has unknown property ${name}`);+ }+ assertAcpxPropertyValue(name, property, value);+ }+ const projected = projectAcpxFixture(params, fixture.canonicalResponse);+ if (canonicalFixtureJson(projected.questionSet) !== canonicalFixtureJson(fixture.canonicalQuestionSet)) {+ throw contractError("invalid_acpx_question_fixture", "native and canonical question sets differ");+ }+ if (canonicalFixtureJson(projected.nativeResponse) !== canonicalFixtureJson(fixture.nativeResponse)) {+ throw contractError("invalid_acpx_question_fixture", "canonical and native responses differ");+ }return fixture;}+function projectAcpxFixture(params, canonicalResponse) {+ const schema = params.requestedSchema;+ const required = normalizedAcpxRequired(schema);+ const bindings = Object.entries(schema.properties).map(([name, property], index) =>+ projectAcpxProperty(name, property, index, required.has(name))+ );+ const title = optionalFixtureText(schema.title) ?? "Additional information needed";+ const descriptions = [+ optionalFixtureText(params.message),+ optionalFixtureText(schema.description),+ ].filter((value, index, all) => value !== undefined && value !== title && all.indexOf(value) === index);+ const questionSet = {+ schema: "paperclip.question_set.v1",+ title,+ ...(descriptions.length > 0 ? { description: descriptions.join("\n\n") } : {}),+ submitLabel: "Submit answers",+ questions: bindings.map((binding) => binding.question),+ };+ const content = {};+ for (const binding of bindings) {+ const answer = canonicalResponse.answers?.[binding.question.id];+ // The production parser accepts explicit empty optional answers and omits+ // them from its normalized response. Mirror that before projecting ACP+ // content so the fixture gate certifies the same wire behavior.+ if (!answer || !fixtureAnswerHasValue(answer)) continue;+ if (binding.type === "string" && binding.question.answerMode === "text") {+ if (answer.text !== undefined) defineAcpxResponseProperty(content, binding.name, answer.text);+ } else if (binding.type === "number" || binding.type === "integer") {+ if (answer.text !== undefined) {+ defineAcpxResponseProperty(content, binding.name, Number(answer.text));+ }+ } else if (binding.type === "boolean") {+ const selected = answer.selectedOptionIds?.[0];+ if (selected !== undefined) {+ defineAcpxResponseProperty(+ content,+ binding.name,+ binding.optionValues.get(selected) === "true",+ );+ }+ } else {+ const selected = (answer.selectedOptionIds ?? []).map((id) => binding.optionValues.get(id));+ if (binding.type === "array") {+ defineAcpxResponseProperty(content, binding.name, selected);+ } else if (selected[0] !== undefined) {+ defineAcpxResponseProperty(content, binding.name, selected[0]);+ }+ }+ }+ return { questionSet, nativeResponse: { action: "accept", content } };+}++function fixtureAnswerHasValue(answer) {+ return (+ (typeof answer.text === "string" && answer.text.trim().length > 0)+ || (typeof answer.customText === "string" && answer.customText.trim().length > 0)+ || (Array.isArray(answer.selectedOptionIds) && answer.selectedOptionIds.length > 0)+ );+}++function defineAcpxResponseProperty(content, name, value) {+ Object.defineProperty(content, name, {+ value,+ enumerable: true,+ configurable: true,+ writable: true,+ });+}++function normalizedAcpxRequired(schema) {+ const propertyNames = new Set(+ isPlainRecord(schema.properties) ? Object.keys(schema.properties) : [],+ );+ return new Set(+ Array.isArray(schema.required)+ ? schema.required+ .filter(+ (value) => typeof value === "string" && propertyNames.has(value),+ )+ : []+ );+}++function projectAcpxProperty(name, property, index, required) {+ const id = stableAcpxFieldId(name, index);+ const header = optionalFixtureText(property.title) ?? name;+ const base = {+ id,+ header,+ prompt: optionalFixtureText(property.description) ?? header,+ required,+ };+ if (property.type === "string") {+ const options = acpxNativeOptions(property, "oneOf");+ if (options.length > 0) return projectedAcpxOptions(name, property.type, base, options, "single_select");+ const pattern = optionalFixtureText(property.pattern);+ return {+ name,+ type: property.type,+ optionValues: new Map(),+ question: {+ ...base,+ answerMode: "text",+ textValidation: {+ inputType: "text",+ ...(finiteNonNegativeFixtureInteger(property.minLength) !== undefined+ ? { minLength: property.minLength }+ : {}),+ ...(finiteNonNegativeFixtureInteger(property.maxLength) !== undefined+ ? { maxLength: property.maxLength }+ : {}),+ ...(pattern !== undefined ? { pattern } : {}),+ },+ },+ };+ }+ if (property.type === "number" || property.type === "integer") {+ return {+ name,+ type: property.type,+ optionValues: new Map(),+ question: {+ ...base,+ answerMode: "text",+ textValidation: {+ inputType: property.type,+ ...(Number.isFinite(property.minimum) ? { minimum: property.minimum } : {}),+ ...(Number.isFinite(property.maximum) ? { maximum: property.maximum } : {}),+ },+ },+ };+ }+ if (property.type === "boolean") {+ return projectedAcpxOptions(name, property.type, base, [+ { value: "true", label: "Yes" },+ { value: "false", label: "No" },+ ], "single_select");+ }+ return projectedAcpxOptions(+ name,+ property.type,+ base,+ acpxNativeOptions(property.items, "anyOf"),+ "multi_select",+ );+}++function testFixturePatternWithDeadline(pattern, value, questionId) {+ const outcome = spawnSync(+ process.execPath,+ [+ "--input-type=module",+ "--eval",+ "const chunks=[];for await(const chunk of process.stdin)chunks.push(chunk);const {pattern,value}=JSON.parse(Buffer.concat(chunks).toString());process.stdout.write(new RegExp(pattern).test(value)?'1':'0');",+ ],+ {+ input: JSON.stringify({ pattern, value }),+ encoding: "utf8",+ timeout: 1_000,+ maxBuffer: 1_024,+ windowsHide: true,+ },+ );+ if (outcome.error || outcome.signal || outcome.status !== 0) {+ throw contractError(+ "invalid_question_adapter_fixture",+ `text validation pattern for ${questionId} could not be evaluated safely`,+ );+ }+ return outcome.stdout === "1";+}++function projectedAcpxOptions(name, type, base, nativeOptions, answerMode) {+ const optionValues = new Map();+ const options = nativeOptions.map((option, index) => {+ const id = `option-${index + 1}`;+ optionValues.set(id, option.value);+ return {+ id,+ label: option.label,+ ...(option.description !== undefined ? { description: option.description } : {}),+ };+ });+ return { name, type, optionValues, question: { ...base, answerMode, options } };+}++function acpxNativeOptions(property, preferredTitledKey) {+ const alternateTitledKey = preferredTitledKey === "anyOf" ? "oneOf" : "anyOf";+ const titled = property[preferredTitledKey]+ ?? property[alternateTitledKey];+ if (Array.isArray(titled)) {+ return titled.map((entry) => ({+ value: entry.const,+ label: optionalFixtureText(entry.title) ?? entry.const,+ ...(optionalFixtureText(entry.description) !== undefined+ ? { description: entry.description }+ : {}),+ }));+ }+ return (Array.isArray(property.enum) ? property.enum : [])+ .map((value) => ({ value, label: value }));+}++function stableAcpxFieldId(value, index) {+ const readable = value+ .trim()+ .replace(/[^A-Za-z0-9._:-]+/g, "-")+ .replace(/^-+|-+$/g, "")+ .slice(0, 96);+ return `field-${index + 1}-${readable || "value"}-${sha256(value).slice(0, 12)}`;+}++function optionalFixtureText(value) {+ return typeof value === "string" && value.length > 0 ? value : undefined;+}++function finiteNonNegativeFixtureInteger(value) {+ return Number.isSafeInteger(value) && value >= 0 ? value : undefined;+}++function canonicalFixtureJson(value) {+ if (Array.isArray(value)) return `[${value.map(canonicalFixtureJson).join(",")}]`;+ if (isPlainRecord(value)) {+ return `{${Object.keys(value).sort().map((key) =>+ `${JSON.stringify(key)}:${canonicalFixtureJson(value[key])}`+ ).join(",")}}`;+ }+ return JSON.stringify(value);+}++function assertAcpxProperty(name, value) {+ if (!isPlainRecord(value)) {+ throw contractError("invalid_acpx_question_fixture", `native property ${name}`);+ }+ if (!["string", "number", "integer", "boolean", "array"].includes(value.type)) {+ throw contractError("invalid_acpx_question_fixture", `unsupported native property ${name}`);+ }+ if (value.type === "array") {+ if (!isPlainRecord(value.items) || acpxEnumValues(value.items, "anyOf").length === 0) {+ throw contractError("invalid_acpx_question_fixture", `native array property ${name} requires options`);+ }+ } else if (value.type === "string") {+ acpxEnumValues(value, "oneOf");+ }+}++function assertAcpxPropertyValue(name, property, value) {+ const typeMatches = property.type === "string"+ ? typeof value === "string"+ : property.type === "number"+ ? typeof value === "number" && Number.isFinite(value)+ : property.type === "integer"+ ? Number.isInteger(value)+ : property.type === "boolean"+ ? typeof value === "boolean"+ : Array.isArray(value) && value.every((item) => typeof item === "string");+ if (!typeMatches) {+ throw contractError("invalid_acpx_question_fixture", `native response type for ${name}`);+ }+ const enumValues = property.type === "array"+ ? acpxEnumValues(property.items, "anyOf")+ : property.type === "string"+ ? acpxEnumValues(property, "oneOf")+ : [];+ const responseValues = Array.isArray(value) ? value : [value];+ if (enumValues.length > 0 && responseValues.some((item) => !enumValues.includes(item))) {+ throw contractError("invalid_acpx_question_fixture", `native response option for ${name}`);+ }+}++function acpxEnumValues(property, preferredTitledKey) {+ if (!isPlainRecord(property)) return [];+ const alternateTitledKey = preferredTitledKey === "anyOf" ? "oneOf" : "anyOf";+ const titled = property[preferredTitledKey]+ ?? property[alternateTitledKey];+ const values = Array.isArray(titled)+ ? titled.map((entry) => isPlainRecord(entry) ? entry.const : undefined)+ : Array.isArray(property.enum)+ ? property.enum+ : [];+ if (+ !Array.isArray(values)+ || values.length > 128+ || values.some((value) => typeof value !== "string" || value.length === 0)+ ) {+ throw contractError("invalid_acpx_question_fixture", "native options must be non-empty bounded strings");+ }+ return values;+}++function isPlainRecord(value) {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}+export function assertConformanceFixturePair(fixture, output) {if (fixture?.schemaVersion !== "paperclip.runner.conformance.fixture.v1") {throw contractError("unsupported_required_schema", `conformance fixture requires ${String(fixture?.schemaVersion)}`);diff --git a/packages/paperclip-runner/src/contracts/question-adapter-conformance.test.ts b/packages/paperclip-runner/src/contracts/question-adapter-conformance.test.tsnew file mode 100644index 0000000000..c43f4c2c46--- /dev/null+++ b/packages/paperclip-runner/src/contracts/question-adapter-conformance.test.ts@@ -0,0 +1,136 @@+import { readFile } from "node:fs/promises";++import type { AcpElicitationRequest } from "acpx/runtime";+import { describe, expect, it } from "vitest";++import { normalizeAcpFormElicitation } from "../drivers/acpx/acp-question-adapter.js";+import {+ createCodexQuestionResponseContext,+ normalizeCodexQuestionSet,+ runtimeRequestResponse,+} from "../drivers/codex/codex-question-adapter.js";+import {+ parsePaperclipQuestionResponse,+ parsePaperclipQuestionSet,+ type PaperclipQuestionSet,+} from "./question-set.js";+import type { HarnessRuntimeRequest } from "./harness-driver.js";++interface QuestionAdapterFixture {+ schema: "paperclip.question_adapter_fixture.v1";+ adapter: "codex" | "acpx";+ nativeRequest: Record<string, unknown>;+ canonicalQuestionSet: unknown;+ canonicalResponse: unknown;+ nativeResponse: unknown;+}++async function fixture(+ adapter: QuestionAdapterFixture["adapter"],+): Promise<QuestionAdapterFixture> {+ const source = await readFile(+ new URL(+ `../../protocol/fixtures/questions/${adapter}.json`,+ import.meta.url,+ ),+ "utf8",+ );+ return JSON.parse(source) as QuestionAdapterFixture;+}++describe("question adapter conformance fixtures", () => {+ it("normalizes equivalent Codex and ACPX forms to one canonical question set", async () => {+ const [codex, acpx] = await Promise.all([+ fixture("codex"),+ fixture("acpx"),+ ]);+ const codexQuestionSet = normalizeCodexQuestionSet(+ String(codex.nativeRequest.method),+ codex.nativeRequest.params as Record<string, unknown>,+ createCodexQuestionResponseContext(),+ );+ const acpxQuestionSet = normalizeAcpFormElicitation(+ acpx.nativeRequest.params as AcpElicitationRequest,+ )?.questionSet;+ const codexExpected = parsePaperclipQuestionSet(codex.canonicalQuestionSet);+ const acpxExpected = parsePaperclipQuestionSet(acpx.canonicalQuestionSet);++ expect(codexQuestionSet).toEqual(codexExpected);+ expect(acpxQuestionSet).toEqual(acpxExpected);+ expect(questionPresentation(acpxExpected)).toEqual(+ questionPresentation(codexExpected),+ );+ expect(+ parsePaperclipQuestionResponse(codexExpected, codex.canonicalResponse),+ ).toEqual(codex.canonicalResponse);+ expect(+ parsePaperclipQuestionResponse(acpxExpected, acpx.canonicalResponse),+ ).toEqual(acpx.canonicalResponse);+ });++ it("converts one canonical response back to each provider shape", async () => {+ const [codex, acpx] = await Promise.all([+ fixture("codex"),+ fixture("acpx"),+ ]);+ const responseContext = createCodexQuestionResponseContext();+ const codexQuestionSet = normalizeCodexQuestionSet(+ String(codex.nativeRequest.method),+ codex.nativeRequest.params as Record<string, unknown>,+ responseContext,+ );+ if (codexQuestionSet === null) {+ throw new Error("Codex conformance fixture did not contain a question set");+ }+ const codexRequest: HarnessRuntimeRequest = {+ requestId: "fixture-request",+ requestKind: "user_input",+ method: String(codex.nativeRequest.method),+ turnId: "fixture-turn",+ itemId: "fixture-item",+ status: "pending",+ prompt: "Deployment input",+ details: {},+ input: codexQuestionSet,+ origin: {+ adapter: "codex",+ method: String(codex.nativeRequest.method),+ },+ };+ const codexResponse = parsePaperclipQuestionResponse(+ codexQuestionSet,+ codex.canonicalResponse,+ );+ const normalizedAcpx = normalizeAcpFormElicitation(+ acpx.nativeRequest.params as AcpElicitationRequest,+ );+ const acpxResponse = parsePaperclipQuestionResponse(+ normalizedAcpx!.questionSet,+ acpx.canonicalResponse,+ );++ expect(+ runtimeRequestResponse(codexRequest, {+ action: "submit",+ response: codexResponse,+ }, responseContext),+ ).toEqual(codex.nativeResponse);+ expect(normalizedAcpx?.accept(acpxResponse)).toEqual(acpx.nativeResponse);+ });+});++function questionPresentation(questionSet: PaperclipQuestionSet): unknown {+ return {+ ...questionSet,+ questions: questionSet.questions.map(+ ({ id: _questionId, options, ...question }) => ({+ ...question,+ ...(options+ ? {+ options: options.map(({ id: _optionId, ...option }) => option),+ }+ : {}),+ }),+ ),+ };+}diff --git a/packages/paperclip-runner/test/protocol-contract.test.mjs b/packages/paperclip-runner/test/protocol-contract.test.mjsindex fb63206d6a..45c3ce4e95 100644--- a/packages/paperclip-runner/test/protocol-contract.test.mjs+++ b/packages/paperclip-runner/test/protocol-contract.test.mjs@@ -1,4 +1,5 @@import assert from "node:assert/strict";+import { createHash } from "node:crypto";import { readFile } from "node:fs/promises";import { dirname, resolve } from "node:path";import test from "node:test";@@ -6,8 +7,10 @@ import { fileURLToPath } from "node:url";import { buildProtocolManifest } from "../scripts/generate-protocol-manifest.mjs";import {+ assertAcpxQuestionFixture,assertCodexQuestionFixture,assertConformanceFixturePair,+ assertQuestionAdapterFixture,assertReplayFixtureCompatibility,assertSchemaInstance,compileProtocolValidators,@@ -112,6 +115,495 @@ test("the Codex question fixture uses stable provider-neutral IDs", async () =>assert.deepEqual(Object.keys(value.canonicalResponse.answers), ["environment"]);});+test("the ACPX question fixture enforces its provider-native contract", async () => {+ const canonical = await fixture("questions/acpx.json");+ assert.doesNotThrow(() => assertAcpxQuestionFixture(canonical));++ const implicitObjectSchema = structuredClone(canonical);+ delete implicitObjectSchema.nativeRequest.params.requestedSchema.type;+ assert.doesNotThrow(() => assertAcpxQuestionFixture(implicitObjectSchema));++ const unionObjectSchema = structuredClone(canonical);+ unionObjectSchema.nativeRequest.params.requestedSchema.type = ["object", "null"];+ assert.doesNotThrow(() => assertAcpxQuestionFixture(unionObjectSchema));++ const malformedRequest = structuredClone(canonical);+ malformedRequest.nativeRequest.params.requestedSchema.properties.environment.type = "object";+ assert.throws(+ () => assertAcpxQuestionFixture(malformedRequest),+ /unsupported native property environment/,+ );++ const malformedResponse = structuredClone(canonical);+ malformedResponse.nativeResponse.content.environment = "unknown";+ assert.throws(+ () => assertAcpxQuestionFixture(malformedResponse),+ /native response option for environment/,+ );++ const patternBearingQuestion = structuredClone(canonical);+ const patternBearingProperty = patternBearingQuestion.nativeRequest.params+ .requestedSchema.properties.environment;+ delete patternBearingProperty.oneOf;+ patternBearingProperty.pattern = "^staging$";+ patternBearingQuestion.canonicalQuestionSet.questions[0] = {+ id: "field-1-environment-ba5285161ba6",+ header: "Environment",+ prompt: "Where should we deploy?",+ required: true,+ answerMode: "text",+ textValidation: { inputType: "text", pattern: "^staging$" },+ };+ patternBearingQuestion.canonicalResponse.answers = {+ "field-1-environment-ba5285161ba6": { text: "staging" },+ };+ assert.throws(+ () => assertAcpxQuestionFixture(patternBearingQuestion),+ /native property environment uses an unsupported pattern/,+ );++ const optionPattern = structuredClone(canonical);+ optionPattern.nativeRequest.params.requestedSchema.properties.environment+ .pattern = "^staging$";+ assert.doesNotThrow(() => assertAcpxQuestionFixture(optionPattern));++ const emptyOptionalAnswer = structuredClone(canonical);+ emptyOptionalAnswer.nativeRequest.params.requestedSchema.required = [];+ emptyOptionalAnswer.nativeResponse.content = {};+ emptyOptionalAnswer.canonicalQuestionSet.questions[0] = {+ ...emptyOptionalAnswer.canonicalQuestionSet.questions[0],+ required: false,+ };+ emptyOptionalAnswer.canonicalResponse.answers = {+ [emptyOptionalAnswer.canonicalQuestionSet.questions[0].id]: {},+ };+ assert.doesNotThrow(() => assertAcpxQuestionFixture(emptyOptionalAnswer));++ const emptyOption = structuredClone(canonical);+ emptyOption.nativeRequest.params.requestedSchema.properties.environment+ .oneOf[0].const = "";+ assert.throws(+ () => assertAcpxQuestionFixture(emptyOption),+ /native options must be non-empty bounded strings/,+ );++ const missingResponse = structuredClone(canonical);+ missingResponse.nativeResponse.content = {};+ assert.throws(+ () => assertAcpxQuestionFixture(missingResponse),+ /native response omits required property environment/,+ );++ const divergentQuestion = structuredClone(canonical);+ divergentQuestion.nativeRequest.params.requestedSchema.required = [];+ assert.throws(+ () => assertAcpxQuestionFixture(divergentQuestion),+ /native and canonical question sets differ/,+ );++ const normalizedRequired = structuredClone(canonical);+ normalizedRequired.nativeRequest.params.requestedSchema.required = [+ "environment",+ "environment",+ 42,+ "not-a-property",+ ];+ assert.doesNotThrow(() => assertAcpxQuestionFixture(normalizedRequired));++ const manyUnknownRequired = structuredClone(canonical);+ manyUnknownRequired.nativeRequest.params.requestedSchema.required = [+ ...Array.from({ length: 80 }, (_, index) => `unknown-${index}`),+ "environment",+ ];+ assert.doesNotThrow(() => assertAcpxQuestionFixture(manyUnknownRequired));++ const divergentResponse = structuredClone(canonical);+ divergentResponse.nativeResponse.content.environment = "production";+ assert.throws(+ () => assertAcpxQuestionFixture(divergentResponse),+ /canonical and native responses differ/,+ );++ const arrayWithBothOptionForms = structuredClone(canonical);+ const environment = arrayWithBothOptionForms.nativeRequest.params+ .requestedSchema.properties.environment;+ environment.type = "array";+ environment.items = {+ anyOf: environment.oneOf,+ oneOf: [{ const: "wrong", title: "Wrong precedence" }],+ };+ delete environment.oneOf;+ arrayWithBothOptionForms.canonicalQuestionSet.questions[0].answerMode = "multi_select";+ arrayWithBothOptionForms.nativeResponse.content.environment = ["staging"];+ assert.doesNotThrow(() => assertAcpxQuestionFixture(arrayWithBothOptionForms));++ const malformedPreferredUnion = structuredClone(arrayWithBothOptionForms);+ malformedPreferredUnion.nativeRequest.params.requestedSchema.properties+ .environment.items.anyOf = { const: "staging" };+ assert.throws(+ () => assertAcpxQuestionFixture(malformedPreferredUnion),+ /native array property environment requires options/,+ );++ const enumFallback = structuredClone(malformedPreferredUnion);+ enumFallback.nativeRequest.params.requestedSchema.properties.environment.items = {+ anyOf: { const: "ignored-malformed-preferred-union" },+ oneOf: [{ const: "wrong", title: "Wrong alternate union" }],+ enum: ["staging", "production"],+ };+ enumFallback.canonicalQuestionSet.questions[0].options = [+ { id: "option-1", label: "staging" },+ { id: "option-2", label: "production" },+ ];+ assert.doesNotThrow(() => assertAcpxQuestionFixture(enumFallback));++ const specialPropertyName = "__proto__";+ const specialProperty = structuredClone(+ canonical.nativeRequest.params.requestedSchema.properties.environment,+ );+ const specialPropertyFixture = structuredClone(canonical);+ const specialProperties = specialPropertyFixture.nativeRequest.params+ .requestedSchema.properties;+ delete specialProperties.environment;+ Object.defineProperty(specialProperties, specialPropertyName, {+ value: specialProperty,+ enumerable: true,+ configurable: true,+ writable: true,+ });+ specialPropertyFixture.nativeRequest.params.requestedSchema.required = [+ specialPropertyName,+ ];+ const specialQuestionId = `field-1-__proto__-${createHash("sha256")+ .update(specialPropertyName)+ .digest("hex")+ .slice(0, 12)}`;+ specialPropertyFixture.canonicalQuestionSet.questions[0].id = specialQuestionId;+ specialPropertyFixture.canonicalResponse.answers = {+ [specialQuestionId]: { selectedOptionIds: ["option-1"] },+ };+ specialPropertyFixture.nativeResponse.content = {};+ Object.defineProperty(+ specialPropertyFixture.nativeResponse.content,+ specialPropertyName,+ {+ value: "staging",+ enumerable: true,+ configurable: true,+ writable: true,+ },+ );+ assert.doesNotThrow(() => assertAcpxQuestionFixture(specialPropertyFixture));+});++test("the ACPX question fixture ignores scalar enums the adapter does not project", async () => {+ const base = await fixture("questions/acpx.json");+ const id = "field-1-environment-ba5285161ba6";+ const cases = [+ {+ type: "boolean",+ property: { type: "boolean", title: "Confirm", enum: [true] },+ question: {+ id,+ header: "Confirm",+ prompt: "Confirm",+ required: true,+ answerMode: "single_select",+ options: [+ { id: "option-1", label: "Yes" },+ { id: "option-2", label: "No" },+ ],+ },+ canonicalAnswer: { selectedOptionIds: ["option-2"] },+ nativeAnswer: false,+ },+ {+ type: "number",+ property: { type: "number", title: "Threshold", enum: [1.5] },+ question: {+ id,+ header: "Threshold",+ prompt: "Threshold",+ required: true,+ answerMode: "text",+ textValidation: { inputType: "number" },+ },+ canonicalAnswer: { text: "2.5" },+ nativeAnswer: 2.5,+ },+ {+ type: "integer",+ property: { type: "integer", title: "Replicas", enum: [1] },+ question: {+ id,+ header: "Replicas",+ prompt: "Replicas",+ required: true,+ answerMode: "text",+ textValidation: { inputType: "integer" },+ },+ canonicalAnswer: { text: "2" },+ nativeAnswer: 2,+ },+ ];++ for (const testCase of cases) {+ const value = structuredClone(base);+ value.nativeRequest.params.requestedSchema.properties.environment = testCase.property;+ value.canonicalQuestionSet.questions[0] = testCase.question;+ value.canonicalResponse.answers[id] = testCase.canonicalAnswer;+ value.nativeResponse.content.environment = testCase.nativeAnswer;+ assert.doesNotThrow(+ () => assertAcpxQuestionFixture(value),+ `${testCase.type} enum must not constrain the adapter's projected scalar answer`,+ );+ }++ const invalidNativeType = structuredClone(base);+ invalidNativeType.nativeRequest.params.requestedSchema.properties.environment =+ cases[0].property;+ invalidNativeType.canonicalQuestionSet.questions[0] = cases[0].question;+ invalidNativeType.canonicalResponse.answers[id] = cases[0].canonicalAnswer;+ invalidNativeType.nativeResponse.content.environment = "false";+ assert.throws(+ () => assertAcpxQuestionFixture(invalidNativeType),+ /native response type for environment/,+ );+});++test("every question adapter fixture satisfies its declared schema", async () => {+ const schemas = await loadSchemaCatalog(resolve(protocolRoot, "schemas"));+ const validators = compileProtocolValidators(schemas);+ for (const adapter of ["codex", "acpx"]) {+ const value = await fixture(`questions/${adapter}.json`);+ assert.doesNotThrow(() =>+ assertSchemaInstance(+ validators.questionAdapterFixture,+ value,+ `${adapter}-question-fixture`,+ ),+ );+ assert.doesNotThrow(() => assertQuestionAdapterFixture(value));+ }++ const malformed = structuredClone(await fixture("questions/acpx.json"));+ delete malformed.canonicalQuestionSet.schema;+ assert.throws(+ () =>+ assertSchemaInstance(+ validators.questionAdapterFixture,+ malformed,+ "malformed-acpx-question-fixture",+ ),+ /schema_validation_failed/,+ );++ const unknownQuestion = structuredClone(await fixture("questions/acpx.json"));+ unknownQuestion.canonicalResponse.answers = {+ "unknown-question": { selectedOptionIds: ["option-1"] },+ };+ assert.throws(+ () => assertQuestionAdapterFixture(unknownQuestion),+ /answer has unknown question ID unknown-question/,+ );++ const unknownOption = structuredClone(await fixture("questions/acpx.json"));+ const [questionId] = Object.keys(unknownOption.canonicalResponse.answers);+ unknownOption.canonicalResponse.answers[questionId].selectedOptionIds = [+ "unknown-option",+ ];+ assert.throws(+ () => assertQuestionAdapterFixture(unknownOption),+ /answer has unknown option ID unknown-option/,+ );++ const canonical = await fixture("questions/acpx.json");+ const [requiredQuestion] = canonical.canonicalQuestionSet.questions;+ const requiredQuestionId = requiredQuestion.id;+ const malformedAnswers = [+ {+ label: "missing required answer",+ answer: undefined,+ pattern: /required question .* is missing/,+ },+ {+ label: "empty required answer",+ answer: {},+ pattern: /required question .* is empty/,+ },+ {+ label: "multiple single-select values",+ answer: { selectedOptionIds: requiredQuestion.options.map((option) => option.id) },+ pattern: /single-select answer .* chooses more than one option/,+ },+ {+ label: "text on a select question",+ answer: { text: "staging" },+ pattern: /select answer .* carries text/,+ },+ {+ label: "disabled custom answer",+ answer: { customText: "canary" },+ pattern: /custom answer .* is not enabled/,+ },+ ];+ for (const malformedAnswer of malformedAnswers) {+ const malformedResponse = structuredClone(canonical);+ malformedResponse.canonicalResponse.answers = malformedAnswer.answer === undefined+ ? {}+ : { [requiredQuestionId]: malformedAnswer.answer };+ assert.throws(+ () => assertQuestionAdapterFixture(malformedResponse),+ malformedAnswer.pattern,+ malformedAnswer.label,+ );+ }++ const emptyOptionalAnswer = structuredClone(canonical);+ emptyOptionalAnswer.canonicalQuestionSet.questions[0] = {+ ...requiredQuestion,+ required: false,+ };+ emptyOptionalAnswer.canonicalResponse.answers = {+ [requiredQuestionId]: {},+ };+ assert.doesNotThrow(() => assertQuestionAdapterFixture(emptyOptionalAnswer));++ const textModeMismatch = structuredClone(canonical);+ textModeMismatch.canonicalQuestionSet.questions[0] = {+ ...requiredQuestion,+ answerMode: "text",+ options: [],+ };+ textModeMismatch.canonicalResponse.answers[requiredQuestionId] = {+ customText: "select-only custom value",+ };+ assert.throws(+ () => assertQuestionAdapterFixture(textModeMismatch),+ /text answer .* carries select-only fields/,+ );++ const invalidTextAnswers = [+ {+ label: "minimum text length",+ validation: { minLength: 4 },+ text: "abc",+ pattern: /must contain at least 4 characters/,+ },+ {+ label: "maximum text length",+ validation: { maxLength: 2 },+ text: "abc",+ pattern: /must contain at most 2 characters/,+ },+ {+ label: "text pattern",+ validation: { pattern: "^z+$" },+ text: "abc",+ pattern: /does not match the required format/,+ },+ {+ label: "numeric input",+ validation: { inputType: "number" },+ text: "not-a-number",+ pattern: /must be a valid number/,+ },+ {+ label: "integer input",+ validation: { inputType: "integer" },+ text: "1.5",+ pattern: /must be a valid integer/,+ },+ {+ label: "numeric minimum",+ validation: { inputType: "number", minimum: 2 },+ text: "1",+ pattern: /must be at least 2/,+ },+ {+ label: "numeric maximum",+ validation: { inputType: "number", maximum: 2 },+ text: "3",+ pattern: /must be at most 2/,+ },+ ];+ for (const invalid of invalidTextAnswers) {+ const malformedResponse = structuredClone(canonical);+ malformedResponse.canonicalQuestionSet.questions[0] = {+ ...requiredQuestion,+ answerMode: "text",+ options: [],+ textValidation: invalid.validation,+ };+ malformedResponse.canonicalResponse.answers[requiredQuestionId] = {+ text: invalid.text,+ };+ assert.throws(+ () => assertQuestionAdapterFixture(malformedResponse),+ invalid.pattern,+ invalid.label,+ );+ }++ const invalidCustomText = structuredClone(canonical);+ invalidCustomText.canonicalQuestionSet.questions[0] = {+ ...requiredQuestion,+ customAnswer: { enabled: true },+ textValidation: { minLength: 2, inputType: "text" },+ };+ invalidCustomText.canonicalResponse.answers[requiredQuestionId] = {+ customText: "x",+ };+ assert.throws(+ () => assertQuestionAdapterFixture(invalidCustomText),+ /must contain at least 2 characters/,+ "select custom text validation",+ );++ for (const invalid of [+ {+ label: "contradictory optional text lengths",+ validation: { minLength: 3, maxLength: 2 },+ pattern: /minLength greater than maxLength/,+ },+ {+ label: "contradictory optional numeric bounds",+ validation: { inputType: "number", minimum: 3, maximum: 2 },+ pattern: /minimum greater than maximum/,+ },+ ]) {+ const malformedQuestion = structuredClone(canonical);+ malformedQuestion.canonicalQuestionSet.questions[0] = {+ ...requiredQuestion,+ required: false,+ answerMode: "text",+ options: [],+ textValidation: invalid.validation,+ };+ malformedQuestion.canonicalResponse.answers = {};+ assert.throws(+ () => assertQuestionAdapterFixture(malformedQuestion),+ invalid.pattern,+ invalid.label,+ );+ }++ const pathologicalPattern = structuredClone(canonical);+ pathologicalPattern.canonicalQuestionSet.questions[0] = {+ ...requiredQuestion,+ answerMode: "text",+ options: [],+ textValidation: { pattern: "(a+)+$" },+ };+ pathologicalPattern.canonicalResponse.answers[requiredQuestionId] = {+ text: `${"a".repeat(100_000)}!`,+ };+ assert.throws(+ () => assertQuestionAdapterFixture(pathologicalPattern),+ /could not be evaluated safely/,+ );+});+test("the cross-language conformance input and output have one stable identity", async () => {const input = await fixture("conformance-minimal-run.json");const output = await fixture("conformance-expected-output.json");