Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): make the attachment lanes fail on what they claim to prove
Alfredo Lopeza34e5629ae9b
diff --git a/Makefile b/Makefileindex 8bf7765b0..0afeb86e3 100644--- a/Makefile+++ b/Makefile@@ -311,6 +311,7 @@ minikube-build-image-capabilities-fixture-body:minikube-run-image-capabilities: ## Run the visible image journey with an isolated external-provider fixture and restore the Host@T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ IMAGE_CAPABILITIES_LANE=image \bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run.PHONY: minikube-run-document-uploaddiff --git a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.tsindex e1b005734..d70615959 100644--- a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts+++ b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts@@ -59,6 +59,34 @@ export const FIXTURE_RESPONSE_KIND = {documentAnswer: 'document-answer',} as const+/**+ * Reasons a rejected row may carry: fixed strings, never request content.+ * Declared independently of the fixture for the same reason as the response+ * kinds; `test/imageCapabilityEvidence.test.ts` checks the two lists agree.+ */+export const FIXTURE_REJECTION_REASONS: readonly string[] = Object.freeze([+ 'provider-request-form-unsupported',+ 'provider-path-not-captured',+ 'provider-method-unsupported',+ 'provider-auth-mismatch',+ 'provider-body-unsupported',+ 'provider-body-invalid',+ 'unsupported-model',+ 'text-model-image-incompatible',+ 'image-not-png-data-uri',+ 'image-part-count',+ 'image-png-malformed',+ 'image-png-unsupported-form',+ 'image-tile-grid-mismatch',+ 'image-pixel-not-a-tile-color',+ 'document-byte-length-malformed',+ 'document-tool-result-count',+ 'document-tool-output-malformed',+ 'document-tool-output-unreadable',+ 'document-page-range-malformed',+ 'document-stream-unsupported',+])+/** The fixture's text-only answer. It names no image and no color. */export const FIXTURE_TEXT_ONLY_CONTENT = 'IMAGE_FIXTURE_TEXT_OK'@@ -275,6 +303,15 @@ export interface FixtureProviderAttempt {* Present only on the read-request row.*/documentByteLength?: number+ /**+ * The page of the file the Host's read tool delivered. Present only on the+ * answer row, so the oracle can hash exactly that slice of the file.+ */+ byteRange?: { offset: number; length: number }+ /** Whether the Host delivered less than the whole file. Answer row only. */+ truncated?: boolean+ /** One of {@link FIXTURE_REJECTION_REASONS}. Present only on rejected rows. */+ reason?: string}export interface FixtureEvidenceCounters {@@ -355,6 +392,48 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu)}+ const row: FixtureProviderAttempt = { model, imageSha256, responseKind }++ // Present on every rejected row and on no other row.+ if (responseKind === FIXTURE_RESPONSE_KIND.rejected) {+ const reason = value.reason+ if (typeof reason !== 'string' || !FIXTURE_REJECTION_REASONS.includes(reason)) {+ throw new Error(+ `${source}: attempts[${index}].reason must be one of ${FIXTURE_REJECTION_REASONS.join(', ')} ` ++ `on a ${FIXTURE_RESPONSE_KIND.rejected} row`+ )+ }+ row.reason = reason+ } else if ('reason' in value) {+ throw new Error(+ `${source}: attempts[${index}].reason is only valid on a ${FIXTURE_RESPONSE_KIND.rejected} row`+ )+ }++ // Present on every answer row and on no other row.+ if (responseKind === FIXTURE_RESPONSE_KIND.documentAnswer) {+ const byteRange = value.byteRange+ if (!isRecord(byteRange) || !isByteCount(byteRange.offset) || !isByteCount(byteRange.length)) {+ throw new Error(+ `${source}: attempts[${index}].byteRange must be { offset, length } of non-negative ` ++ `integers on a ${FIXTURE_RESPONSE_KIND.documentAnswer} row`+ )+ }+ if (typeof value.truncated !== 'boolean') {+ throw new Error(+ `${source}: attempts[${index}].truncated must be a boolean on a ` ++ `${FIXTURE_RESPONSE_KIND.documentAnswer} row`+ )+ }+ row.byteRange = { offset: byteRange.offset, length: byteRange.length }+ row.truncated = value.truncated+ } else if ('byteRange' in value || 'truncated' in value) {+ throw new Error(+ `${source}: attempts[${index}].byteRange and truncated are only valid on a ` ++ `${FIXTURE_RESPONSE_KIND.documentAnswer} row`+ )+ }+// Present only on document rows; an image row must not grow the field.if ('documentSha256' in value) {const documentSha256 = value.documentSha256@@ -366,6 +445,7 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu`${source}: attempts[${index}].documentSha256 must be a sha256 hex digest or null`)}+ row.documentSha256 = documentSha256// Present only on the read-request row.if ('documentByteLength' in value) {const documentByteLength = value.documentByteLength@@ -380,9 +460,9 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu`${FIXTURE_RESPONSE_KIND.documentReadRequested} row`)}- return { model, imageSha256, responseKind, documentSha256, documentByteLength }+ row.documentByteLength = documentByteLength}- return { model, imageSha256, responseKind, documentSha256 }+ return row}if ('documentByteLength' in value) {throw new Error(@@ -390,11 +470,15 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu)}- return { model, imageSha256, responseKind }+ return row+}++function isByteCount(value: unknown): value is number {+ return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0}/** Parses and validates the ledger; fixture fields this lane does not use are ignored. */-function parseEvidenceSnapshot(+export function parseEvidenceSnapshot(rawText: string,source: string,expected: { runId: string; profile: string }@@ -535,7 +619,11 @@ export function appendedAttempts(current.imageSha256 !== row.imageSha256 ||current.responseKind !== row.responseKind ||current.documentSha256 !== row.documentSha256 ||- current.documentByteLength !== row.documentByteLength+ current.documentByteLength !== row.documentByteLength ||+ current.reason !== row.reason ||+ current.byteRange?.offset !== row.byteRange?.offset ||+ current.byteRange?.length !== row.byteRange?.length ||+ current.truncated !== row.truncated) {throw new Error(`[image-capabilities] ledger row ${index} changed between reads; the ledger must be append-only.`diff --git a/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts b/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.tsindex afada39b2..80821e8a7 100644--- a/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts+++ b/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts@@ -157,9 +157,12 @@ function fileChips(page: Page) {return page.getByTestId('composer-file-chip')}-/** The attachment list a sent user message renders (`MessageAttachmentList`). */+/**+ * The attachment items a sent message renders (`MessageAttachmentList`). The+ * items are counted, not the list container, so a second file in one list fails.+ */function sentMessageAttachments(page: Page) {- return page.getByLabel('Message attachments')+ return page.getByLabel('Message attachments').locator('.message-attachment-chip')}/**@@ -178,11 +181,15 @@ const LARGE_DOCUMENT_BYTES = 6 * 1024 * 1024/*** Default page `clerum__attachment_read` returns* (`CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES`). The fixture digests the text of- * the first page it receives, so a larger document is identified by the digest- * of its first page.+ * the page it receives and records that page's `byteRange`, so the oracle+ * hashes exactly that slice of the file. This value is only the upper bound a+ * first page may have; the slice itself comes from the ledger.*/const ATTACHMENT_READ_PAGE_BYTES = 65_536+/** How much of the file the Host's read tool must deliver on the first read. */+type ExpectedRead = 'whole-file' | 'first-page'+/** Start a blank chat and prove the thread is empty before anything is sent. */async function startBlankChat(page: Page) {await page.getByTestId('nav-new-chat').click()@@ -274,23 +281,34 @@ async function attachDocument(* which may treat intraword underscores as emphasis; the ledger below pins the* exact digest regardless.*/+const DOCUMENT_ANSWER_PREFIX_PATTERN = FIXTURE_DOCUMENT_ANSWER_PREFIX.replace(/_/g, '[_-]?')+function documentAnswerRegex(documentSha: string): RegExp {- const prefix = FIXTURE_DOCUMENT_ANSWER_PREFIX.replace(/_/g, '[_-]?')- return new RegExp(`${prefix}${documentSha.slice(0, FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS)}`, 'i')+ return new RegExp(+ `${DOCUMENT_ANSWER_PREFIX_PATTERN}${documentSha.slice(0, FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS)}`,+ 'i'+ )}+/** Any fixture document answer: proves the turn finished before the ledger is read. */+const ANY_DOCUMENT_ANSWER = new RegExp(+ `${DOCUMENT_ANSWER_PREFIX_PATTERN}[a-f0-9]{${FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS}}`,+ 'i'+)+/*** Sends the composer's single ready document and proves the round trip: the- * sent bubble lists the file, the answer carries the digest of the text the- * read tool returned, and the fixture ledger shows one read request carrying- * the byte length the Host verified and one answer carrying that digest.+ * sent bubble lists the file, the fixture ledger shows one read request+ * carrying the byte length the Host verified and one answer carrying the page+ * the read tool delivered, and the answer carries the digest of exactly that+ * slice of the attached bytes.*/async function sendDocumentAndVerifyAnswer(page: Page,env: ReturnType<typeof requireImageCapabilitiesFixtureEnv>,assertBinding: () => Promise<void>,document: { fileName: string; buffer: Buffer },- readTextSha: string+ expectedRead: ExpectedRead) {const before = readImageCapabilityEvidence(env)await assertBinding()@@ -304,8 +322,7 @@ async function sendDocumentAndVerifyAnswer(await expect(sentMessageAttachments(page)).toContainText(document.fileName)const response = page.getByTestId('agent-response').locator('.message-block.markdown-content')await expect(response).toHaveCount(1, { timeout: 120_000 })- await expect(response).toContainText(documentAnswerRegex(readTextSha), { timeout: 120_000 })- await observeCompletedTask(page, env.hostRef, documentAnswerRegex(readTextSha))+ await expect(response).toContainText(ANY_DOCUMENT_ANSWER, { timeout: 120_000 })const after = readImageCapabilityEvidence(env)const appended = appendedAttempts(before, after)@@ -322,10 +339,30 @@ async function sendDocumentAndVerifyAnswer(expect(readRows[0]?.documentSha256).toBeNull()// The Host announced the whole file: the byte length it verified on admission.expect(readRows[0]?.documentByteLength).toBe(document.buffer.length)- // Turn 2: the tool result reached the wire carrying exactly the attached bytes.+ // Turn 2: the tool result reached the wire carrying exactly the attached bytes+ // of the page the Host reports it delivered.expect(answerRows).toHaveLength(1)- expect(answerRows[0]?.model).toBe(IMAGE_CAPABILITY_FIXTURE_MODELS.supported)- expect(answerRows[0]?.documentSha256).toBe(readTextSha)+ const answer = answerRows[0]+ expect(answer?.model).toBe(IMAGE_CAPABILITY_FIXTURE_MODELS.supported)+ const range = answer?.byteRange+ if (!range || answer?.truncated === undefined) {+ throw new Error('[document-upload] the answer row carries no delivered page')+ }+ expect(range.offset).toBe(0)+ if (expectedRead === 'whole-file') {+ expect(answer.truncated).toBe(false)+ expect(range.length).toBe(document.buffer.length)+ } else {+ expect(answer.truncated).toBe(true)+ expect(range.length).toBeGreaterThan(0)+ expect(range.length).toBeLessThanOrEqual(ATTACHMENT_READ_PAGE_BYTES)+ expect(range.length).toBeLessThan(document.buffer.length)+ }+ const pageSha = sha256Hex(document.buffer.subarray(range.offset, range.offset + range.length))+ expect(answer.documentSha256).toBe(pageSha)+ // The visible answer and the recorded task name that same digest.+ await expect(response).toContainText(documentAnswerRegex(pageSha))+ await observeCompletedTask(page, env.hostRef, documentAnswerRegex(pageSha))// A document is never an image: no pixels moved, and the fixture refused nothing.expect(appended.filter(row => row.imageSha256 !== null)).toHaveLength(0)@@ -359,7 +396,6 @@ test('document-upload fixture: an attached text file reaches the model through tmimeType: 'text/plain',buffer: Buffer.from(`document-upload ${randomBytes(6).toString('hex')}\nsecond line\n`, 'utf8'),}- const documentSha = sha256Hex(document.buffer)// Run-unique ASCII (hex of random bytes): its first page differs from every// other run's, and in ASCII the byte cut and the text cut of a page coincide.@@ -369,12 +405,6 @@ test('document-upload fixture: an attached text file reaches the model through tbuffer: Buffer.from(randomBytes(LARGE_DOCUMENT_BYTES / 2).toString('hex'), 'ascii'),}expect(largeDocument.buffer.length).toBe(LARGE_DOCUMENT_BYTES)- const largeDocumentPageSha = sha256Hex(- largeDocument.buffer.subarray(0, ATTACHMENT_READ_PAGE_BYTES)- )-- fs.mkdirSync(testInfo.outputPath(), { recursive: true })- fs.writeFileSync(testInfo.outputPath(document.fileName), document.buffer)let app: ElectronApplication | undefinedlet recordedPage: Page | undefined@@ -441,7 +471,7 @@ test('document-upload fixture: an attached text file reaches the model through t})await test.step('the answer carries the digest of the delivered text and the ledger shows both provider turns', async () => {- await sendDocumentAndVerifyAnswer(page, env, assertBinding, document, documentSha)+ await sendDocumentAndVerifyAnswer(page, env, assertBinding, document, 'whole-file')})await test.step('a 6 MiB document above the old text share is admitted as a ready chip', async () => {@@ -475,13 +505,7 @@ test('document-upload fixture: an attached text file reaches the model through t})await test.step('the 6 MiB document is sent, read inline and answered with the digest of its first page', async () => {- await sendDocumentAndVerifyAnswer(- page,- env,- assertBinding,- largeDocument,- largeDocumentPageSha- )+ await sendDocumentAndVerifyAnswer(page, env, assertBinding, largeDocument, 'first-page')})await screenshotAndLog(page, testInfo, 'desktop-document-upload-fixture')diff --git a/desktop-app/test/e2e-playwright/qa-recorder-image-capabilities.spec.ts b/desktop-app/test/e2e-playwright/qa-recorder-image-capabilities.spec.tsindex b74809b36..3cadbc41e 100644--- a/desktop-app/test/e2e-playwright/qa-recorder-image-capabilities.spec.ts+++ b/desktop-app/test/e2e-playwright/qa-recorder-image-capabilities.spec.ts@@ -370,10 +370,20 @@ async function expectImageSendBlocked(// Enter must not reach the controller either: nothing is delivered, the draft// survives, and the chip is still pending.await composer(page).press('Enter')+ // Witness that the key landed where a send would start.+ await expect(composer(page)).toBeFocused()await expect(page.locator('[data-chat-message-id]')).toHaveCount(0)+ await expect(sendButton(page)).toBeDisabled()+ // Liveness witness, after the negatives: the blocked state the refusal had to+ // leave behind is still on screen, so the empty thread is not an unmounted chat.+ await expect(modelChip(page)).toHaveAttribute('data-model-id', model)await expect(composer(page)).toHaveValue(draft)await expect(attachmentChips(page).filter({ hasText: attachmentName })).toBeVisible()- await expect(sendButton(page)).toBeDisabled()+ await expect(notice).toBeVisible()+ await expect(notice).toContainText(model)+ await expect(notice).toContainText(messagePattern)+ // The witnesses above took real time; a late delivery would show up here.+ await expect(page.locator('[data-chat-message-id]')).toHaveCount(0)}/** Exactly one exchange in this chat, and the answer is not empty. */diff --git a/desktop-app/test/imageCapabilityEvidence.test.ts b/desktop-app/test/imageCapabilityEvidence.test.tsnew file mode 100644index 000000000..7912bc0d6--- /dev/null+++ b/desktop-app/test/imageCapabilityEvidence.test.ts@@ -0,0 +1,156 @@+import { describe, expect, it } from 'vitest'+import path from 'node:path'+import { pathToFileURL } from 'node:url'+import * as evidenceModule from './e2e-playwright/helpers/imageCapabilityEvidence.js'+import {+ FIXTURE_RESPONSE_KIND,+ appendedAttempts,+ parseEvidenceSnapshot,+} from './e2e-playwright/helpers/imageCapabilityEvidence.js'++// The fixture is untyped ESM outside this package; only its reason list is read,+// through a typed dynamic import so the compiler does not resolve the module.+const PROVIDER_MODULE_URL = pathToFileURL(+ path.resolve(__dirname, '../../tests/e2e/fixtures/image-capabilities/provider.mjs')+).href++const RUN_ID = 'image-capabilities-0123456789ab'+const PROFILE = 'clerum-image-fixture-12345678'+const SOURCE = 'unit-test ledger'+const DIGEST = 'a'.repeat(64)++type Row = Record<string, unknown>++const rows = {+ rejected: (): Row => ({+ model: 'glm-5.3',+ imageSha256: null,+ responseKind: 'rejected',+ reason: 'text-model-image-incompatible',+ }),+ read: (): Row => ({+ model: 'glm-5.3-flash',+ imageSha256: null,+ responseKind: 'document-read-requested',+ documentSha256: null,+ documentByteLength: 6_291_456,+ }),+ answer: (): Row => ({+ model: 'glm-5.3-flash',+ imageSha256: null,+ responseKind: 'document-answer',+ documentSha256: DIGEST,+ byteRange: { offset: 0, length: 65_536 },+ truncated: true,+ }),+ textOnly: (): Row => ({ model: 'glm-5.3-flash', imageSha256: null, responseKind: 'text-only' }),+}++function ledger(attempts: Row[]): string {+ return JSON.stringify({+ runId: RUN_ID,+ profile: PROFILE,+ pid: 41,+ counters: {+ totalAttempts: attempts.length,+ imageAttempts: 0,+ textAttempts: 0,+ rejectedAttempts: 0,+ tileColorResponses: 0,+ textOnlyResponses: 0,+ textModelImageRefusals: 0,+ blockedEgress: 0,+ documentReadRequests: 0,+ documentAnswers: 0,+ documentFailures: 0,+ },+ attempts,+ })+}++const parse = (attempts: Row[]) =>+ parseEvidenceSnapshot(ledger(attempts), SOURCE, { runId: RUN_ID, profile: PROFILE })++describe('imageCapabilityEvidence ledger rows', () => {+ it('keeps the rejection reason and the delivered page on the rows that carry them', () => {+ const snapshot = parse([rows.rejected(), rows.read(), rows.answer(), rows.textOnly()])+ expect(snapshot.attempts).toEqual([+ rows.rejected(),+ rows.read(),+ rows.answer(),+ rows.textOnly(),+ ])+ })++ it('requires a closed-set reason on every rejected row and nowhere else', () => {+ // Witness: the well-formed rejected row parses.+ expect(parse([rows.rejected()]).attempts[0]?.reason).toBe('text-model-image-incompatible')+ const { reason: _omitted, ...withoutReason } = rows.rejected()+ for (const [label, row] of [+ ['missing reason', withoutReason],+ ['free-text reason', { ...rows.rejected(), reason: 'prompt said: draw a cat' }],+ ['non-string reason', { ...rows.rejected(), reason: 7 }],+ ['reason on an accepted row', { ...rows.textOnly(), reason: 'unsupported-model' }],+ ] as const) {+ expect(() => parse([row]), label).toThrow(/reason/)+ }+ })++ it('requires the delivered page on every document answer and nowhere else', () => {+ // Witness: the well-formed answer row parses with its page.+ expect(parse([rows.answer()]).attempts[0]?.byteRange).toEqual({ offset: 0, length: 65_536 })+ const { byteRange: _range, ...withoutRange } = rows.answer()+ const { truncated: _flag, ...withoutTruncated } = rows.answer()+ for (const [label, row] of [+ ['missing byteRange', withoutRange],+ ['null byteRange', { ...rows.answer(), byteRange: null }],+ ['array byteRange', { ...rows.answer(), byteRange: [0, 65_536] }],+ ['missing length', { ...rows.answer(), byteRange: { offset: 0 } }],+ ['negative offset', { ...rows.answer(), byteRange: { offset: -1, length: 10 } }],+ ['fractional length', { ...rows.answer(), byteRange: { offset: 0, length: 1.5 } }],+ ['string length', { ...rows.answer(), byteRange: { offset: 0, length: '10' } }],+ ['missing truncated', withoutTruncated],+ ['string truncated', { ...rows.answer(), truncated: 'true' }],+ ['byteRange on a read row', { ...rows.read(), byteRange: { offset: 0, length: 10 } }],+ ['truncated on a read row', { ...rows.read(), truncated: false }],+ ['byteRange on a text row', { ...rows.textOnly(), byteRange: { offset: 0, length: 10 } }],+ ] as const) {+ expect(() => parse([row]), label).toThrow(/byteRange|truncated/)+ }+ })+})++describe('appendedAttempts', () => {+ it('refuses a row whose reason or delivered page changed between reads', () => {+ const before = parse([rows.rejected(), rows.answer()])+ // Witness: an unchanged prefix yields the appended row.+ expect(+ appendedAttempts(before, parse([rows.rejected(), rows.answer(), rows.textOnly()]))+ ).toEqual([rows.textOnly()])+ for (const [label, changed] of [+ ['reason', [{ ...rows.rejected(), reason: 'unsupported-model' }, rows.answer()]],+ ['offset', [rows.rejected(), { ...rows.answer(), byteRange: { offset: 1, length: 65_536 } }]],+ ['length', [rows.rejected(), { ...rows.answer(), byteRange: { offset: 0, length: 65_535 } }]],+ ['truncated', [rows.rejected(), { ...rows.answer(), truncated: false }]],+ ] as const) {+ expect(() => appendedAttempts(before, parse([...changed])), label).toThrow(/append-only/)+ }+ })+})++describe('rejection reason parity', () => {+ it('declares the same closed set as the fixture, independently', async () => {+ const providerModule = (await import(PROVIDER_MODULE_URL)) as {+ FIXTURE_REJECTION_REASONS: readonly string[]+ }+ const declared = (evidenceModule as Record<string, unknown>).FIXTURE_REJECTION_REASONS+ expect(Array.isArray(declared)).toBe(true)+ expect(declared).not.toBe(providerModule.FIXTURE_REJECTION_REASONS)+ expect([...(declared as readonly string[])].sort()).toEqual(+ [...providerModule.FIXTURE_REJECTION_REASONS].sort()+ )+ // Witness: the set is not empty, so equality is not two empty lists.+ expect((declared as readonly string[]).length).toBeGreaterThan(0)+ expect(FIXTURE_RESPONSE_KIND.rejected).toBe('rejected')+ })+})diff --git a/docs/testing/desktop-headful-journeys.md b/docs/testing/desktop-headful-journeys.mdindex ac0e734ff..b8e41d69c 100644--- a/docs/testing/desktop-headful-journeys.md+++ b/docs/testing/desktop-headful-journeys.md@@ -18,22 +18,22 @@ QA identity from `.env.qa-recorder` (no first-available fallback), shares launchlogin / proof plumbing in `qa-recorder-helpers.ts`, and records a WebM + PNG pertest under `.local-notes/qa-recorder/runs/desktop-app/` (git-ignored).-| Journey | Spec | Tests | Confirm flag | Status |-| --------------------- | ---------------------------------------- | ----------------------------------------------------------------------------------- | ---------------------------------------------- | ------ |-| Auth & session | `qa-recorder-auth.spec.ts` | 3 — sign-in screen, authenticated shell, logout | — | ✅ |-| Navigation | `qa-recorder-navigation.spec.ts` | 3 — primary sidebar, footer settings menu, resources pages | — | ✅ |-| Agents | `qa-recorder-agents.spec.ts` | 3 — fleet, chat workspace, workspace routes | — | ✅ |-| Connectors | `qa-recorder-connectors.spec.ts` | 1 — MCP/connector health inventory | — | ✅ |-| Shared files | `qa-recorder-shared-files.spec.ts` | 1 — filesystem + directory browser | — | ✅ |-| Plugins & workflows | `qa-recorder-plugins.spec.ts` | 1 — inventory, detail, runs/artifacts (+optional trigger) | `QA_RECORDER_CONFIRM_MUTATIONS` (trigger only) | ✅ |-| Apps | `qa-recorder-apps.spec.ts` | 1 — catalog + embedded session | — | ✅ |-| Settings | `qa-recorder-settings.spec.ts` | 2 — appearance/notifications, information/configuration (+optional endpoint switch) | `QA_RECORDER_CONFIRM_MUTATIONS` (switch only) | ✅ |-| Inbox & search | `qa-recorder-inbox-search.spec.ts` | 2 — inbox, global search | — | ✅ |-| Chat | `qa-recorder-chat.spec.ts` | 1 — composer, thread, task progress | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |-| Settings + chat smoke | `qa-recorder-settings-chat.spec.ts` | 1 — Settings tabs + one chat message | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |-| Image capabilities | `qa-recorder-image-capabilities.spec.ts` | 2 — capability gate on a text-only model, fixture-backed image answer | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |-| Model selector | `qa-recorder-model-selector.spec.ts` | 1 — a multi-model catalog lists rows without capability tags | — | ✅ |-| Document upload | `qa-recorder-document-upload.spec.ts` | 1 — attached text file read through the read tool (fixture lane only) | `IMAGE_CAPABILITIES_RUN_ID` (runner-set) | ✅ |+| Journey | Spec | Tests | Confirm flag | Status |+| --------------------- | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | ------ |+| Auth & session | `qa-recorder-auth.spec.ts` | 3 — sign-in screen, authenticated shell, logout | — | ✅ |+| Navigation | `qa-recorder-navigation.spec.ts` | 3 — primary sidebar, footer settings menu, resources pages | — | ✅ |+| Agents | `qa-recorder-agents.spec.ts` | 3 — fleet, chat workspace, workspace routes | — | ✅ |+| Connectors | `qa-recorder-connectors.spec.ts` | 1 — MCP/connector health inventory | — | ✅ |+| Shared files | `qa-recorder-shared-files.spec.ts` | 1 — filesystem + directory browser | — | ✅ |+| Plugins & workflows | `qa-recorder-plugins.spec.ts` | 1 — inventory, detail, runs/artifacts (+optional trigger) | `QA_RECORDER_CONFIRM_MUTATIONS` (trigger only) | ✅ |+| Apps | `qa-recorder-apps.spec.ts` | 1 — catalog + embedded session | — | ✅ |+| Settings | `qa-recorder-settings.spec.ts` | 2 — appearance/notifications, information/configuration (+optional endpoint switch) | `QA_RECORDER_CONFIRM_MUTATIONS` (switch only) | ✅ |+| Inbox & search | `qa-recorder-inbox-search.spec.ts` | 2 — inbox, global search | — | ✅ |+| Chat | `qa-recorder-chat.spec.ts` | 1 — composer, thread, task progress | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |+| Settings + chat smoke | `qa-recorder-settings-chat.spec.ts` | 1 — Settings tabs + one chat message | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |+| Image capabilities | `qa-recorder-image-capabilities.spec.ts` | 2 — capability gate on a text-only model, fixture-backed image answer | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |+| Model selector | `qa-recorder-model-selector.spec.ts` | 1 — a multi-model catalog lists rows without capability tags | — | ✅ |+| Document upload | `qa-recorder-document-upload.spec.ts` | 1 — attached text file read through the read tool (fixture lane only); a small file is answered with its whole-file digest, and a 6 MiB file with the digest of the first page the Host delivered (`byteRange` and `truncated: true` from the fixture ledger) | `IMAGE_CAPABILITIES_RUN_ID` (runner-set) | ✅ |**Totals: 14 specs, 23 tests.** Read-only journeys need no confirmation flag; everyjourney still calls the loopback health guard on both API URLs — except against adiff --git a/scripts/e2e/image-capabilities-fixture.mjs b/scripts/e2e/image-capabilities-fixture.mjsindex e0e49e59a..7103ba830 100644--- a/scripts/e2e/image-capabilities-fixture.mjs+++ b/scripts/e2e/image-capabilities-fixture.mjs@@ -6,6 +6,7 @@ import { pathToFileURL } from 'node:url'import { isDeepStrictEqual } from 'node:util'import {openOwnedFile,+ readOwnedDescriptor,readOwnedFile,validateRunDirectory,} from './prepare-codex-approved-tools.mjs'@@ -28,13 +29,18 @@ export const fixtureConfigKeys = [* an environment value can never name an arbitrary config path.*/export const playwrightLanes = {+ // `grep` selects the lane's fixture journey. The image spec also holds the+ // real-provider smoke, which skips itself in fixture mode; selecting by title+ // keeps it out of the run instead of reporting it as skipped.image: {config: 'desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',label: 'image-capabilities-playwright',+ grep: 'image-capabilities fixture: ',},'document-upload': {config: 'desktop-app/test/e2e-playwright/playwright.document-upload.config.ts',label: 'document-upload-playwright',+ grep: 'document-upload fixture: ',},}@@ -47,6 +53,61 @@ export function playwrightLaneConfig(env) {return { lane, ...playwrightLanes[lane] }}+/**+ * Only `run` drives Playwright, so only `run` resolves the lane. `restore`+ * must reach the owned runtime even when the lane variable is stale or wrong.+ */+export function selectFixtureAction(argument, env) {+ const action = argument ?? 'run'+ if (!['run', 'restore'].includes(action)) throw new Error('Expected run or restore')+ if (action === 'restore') return { action }+ return { action, ...playwrightLaneConfig(env) }+}++const isCount = value => Number.isSafeInteger(value) && value >= 0++/**+ * A zero exit from the Playwright CLI is not a verdict on its own: a run that+ * collected nothing, skipped everything or retried into a pass also exits 0.+ * PASS needs the JSON report of this lane's config with at least one expected+ * test and no unexpected, skipped or flaky outcome.+ */+export function playwrightVerdict(report, { lane, configFile }) {+ const { grep } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })+ const reported = report?.config?.configFile+ if (typeof reported !== 'string' || fs.realpathSync(reported) !== fs.realpathSync(configFile))+ throw new Error('Playwright report config mismatch')+ const stats = report.stats ?? {}+ if (!isCount(stats.expected) || stats.expected < 1)+ throw new Error('Playwright ran no expected tests')+ for (const outcome of ['unexpected', 'skipped', 'flaky'])+ if (stats[outcome] !== 0)+ throw new Error(`Playwright report is not clean: ${outcome}=${String(stats[outcome])}`)+ if (!Array.isArray(report.errors) || report.errors.length)+ throw new Error('Playwright report has global errors')+ const titles = []+ const collect = suites => {+ if (!Array.isArray(suites)) throw new Error('Incomplete Playwright suites')+ for (const suite of suites) {+ for (const spec of suite.specs ?? []) titles.push(spec.title)+ collect(suite.suites ?? [])+ }+ }+ collect(report.suites)+ if (titles.length !== stats.expected || titles.some(title => !title.startsWith(grep)))+ throw new Error(`Playwright report has specs outside the ${lane} lane`)+ return 'PASS'+}++/** The PASS line is printed from the persisted state, never from memory. */+export function fixturePassLine(state, { lane, evidence }) {+ if (!Object.hasOwn(state, 'lane') || state.lane !== lane)+ throw new Error('Run state lane mismatch')+ if (state.playwright !== 'PASS') throw new Error('Run state has no Playwright PASS')+ if (state.restored !== true) throw new Error('Run state was not restored')+ return `IMAGE_CAPABILITIES_E2E_PASS lane=${lane} evidence=${evidence}\n`+}+export function proveImages(manifest, observed, head, profile) {if (!/^[a-f0-9]{40}$/.test(head) || manifest.profile !== profile)throw new Error('Fixture image ownership mismatch')@@ -182,7 +243,13 @@ async function main() {)throw new Error('Matching branch profile and context required')command('bash', ['scripts/minikube/require-t2-mutation-lock.sh'])- const { lane, config: laneConfig, label: laneLabel } = playwrightLaneConfig(process.env)+ const {+ action,+ lane,+ config: laneConfig,+ label: laneLabel,+ grep: laneGrep,+ } = selectFixtureAction(process.argv[2], process.env)const head = command('git', ['rev-parse', 'HEAD']).trim()const branch = command('git', ['branch', '--show-current']).trim()if (command('git', ['status', '--porcelain']).trim()) throw new Error('Clean checkout required')@@ -248,8 +315,6 @@ async function main() {}throw new Error('HCC did not reconcile the expected Host image')}- const action = process.argv[2] ?? 'run'- if (!['run', 'restore'].includes(action)) throw new Error('Expected run or restore')const runRoot = path.join(canonical, '.local-notes/infra/runs')fs.mkdirSync(runRoot, { recursive: true })const runId = `image-capabilities-${randomBytes(6).toString('hex')}`@@ -263,6 +328,7 @@ async function main() {? JSON.parse(readOwnedFile(evidence, 'state.json')): {runId,+ lane,branch,profile,head,@@ -742,28 +808,50 @@ async function main() {}command('npm', ['run', 'verify:electron', '--prefix', 'desktop-app'])process.stdout.write(`Image capability fixture ready: ${state.runId} (lane ${lane})\n`)- const output = command(- 'node',- [- 'scripts/minikube/run-with-deadline.mjs',- '--timeout-seconds',- '900',- '--heartbeat-seconds',- '20',- '--kill-grace-seconds',- '5',- '--label',- laneLabel,- '--',+ // Playwright's JSON reporter writes this reserved inode with writeFile. Keep+ // the descriptor until the verdict; a replacement path cannot supply proof.+ const reportFile = openOwnedFile(evidence, 'playwright-report.json', {+ create: true,+ maxBytes: 16 * 1024 * 1024,+ })+ try {+ const output = command('node',- 'desktop-app/node_modules/@playwright/test/cli.js',- 'test',- `--config=${laneConfig}`,- ],- { env: environment, timeout: 930_000 }- )- report(output)- state.playwright = 'PASS'+ [+ 'scripts/minikube/run-with-deadline.mjs',+ '--timeout-seconds',+ '900',+ '--heartbeat-seconds',+ '20',+ '--kill-grace-seconds',+ '5',+ '--label',+ laneLabel,+ '--',+ 'node',+ 'desktop-app/node_modules/@playwright/test/cli.js',+ 'test',+ `--config=${laneConfig}`,+ `--grep=${laneGrep}`,+ // `list` keeps the configured console output; `json` is the verdict.+ '--reporter=list,json',+ ],+ {+ env: {+ ...environment,+ PLAYWRIGHT_JSON_OUTPUT_FILE: path.join(evidence, 'playwright-report.json'),+ },+ timeout: 930_000,+ }+ )+ report(output)+ state.playwright = playwrightVerdict(JSON.parse(readOwnedDescriptor(reportFile)), {+ lane,+ configFile: path.join(root, laneConfig),+ })+ } finally {+ fs.closeSync(reportFile.fd)+ }save()} catch (error) {failure = error@@ -809,7 +897,9 @@ async function main() {}uninstallSignalRestore()if (failure) throw failure- process.stdout.write(`IMAGE_CAPABILITIES_E2E_PASS evidence=${evidence}\n`)+ process.stdout.write(+ fixturePassLine(JSON.parse(readOwnedFile(evidence, 'state.json')), { lane, evidence })+ )}if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {diff --git a/scripts/e2e/image-capabilities-fixture.test.mjs b/scripts/e2e/image-capabilities-fixture.test.mjsindex 22c5e9c6b..d58565884 100644--- a/scripts/e2e/image-capabilities-fixture.test.mjs+++ b/scripts/e2e/image-capabilities-fixture.test.mjs@@ -1,18 +1,43 @@import assert from 'node:assert/strict'import { EventEmitter } from 'node:events'+import path from 'node:path'import test from 'node:test'+import { fileURLToPath } from 'node:url'import {baseImage,fixtureImage,+ fixturePassLine,installSignalRestore,modelInputs,playwrightLaneConfig,+ playwrightVerdict,proveImages,requireOwnedResource,runAnnotation,sanitizeFixtureReport,+ selectFixtureAction,} from './image-capabilities-fixture.mjs'+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')+const laneConfigFile = lane =>+ path.join(repoRoot, playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane }).config)+function playwrightReport({ lane = 'image', stats = {}, errors = [], titles } = {}) {+ const { grep } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })+ const specTitles = titles ?? [`${grep}a journey`]+ return {+ config: { configFile: laneConfigFile(lane) },+ suites: [+ {+ title: 'qa-recorder.spec.ts',+ specs: specTitles.map(title => ({ title, ok: true })),+ suites: [],+ },+ ],+ errors,+ stats: { expected: specTitles.length, skipped: 0, unexpected: 0, flaky: 0, ...stats },+ }+}+const head = 'a'.repeat(40)const profile = 'clerum-image-fixture-12345678'const ids = { [baseImage]: `sha256:${'b'.repeat(64)}`, [fixtureImage]: `sha256:${'c'.repeat(64)}` }@@ -30,11 +55,13 @@ test('the Playwright lane defaults to the image journey and accepts only known llane: 'image',config: 'desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',label: 'image-capabilities-playwright',+ grep: 'image-capabilities fixture: ',})assert.deepEqual(playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'document-upload' }), {lane: 'document-upload',config: 'desktop-app/test/e2e-playwright/playwright.document-upload.config.ts',label: 'document-upload-playwright',+ grep: 'document-upload fixture: ',})for (const lane of ['', 'documents', '../evil.config.ts', '__proto__', 'toString']) {assert.throws(@@ -45,6 +72,127 @@ test('the Playwright lane defaults to the image journey and accepts only known l}})+test('restore never resolves the Playwright lane, so a stale or invalid lane cannot block restoration', () => {+ for (const lane of [undefined, 'image', 'typo', '', '__proto__', '../evil.config.ts']) {+ const env = lane === undefined ? {} : { IMAGE_CAPABILITIES_LANE: lane }+ // Witness: the restore path was selected, and it carries no lane at all.+ assert.deepEqual(selectFixtureAction('restore', env), { action: 'restore' }, String(lane))+ }+})++test('run still resolves and validates the Playwright lane', () => {+ assert.deepEqual(selectFixtureAction(undefined, {}), {+ action: 'run',+ ...playwrightLaneConfig({}),+ })+ assert.deepEqual(selectFixtureAction('run', { IMAGE_CAPABILITIES_LANE: 'document-upload' }), {+ action: 'run',+ ...playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'document-upload' }),+ })+ for (const lane of ['typo', '', '__proto__']) {+ assert.throws(+ () => selectFixtureAction('run', { IMAGE_CAPABILITIES_LANE: lane }),+ /IMAGE_CAPABILITIES_LANE must be one of image, document-upload/,+ lane+ )+ }+ assert.throws(() => selectFixtureAction('cleanup', {}), /Expected run or restore/)+})++test('the PASS line names the lane only when the persisted state was produced by that lane', () => {+ const evidence = '/runs/image-capabilities-0123456789ab'+ const state = { lane: 'document-upload', playwright: 'PASS', restored: true }+ // Witness: a state written by the requested lane yields the lane-bearing PASS line.+ assert.equal(+ fixturePassLine(state, { lane: 'document-upload', evidence }),+ `IMAGE_CAPABILITIES_E2E_PASS lane=document-upload evidence=${evidence}\n`+ )+ for (const [label, mutated] of [+ ['other lane', { ...state, lane: 'image' }],+ ['missing lane', { playwright: 'PASS', restored: true }],+ [+ 'inherited lane',+ Object.assign(Object.create({ lane: 'document-upload' }), {+ playwright: 'PASS',+ restored: true,+ }),+ ],+ ]) {+ assert.throws(+ () => fixturePassLine(mutated, { lane: 'document-upload', evidence }),+ /Run state lane mismatch/,+ label+ )+ }+ assert.throws(+ () =>+ fixturePassLine({ ...state, playwright: undefined }, { lane: 'document-upload', evidence }),+ /Run state has no Playwright PASS/+ )+ assert.throws(+ () => fixturePassLine({ ...state, restored: false }, { lane: 'document-upload', evidence }),+ /Run state was not restored/+ )+})++test('the Playwright verdict is PASS only for a complete, clean report from the lane config', () => {+ // Witness: a clean run of the requested lane is PASS.+ assert.equal(+ playwrightVerdict(playwrightReport(), { lane: 'image', configFile: laneConfigFile('image') }),+ 'PASS'+ )+ assert.equal(+ playwrightVerdict(playwrightReport({ lane: 'document-upload' }), {+ lane: 'document-upload',+ configFile: laneConfigFile('document-upload'),+ }),+ 'PASS'+ )+ const options = { lane: 'image', configFile: laneConfigFile('image') }+ for (const [label, report, pattern] of [+ ['zero tests', playwrightReport({ titles: [] }), /Playwright ran no expected tests/],+ [+ 'all skipped',+ playwrightReport({ stats: { expected: 0, skipped: 1 } }),+ /Playwright ran no expected tests/,+ ],+ ['one skipped', playwrightReport({ stats: { skipped: 1 } }), /skipped=1/],+ ['one unexpected', playwrightReport({ stats: { unexpected: 1 } }), /unexpected=1/],+ ['one flaky', playwrightReport({ stats: { flaky: 1 } }), /flaky=1/],+ [+ 'non-integer count',+ playwrightReport({ stats: { expected: '1' } }),+ /Playwright ran no expected tests/,+ ],+ [+ 'missing stats',+ { ...playwrightReport(), stats: undefined },+ /Playwright ran no expected tests/,+ ],+ ['global error', playwrightReport({ errors: [{ message: 'boom' }] }), /global errors/],+ [+ 'other lane config',+ { ...playwrightReport(), config: { configFile: laneConfigFile('document-upload') } },+ /config mismatch/,+ ],+ ['missing config', { ...playwrightReport(), config: {} }, /config mismatch/],+ [+ 'spec outside the lane selection',+ playwrightReport({+ titles: ['image-capabilities fixture: a journey', 'optional QA recorder: real'],+ }),+ /outside the image lane/,+ ],+ [+ 'malformed suites',+ { ...playwrightReport(), suites: undefined },+ /Incomplete Playwright suites/,+ ],+ ]) {+ assert.throws(() => playwrightVerdict(report, options), pattern, label)+ }+})+test('requires both exact source revisions and the live derived-base identity', () => {assert.deepEqual(proveImages(manifest(), ids, head, profile), { head, profile, images: ids })for (const mutate of [diff --git a/scripts/tests/test-minikube-pre-gate-shadow.sh b/scripts/tests/test-minikube-pre-gate-shadow.shindex 8dd9a1126..ad14139ca 100755--- a/scripts/tests/test-minikube-pre-gate-shadow.sh+++ b/scripts/tests/test-minikube-pre-gate-shadow.sh@@ -451,11 +451,16 @@ STUBcleanup_fixture "$d"}-# minikube-verify-images only reads the manifest and the daemon; its recipe has-# no lock wrapper, so T2_SKIP_LOCK / T2_LOCK_TOKEN on that call do nothing and-# read as if it needed the lease. It is called without them. If the recipe ever-# gains a lock wrapper, the Makefile check below fails first and the call must-# pass the lease like the build and pull calls above.+# The minikube-verify-images recipe runs `build-images.sh --verify-only` with no+# mutation-lock wrapper, and build-images.sh returns from+# require_inherited_mutation_lease before any lease check when --verify-only is+# set. T2_SKIP_LOCK / T2_LOCK_TOKEN on that call would therefore be ignored and+# would read as if it needed the lease, so it is called without them. This test+# checks the call sites and the Makefile recipe text only (the --verify-only+# flag is present and no mutation-lock wrapper is); it does not run+# build-images.sh, so the script-side skip is not asserted here. If the recipe+# ever gains a lock wrapper, the recipe check below fails and the call must pass+# the lease like the build and pull calls above.assert_verify_images_is_called_without_the_lock_lease() {local d out_local out_ghcr rc_local rc_ghcr verify_calls bare_calls reciperecipe="$(awk '/^minikube-verify-images:/{f=1;next} f&&/^[^\t]/{exit} f' "$REPO_ROOT/Makefile")"diff --git a/tests/e2e/fixtures/image-capabilities/provider.mjs b/tests/e2e/fixtures/image-capabilities/provider.mjsindex cc6ab178b..85bf71b60 100644--- a/tests/e2e/fixtures/image-capabilities/provider.mjs+++ b/tests/e2e/fixtures/image-capabilities/provider.mjs@@ -32,7 +32,11 @@* carries the tool result, and the answer is `DOCUMENT_FIXTURE_SHA256:` plus the* first 16 hex characters of the SHA-256 of the delivered text. A run whose* document never reached the provider cannot produce that digest. Document rows- * add `documentSha256` (the digest of the delivered text, never the text).+ * add `documentSha256` (the digest of the delivered text, never the text); the+ * answer row also records the page the Host delivered (`byteRange` and+ * `truncated`), so the oracle hashes that exact slice of the file. Rejected rows+ * add a `reason` drawn from `FIXTURE_REJECTION_REASONS`, a closed set of fixed+ * strings that never carries request content.** Determinism and sanitation. Responses carry a synthetic completion id derived* from the run id, a fixed test usage block, and `finish_reason: 'stop'` (or@@ -131,6 +135,33 @@ const TILE_COLOR_NAMES = new Map(TILE_PALETTE.map(([name, red, green, blue]) => [`${red},${green},${blue}`, name]))+/**+ * Every reason a rejected ledger row may carry. Each is a fixed string chosen by+ * the fixture, so a rejected row can never echo a header, prompt or payload.+ */+export const FIXTURE_REJECTION_REASONS = Object.freeze([+ 'provider-request-form-unsupported',+ 'provider-path-not-captured',+ 'provider-method-unsupported',+ 'provider-auth-mismatch',+ 'provider-body-unsupported',+ 'provider-body-invalid',+ 'unsupported-model',+ 'text-model-image-incompatible',+ 'image-not-png-data-uri',+ 'image-part-count',+ 'image-png-malformed',+ 'image-png-unsupported-form',+ 'image-tile-grid-mismatch',+ 'image-pixel-not-a-tile-color',+ 'document-byte-length-malformed',+ 'document-tool-result-count',+ 'document-tool-output-malformed',+ 'document-tool-output-unreadable',+ 'document-page-range-malformed',+ 'document-stream-unsupported',+])+/** A refusal with a stable machine-readable reason code for the evidence tally. */class FixtureImageError extends Error {constructor(code, message) {@@ -459,9 +490,30 @@ function classifyDocumentTurn(body) {'document-tool-output-unreadable','the tool result does not carry the document text')- return { stage: 'answer', text: result.text }+ const range = result.byteRange+ if (+ !range ||+ typeof range !== 'object' ||+ Array.isArray(range) ||+ !isByteCount(range.offset) ||+ !isByteCount(range.length) ||+ typeof result.truncated !== 'boolean'+ )+ throw new FixtureDocumentError(+ 'document-page-range-malformed',+ 'the tool result does not carry a readable byteRange and truncated flag'+ )+ // Only the two integers and the flag are copied: the ledger never holds text.+ return {+ stage: 'answer',+ text: result.text,+ byteRange: { offset: range.offset, length: range.length },+ truncated: result.truncated,+ }}+const isByteCount = value => Number.isSafeInteger(value) && value >= 0+const INTERNAL_HOST_SUFFIXES = ['.svc.cluster.local','.svc',@@ -720,18 +772,17 @@ export function createImageFixtureFetch(* request was refused before its body was parsed, and `imageSha256` is null* whenever the attempt carried no usable image.*/- const recordCall = (model, imageSha256, responseKind, documentSha256, documentByteLength) => {- // Only document rows carry `documentSha256`, and only the read row carries- // `documentByteLength`, so the image rows keep the exact shape they always had.- const row =- documentSha256 === undefined- ? { model, imageSha256, responseKind }- : { model, imageSha256, responseKind, documentSha256 }- state.attempts.push(documentByteLength === undefined ? row : { ...row, documentByteLength })+ const recordCall = (model, imageSha256, responseKind, fields = {}) => {+ // Only document rows carry document fields and only rejected rows carry a+ // `reason`, so the image rows keep the exact shape they always had.+ state.attempts.push({ model, imageSha256, responseKind, ...fields })}const refuse = (model, reason, { status = 400, code = reason, imageSha256 = null } = {}) => {+ // The ledger records a fixed string from the closed set, never request text.+ if (!FIXTURE_REJECTION_REASONS.includes(reason))+ throw new Error(`Fixture rejection reason is not in the closed set: ${reason}`)state.counters.rejectedAttempts += 1- recordCall(model, imageSha256, 'rejected')+ recordCall(model, imageSha256, 'rejected', { reason })publish()return fixtureErrorResponse(status, code, reason)}@@ -739,11 +790,11 @@ export function createImageFixtureFetch(model,content,responseKind,- { imageSha256 = null, documentSha256, stream = false } = {}+ { imageSha256 = null, document, stream = false } = {}) => {sequence += 1const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`- recordCall(model, imageSha256, responseKind, documentSha256)+ recordCall(model, imageSha256, responseKind, document)publish()return stream? streamResponse(completionId, model, content)@@ -831,7 +882,10 @@ export function createImageFixtureFetch(state.counters.documentReadRequests += 1sequence += 1const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`- recordCall(model, null, 'document-read-requested', null, documentTurn.byteLength)+ recordCall(model, null, 'document-read-requested', {+ documentSha256: null,+ documentByteLength: documentTurn.byteLength,+ })publish()return jsonResponse(toolCallCompletionBody(completionId, model, ATTACHMENT_READ_TOOL, {@@ -847,7 +901,14 @@ export function createImageFixtureFetch(model,`${DOCUMENT_ANSWER_PREFIX}${documentSha256.slice(0, DOCUMENT_ANSWER_DIGEST_CHARS)}`,'document-answer',- { documentSha256 }+ {+ // The page the Host delivered, so the oracle hashes that exact slice.+ document: {+ documentSha256,+ byteRange: documentTurn.byteRange,+ truncated: documentTurn.truncated,+ },+ })}state.counters.textOnlyResponses += 1diff --git a/tests/e2e/fixtures/image-capabilities/provider.test.mjs b/tests/e2e/fixtures/image-capabilities/provider.test.mjsindex 5e74b3c20..a527950d2 100644--- a/tests/e2e/fixtures/image-capabilities/provider.test.mjs+++ b/tests/e2e/fixtures/image-capabilities/provider.test.mjs@@ -16,6 +16,7 @@ import { createHash, randomBytes } from 'node:crypto'import { readFileSync } from 'node:fs'import test from 'node:test'import { crc32, deflateSync } from 'node:zlib'+import * as providerModule from './provider.mjs'import { createImageFixtureFetch, validateFixtureEnvironment } from './provider.mjs'/**@@ -383,7 +384,12 @@ test('the text-only model refuses an image and records the incompatible attempt'assert.equal(evidence.counters.rejectedAttempts, 1)assert.equal(evidence.counters.tileColorResponses, 0)assert.deepEqual(evidence.attempts, [- { model: TEXT_MODEL, imageSha256: null, responseKind: 'rejected' },+ {+ model: TEXT_MODEL,+ imageSha256: null,+ responseKind: 'rejected',+ reason: 'text-model-image-incompatible',+ },])})@@ -867,11 +873,100 @@ test('document turn 2: the answer is a digest of the delivered text and the ledgimageSha256: null,responseKind: 'document-answer',documentSha256: digest,+ byteRange: { offset: 0, length: Buffer.byteLength(DOCUMENT_TEXT) },+ truncated: false,},])assert.equal(h.getEvidence().counters.documentAnswers, 1)})+test('document turn 2: the answer row records the page the Host delivered, not a constant', async () => {+ const rows = []+ for (const page of [+ { offset: 0, length: 65_536, truncated: true },+ { offset: 0, length: 4_096, truncated: true },+ { offset: 65_536, length: 2_048, truncated: false },+ ]) {+ const h = harness()+ const result = {+ ...documentReadResult(),+ byteRange: { offset: page.offset, length: page.length },+ truncated: page.truncated,+ }+ const response = await call(+ h,+ chatBody(VISUAL_MODEL, documentAnswerMessages(wrappedToolOutput(result)), {+ tools: [readToolDefinition],+ })+ )+ assert.equal(response.status, 200)+ const [row] = h.getEvidence().attempts+ assert.equal(row.responseKind, 'document-answer')+ rows.push({ byteRange: row.byteRange, truncated: row.truncated })+ }+ assert.deepEqual(rows, [+ { byteRange: { offset: 0, length: 65_536 }, truncated: true },+ { byteRange: { offset: 0, length: 4_096 }, truncated: true },+ { byteRange: { offset: 65_536, length: 2_048 }, truncated: false },+ ])+})++test('document turn 2: a page range or truncation flag the fixture cannot read is refused', async () => {+ // Liveness witness: the well-formed result is answered.+ const twin = harness()+ const twinResponse = await call(+ twin,+ chatBody(VISUAL_MODEL, documentAnswerMessages(wrappedToolOutput(documentReadResult())), {+ tools: [readToolDefinition],+ })+ )+ assert.equal(twinResponse.status, 200)+ assert.equal(twin.getEvidence().counters.documentAnswers, 1)++ const valid = documentReadResult()+ const withoutTruncated = { ...valid }+ delete withoutTruncated.truncated+ const withoutRange = { ...valid }+ delete withoutRange.byteRange+ for (const [label, result] of [+ ['missing byteRange', withoutRange],+ ['null byteRange', { ...valid, byteRange: null }],+ ['array byteRange', { ...valid, byteRange: [0, 34] }],+ ['missing offset', { ...valid, byteRange: { length: 34 } }],+ ['negative offset', { ...valid, byteRange: { offset: -1, length: 34 } }],+ ['fractional length', { ...valid, byteRange: { offset: 0, length: 1.5 } }],+ ['string length', { ...valid, byteRange: { offset: 0, length: '34' } }],+ ['unsafe length', { ...valid, byteRange: { offset: 0, length: 2 ** 53 } }],+ ['missing truncated', withoutTruncated],+ ['string truncated', { ...valid, truncated: 'false' }],+ ]) {+ const h = harness()+ const response = await call(+ h,+ chatBody(VISUAL_MODEL, documentAnswerMessages(wrappedToolOutput(result)), {+ tools: [readToolDefinition],+ })+ )+ assert.equal(response.status, 400, label)+ assert.equal((await response.json()).error.code, 'document-page-range-malformed', label)+ const evidence = h.getEvidence()+ assert.equal(evidence.counters.documentFailures, 1, label)+ assert.equal(evidence.counters.documentAnswers, 0, label)+ assert.deepEqual(+ evidence.attempts,+ [+ {+ model: VISUAL_MODEL,+ imageSha256: null,+ responseKind: 'rejected',+ reason: 'document-page-range-malformed',+ },+ ],+ label+ )+ }+})+test('document turn 2: a different delivered text gives a different answer (not a constant)', async () => {const answers = []for (const text of [DOCUMENT_TEXT, `${DOCUMENT_TEXT}changed\n`]) {@@ -978,3 +1073,157 @@ test('image rows keep their original shape (no documentSha256)', async () => {assert.equal('documentSha256' in h.getEvidence().attempts[0], false)})++/**+ * Declared here, not imported, so the oracle stays independent of the producer:+ * a reason added to the fixture without a test fails the parity check below.+ */+const EXPECTED_REJECTION_REASONS = [+ 'document-byte-length-malformed',+ 'document-page-range-malformed',+ 'document-stream-unsupported',+ 'document-tool-output-malformed',+ 'document-tool-output-unreadable',+ 'document-tool-result-count',+ 'image-not-png-data-uri',+ 'image-part-count',+ 'image-pixel-not-a-tile-color',+ 'image-png-malformed',+ 'image-png-unsupported-form',+ 'image-tile-grid-mismatch',+ 'provider-auth-mismatch',+ 'provider-body-invalid',+ 'provider-body-unsupported',+ 'provider-method-unsupported',+ 'provider-path-not-captured',+ 'provider-request-form-unsupported',+ 'text-model-image-incompatible',+ 'unsupported-model',+]++test('the fixture publishes exactly the closed set of rejection reasons, frozen', () => {+ const reasons = providerModule.FIXTURE_REJECTION_REASONS+ assert.ok(Array.isArray(reasons), 'FIXTURE_REJECTION_REASONS is exported as an array')+ assert.deepEqual([...reasons].sort(), EXPECTED_REJECTION_REASONS)+ assert.equal(Object.isFrozen(reasons), true)+})++test('every rejected row records its closed-set reason and no request content', async () => {+ const { png } = palettePng()+ const validBody = JSON.stringify(chatBody(VISUAL_MODEL, imageMessages(png)))+ const twoImages = imageMessages(png)+ twoImages[0].content.push(twoImages[0].content[1])+ const fetchWith = (url, init) => h => h.fetch(url, init)+ const cases = [+ [+ 'request form',+ h => h.fetch(new Request(PROVIDER_URL, { method: 'POST', body: validBody })),+ null,+ 'provider-request-form-unsupported',+ ],+ [+ 'GET method',+ fetchWith(PROVIDER_URL, { method: 'GET', headers: authHeaders(), body: validBody }),+ null,+ 'provider-method-unsupported',+ ],+ [+ 'query string',+ fetchWith(`${PROVIDER_URL}?x=1`, { method: 'POST', headers: authHeaders(), body: validBody }),+ null,+ 'provider-path-not-captured',+ ],+ [+ 'wrong key',+ fetchWith(PROVIDER_URL, {+ method: 'POST',+ headers: { authorization: 'Bearer not-the-run-key' },+ body: validBody,+ }),+ null,+ 'provider-auth-mismatch',+ ],+ [+ 'buffer body',+ fetchWith(PROVIDER_URL, {+ method: 'POST',+ headers: authHeaders(),+ body: Buffer.from(validBody),+ }),+ null,+ 'provider-body-unsupported',+ ],+ [+ 'invalid json',+ fetchWith(PROVIDER_URL, { method: 'POST', headers: authHeaders(), body: '{' }),+ null,+ 'provider-body-invalid',+ ],+ [+ 'unknown model',+ h => call(h, chatBody('glm-5.2', textMessages())),+ 'glm-5.2',+ 'unsupported-model',+ ],+ [+ 'text model with an image',+ h => call(h, chatBody(TEXT_MODEL, imageMessages(png))),+ TEXT_MODEL,+ 'text-model-image-incompatible',+ ],+ [+ 'two image parts',+ h => call(h, chatBody(VISUAL_MODEL, twoImages)),+ VISUAL_MODEL,+ 'image-part-count',+ ],+ [+ 'streamed document turn',+ h =>+ call(+ h,+ chatBody(VISUAL_MODEL, [documentUserMessage()], {+ tools: [readToolDefinition],+ stream: true,+ })+ ),+ VISUAL_MODEL,+ 'document-stream-unsupported',+ ],+ [+ 'unreadable tool result',+ h =>+ call(+ h,+ chatBody(+ VISUAL_MODEL,+ documentAnswerMessages(wrappedToolOutput({ error: 'attachment_not_found' })),+ { tools: [readToolDefinition] }+ )+ ),+ VISUAL_MODEL,+ 'document-tool-output-unreadable',+ ],+ ]+ for (const [label, send, model, reason] of cases) {+ const h = harness()+ const response = await send(h)+ assert.ok(response.status >= 400, label)+ const evidence = h.getEvidence()+ assert.equal(evidence.counters.rejectedAttempts, 1, label)+ // The row carries the fixed reason string and nothing from the request.+ assert.deepEqual(+ evidence.attempts,+ [{ model, imageSha256: null, responseKind: 'rejected', reason }],+ label+ )+ assert.ok(EXPECTED_REJECTION_REASONS.includes(reason), label)+ }++ // Accepted rows never carry a reason; witness: the accepted row exists.+ const accepted = harness()+ assert.equal((await call(accepted, chatBody(VISUAL_MODEL, textMessages()))).status, 200)+ assert.equal(accepted.getEvidence().attempts.length, 1)+ assert.equal(accepted.getEvidence().attempts[0].responseKind, 'text-only')+ assert.equal(Object.hasOwn(accepted.getEvidence().attempts[0], 'reason'), false)+})diff --git a/tests/e2e/fixtures/image-capabilities/sdk.test.mjs b/tests/e2e/fixtures/image-capabilities/sdk.test.mjsindex c2c27713e..fa5fa6a25 100644--- a/tests/e2e/fixtures/image-capabilities/sdk.test.mjs+++ b/tests/e2e/fixtures/image-capabilities/sdk.test.mjs@@ -150,6 +150,8 @@ test('the production ZAI serializer and pinned SDK complete an attachment read aimageSha256: null,responseKind: 'document-answer',documentSha256: digest,+ byteRange: { offset: 0, length: Buffer.byteLength(documentText) },+ truncated: false,},])})diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex fd47cc8f4..c400c415d 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.104",+ "version": "1.0.105","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.104",+ "version": "1.0.105","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex f3b79da39..1f3b95d8d 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.104",+ "version": "1.0.105","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",