Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): make the attachment lanes fail on what they claim to prove

Alfredo Lopeza34e5629ae9b
diff --git a/Makefile b/Makefile
index 8bf7765b0..0afeb86e3 100644
--- a/Makefile
+++ b/Makefile
@@ -311,6 +311,7 @@ minikube-build-image-capabilities-fixture-body:
minikube-run-image-capabilities: ## Run the visible image journey with an isolated external-provider fixture and restore the Host
@T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
+ IMAGE_CAPABILITIES_LANE=image \
bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run
.PHONY: minikube-run-document-upload
diff --git a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
index e1b005734..d70615959 100644
--- a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
+++ b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
@@ -59,6 +59,34 @@ export const FIXTURE_RESPONSE_KIND = {
documentAnswer: 'document-answer',
} as const
+/**
+ * Reasons a rejected row may carry: fixed strings, never request content.
+ * Declared independently of the fixture for the same reason as the response
+ * kinds; `test/imageCapabilityEvidence.test.ts` checks the two lists agree.
+ */
+export const FIXTURE_REJECTION_REASONS: readonly string[] = Object.freeze([
+ 'provider-request-form-unsupported',
+ 'provider-path-not-captured',
+ 'provider-method-unsupported',
+ 'provider-auth-mismatch',
+ 'provider-body-unsupported',
+ 'provider-body-invalid',
+ 'unsupported-model',
+ 'text-model-image-incompatible',
+ 'image-not-png-data-uri',
+ 'image-part-count',
+ 'image-png-malformed',
+ 'image-png-unsupported-form',
+ 'image-tile-grid-mismatch',
+ 'image-pixel-not-a-tile-color',
+ 'document-byte-length-malformed',
+ 'document-tool-result-count',
+ 'document-tool-output-malformed',
+ 'document-tool-output-unreadable',
+ 'document-page-range-malformed',
+ 'document-stream-unsupported',
+])
+
/** The fixture's text-only answer. It names no image and no color. */
export const FIXTURE_TEXT_ONLY_CONTENT = 'IMAGE_FIXTURE_TEXT_OK'
@@ -275,6 +303,15 @@ export interface FixtureProviderAttempt {
* Present only on the read-request row.
*/
documentByteLength?: number
+ /**
+ * The page of the file the Host's read tool delivered. Present only on the
+ * answer row, so the oracle can hash exactly that slice of the file.
+ */
+ byteRange?: { offset: number; length: number }
+ /** Whether the Host delivered less than the whole file. Answer row only. */
+ truncated?: boolean
+ /** One of {@link FIXTURE_REJECTION_REASONS}. Present only on rejected rows. */
+ reason?: string
}
export interface FixtureEvidenceCounters {
@@ -355,6 +392,48 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu
)
}
+ const row: FixtureProviderAttempt = { model, imageSha256, responseKind }
+
+ // Present on every rejected row and on no other row.
+ if (responseKind === FIXTURE_RESPONSE_KIND.rejected) {
+ const reason = value.reason
+ if (typeof reason !== 'string' || !FIXTURE_REJECTION_REASONS.includes(reason)) {
+ throw new Error(
+ `${source}: attempts[${index}].reason must be one of ${FIXTURE_REJECTION_REASONS.join(', ')} ` +
+ `on a ${FIXTURE_RESPONSE_KIND.rejected} row`
+ )
+ }
+ row.reason = reason
+ } else if ('reason' in value) {
+ throw new Error(
+ `${source}: attempts[${index}].reason is only valid on a ${FIXTURE_RESPONSE_KIND.rejected} row`
+ )
+ }
+
+ // Present on every answer row and on no other row.
+ if (responseKind === FIXTURE_RESPONSE_KIND.documentAnswer) {
+ const byteRange = value.byteRange
+ if (!isRecord(byteRange) || !isByteCount(byteRange.offset) || !isByteCount(byteRange.length)) {
+ throw new Error(
+ `${source}: attempts[${index}].byteRange must be { offset, length } of non-negative ` +
+ `integers on a ${FIXTURE_RESPONSE_KIND.documentAnswer} row`
+ )
+ }
+ if (typeof value.truncated !== 'boolean') {
+ throw new Error(
+ `${source}: attempts[${index}].truncated must be a boolean on a ` +
+ `${FIXTURE_RESPONSE_KIND.documentAnswer} row`
+ )
+ }
+ row.byteRange = { offset: byteRange.offset, length: byteRange.length }
+ row.truncated = value.truncated
+ } else if ('byteRange' in value || 'truncated' in value) {
+ throw new Error(
+ `${source}: attempts[${index}].byteRange and truncated are only valid on a ` +
+ `${FIXTURE_RESPONSE_KIND.documentAnswer} row`
+ )
+ }
+
// Present only on document rows; an image row must not grow the field.
if ('documentSha256' in value) {
const documentSha256 = value.documentSha256
@@ -366,6 +445,7 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu
`${source}: attempts[${index}].documentSha256 must be a sha256 hex digest or null`
)
}
+ row.documentSha256 = documentSha256
// Present only on the read-request row.
if ('documentByteLength' in value) {
const documentByteLength = value.documentByteLength
@@ -380,9 +460,9 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu
`${FIXTURE_RESPONSE_KIND.documentReadRequested} row`
)
}
- return { model, imageSha256, responseKind, documentSha256, documentByteLength }
+ row.documentByteLength = documentByteLength
}
- return { model, imageSha256, responseKind, documentSha256 }
+ return row
}
if ('documentByteLength' in value) {
throw new Error(
@@ -390,11 +470,15 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu
)
}
- return { model, imageSha256, responseKind }
+ return row
+}
+
+function isByteCount(value: unknown): value is number {
+ return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0
}
/** Parses and validates the ledger; fixture fields this lane does not use are ignored. */
-function parseEvidenceSnapshot(
+export function parseEvidenceSnapshot(
rawText: string,
source: string,
expected: { runId: string; profile: string }
@@ -535,7 +619,11 @@ export function appendedAttempts(
current.imageSha256 !== row.imageSha256 ||
current.responseKind !== row.responseKind ||
current.documentSha256 !== row.documentSha256 ||
- current.documentByteLength !== row.documentByteLength
+ current.documentByteLength !== row.documentByteLength ||
+ current.reason !== row.reason ||
+ current.byteRange?.offset !== row.byteRange?.offset ||
+ current.byteRange?.length !== row.byteRange?.length ||
+ current.truncated !== row.truncated
) {
throw new Error(
`[image-capabilities] ledger row ${index} changed between reads; the ledger must be append-only.`
diff --git a/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts b/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts
index afada39b2..80821e8a7 100644
--- a/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts
+++ b/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts
@@ -157,9 +157,12 @@ function fileChips(page: Page) {
return page.getByTestId('composer-file-chip')
}
-/** The attachment list a sent user message renders (`MessageAttachmentList`). */
+/**
+ * The attachment items a sent message renders (`MessageAttachmentList`). The
+ * items are counted, not the list container, so a second file in one list fails.
+ */
function sentMessageAttachments(page: Page) {
- return page.getByLabel('Message attachments')
+ return page.getByLabel('Message attachments').locator('.message-attachment-chip')
}
/**
@@ -178,11 +181,15 @@ const LARGE_DOCUMENT_BYTES = 6 * 1024 * 1024
/**
* Default page `clerum__attachment_read` returns
* (`CLERUM_ATTACHMENT_TEXT_READ_MAX_BYTES`). The fixture digests the text of
- * the first page it receives, so a larger document is identified by the digest
- * of its first page.
+ * the page it receives and records that page's `byteRange`, so the oracle
+ * hashes exactly that slice of the file. This value is only the upper bound a
+ * first page may have; the slice itself comes from the ledger.
*/
const ATTACHMENT_READ_PAGE_BYTES = 65_536
+/** How much of the file the Host's read tool must deliver on the first read. */
+type ExpectedRead = 'whole-file' | 'first-page'
+
/** Start a blank chat and prove the thread is empty before anything is sent. */
async function startBlankChat(page: Page) {
await page.getByTestId('nav-new-chat').click()
@@ -274,23 +281,34 @@ async function attachDocument(
* which may treat intraword underscores as emphasis; the ledger below pins the
* exact digest regardless.
*/
+const DOCUMENT_ANSWER_PREFIX_PATTERN = FIXTURE_DOCUMENT_ANSWER_PREFIX.replace(/_/g, '[_-]?')
+
function documentAnswerRegex(documentSha: string): RegExp {
- const prefix = FIXTURE_DOCUMENT_ANSWER_PREFIX.replace(/_/g, '[_-]?')
- return new RegExp(`${prefix}${documentSha.slice(0, FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS)}`, 'i')
+ return new RegExp(
+ `${DOCUMENT_ANSWER_PREFIX_PATTERN}${documentSha.slice(0, FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS)}`,
+ 'i'
+ )
}
+/** Any fixture document answer: proves the turn finished before the ledger is read. */
+const ANY_DOCUMENT_ANSWER = new RegExp(
+ `${DOCUMENT_ANSWER_PREFIX_PATTERN}[a-f0-9]{${FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS}}`,
+ 'i'
+)
+
/**
* Sends the composer's single ready document and proves the round trip: the
- * sent bubble lists the file, the answer carries the digest of the text the
- * read tool returned, and the fixture ledger shows one read request carrying
- * the byte length the Host verified and one answer carrying that digest.
+ * sent bubble lists the file, the fixture ledger shows one read request
+ * carrying the byte length the Host verified and one answer carrying the page
+ * the read tool delivered, and the answer carries the digest of exactly that
+ * slice of the attached bytes.
*/
async function sendDocumentAndVerifyAnswer(
page: Page,
env: ReturnType<typeof requireImageCapabilitiesFixtureEnv>,
assertBinding: () => Promise<void>,
document: { fileName: string; buffer: Buffer },
- readTextSha: string
+ expectedRead: ExpectedRead
) {
const before = readImageCapabilityEvidence(env)
await assertBinding()
@@ -304,8 +322,7 @@ async function sendDocumentAndVerifyAnswer(
await expect(sentMessageAttachments(page)).toContainText(document.fileName)
const response = page.getByTestId('agent-response').locator('.message-block.markdown-content')
await expect(response).toHaveCount(1, { timeout: 120_000 })
- await expect(response).toContainText(documentAnswerRegex(readTextSha), { timeout: 120_000 })
- await observeCompletedTask(page, env.hostRef, documentAnswerRegex(readTextSha))
+ await expect(response).toContainText(ANY_DOCUMENT_ANSWER, { timeout: 120_000 })
const after = readImageCapabilityEvidence(env)
const appended = appendedAttempts(before, after)
@@ -322,10 +339,30 @@ async function sendDocumentAndVerifyAnswer(
expect(readRows[0]?.documentSha256).toBeNull()
// The Host announced the whole file: the byte length it verified on admission.
expect(readRows[0]?.documentByteLength).toBe(document.buffer.length)
- // Turn 2: the tool result reached the wire carrying exactly the attached bytes.
+ // Turn 2: the tool result reached the wire carrying exactly the attached bytes
+ // of the page the Host reports it delivered.
expect(answerRows).toHaveLength(1)
- expect(answerRows[0]?.model).toBe(IMAGE_CAPABILITY_FIXTURE_MODELS.supported)
- expect(answerRows[0]?.documentSha256).toBe(readTextSha)
+ const answer = answerRows[0]
+ expect(answer?.model).toBe(IMAGE_CAPABILITY_FIXTURE_MODELS.supported)
+ const range = answer?.byteRange
+ if (!range || answer?.truncated === undefined) {
+ throw new Error('[document-upload] the answer row carries no delivered page')
+ }
+ expect(range.offset).toBe(0)
+ if (expectedRead === 'whole-file') {
+ expect(answer.truncated).toBe(false)
+ expect(range.length).toBe(document.buffer.length)
+ } else {
+ expect(answer.truncated).toBe(true)
+ expect(range.length).toBeGreaterThan(0)
+ expect(range.length).toBeLessThanOrEqual(ATTACHMENT_READ_PAGE_BYTES)
+ expect(range.length).toBeLessThan(document.buffer.length)
+ }
+ const pageSha = sha256Hex(document.buffer.subarray(range.offset, range.offset + range.length))
+ expect(answer.documentSha256).toBe(pageSha)
+ // The visible answer and the recorded task name that same digest.
+ await expect(response).toContainText(documentAnswerRegex(pageSha))
+ await observeCompletedTask(page, env.hostRef, documentAnswerRegex(pageSha))
// A document is never an image: no pixels moved, and the fixture refused nothing.
expect(appended.filter(row => row.imageSha256 !== null)).toHaveLength(0)
@@ -359,7 +396,6 @@ test('document-upload fixture: an attached text file reaches the model through t
mimeType: 'text/plain',
buffer: Buffer.from(`document-upload ${randomBytes(6).toString('hex')}\nsecond line\n`, 'utf8'),
}
- const documentSha = sha256Hex(document.buffer)
// Run-unique ASCII (hex of random bytes): its first page differs from every
// other run's, and in ASCII the byte cut and the text cut of a page coincide.
@@ -369,12 +405,6 @@ test('document-upload fixture: an attached text file reaches the model through t
buffer: Buffer.from(randomBytes(LARGE_DOCUMENT_BYTES / 2).toString('hex'), 'ascii'),
}
expect(largeDocument.buffer.length).toBe(LARGE_DOCUMENT_BYTES)
- const largeDocumentPageSha = sha256Hex(
- largeDocument.buffer.subarray(0, ATTACHMENT_READ_PAGE_BYTES)
- )
-
- fs.mkdirSync(testInfo.outputPath(), { recursive: true })
- fs.writeFileSync(testInfo.outputPath(document.fileName), document.buffer)
let app: ElectronApplication | undefined
let recordedPage: Page | undefined
@@ -441,7 +471,7 @@ test('document-upload fixture: an attached text file reaches the model through t
})
await test.step('the answer carries the digest of the delivered text and the ledger shows both provider turns', async () => {
- await sendDocumentAndVerifyAnswer(page, env, assertBinding, document, documentSha)
+ await sendDocumentAndVerifyAnswer(page, env, assertBinding, document, 'whole-file')
})
await test.step('a 6 MiB document above the old text share is admitted as a ready chip', async () => {
@@ -475,13 +505,7 @@ test('document-upload fixture: an attached text file reaches the model through t
})
await test.step('the 6 MiB document is sent, read inline and answered with the digest of its first page', async () => {
- await sendDocumentAndVerifyAnswer(
- page,
- env,
- assertBinding,
- largeDocument,
- largeDocumentPageSha
- )
+ await sendDocumentAndVerifyAnswer(page, env, assertBinding, largeDocument, 'first-page')
})
await screenshotAndLog(page, testInfo, 'desktop-document-upload-fixture')
diff --git a/desktop-app/test/e2e-playwright/qa-recorder-image-capabilities.spec.ts b/desktop-app/test/e2e-playwright/qa-recorder-image-capabilities.spec.ts
index b74809b36..3cadbc41e 100644
--- a/desktop-app/test/e2e-playwright/qa-recorder-image-capabilities.spec.ts
+++ b/desktop-app/test/e2e-playwright/qa-recorder-image-capabilities.spec.ts
@@ -370,10 +370,20 @@ async function expectImageSendBlocked(
// Enter must not reach the controller either: nothing is delivered, the draft
// survives, and the chip is still pending.
await composer(page).press('Enter')
+ // Witness that the key landed where a send would start.
+ await expect(composer(page)).toBeFocused()
await expect(page.locator('[data-chat-message-id]')).toHaveCount(0)
+ await expect(sendButton(page)).toBeDisabled()
+ // Liveness witness, after the negatives: the blocked state the refusal had to
+ // leave behind is still on screen, so the empty thread is not an unmounted chat.
+ await expect(modelChip(page)).toHaveAttribute('data-model-id', model)
await expect(composer(page)).toHaveValue(draft)
await expect(attachmentChips(page).filter({ hasText: attachmentName })).toBeVisible()
- await expect(sendButton(page)).toBeDisabled()
+ await expect(notice).toBeVisible()
+ await expect(notice).toContainText(model)
+ await expect(notice).toContainText(messagePattern)
+ // The witnesses above took real time; a late delivery would show up here.
+ await expect(page.locator('[data-chat-message-id]')).toHaveCount(0)
}
/** Exactly one exchange in this chat, and the answer is not empty. */
diff --git a/desktop-app/test/imageCapabilityEvidence.test.ts b/desktop-app/test/imageCapabilityEvidence.test.ts
new file mode 100644
index 000000000..7912bc0d6
--- /dev/null
+++ b/desktop-app/test/imageCapabilityEvidence.test.ts
@@ -0,0 +1,156 @@
+import { describe, expect, it } from 'vitest'
+import path from 'node:path'
+import { pathToFileURL } from 'node:url'
+import * as evidenceModule from './e2e-playwright/helpers/imageCapabilityEvidence.js'
+import {
+ FIXTURE_RESPONSE_KIND,
+ appendedAttempts,
+ parseEvidenceSnapshot,
+} from './e2e-playwright/helpers/imageCapabilityEvidence.js'
+
+// The fixture is untyped ESM outside this package; only its reason list is read,
+// through a typed dynamic import so the compiler does not resolve the module.
+const PROVIDER_MODULE_URL = pathToFileURL(
+ path.resolve(__dirname, '../../tests/e2e/fixtures/image-capabilities/provider.mjs')
+).href
+
+const RUN_ID = 'image-capabilities-0123456789ab'
+const PROFILE = 'clerum-image-fixture-12345678'
+const SOURCE = 'unit-test ledger'
+const DIGEST = 'a'.repeat(64)
+
+type Row = Record<string, unknown>
+
+const rows = {
+ rejected: (): Row => ({
+ model: 'glm-5.3',
+ imageSha256: null,
+ responseKind: 'rejected',
+ reason: 'text-model-image-incompatible',
+ }),
+ read: (): Row => ({
+ model: 'glm-5.3-flash',
+ imageSha256: null,
+ responseKind: 'document-read-requested',
+ documentSha256: null,
+ documentByteLength: 6_291_456,
+ }),
+ answer: (): Row => ({
+ model: 'glm-5.3-flash',
+ imageSha256: null,
+ responseKind: 'document-answer',
+ documentSha256: DIGEST,
+ byteRange: { offset: 0, length: 65_536 },
+ truncated: true,
+ }),
+ textOnly: (): Row => ({ model: 'glm-5.3-flash', imageSha256: null, responseKind: 'text-only' }),
+}
+
+function ledger(attempts: Row[]): string {
+ return JSON.stringify({
+ runId: RUN_ID,
+ profile: PROFILE,
+ pid: 41,
+ counters: {
+ totalAttempts: attempts.length,
+ imageAttempts: 0,
+ textAttempts: 0,
+ rejectedAttempts: 0,
+ tileColorResponses: 0,
+ textOnlyResponses: 0,
+ textModelImageRefusals: 0,
+ blockedEgress: 0,
+ documentReadRequests: 0,
+ documentAnswers: 0,
+ documentFailures: 0,
+ },
+ attempts,
+ })
+}
+
+const parse = (attempts: Row[]) =>
+ parseEvidenceSnapshot(ledger(attempts), SOURCE, { runId: RUN_ID, profile: PROFILE })
+
+describe('imageCapabilityEvidence ledger rows', () => {
+ it('keeps the rejection reason and the delivered page on the rows that carry them', () => {
+ const snapshot = parse([rows.rejected(), rows.read(), rows.answer(), rows.textOnly()])
+ expect(snapshot.attempts).toEqual([
+ rows.rejected(),
+ rows.read(),
+ rows.answer(),
+ rows.textOnly(),
+ ])
+ })
+
+ it('requires a closed-set reason on every rejected row and nowhere else', () => {
+ // Witness: the well-formed rejected row parses.
+ expect(parse([rows.rejected()]).attempts[0]?.reason).toBe('text-model-image-incompatible')
+ const { reason: _omitted, ...withoutReason } = rows.rejected()
+ for (const [label, row] of [
+ ['missing reason', withoutReason],
+ ['free-text reason', { ...rows.rejected(), reason: 'prompt said: draw a cat' }],
+ ['non-string reason', { ...rows.rejected(), reason: 7 }],
+ ['reason on an accepted row', { ...rows.textOnly(), reason: 'unsupported-model' }],
+ ] as const) {
+ expect(() => parse([row]), label).toThrow(/reason/)
+ }
+ })
+
+ it('requires the delivered page on every document answer and nowhere else', () => {
+ // Witness: the well-formed answer row parses with its page.
+ expect(parse([rows.answer()]).attempts[0]?.byteRange).toEqual({ offset: 0, length: 65_536 })
+ const { byteRange: _range, ...withoutRange } = rows.answer()
+ const { truncated: _flag, ...withoutTruncated } = rows.answer()
+ for (const [label, row] of [
+ ['missing byteRange', withoutRange],
+ ['null byteRange', { ...rows.answer(), byteRange: null }],
+ ['array byteRange', { ...rows.answer(), byteRange: [0, 65_536] }],
+ ['missing length', { ...rows.answer(), byteRange: { offset: 0 } }],
+ ['negative offset', { ...rows.answer(), byteRange: { offset: -1, length: 10 } }],
+ ['fractional length', { ...rows.answer(), byteRange: { offset: 0, length: 1.5 } }],
+ ['string length', { ...rows.answer(), byteRange: { offset: 0, length: '10' } }],
+ ['missing truncated', withoutTruncated],
+ ['string truncated', { ...rows.answer(), truncated: 'true' }],
+ ['byteRange on a read row', { ...rows.read(), byteRange: { offset: 0, length: 10 } }],
+ ['truncated on a read row', { ...rows.read(), truncated: false }],
+ ['byteRange on a text row', { ...rows.textOnly(), byteRange: { offset: 0, length: 10 } }],
+ ] as const) {
+ expect(() => parse([row]), label).toThrow(/byteRange|truncated/)
+ }
+ })
+})
+
+describe('appendedAttempts', () => {
+ it('refuses a row whose reason or delivered page changed between reads', () => {
+ const before = parse([rows.rejected(), rows.answer()])
+ // Witness: an unchanged prefix yields the appended row.
+ expect(
+ appendedAttempts(before, parse([rows.rejected(), rows.answer(), rows.textOnly()]))
+ ).toEqual([rows.textOnly()])
+ for (const [label, changed] of [
+ ['reason', [{ ...rows.rejected(), reason: 'unsupported-model' }, rows.answer()]],
+ ['offset', [rows.rejected(), { ...rows.answer(), byteRange: { offset: 1, length: 65_536 } }]],
+ ['length', [rows.rejected(), { ...rows.answer(), byteRange: { offset: 0, length: 65_535 } }]],
+ ['truncated', [rows.rejected(), { ...rows.answer(), truncated: false }]],
+ ] as const) {
+ expect(() => appendedAttempts(before, parse([...changed])), label).toThrow(/append-only/)
+ }
+ })
+})
+
+describe('rejection reason parity', () => {
+ it('declares the same closed set as the fixture, independently', async () => {
+ const providerModule = (await import(PROVIDER_MODULE_URL)) as {
+ FIXTURE_REJECTION_REASONS: readonly string[]
+ }
+ const declared = (evidenceModule as Record<string, unknown>).FIXTURE_REJECTION_REASONS
+ expect(Array.isArray(declared)).toBe(true)
+ expect(declared).not.toBe(providerModule.FIXTURE_REJECTION_REASONS)
+ expect([...(declared as readonly string[])].sort()).toEqual(
+ [...providerModule.FIXTURE_REJECTION_REASONS].sort()
+ )
+ // Witness: the set is not empty, so equality is not two empty lists.
+ expect((declared as readonly string[]).length).toBeGreaterThan(0)
+ expect(FIXTURE_RESPONSE_KIND.rejected).toBe('rejected')
+ })
+})
diff --git a/docs/testing/desktop-headful-journeys.md b/docs/testing/desktop-headful-journeys.md
index ac0e734ff..b8e41d69c 100644
--- a/docs/testing/desktop-headful-journeys.md
+++ b/docs/testing/desktop-headful-journeys.md
@@ -18,22 +18,22 @@ QA identity from `.env.qa-recorder` (no first-available fallback), shares launch
login / proof plumbing in `qa-recorder-helpers.ts`, and records a WebM + PNG per
test under `.local-notes/qa-recorder/runs/desktop-app/` (git-ignored).
-| Journey | Spec | Tests | Confirm flag | Status |
-| --------------------- | ---------------------------------------- | ----------------------------------------------------------------------------------- | ---------------------------------------------- | ------ |
-| Auth & session | `qa-recorder-auth.spec.ts` | 3 — sign-in screen, authenticated shell, logout | — | ✅ |
-| Navigation | `qa-recorder-navigation.spec.ts` | 3 — primary sidebar, footer settings menu, resources pages | — | ✅ |
-| Agents | `qa-recorder-agents.spec.ts` | 3 — fleet, chat workspace, workspace routes | — | ✅ |
-| Connectors | `qa-recorder-connectors.spec.ts` | 1 — MCP/connector health inventory | — | ✅ |
-| Shared files | `qa-recorder-shared-files.spec.ts` | 1 — filesystem + directory browser | — | ✅ |
-| Plugins & workflows | `qa-recorder-plugins.spec.ts` | 1 — inventory, detail, runs/artifacts (+optional trigger) | `QA_RECORDER_CONFIRM_MUTATIONS` (trigger only) | ✅ |
-| Apps | `qa-recorder-apps.spec.ts` | 1 — catalog + embedded session | — | ✅ |
-| Settings | `qa-recorder-settings.spec.ts` | 2 — appearance/notifications, information/configuration (+optional endpoint switch) | `QA_RECORDER_CONFIRM_MUTATIONS` (switch only) | ✅ |
-| Inbox & search | `qa-recorder-inbox-search.spec.ts` | 2 — inbox, global search | — | ✅ |
-| Chat | `qa-recorder-chat.spec.ts` | 1 — composer, thread, task progress | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |
-| Settings + chat smoke | `qa-recorder-settings-chat.spec.ts` | 1 — Settings tabs + one chat message | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |
-| Image capabilities | `qa-recorder-image-capabilities.spec.ts` | 2 — capability gate on a text-only model, fixture-backed image answer | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |
-| Model selector | `qa-recorder-model-selector.spec.ts` | 1 — a multi-model catalog lists rows without capability tags | — | ✅ |
-| Document upload | `qa-recorder-document-upload.spec.ts` | 1 — attached text file read through the read tool (fixture lane only) | `IMAGE_CAPABILITIES_RUN_ID` (runner-set) | ✅ |
+| Journey | Spec | Tests | Confirm flag | Status |
+| --------------------- | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | ------ |
+| Auth & session | `qa-recorder-auth.spec.ts` | 3 — sign-in screen, authenticated shell, logout | — | ✅ |
+| Navigation | `qa-recorder-navigation.spec.ts` | 3 — primary sidebar, footer settings menu, resources pages | — | ✅ |
+| Agents | `qa-recorder-agents.spec.ts` | 3 — fleet, chat workspace, workspace routes | — | ✅ |
+| Connectors | `qa-recorder-connectors.spec.ts` | 1 — MCP/connector health inventory | — | ✅ |
+| Shared files | `qa-recorder-shared-files.spec.ts` | 1 — filesystem + directory browser | — | ✅ |
+| Plugins & workflows | `qa-recorder-plugins.spec.ts` | 1 — inventory, detail, runs/artifacts (+optional trigger) | `QA_RECORDER_CONFIRM_MUTATIONS` (trigger only) | ✅ |
+| Apps | `qa-recorder-apps.spec.ts` | 1 — catalog + embedded session | — | ✅ |
+| Settings | `qa-recorder-settings.spec.ts` | 2 — appearance/notifications, information/configuration (+optional endpoint switch) | `QA_RECORDER_CONFIRM_MUTATIONS` (switch only) | ✅ |
+| Inbox & search | `qa-recorder-inbox-search.spec.ts` | 2 — inbox, global search | — | ✅ |
+| Chat | `qa-recorder-chat.spec.ts` | 1 — composer, thread, task progress | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |
+| Settings + chat smoke | `qa-recorder-settings-chat.spec.ts` | 1 — Settings tabs + one chat message | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |
+| Image capabilities | `qa-recorder-image-capabilities.spec.ts` | 2 — capability gate on a text-only model, fixture-backed image answer | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |
+| Model selector | `qa-recorder-model-selector.spec.ts` | 1 — a multi-model catalog lists rows without capability tags | — | ✅ |
+| Document upload | `qa-recorder-document-upload.spec.ts` | 1 — attached text file read through the read tool (fixture lane only); a small file is answered with its whole-file digest, and a 6 MiB file with the digest of the first page the Host delivered (`byteRange` and `truncated: true` from the fixture ledger) | `IMAGE_CAPABILITIES_RUN_ID` (runner-set) | ✅ |
**Totals: 14 specs, 23 tests.** Read-only journeys need no confirmation flag; every
journey still calls the loopback health guard on both API URLs — except against a
diff --git a/scripts/e2e/image-capabilities-fixture.mjs b/scripts/e2e/image-capabilities-fixture.mjs
index e0e49e59a..7103ba830 100644
--- a/scripts/e2e/image-capabilities-fixture.mjs
+++ b/scripts/e2e/image-capabilities-fixture.mjs
@@ -6,6 +6,7 @@ import { pathToFileURL } from 'node:url'
import { isDeepStrictEqual } from 'node:util'
import {
openOwnedFile,
+ readOwnedDescriptor,
readOwnedFile,
validateRunDirectory,
} from './prepare-codex-approved-tools.mjs'
@@ -28,13 +29,18 @@ export const fixtureConfigKeys = [
* an environment value can never name an arbitrary config path.
*/
export const playwrightLanes = {
+ // `grep` selects the lane's fixture journey. The image spec also holds the
+ // real-provider smoke, which skips itself in fixture mode; selecting by title
+ // keeps it out of the run instead of reporting it as skipped.
image: {
config: 'desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',
label: 'image-capabilities-playwright',
+ grep: 'image-capabilities fixture: ',
},
'document-upload': {
config: 'desktop-app/test/e2e-playwright/playwright.document-upload.config.ts',
label: 'document-upload-playwright',
+ grep: 'document-upload fixture: ',
},
}
@@ -47,6 +53,61 @@ export function playwrightLaneConfig(env) {
return { lane, ...playwrightLanes[lane] }
}
+/**
+ * Only `run` drives Playwright, so only `run` resolves the lane. `restore`
+ * must reach the owned runtime even when the lane variable is stale or wrong.
+ */
+export function selectFixtureAction(argument, env) {
+ const action = argument ?? 'run'
+ if (!['run', 'restore'].includes(action)) throw new Error('Expected run or restore')
+ if (action === 'restore') return { action }
+ return { action, ...playwrightLaneConfig(env) }
+}
+
+const isCount = value => Number.isSafeInteger(value) && value >= 0
+
+/**
+ * A zero exit from the Playwright CLI is not a verdict on its own: a run that
+ * collected nothing, skipped everything or retried into a pass also exits 0.
+ * PASS needs the JSON report of this lane's config with at least one expected
+ * test and no unexpected, skipped or flaky outcome.
+ */
+export function playwrightVerdict(report, { lane, configFile }) {
+ const { grep } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })
+ const reported = report?.config?.configFile
+ if (typeof reported !== 'string' || fs.realpathSync(reported) !== fs.realpathSync(configFile))
+ throw new Error('Playwright report config mismatch')
+ const stats = report.stats ?? {}
+ if (!isCount(stats.expected) || stats.expected < 1)
+ throw new Error('Playwright ran no expected tests')
+ for (const outcome of ['unexpected', 'skipped', 'flaky'])
+ if (stats[outcome] !== 0)
+ throw new Error(`Playwright report is not clean: ${outcome}=${String(stats[outcome])}`)
+ if (!Array.isArray(report.errors) || report.errors.length)
+ throw new Error('Playwright report has global errors')
+ const titles = []
+ const collect = suites => {
+ if (!Array.isArray(suites)) throw new Error('Incomplete Playwright suites')
+ for (const suite of suites) {
+ for (const spec of suite.specs ?? []) titles.push(spec.title)
+ collect(suite.suites ?? [])
+ }
+ }
+ collect(report.suites)
+ if (titles.length !== stats.expected || titles.some(title => !title.startsWith(grep)))
+ throw new Error(`Playwright report has specs outside the ${lane} lane`)
+ return 'PASS'
+}
+
+/** The PASS line is printed from the persisted state, never from memory. */
+export function fixturePassLine(state, { lane, evidence }) {
+ if (!Object.hasOwn(state, 'lane') || state.lane !== lane)
+ throw new Error('Run state lane mismatch')
+ if (state.playwright !== 'PASS') throw new Error('Run state has no Playwright PASS')
+ if (state.restored !== true) throw new Error('Run state was not restored')
+ return `IMAGE_CAPABILITIES_E2E_PASS lane=${lane} evidence=${evidence}\n`
+}
+
export function proveImages(manifest, observed, head, profile) {
if (!/^[a-f0-9]{40}$/.test(head) || manifest.profile !== profile)
throw new Error('Fixture image ownership mismatch')
@@ -182,7 +243,13 @@ async function main() {
)
throw new Error('Matching branch profile and context required')
command('bash', ['scripts/minikube/require-t2-mutation-lock.sh'])
- const { lane, config: laneConfig, label: laneLabel } = playwrightLaneConfig(process.env)
+ const {
+ action,
+ lane,
+ config: laneConfig,
+ label: laneLabel,
+ grep: laneGrep,
+ } = selectFixtureAction(process.argv[2], process.env)
const head = command('git', ['rev-parse', 'HEAD']).trim()
const branch = command('git', ['branch', '--show-current']).trim()
if (command('git', ['status', '--porcelain']).trim()) throw new Error('Clean checkout required')
@@ -248,8 +315,6 @@ async function main() {
}
throw new Error('HCC did not reconcile the expected Host image')
}
- const action = process.argv[2] ?? 'run'
- if (!['run', 'restore'].includes(action)) throw new Error('Expected run or restore')
const runRoot = path.join(canonical, '.local-notes/infra/runs')
fs.mkdirSync(runRoot, { recursive: true })
const runId = `image-capabilities-${randomBytes(6).toString('hex')}`
@@ -263,6 +328,7 @@ async function main() {
? JSON.parse(readOwnedFile(evidence, 'state.json'))
: {
runId,
+ lane,
branch,
profile,
head,
@@ -742,28 +808,50 @@ async function main() {
}
command('npm', ['run', 'verify:electron', '--prefix', 'desktop-app'])
process.stdout.write(`Image capability fixture ready: ${state.runId} (lane ${lane})\n`)
- const output = command(
- 'node',
- [
- 'scripts/minikube/run-with-deadline.mjs',
- '--timeout-seconds',
- '900',
- '--heartbeat-seconds',
- '20',
- '--kill-grace-seconds',
- '5',
- '--label',
- laneLabel,
- '--',
+ // Playwright's JSON reporter writes this reserved inode with writeFile. Keep
+ // the descriptor until the verdict; a replacement path cannot supply proof.
+ const reportFile = openOwnedFile(evidence, 'playwright-report.json', {
+ create: true,
+ maxBytes: 16 * 1024 * 1024,
+ })
+ try {
+ const output = command(
'node',
- 'desktop-app/node_modules/@playwright/test/cli.js',
- 'test',
- `--config=${laneConfig}`,
- ],
- { env: environment, timeout: 930_000 }
- )
- report(output)
- state.playwright = 'PASS'
+ [
+ 'scripts/minikube/run-with-deadline.mjs',
+ '--timeout-seconds',
+ '900',
+ '--heartbeat-seconds',
+ '20',
+ '--kill-grace-seconds',
+ '5',
+ '--label',
+ laneLabel,
+ '--',
+ 'node',
+ 'desktop-app/node_modules/@playwright/test/cli.js',
+ 'test',
+ `--config=${laneConfig}`,
+ `--grep=${laneGrep}`,
+ // `list` keeps the configured console output; `json` is the verdict.
+ '--reporter=list,json',
+ ],
+ {
+ env: {
+ ...environment,
+ PLAYWRIGHT_JSON_OUTPUT_FILE: path.join(evidence, 'playwright-report.json'),
+ },
+ timeout: 930_000,
+ }
+ )
+ report(output)
+ state.playwright = playwrightVerdict(JSON.parse(readOwnedDescriptor(reportFile)), {
+ lane,
+ configFile: path.join(root, laneConfig),
+ })
+ } finally {
+ fs.closeSync(reportFile.fd)
+ }
save()
} catch (error) {
failure = error
@@ -809,7 +897,9 @@ async function main() {
}
uninstallSignalRestore()
if (failure) throw failure
- process.stdout.write(`IMAGE_CAPABILITIES_E2E_PASS evidence=${evidence}\n`)
+ process.stdout.write(
+ fixturePassLine(JSON.parse(readOwnedFile(evidence, 'state.json')), { lane, evidence })
+ )
}
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
diff --git a/scripts/e2e/image-capabilities-fixture.test.mjs b/scripts/e2e/image-capabilities-fixture.test.mjs
index 22c5e9c6b..d58565884 100644
--- a/scripts/e2e/image-capabilities-fixture.test.mjs
+++ b/scripts/e2e/image-capabilities-fixture.test.mjs
@@ -1,18 +1,43 @@
import assert from 'node:assert/strict'
import { EventEmitter } from 'node:events'
+import path from 'node:path'
import test from 'node:test'
+import { fileURLToPath } from 'node:url'
import {
baseImage,
fixtureImage,
+ fixturePassLine,
installSignalRestore,
modelInputs,
playwrightLaneConfig,
+ playwrightVerdict,
proveImages,
requireOwnedResource,
runAnnotation,
sanitizeFixtureReport,
+ selectFixtureAction,
} from './image-capabilities-fixture.mjs'
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
+const laneConfigFile = lane =>
+ path.join(repoRoot, playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane }).config)
+function playwrightReport({ lane = 'image', stats = {}, errors = [], titles } = {}) {
+ const { grep } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })
+ const specTitles = titles ?? [`${grep}a journey`]
+ return {
+ config: { configFile: laneConfigFile(lane) },
+ suites: [
+ {
+ title: 'qa-recorder.spec.ts',
+ specs: specTitles.map(title => ({ title, ok: true })),
+ suites: [],
+ },
+ ],
+ errors,
+ stats: { expected: specTitles.length, skipped: 0, unexpected: 0, flaky: 0, ...stats },
+ }
+}
+
const head = 'a'.repeat(40)
const profile = 'clerum-image-fixture-12345678'
const ids = { [baseImage]: `sha256:${'b'.repeat(64)}`, [fixtureImage]: `sha256:${'c'.repeat(64)}` }
@@ -30,11 +55,13 @@ test('the Playwright lane defaults to the image journey and accepts only known l
lane: 'image',
config: 'desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',
label: 'image-capabilities-playwright',
+ grep: 'image-capabilities fixture: ',
})
assert.deepEqual(playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'document-upload' }), {
lane: 'document-upload',
config: 'desktop-app/test/e2e-playwright/playwright.document-upload.config.ts',
label: 'document-upload-playwright',
+ grep: 'document-upload fixture: ',
})
for (const lane of ['', 'documents', '../evil.config.ts', '__proto__', 'toString']) {
assert.throws(
@@ -45,6 +72,127 @@ test('the Playwright lane defaults to the image journey and accepts only known l
}
})
+test('restore never resolves the Playwright lane, so a stale or invalid lane cannot block restoration', () => {
+ for (const lane of [undefined, 'image', 'typo', '', '__proto__', '../evil.config.ts']) {
+ const env = lane === undefined ? {} : { IMAGE_CAPABILITIES_LANE: lane }
+ // Witness: the restore path was selected, and it carries no lane at all.
+ assert.deepEqual(selectFixtureAction('restore', env), { action: 'restore' }, String(lane))
+ }
+})
+
+test('run still resolves and validates the Playwright lane', () => {
+ assert.deepEqual(selectFixtureAction(undefined, {}), {
+ action: 'run',
+ ...playwrightLaneConfig({}),
+ })
+ assert.deepEqual(selectFixtureAction('run', { IMAGE_CAPABILITIES_LANE: 'document-upload' }), {
+ action: 'run',
+ ...playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'document-upload' }),
+ })
+ for (const lane of ['typo', '', '__proto__']) {
+ assert.throws(
+ () => selectFixtureAction('run', { IMAGE_CAPABILITIES_LANE: lane }),
+ /IMAGE_CAPABILITIES_LANE must be one of image, document-upload/,
+ lane
+ )
+ }
+ assert.throws(() => selectFixtureAction('cleanup', {}), /Expected run or restore/)
+})
+
+test('the PASS line names the lane only when the persisted state was produced by that lane', () => {
+ const evidence = '/runs/image-capabilities-0123456789ab'
+ const state = { lane: 'document-upload', playwright: 'PASS', restored: true }
+ // Witness: a state written by the requested lane yields the lane-bearing PASS line.
+ assert.equal(
+ fixturePassLine(state, { lane: 'document-upload', evidence }),
+ `IMAGE_CAPABILITIES_E2E_PASS lane=document-upload evidence=${evidence}\n`
+ )
+ for (const [label, mutated] of [
+ ['other lane', { ...state, lane: 'image' }],
+ ['missing lane', { playwright: 'PASS', restored: true }],
+ [
+ 'inherited lane',
+ Object.assign(Object.create({ lane: 'document-upload' }), {
+ playwright: 'PASS',
+ restored: true,
+ }),
+ ],
+ ]) {
+ assert.throws(
+ () => fixturePassLine(mutated, { lane: 'document-upload', evidence }),
+ /Run state lane mismatch/,
+ label
+ )
+ }
+ assert.throws(
+ () =>
+ fixturePassLine({ ...state, playwright: undefined }, { lane: 'document-upload', evidence }),
+ /Run state has no Playwright PASS/
+ )
+ assert.throws(
+ () => fixturePassLine({ ...state, restored: false }, { lane: 'document-upload', evidence }),
+ /Run state was not restored/
+ )
+})
+
+test('the Playwright verdict is PASS only for a complete, clean report from the lane config', () => {
+ // Witness: a clean run of the requested lane is PASS.
+ assert.equal(
+ playwrightVerdict(playwrightReport(), { lane: 'image', configFile: laneConfigFile('image') }),
+ 'PASS'
+ )
+ assert.equal(
+ playwrightVerdict(playwrightReport({ lane: 'document-upload' }), {
+ lane: 'document-upload',
+ configFile: laneConfigFile('document-upload'),
+ }),
+ 'PASS'
+ )
+ const options = { lane: 'image', configFile: laneConfigFile('image') }
+ for (const [label, report, pattern] of [
+ ['zero tests', playwrightReport({ titles: [] }), /Playwright ran no expected tests/],
+ [
+ 'all skipped',
+ playwrightReport({ stats: { expected: 0, skipped: 1 } }),
+ /Playwright ran no expected tests/,
+ ],
+ ['one skipped', playwrightReport({ stats: { skipped: 1 } }), /skipped=1/],
+ ['one unexpected', playwrightReport({ stats: { unexpected: 1 } }), /unexpected=1/],
+ ['one flaky', playwrightReport({ stats: { flaky: 1 } }), /flaky=1/],
+ [
+ 'non-integer count',
+ playwrightReport({ stats: { expected: '1' } }),
+ /Playwright ran no expected tests/,
+ ],
+ [
+ 'missing stats',
+ { ...playwrightReport(), stats: undefined },
+ /Playwright ran no expected tests/,
+ ],
+ ['global error', playwrightReport({ errors: [{ message: 'boom' }] }), /global errors/],
+ [
+ 'other lane config',
+ { ...playwrightReport(), config: { configFile: laneConfigFile('document-upload') } },
+ /config mismatch/,
+ ],
+ ['missing config', { ...playwrightReport(), config: {} }, /config mismatch/],
+ [
+ 'spec outside the lane selection',
+ playwrightReport({
+ titles: ['image-capabilities fixture: a journey', 'optional QA recorder: real'],
+ }),
+ /outside the image lane/,
+ ],
+ [
+ 'malformed suites',
+ { ...playwrightReport(), suites: undefined },
+ /Incomplete Playwright suites/,
+ ],
+ ]) {
+ assert.throws(() => playwrightVerdict(report, options), pattern, label)
+ }
+})
+
test('requires both exact source revisions and the live derived-base identity', () => {
assert.deepEqual(proveImages(manifest(), ids, head, profile), { head, profile, images: ids })
for (const mutate of [
diff --git a/scripts/tests/test-minikube-pre-gate-shadow.sh b/scripts/tests/test-minikube-pre-gate-shadow.sh
index 8dd9a1126..ad14139ca 100755
--- a/scripts/tests/test-minikube-pre-gate-shadow.sh
+++ b/scripts/tests/test-minikube-pre-gate-shadow.sh
@@ -451,11 +451,16 @@ STUB
cleanup_fixture "$d"
}
-# minikube-verify-images only reads the manifest and the daemon; its recipe has
-# no lock wrapper, so T2_SKIP_LOCK / T2_LOCK_TOKEN on that call do nothing and
-# read as if it needed the lease. It is called without them. If the recipe ever
-# gains a lock wrapper, the Makefile check below fails first and the call must
-# pass the lease like the build and pull calls above.
+# The minikube-verify-images recipe runs `build-images.sh --verify-only` with no
+# mutation-lock wrapper, and build-images.sh returns from
+# require_inherited_mutation_lease before any lease check when --verify-only is
+# set. T2_SKIP_LOCK / T2_LOCK_TOKEN on that call would therefore be ignored and
+# would read as if it needed the lease, so it is called without them. This test
+# checks the call sites and the Makefile recipe text only (the --verify-only
+# flag is present and no mutation-lock wrapper is); it does not run
+# build-images.sh, so the script-side skip is not asserted here. If the recipe
+# ever gains a lock wrapper, the recipe check below fails and the call must pass
+# the lease like the build and pull calls above.
assert_verify_images_is_called_without_the_lock_lease() {
local d out_local out_ghcr rc_local rc_ghcr verify_calls bare_calls recipe
recipe="$(awk '/^minikube-verify-images:/{f=1;next} f&&/^[^\t]/{exit} f' "$REPO_ROOT/Makefile")"
diff --git a/tests/e2e/fixtures/image-capabilities/provider.mjs b/tests/e2e/fixtures/image-capabilities/provider.mjs
index cc6ab178b..85bf71b60 100644
--- a/tests/e2e/fixtures/image-capabilities/provider.mjs
+++ b/tests/e2e/fixtures/image-capabilities/provider.mjs
@@ -32,7 +32,11 @@
* carries the tool result, and the answer is `DOCUMENT_FIXTURE_SHA256:` plus the
* first 16 hex characters of the SHA-256 of the delivered text. A run whose
* document never reached the provider cannot produce that digest. Document rows
- * add `documentSha256` (the digest of the delivered text, never the text).
+ * add `documentSha256` (the digest of the delivered text, never the text); the
+ * answer row also records the page the Host delivered (`byteRange` and
+ * `truncated`), so the oracle hashes that exact slice of the file. Rejected rows
+ * add a `reason` drawn from `FIXTURE_REJECTION_REASONS`, a closed set of fixed
+ * strings that never carries request content.
*
* Determinism and sanitation. Responses carry a synthetic completion id derived
* from the run id, a fixed test usage block, and `finish_reason: 'stop'` (or
@@ -131,6 +135,33 @@ const TILE_COLOR_NAMES = new Map(
TILE_PALETTE.map(([name, red, green, blue]) => [`${red},${green},${blue}`, name])
)
+/**
+ * Every reason a rejected ledger row may carry. Each is a fixed string chosen by
+ * the fixture, so a rejected row can never echo a header, prompt or payload.
+ */
+export const FIXTURE_REJECTION_REASONS = Object.freeze([
+ 'provider-request-form-unsupported',
+ 'provider-path-not-captured',
+ 'provider-method-unsupported',
+ 'provider-auth-mismatch',
+ 'provider-body-unsupported',
+ 'provider-body-invalid',
+ 'unsupported-model',
+ 'text-model-image-incompatible',
+ 'image-not-png-data-uri',
+ 'image-part-count',
+ 'image-png-malformed',
+ 'image-png-unsupported-form',
+ 'image-tile-grid-mismatch',
+ 'image-pixel-not-a-tile-color',
+ 'document-byte-length-malformed',
+ 'document-tool-result-count',
+ 'document-tool-output-malformed',
+ 'document-tool-output-unreadable',
+ 'document-page-range-malformed',
+ 'document-stream-unsupported',
+])
+
/** A refusal with a stable machine-readable reason code for the evidence tally. */
class FixtureImageError extends Error {
constructor(code, message) {
@@ -459,9 +490,30 @@ function classifyDocumentTurn(body) {
'document-tool-output-unreadable',
'the tool result does not carry the document text'
)
- return { stage: 'answer', text: result.text }
+ const range = result.byteRange
+ if (
+ !range ||
+ typeof range !== 'object' ||
+ Array.isArray(range) ||
+ !isByteCount(range.offset) ||
+ !isByteCount(range.length) ||
+ typeof result.truncated !== 'boolean'
+ )
+ throw new FixtureDocumentError(
+ 'document-page-range-malformed',
+ 'the tool result does not carry a readable byteRange and truncated flag'
+ )
+ // Only the two integers and the flag are copied: the ledger never holds text.
+ return {
+ stage: 'answer',
+ text: result.text,
+ byteRange: { offset: range.offset, length: range.length },
+ truncated: result.truncated,
+ }
}
+const isByteCount = value => Number.isSafeInteger(value) && value >= 0
+
const INTERNAL_HOST_SUFFIXES = [
'.svc.cluster.local',
'.svc',
@@ -720,18 +772,17 @@ export function createImageFixtureFetch(
* request was refused before its body was parsed, and `imageSha256` is null
* whenever the attempt carried no usable image.
*/
- const recordCall = (model, imageSha256, responseKind, documentSha256, documentByteLength) => {
- // Only document rows carry `documentSha256`, and only the read row carries
- // `documentByteLength`, so the image rows keep the exact shape they always had.
- const row =
- documentSha256 === undefined
- ? { model, imageSha256, responseKind }
- : { model, imageSha256, responseKind, documentSha256 }
- state.attempts.push(documentByteLength === undefined ? row : { ...row, documentByteLength })
+ const recordCall = (model, imageSha256, responseKind, fields = {}) => {
+ // Only document rows carry document fields and only rejected rows carry a
+ // `reason`, so the image rows keep the exact shape they always had.
+ state.attempts.push({ model, imageSha256, responseKind, ...fields })
}
const refuse = (model, reason, { status = 400, code = reason, imageSha256 = null } = {}) => {
+ // The ledger records a fixed string from the closed set, never request text.
+ if (!FIXTURE_REJECTION_REASONS.includes(reason))
+ throw new Error(`Fixture rejection reason is not in the closed set: ${reason}`)
state.counters.rejectedAttempts += 1
- recordCall(model, imageSha256, 'rejected')
+ recordCall(model, imageSha256, 'rejected', { reason })
publish()
return fixtureErrorResponse(status, code, reason)
}
@@ -739,11 +790,11 @@ export function createImageFixtureFetch(
model,
content,
responseKind,
- { imageSha256 = null, documentSha256, stream = false } = {}
+ { imageSha256 = null, document, stream = false } = {}
) => {
sequence += 1
const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`
- recordCall(model, imageSha256, responseKind, documentSha256)
+ recordCall(model, imageSha256, responseKind, document)
publish()
return stream
? streamResponse(completionId, model, content)
@@ -831,7 +882,10 @@ export function createImageFixtureFetch(
state.counters.documentReadRequests += 1
sequence += 1
const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`
- recordCall(model, null, 'document-read-requested', null, documentTurn.byteLength)
+ recordCall(model, null, 'document-read-requested', {
+ documentSha256: null,
+ documentByteLength: documentTurn.byteLength,
+ })
publish()
return jsonResponse(
toolCallCompletionBody(completionId, model, ATTACHMENT_READ_TOOL, {
@@ -847,7 +901,14 @@ export function createImageFixtureFetch(
model,
`${DOCUMENT_ANSWER_PREFIX}${documentSha256.slice(0, DOCUMENT_ANSWER_DIGEST_CHARS)}`,
'document-answer',
- { documentSha256 }
+ {
+ // The page the Host delivered, so the oracle hashes that exact slice.
+ document: {
+ documentSha256,
+ byteRange: documentTurn.byteRange,
+ truncated: documentTurn.truncated,
+ },
+ }
)
}
state.counters.textOnlyResponses += 1
diff --git a/tests/e2e/fixtures/image-capabilities/provider.test.mjs b/tests/e2e/fixtures/image-capabilities/provider.test.mjs
index 5e74b3c20..a527950d2 100644
--- a/tests/e2e/fixtures/image-capabilities/provider.test.mjs
+++ b/tests/e2e/fixtures/image-capabilities/provider.test.mjs
@@ -16,6 +16,7 @@ import { createHash, randomBytes } from 'node:crypto'
import { readFileSync } from 'node:fs'
import test from 'node:test'
import { crc32, deflateSync } from 'node:zlib'
+import * as providerModule from './provider.mjs'
import { createImageFixtureFetch, validateFixtureEnvironment } from './provider.mjs'
/**
@@ -383,7 +384,12 @@ test('the text-only model refuses an image and records the incompatible attempt'
assert.equal(evidence.counters.rejectedAttempts, 1)
assert.equal(evidence.counters.tileColorResponses, 0)
assert.deepEqual(evidence.attempts, [
- { model: TEXT_MODEL, imageSha256: null, responseKind: 'rejected' },
+ {
+ model: TEXT_MODEL,
+ imageSha256: null,
+ responseKind: 'rejected',
+ reason: 'text-model-image-incompatible',
+ },
])
})
@@ -867,11 +873,100 @@ test('document turn 2: the answer is a digest of the delivered text and the ledg
imageSha256: null,
responseKind: 'document-answer',
documentSha256: digest,
+ byteRange: { offset: 0, length: Buffer.byteLength(DOCUMENT_TEXT) },
+ truncated: false,
},
])
assert.equal(h.getEvidence().counters.documentAnswers, 1)
})
+test('document turn 2: the answer row records the page the Host delivered, not a constant', async () => {
+ const rows = []
+ for (const page of [
+ { offset: 0, length: 65_536, truncated: true },
+ { offset: 0, length: 4_096, truncated: true },
+ { offset: 65_536, length: 2_048, truncated: false },
+ ]) {
+ const h = harness()
+ const result = {
+ ...documentReadResult(),
+ byteRange: { offset: page.offset, length: page.length },
+ truncated: page.truncated,
+ }
+ const response = await call(
+ h,
+ chatBody(VISUAL_MODEL, documentAnswerMessages(wrappedToolOutput(result)), {
+ tools: [readToolDefinition],
+ })
+ )
+ assert.equal(response.status, 200)
+ const [row] = h.getEvidence().attempts
+ assert.equal(row.responseKind, 'document-answer')
+ rows.push({ byteRange: row.byteRange, truncated: row.truncated })
+ }
+ assert.deepEqual(rows, [
+ { byteRange: { offset: 0, length: 65_536 }, truncated: true },
+ { byteRange: { offset: 0, length: 4_096 }, truncated: true },
+ { byteRange: { offset: 65_536, length: 2_048 }, truncated: false },
+ ])
+})
+
+test('document turn 2: a page range or truncation flag the fixture cannot read is refused', async () => {
+ // Liveness witness: the well-formed result is answered.
+ const twin = harness()
+ const twinResponse = await call(
+ twin,
+ chatBody(VISUAL_MODEL, documentAnswerMessages(wrappedToolOutput(documentReadResult())), {
+ tools: [readToolDefinition],
+ })
+ )
+ assert.equal(twinResponse.status, 200)
+ assert.equal(twin.getEvidence().counters.documentAnswers, 1)
+
+ const valid = documentReadResult()
+ const withoutTruncated = { ...valid }
+ delete withoutTruncated.truncated
+ const withoutRange = { ...valid }
+ delete withoutRange.byteRange
+ for (const [label, result] of [
+ ['missing byteRange', withoutRange],
+ ['null byteRange', { ...valid, byteRange: null }],
+ ['array byteRange', { ...valid, byteRange: [0, 34] }],
+ ['missing offset', { ...valid, byteRange: { length: 34 } }],
+ ['negative offset', { ...valid, byteRange: { offset: -1, length: 34 } }],
+ ['fractional length', { ...valid, byteRange: { offset: 0, length: 1.5 } }],
+ ['string length', { ...valid, byteRange: { offset: 0, length: '34' } }],
+ ['unsafe length', { ...valid, byteRange: { offset: 0, length: 2 ** 53 } }],
+ ['missing truncated', withoutTruncated],
+ ['string truncated', { ...valid, truncated: 'false' }],
+ ]) {
+ const h = harness()
+ const response = await call(
+ h,
+ chatBody(VISUAL_MODEL, documentAnswerMessages(wrappedToolOutput(result)), {
+ tools: [readToolDefinition],
+ })
+ )
+ assert.equal(response.status, 400, label)
+ assert.equal((await response.json()).error.code, 'document-page-range-malformed', label)
+ const evidence = h.getEvidence()
+ assert.equal(evidence.counters.documentFailures, 1, label)
+ assert.equal(evidence.counters.documentAnswers, 0, label)
+ assert.deepEqual(
+ evidence.attempts,
+ [
+ {
+ model: VISUAL_MODEL,
+ imageSha256: null,
+ responseKind: 'rejected',
+ reason: 'document-page-range-malformed',
+ },
+ ],
+ label
+ )
+ }
+})
+
test('document turn 2: a different delivered text gives a different answer (not a constant)', async () => {
const answers = []
for (const text of [DOCUMENT_TEXT, `${DOCUMENT_TEXT}changed\n`]) {
@@ -978,3 +1073,157 @@ test('image rows keep their original shape (no documentSha256)', async () => {
assert.equal('documentSha256' in h.getEvidence().attempts[0], false)
})
+
+/**
+ * Declared here, not imported, so the oracle stays independent of the producer:
+ * a reason added to the fixture without a test fails the parity check below.
+ */
+const EXPECTED_REJECTION_REASONS = [
+ 'document-byte-length-malformed',
+ 'document-page-range-malformed',
+ 'document-stream-unsupported',
+ 'document-tool-output-malformed',
+ 'document-tool-output-unreadable',
+ 'document-tool-result-count',
+ 'image-not-png-data-uri',
+ 'image-part-count',
+ 'image-pixel-not-a-tile-color',
+ 'image-png-malformed',
+ 'image-png-unsupported-form',
+ 'image-tile-grid-mismatch',
+ 'provider-auth-mismatch',
+ 'provider-body-invalid',
+ 'provider-body-unsupported',
+ 'provider-method-unsupported',
+ 'provider-path-not-captured',
+ 'provider-request-form-unsupported',
+ 'text-model-image-incompatible',
+ 'unsupported-model',
+]
+
+test('the fixture publishes exactly the closed set of rejection reasons, frozen', () => {
+ const reasons = providerModule.FIXTURE_REJECTION_REASONS
+ assert.ok(Array.isArray(reasons), 'FIXTURE_REJECTION_REASONS is exported as an array')
+ assert.deepEqual([...reasons].sort(), EXPECTED_REJECTION_REASONS)
+ assert.equal(Object.isFrozen(reasons), true)
+})
+
+test('every rejected row records its closed-set reason and no request content', async () => {
+ const { png } = palettePng()
+ const validBody = JSON.stringify(chatBody(VISUAL_MODEL, imageMessages(png)))
+ const twoImages = imageMessages(png)
+ twoImages[0].content.push(twoImages[0].content[1])
+ const fetchWith = (url, init) => h => h.fetch(url, init)
+ const cases = [
+ [
+ 'request form',
+ h => h.fetch(new Request(PROVIDER_URL, { method: 'POST', body: validBody })),
+ null,
+ 'provider-request-form-unsupported',
+ ],
+ [
+ 'GET method',
+ fetchWith(PROVIDER_URL, { method: 'GET', headers: authHeaders(), body: validBody }),
+ null,
+ 'provider-method-unsupported',
+ ],
+ [
+ 'query string',
+ fetchWith(`${PROVIDER_URL}?x=1`, { method: 'POST', headers: authHeaders(), body: validBody }),
+ null,
+ 'provider-path-not-captured',
+ ],
+ [
+ 'wrong key',
+ fetchWith(PROVIDER_URL, {
+ method: 'POST',
+ headers: { authorization: 'Bearer not-the-run-key' },
+ body: validBody,
+ }),
+ null,
+ 'provider-auth-mismatch',
+ ],
+ [
+ 'buffer body',
+ fetchWith(PROVIDER_URL, {
+ method: 'POST',
+ headers: authHeaders(),
+ body: Buffer.from(validBody),
+ }),
+ null,
+ 'provider-body-unsupported',
+ ],
+ [
+ 'invalid json',
+ fetchWith(PROVIDER_URL, { method: 'POST', headers: authHeaders(), body: '{' }),
+ null,
+ 'provider-body-invalid',
+ ],
+ [
+ 'unknown model',
+ h => call(h, chatBody('glm-5.2', textMessages())),
+ 'glm-5.2',
+ 'unsupported-model',
+ ],
+ [
+ 'text model with an image',
+ h => call(h, chatBody(TEXT_MODEL, imageMessages(png))),
+ TEXT_MODEL,
+ 'text-model-image-incompatible',
+ ],
+ [
+ 'two image parts',
+ h => call(h, chatBody(VISUAL_MODEL, twoImages)),
+ VISUAL_MODEL,
+ 'image-part-count',
+ ],
+ [
+ 'streamed document turn',
+ h =>
+ call(
+ h,
+ chatBody(VISUAL_MODEL, [documentUserMessage()], {
+ tools: [readToolDefinition],
+ stream: true,
+ })
+ ),
+ VISUAL_MODEL,
+ 'document-stream-unsupported',
+ ],
+ [
+ 'unreadable tool result',
+ h =>
+ call(
+ h,
+ chatBody(
+ VISUAL_MODEL,
+ documentAnswerMessages(wrappedToolOutput({ error: 'attachment_not_found' })),
+ { tools: [readToolDefinition] }
+ )
+ ),
+ VISUAL_MODEL,
+ 'document-tool-output-unreadable',
+ ],
+ ]
+ for (const [label, send, model, reason] of cases) {
+ const h = harness()
+ const response = await send(h)
+ assert.ok(response.status >= 400, label)
+ const evidence = h.getEvidence()
+ assert.equal(evidence.counters.rejectedAttempts, 1, label)
+ // The row carries the fixed reason string and nothing from the request.
+ assert.deepEqual(
+ evidence.attempts,
+ [{ model, imageSha256: null, responseKind: 'rejected', reason }],
+ label
+ )
+ assert.ok(EXPECTED_REJECTION_REASONS.includes(reason), label)
+ }
+
+ // Accepted rows never carry a reason; witness: the accepted row exists.
+ const accepted = harness()
+ assert.equal((await call(accepted, chatBody(VISUAL_MODEL, textMessages()))).status, 200)
+ assert.equal(accepted.getEvidence().attempts.length, 1)
+ assert.equal(accepted.getEvidence().attempts[0].responseKind, 'text-only')
+ assert.equal(Object.hasOwn(accepted.getEvidence().attempts[0], 'reason'), false)
+})
diff --git a/tests/e2e/fixtures/image-capabilities/sdk.test.mjs b/tests/e2e/fixtures/image-capabilities/sdk.test.mjs
index c2c27713e..fa5fa6a25 100644
--- a/tests/e2e/fixtures/image-capabilities/sdk.test.mjs
+++ b/tests/e2e/fixtures/image-capabilities/sdk.test.mjs
@@ -150,6 +150,8 @@ test('the production ZAI serializer and pinned SDK complete an attachment read a
imageSha256: null,
responseKind: 'document-answer',
documentSha256: digest,
+ byteRange: { offset: 0, length: Buffer.byteLength(documentText) },
+ truncated: false,
},
])
})
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index fd47cc8f4..c400c415d 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.104",
+ "version": "1.0.105",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.104",
+ "version": "1.0.105",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index f3b79da39..1f3b95d8d 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.104",
+ "version": "1.0.105",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",