Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): add a document-upload lane to the image-capabilities provider fixture
Alfredo Lopezbe37b70fe24a
diff --git a/Makefile b/Makefileindex ecddfb408..7ecb70530 100644--- a/Makefile+++ b/Makefile@@ -308,6 +308,13 @@ minikube-run-image-capabilities: ## Run the visible image journey with an isolatT2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run+.PHONY: minikube-run-document-upload+minikube-run-document-upload: ## Run the visible Desktop document-upload journey (issue #678) with the isolated external-provider fixture and restore the Host+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ IMAGE_CAPABILITIES_LANE=document-upload \+ bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run+minikube-restore-image-capabilities: ## Resume restoration of a recorded image fixture run (IMAGE_CAPABILITIES_RUN_DIR=<dir printed by minikube-run-image-capabilities>)@test -n "$(IMAGE_CAPABILITIES_RUN_DIR)" || { echo "IMAGE_CAPABILITIES_RUN_DIR is required: the run directory printed by 'make minikube-run-image-capabilities'"; exit 1; }@T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \diff --git a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.tsindex 47582627a..50e53a8c0 100644--- a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts+++ b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts@@ -52,11 +52,25 @@ export const FIXTURE_RESPONSE_KIND = {tileColors: 'tile-colors',textOnly: 'text-only',rejected: 'rejected',+ // Issue #678: the two provider turns of an attachment read. The first asks for+ // the `clerum__attachment_read` tool; the second answers from the text the tool+ // delivered.+ documentReadRequested: 'document-read-requested',+ documentAnswer: 'document-answer',} as const/** The fixture's text-only answer. It names no image and no color. */export const FIXTURE_TEXT_ONLY_CONTENT = 'IMAGE_FIXTURE_TEXT_OK'+/**+ * Prefix of the fixture's document answer. The answer is this prefix plus the+ * first {@link FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS} hex characters of the+ * sha256 of the text the Host delivered to the model, so it cannot be produced+ * from the prompt alone.+ */+export const FIXTURE_DOCUMENT_ANSWER_PREFIX = 'DOCUMENT_FIXTURE_SHA256:'+export const FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS = 16+const SHA256_HEX = /^[a-f0-9]{64}$/const MINIKUBE_PROFILE_PATTERN = /^[a-z0-9][a-z0-9-]{0,62}$/const LOOPBACK_HOSTS = new Set(['127.0.0.1', 'localhost', '::1'])@@ -250,6 +264,12 @@ export interface FixtureProviderAttempt {/** Digest of the exact PNG bytes that reached the wire, or null when none did. */imageSha256: string | nullresponseKind: string+ /**+ * Digest of the document text that reached the wire in a tool-result message.+ * The fixture omits the field on image rows, so it is `undefined` there and+ * `null` on the read-request row, which carries no document yet.+ */+ documentSha256?: string | null}export interface FixtureEvidenceCounters {@@ -261,6 +281,9 @@ export interface FixtureEvidenceCounters {textOnlyResponses: numbertextModelImageRefusals: numberblockedEgress: number+ documentReadRequests: number+ documentAnswers: number+ documentFailures: number}export interface ImageCapabilityEvidenceSnapshot {@@ -284,6 +307,9 @@ const COUNTER_KEYS: readonly (keyof FixtureEvidenceCounters)[] = ['textOnlyResponses','textModelImageRefusals','blockedEgress',+ 'documentReadRequests',+ 'documentAnswers',+ 'documentFailures',]function requireCounters(value: unknown, source: string): FixtureEvidenceCounters {@@ -324,6 +350,20 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu)}+ // Present only on document rows; an image row must not grow the field.+ if ('documentSha256' in value) {+ const documentSha256 = value.documentSha256+ if (+ documentSha256 !== null &&+ !(typeof documentSha256 === 'string' && SHA256_HEX.test(documentSha256))+ ) {+ throw new Error(+ `${source}: attempts[${index}].documentSha256 must be a sha256 hex digest or null`+ )+ }+ return { model, imageSha256, responseKind, documentSha256 }+ }+return { model, imageSha256, responseKind }}@@ -467,7 +507,8 @@ export function appendedAttempts(!current ||current.model !== row.model ||current.imageSha256 !== row.imageSha256 ||- current.responseKind !== row.responseKind+ current.responseKind !== row.responseKind ||+ current.documentSha256 !== row.documentSha256) {throw new Error(`[image-capabilities] ledger row ${index} changed between reads; the ledger must be append-only.`diff --git a/desktop-app/test/e2e-playwright/playwright.document-upload.config.ts b/desktop-app/test/e2e-playwright/playwright.document-upload.config.tsnew file mode 100644index 000000000..b37bee4e1--- /dev/null+++ b/desktop-app/test/e2e-playwright/playwright.document-upload.config.ts@@ -0,0 +1,54 @@+import { defineConfig } from '@playwright/test'+import path from 'node:path'+import { requireImageCapabilitiesFixtureEnv } from './helpers/imageCapabilityEvidence'++// E2E_GUARDIAN_IPC_FLOW: this file only configures the runner. The spec owns+// visible IPC transitions and reads the external provider ledger as its wire oracle.++/*+ * Desktop App document-upload E2E lane (issue #678).+ *+ * It reuses the image-capabilities fixture stack (derived Host image, provider+ * fixture, seeded catalog and login), so it has the same contract as+ * playwright.image-capabilities.config.ts and for the same reasons:+ *+ * - It loads NO env file. Every binding is exported by the runner+ * (`make minikube-run-document-upload`) and a missing or inconsistent one+ * fails here, at config load, before Playwright launches anything.+ * - It matches only the document-upload journey, so no recorder journey that+ * talks to a real provider can run from this lane.+ *+ * The journey attaches a text document through the visible composer picker and+ * asserts, through the provider fixture's ledger, that the document text reached+ * the model through the `clerum__attachment_read` tool and not through the prompt.+ */++const repoRoot = path.resolve(__dirname, '../../..')++if (process.env.NODE_ENV !== undefined && process.env.NODE_ENV !== 'test') {+ throw new Error(+ `NODE_ENV must be "test" (or unset) for the document-upload fixture lane; ` ++ `received "${process.env.NODE_ENV}".`+ )+}+process.env.NODE_ENV = 'test'++// Fail at config load, with every missing binding listed at once.+requireImageCapabilitiesFixtureEnv(process.env)++const recorderRoot = process.env.QA_RECORDER_ROOT+ ? path.resolve(process.env.QA_RECORDER_ROOT)+ : path.join(repoRoot, '.local-notes', 'qa-recorder')++export default defineConfig({+ testDir: '.',+ testMatch: /qa-recorder-document-upload\.spec\.ts/,+ // One real Electron launch, one sign-in and one chat with two provider turns.+ timeout: 300_000,+ expect: { timeout: 30_000 },+ retries: 0,+ workers: 1,+ reporter: [['list']],+ outputDir: path.join(recorderRoot, 'runs', 'desktop-app-document-upload'),+ preserveOutput: 'always',+})diff --git a/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts b/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.tsnew file mode 100644index 000000000..4e7ab6b3c--- /dev/null+++ b/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts@@ -0,0 +1,358 @@+import { type ElectronApplication, type Page, type TestInfo, expect, test } from '@playwright/test'+import { randomBytes } from 'node:crypto'+import fs from 'node:fs'+import { createRequire } from 'node:module'+import path from 'node:path'+import {+ FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS,+ FIXTURE_DOCUMENT_ANSWER_PREFIX,+ FIXTURE_RESPONSE_KIND,+ IMAGE_CAPABILITY_FIXTURE_MODELS,+ appendedAttempts,+ readImageCapabilityEvidence,+ requireImageCapabilitiesFixtureEnv,+ resolveImageCapabilitiesMode,+ sha256Hex,+} from './helpers/imageCapabilityEvidence'+import {+ DESKTOP_APP_ROOT,+ EXTERNAL_REST_API_BASE_URL,+ MAIN_ENTRY,+ RPC_PROXY_BASE_URL,+ assertAllowedTarget,+ desktopCredentials,+ finalizeRecording,+ launchDesktopApp,+ login,+ screenshotAndLog,+} from './qa-recorder-helpers'++/*+ * E2E_GUARDIAN_IPC_FLOW: Desktop chat, the model catalog, the agent list and the+ * composer all travel over Electron IPC (main-process handlers in desktop-app/src),+ * so the renderer has no HTTP request to await for any transition in this+ * journey. The visible/business oracles are the composer's own DOM contracts,+ * the persisted thread, and the provider-boundary ledger of the derived in-cluster+ * fixture.+ *+ * Issue #678: any document attached from the composer travels inline to the Host+ * as a `kind:'file'` attachment. The Host never puts its text in the prompt: it+ * announces the file in the turn context and the model reads it through the+ * `clerum__attachment_read` tool. The provider fixture asks for that tool call,+ * then answers with a digest of the text it received in the tool result. So the+ * answer on screen can only be produced if the exact bytes attached here crossed+ * the composer, IPC, the Host admission, the read tool and the provider wire.+ *+ * Only the external provider peer is simulated. The run is selected by the+ * fixture lane's own run id and never reaches a real provider.+ */++const MODE = resolveImageCapabilitiesMode()++/**+ * The prompt names neither the file content nor its digest: the answer cannot be+ * derived from it.+ */+const DOCUMENT_PROMPT = 'Summarize the attached file.'++function instanceBindingCheck(app: ElectronApplication, page: Page, testInfo: TestInfo) {+ const expectedRun = path.resolve(testInfo.outputPath('electron-isolation'))+ const expectedData = path.join(expectedRun, 'user-data')+ const expectedConfig = path.join(expectedRun, 'runtime-config.json')+ const expectedExecutable = createRequire(path.join(DESKTOP_APP_ROOT, 'package.json'))(+ 'electron'+ ) as string+ const restOrigin = new URL(EXTERNAL_REST_API_BASE_URL).origin+ const rpcOrigin = new URL(RPC_PROXY_BASE_URL).origin+ const slug = `${restOrigin}_${rpcOrigin}`+ .toLowerCase()+ .replace(/[^a-z0-9]+/g, '_')+ .replace(/^_+|_+$/g, '')+ .slice(0, 48)+ const expectedEnvKey = `${slug}-${sha256Hex(Buffer.from(`${restOrigin}|rpc=${rpcOrigin}`)).slice(0, 12)}`+ return async () => {+ // Observe the running processes and IPC configuration, rather than trusting+ // launch environment variables. This never changes selection or auth state.+ const identity = await app.evaluate(({ app: electronApp }) => ({+ pid: process.pid,+ argv: process.argv,+ executable: electronApp.getPath('exe'),+ userData: electronApp.getPath('userData'),+ }))+ const runtime = await page.evaluate(async () => {+ const state = await window.clerum.auth.getRuntimeConfigState()+ return { storagePath: state.storagePath, envKey: state.envKey }+ })+ expect(identity.pid).toBe(app.process().pid)+ expect(identity.argv.includes(MAIN_ENTRY)).toBe(true)+ expect(fs.realpathSync(identity.executable)).toBe(fs.realpathSync(expectedExecutable))+ expect(path.resolve(identity.userData)).toBe(expectedData)+ expect(path.resolve(runtime.storagePath)).toBe(expectedConfig)+ expect(runtime.envKey).toBe(expectedEnvKey)+ }+}++/**+ * Correlates the two rendered messages with one durable Host task and returns+ * its id once the task result matches the expected answer. Read-only IPC reads+ * after the visible send; they log in, select and send nothing.+ */+async function observeCompletedTask(+ page: Page,+ host: string,+ expectedAnswer: RegExp+): Promise<string> {+ const visibleIds = await page+ .locator('[data-chat-message-id]')+ .evaluateAll(elements => elements.map(element => element.getAttribute('data-chat-message-id')))+ expect(visibleIds).toHaveLength(2)+ let observed: { taskId: string; result: string } | null = null+ await expect+ .poll(+ async () => {+ observed = await page.evaluate(+ async ({ hostRef, ids }) => {+ const chats = await window.clerum.chat.list(hostRef)+ for (const chat of chats.slice(0, 20)) {+ const messages = await window.clerum.chat.loadMessages(hostRef, chat.id)+ const pair = ids.map(id => messages.find(message => message.id === id))+ const taskId = pair[0]?.task_id+ if (!taskId || pair[1]?.task_id !== taskId) continue+ const result = await window.clerum.rpc.getTaskResult(hostRef, taskId)+ return { taskId, result: JSON.stringify(result) }+ }+ return null+ },+ { hostRef: host, ids: visibleIds }+ )+ return observed?.result ?? ''+ },+ { timeout: 30_000 }+ )+ .toMatch(expectedAnswer)+ expect(observed).not.toBeNull()+ return observed!.taskId+}++function composer(page: Page) {+ return page.getByRole('textbox', { name: 'Agent message composer' })+}++function sendButton(page: Page) {+ return page.getByTestId('send-button')+}++function modelChip(page: Page) {+ return page.getByTestId('selected-chat-model')+}++function fileChips(page: Page) {+ return page.getByTestId('composer-file-chip')+}++/** Start a blank chat and prove the thread is empty before anything is sent. */+async function startBlankChat(page: Page) {+ await page.getByTestId('nav-new-chat').click()+ await expect(composer(page)).toBeVisible({ timeout: 30_000 })+ await expect(page.locator('[data-chat-message-id]')).toHaveCount(0, { timeout: 20_000 })+}++/**+ * Land on the exact configured agent through the visible Switch chat agent+ * menu, and trust the composer only once the switch control reports it.+ */+async function openConfiguredAgentChat(page: Page, hostRef: string) {+ const escaped = hostRef.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')+ await page.getByTestId('nav-chat').click()+ const switchAgent = page.getByRole('button', { name: 'Switch chat agent' })+ await expect(switchAgent).toBeVisible({ timeout: 30_000 })+ await switchAgent.click()+ const agentMenuItem = page.getByRole('menuitem', { name: new RegExp(`^${escaped}$`, 'i') })+ await expect(agentMenuItem).toHaveCount(1)+ await agentMenuItem.click()+ await expect(switchAgent).toContainText(new RegExp(escaped, 'i'))+}++/** Selects the model through the visible popover and asserts the chip reports it. */+async function selectSupportedModel(page: Page, model: string) {+ await expect(modelChip(page)).toBeVisible({ timeout: 30_000 })+ await modelChip(page).click()+ const menu = page.getByRole('menu', { name: 'Select model' })+ await expect(menu).toBeVisible({ timeout: 20_000 })+ const row = page.getByTestId(`model-option-${model}`)+ await expect(row).toHaveCount(1)+ await row.click()+ await expect(modelChip(page)).toHaveAttribute('data-model-id', model)+}++/**+ * Reach the authenticated shell through the real form. While the form is still+ * unsent, the authenticated surface must not be mounted, otherwise the journey+ * could pass without a real sign-in.+ */+async function signIn(page: Page) {+ await expect(page.locator('.boot-overlay')).toBeHidden({ timeout: 30_000 })++ const emailInput = page.locator('#email-input')+ const settingsMenu = page.getByTestId('nav-settings-menu')+ await expect(emailInput.or(settingsMenu)).toBeVisible({ timeout: 30_000 })+ if (await settingsMenu.isVisible()) {+ if ((await settingsMenu.getAttribute('aria-expanded')) !== 'true') {+ await settingsMenu.click()+ }+ await expect(settingsMenu).toHaveAttribute('aria-expanded', 'true')+ await page.getByTestId('logout-btn').click()+ }++ await expect(emailInput).toBeVisible({ timeout: 20_000 })+ await expect(composer(page)).toHaveCount(0)+ await expect(sendButton(page)).toHaveCount(0)+ await expect(page.getByTestId('nav-chat')).toHaveCount(0)++ await login(page, desktopCredentials())+ await expect(page.getByTestId('nav-chat')).toBeVisible({ timeout: 30_000 })+}++/** Attach a document the way a user does: Add context -> Upload Files -> native picker. */+async function attachDocument(+ page: Page,+ document: { fileName: string; mimeType: string; buffer: Buffer }+) {+ const attachButton = page.getByRole('button', { name: 'Add context' })+ await expect(attachButton).toBeEnabled()+ await attachButton.click()+ const uploadItem = page.getByRole('menuitem', { name: 'Upload Files' })+ await expect(uploadItem).toBeEnabled({ timeout: 15_000 })++ const [fileChooser] = await Promise.all([+ page.waitForEvent('filechooser', { timeout: 15_000 }),+ uploadItem.click(),+ ])+ await fileChooser.setFiles({+ name: document.fileName,+ mimeType: document.mimeType,+ buffer: document.buffer,+ })+}++/**+ * The fixture's document answer, matched from the exported marker. The+ * underscores are optional in the pattern because the bubble renders Markdown,+ * which may treat intraword underscores as emphasis; the ledger below pins the+ * exact digest regardless.+ */+function documentAnswerRegex(documentSha: string): RegExp {+ const prefix = FIXTURE_DOCUMENT_ANSWER_PREFIX.replace(/_/g, '[_-]?')+ return new RegExp(`${prefix}${documentSha.slice(0, FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS)}`, 'i')+}++test('document-upload fixture: an attached text file reaches the model through the read tool', async ({}, testInfo) => {+ test.skip(+ MODE !== 'fixture',+ 'Set IMAGE_CAPABILITIES_RUN_ID=image-capabilities-<12 hex> (the runner does) to run the ' ++ 'derived-provider fixture journey.'+ )++ const env = requireImageCapabilitiesFixtureEnv()+ await assertAllowedTarget('EXTERNAL_REST_API_BASE_URL', env.externalRestApiBaseUrl)+ await assertAllowedTarget('RPC_PROXY_BASE_URL', env.rpcProxyBaseUrl)++ // Precondition read before anything is sent: the derived fixture must be live+ // in the deployment that serves this host, for this run and this profile.+ const baseline = readImageCapabilityEvidence(env)+ expect(baseline.runId).toBe(env.runId)+ expect(baseline.profile).toBe(env.profile)++ // A random token the prompt never mentions: the digest of these exact bytes is+ // the only thing that can appear in the answer.+ const document = {+ fileName: `notes-${randomBytes(3).toString('hex')}.txt`,+ mimeType: 'text/plain',+ buffer: Buffer.from(`document-upload ${randomBytes(6).toString('hex')}\nsecond line\n`, 'utf8'),+ }+ const documentSha = sha256Hex(document.buffer)+ fs.mkdirSync(testInfo.outputPath(), { recursive: true })+ fs.writeFileSync(testInfo.outputPath(document.fileName), document.buffer)++ let app: ElectronApplication | undefined+ let recordedPage: Page | undefined++ try {+ const launched = await launchDesktopApp(+ testInfo,+ `issue678-minikube${new URL(EXTERNAL_REST_API_BASE_URL).port}`+ )+ app = launched.app+ recordedPage = launched.page+ const page = launched.page++ const assertBinding = instanceBindingCheck(app, page, testInfo)+ await assertBinding()+ await signIn(page)+ await assertBinding()+ await openConfiguredAgentChat(page, env.hostRef)++ await test.step('the document is admitted as a ready file chip and the send is enabled', async () => {+ await startBlankChat(page)+ await selectSupportedModel(page, IMAGE_CAPABILITY_FIXTURE_MODELS.supported)++ await attachDocument(page, document)+ const chip = fileChips(page).filter({ hasText: document.fileName })+ await expect(chip).toHaveCount(1, { timeout: 20_000 })+ await expect(chip).toHaveAttribute('data-file-status', 'ready', { timeout: 20_000 })++ await composer(page).fill(DOCUMENT_PROMPT)+ await expect(sendButton(page)).toBeEnabled({ timeout: 20_000 })+ })++ await test.step('the answer carries the digest of the delivered text and the ledger shows both provider turns', async () => {+ const before = readImageCapabilityEvidence(env)+ await assertBinding()+ await sendButton(page).click()+ await expect(fileChips(page)).toHaveCount(0, { timeout: 30_000 })++ await expect(page.getByTestId('message-list')).toBeVisible({ timeout: 20_000 })+ await expect(page.locator('[data-chat-message-id]')).toHaveCount(2, { timeout: 120_000 })+ const response = page.getByTestId('agent-response').locator('.message-block.markdown-content')+ await expect(response).toHaveCount(1, { timeout: 120_000 })+ await expect(response).toContainText(documentAnswerRegex(documentSha), { timeout: 120_000 })+ await observeCompletedTask(page, env.hostRef, documentAnswerRegex(documentSha))++ const after = readImageCapabilityEvidence(env)+ const appended = appendedAttempts(before, after)+ const readRows = appended.filter(+ row => row.responseKind === FIXTURE_RESPONSE_KIND.documentReadRequested+ )+ const answerRows = appended.filter(+ row => row.responseKind === FIXTURE_RESPONSE_KIND.documentAnswer+ )++ // Turn 1: the model asked for the read tool. The prompt alone produced no answer.+ expect(readRows).toHaveLength(1)+ expect(readRows[0]?.model).toBe(IMAGE_CAPABILITY_FIXTURE_MODELS.supported)+ expect(readRows[0]?.documentSha256).toBeNull()+ // Turn 2: the tool result reached the wire carrying exactly the attached bytes.+ expect(answerRows).toHaveLength(1)+ expect(answerRows[0]?.model).toBe(IMAGE_CAPABILITY_FIXTURE_MODELS.supported)+ expect(answerRows[0]?.documentSha256).toBe(documentSha)++ // A document is never an image: no pixels moved, and the fixture refused nothing.+ expect(appended.filter(row => row.imageSha256 !== null)).toHaveLength(0)+ expect(after.counters.imageAttempts - before.counters.imageAttempts).toBe(0)+ expect(after.counters.documentReadRequests - before.counters.documentReadRequests).toBe(1)+ expect(after.counters.documentAnswers - before.counters.documentAnswers).toBe(1)+ expect(after.counters.documentFailures - before.counters.documentFailures).toBe(0)+ })++ await screenshotAndLog(page, testInfo, 'desktop-document-upload-fixture')+ } catch (error) {+ if (recordedPage && !recordedPage.isClosed()) {+ await recordedPage+ .screenshot({ path: testInfo.outputPath('failure.png') })+ .catch(() => undefined)+ }+ throw error+ } finally {+ await finalizeRecording(app, recordedPage)+ }+})diff --git a/docs/testing/desktop-headful-journeys.md b/docs/testing/desktop-headful-journeys.mdindex 6deec7505..ac0e734ff 100644--- a/docs/testing/desktop-headful-journeys.md+++ b/docs/testing/desktop-headful-journeys.md@@ -33,8 +33,9 @@ test under `.local-notes/qa-recorder/runs/desktop-app/` (git-ignored).| Settings + chat smoke | `qa-recorder-settings-chat.spec.ts` | 1 — Settings tabs + one chat message | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ || Image capabilities | `qa-recorder-image-capabilities.spec.ts` | 2 — capability gate on a text-only model, fixture-backed image answer | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ || Model selector | `qa-recorder-model-selector.spec.ts` | 1 — a multi-model catalog lists rows without capability tags | — | ✅ |+| Document upload | `qa-recorder-document-upload.spec.ts` | 1 — attached text file read through the read tool (fixture lane only) | `IMAGE_CAPABILITIES_RUN_ID` (runner-set) | ✅ |-**Totals: 13 specs, 22 tests.** Read-only journeys need no confirmation flag; every+**Totals: 14 specs, 23 tests.** Read-only journeys need no confirmation flag; everyjourney still calls the loopback health guard on both API URLs — except against anon-loopback target, where `QA_RECORDER_ALLOW_REMOTE=1` skips the precheck, so awrong URL surfaces as an Electron timeout instead of a fast refusal. Mutating / paiddiff --git a/scripts/e2e/image-capabilities-fixture.mjs b/scripts/e2e/image-capabilities-fixture.mjsindex dd6e97f0f..a49505eec 100644--- a/scripts/e2e/image-capabilities-fixture.mjs+++ b/scripts/e2e/image-capabilities-fixture.mjs@@ -22,6 +22,31 @@ export const fixtureConfigKeys = ['CONTROL_API_REAL_PG_CONTEXT',]+/**+ * The Playwright journeys this runner can drive against the swapped Host. The+ * lane is chosen by `IMAGE_CAPABILITIES_LANE` and only from this closed set, so+ * an environment value can never name an arbitrary config path.+ */+export const playwrightLanes = {+ image: {+ config: 'desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',+ label: 'image-capabilities-playwright',+ },+ 'document-upload': {+ config: 'desktop-app/test/e2e-playwright/playwright.document-upload.config.ts',+ label: 'document-upload-playwright',+ },+}++export function playwrightLaneConfig(env) {+ const lane = env.IMAGE_CAPABILITIES_LANE ?? 'image'+ if (!Object.hasOwn(playwrightLanes, lane))+ throw new Error(+ `IMAGE_CAPABILITIES_LANE must be one of ${Object.keys(playwrightLanes).join(', ')} (received ${JSON.stringify(lane)})`+ )+ return { lane, ...playwrightLanes[lane] }+}+export function proveImages(manifest, observed, head, profile) {if (!/^[a-f0-9]{40}$/.test(head) || manifest.profile !== profile)throw new Error('Fixture image ownership mismatch')@@ -157,6 +182,7 @@ async function main() {)throw new Error('Matching branch profile and context required')command('bash', ['scripts/minikube/require-t2-mutation-lock.sh'])+ const { lane, config: laneConfig, label: laneLabel } = playwrightLaneConfig(process.env)const head = command('git', ['rev-parse', 'HEAD']).trim()const branch = command('git', ['branch', '--show-current']).trim()if (command('git', ['status', '--porcelain']).trim()) throw new Error('Clean checkout required')@@ -715,7 +741,7 @@ async function main() {QA_RECORDER_ROOT: evidence,}command('npm', ['run', 'verify:electron', '--prefix', 'desktop-app'])- process.stdout.write(`Image capability fixture ready: ${state.runId}\n`)+ process.stdout.write(`Image capability fixture ready: ${state.runId} (lane ${lane})\n`)const output = command('node',[@@ -727,12 +753,12 @@ async function main() {'--kill-grace-seconds','5','--label',- 'image-capabilities-playwright',+ laneLabel,'--','node','desktop-app/node_modules/@playwright/test/cli.js','test',- '--config=desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',+ `--config=${laneConfig}`,],{ env: environment, timeout: 930_000 })diff --git a/scripts/e2e/image-capabilities-fixture.test.mjs b/scripts/e2e/image-capabilities-fixture.test.mjsindex d702274d7..22c5e9c6b 100644--- a/scripts/e2e/image-capabilities-fixture.test.mjs+++ b/scripts/e2e/image-capabilities-fixture.test.mjs@@ -6,6 +6,7 @@ import {fixtureImage,installSignalRestore,modelInputs,+ playwrightLaneConfig,proveImages,requireOwnedResource,runAnnotation,@@ -24,6 +25,26 @@ function manifest() {}}+test('the Playwright lane defaults to the image journey and accepts only known lanes', () => {+ assert.deepEqual(playwrightLaneConfig({}), {+ lane: 'image',+ config: 'desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',+ label: 'image-capabilities-playwright',+ })+ assert.deepEqual(playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'document-upload' }), {+ lane: 'document-upload',+ config: 'desktop-app/test/e2e-playwright/playwright.document-upload.config.ts',+ label: 'document-upload-playwright',+ })+ for (const lane of ['', 'documents', '../evil.config.ts', '__proto__', 'toString']) {+ assert.throws(+ () => playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane }),+ /IMAGE_CAPABILITIES_LANE must be one of image, document-upload/,+ lane+ )+ }+})+test('requires both exact source revisions and the live derived-base identity', () => {assert.deepEqual(proveImages(manifest(), ids, head, profile), { head, profile, images: ids })for (const mutate of [diff --git a/tests/e2e/fixtures/image-capabilities/README.md b/tests/e2e/fixtures/image-capabilities/README.mdindex 45dc63463..f74ec51f1 100644--- a/tests/e2e/fixtures/image-capabilities/README.md+++ b/tests/e2e/fixtures/image-capabilities/README.md@@ -18,6 +18,41 @@ compiled ZAI provider with the PNG generated by the browser journey.The preload runs with Node's `--import`; the production `main.js` remains themain program so its normal startup guard and crash handlers execute.+## Documents (issue #678)++The same fixture also answers the two provider turns of a document attachment.+The Host never puts a file's text in the prompt: it announces the file in the+turn context and the model reads it through the `clerum__attachment_read` tool.+When that tool is offered and the last user message announces an+`attached_file`, the fixture first replies with a tool call for it. After the+tool result arrives, it replies `DOCUMENT_FIXTURE_SHA256:` plus the first 16 hex+characters of the SHA-256 of the text in that result. The prompt alone can never+produce that answer.++The ledger records `document-read-requested` (no digest yet) and+`document-answer` (with `documentSha256`, never the text). Malformed, non-text+or repeated tool results and streamed document turns are refused and counted in+`documentFailures`. Image rows keep their previous shape.++`sdk.test.mjs` drives both turns through the compiled Host: the turn-context+block, the safety wrapper and the ZAI provider are the production ones.++The browser journey has its own lane, selected by the runner:++```sh+make minikube-build-image-capabilities-fixture+make minikube-t2+make minikube-run-document-upload+```++`make minikube-run-document-upload` is `minikube-run-image-capabilities` with+`IMAGE_CAPABILITIES_LANE=document-upload`; the config is+`desktop-app/test/e2e-playwright/playwright.document-upload.config.ts`. It+attaches a text file through Add context -> Upload Files, waits for a `ready`+file chip, sends, and requires the digest of the attached bytes in the answer,+one `document-read-requested` row, one `document-answer` row carrying that+digest, no image row and no `documentFailures`.+## Local executionUse Node 24 and Bash 5, a clean development branch based on current `origin/dev`,diff --git a/tests/e2e/fixtures/image-capabilities/provider.mjs b/tests/e2e/fixtures/image-capabilities/provider.mjsindex 7165f2776..c25139bde 100644--- a/tests/e2e/fixtures/image-capabilities/provider.mjs+++ b/tests/e2e/fixtures/image-capabilities/provider.mjs@@ -25,15 +25,25 @@* the six tile centers. A run that never delivered the image cannot pass by* guessing, and a request that carries no image never receives a color list.*+ * Documents (issue #678). The Host never puts an attached document's text in the+ * prompt: the first request lists the file by id and offers the+ * `clerum__attachment_read` tool. So the one tool call this fixture ever makes+ * is that read, answered with `finish_reason: 'tool_calls'`; the second request+ * carries the tool result, and the answer is `DOCUMENT_FIXTURE_SHA256:` plus the+ * first 16 hex characters of the SHA-256 of the delivered text. A run whose+ * document never reached the provider cannot produce that digest. Document rows+ * add `documentSha256` (the digest of the delivered text, never the text).+ ** Determinism and sanitation. Responses carry a synthetic completion id derived- * from the run id, a fixed test usage block, `finish_reason: 'stop'`, and never a- * tool call. Both accepted models answer a text-only request with- * `IMAGE_FIXTURE_TEXT_OK`, so a legitimate title or summary call is never turned- * into a false failure; only a delivered image can produce a color list. Evidence- * is limited to the run id, counters, per-model counts, a per-call ledger- * (model, nullable image digest, response kind), and SHA-256 digests of the- * delivered image bytes. Headers, auth values, prompts, and raw image bytes are- * never recorded.+ * from the run id, a fixed test usage block, and `finish_reason: 'stop'` (or+ * `'tool_calls'` for the document read). Both accepted models answer a text-only+ * request with `IMAGE_FIXTURE_TEXT_OK`, so a legitimate title or summary call is+ * never turned into a false failure; only a delivered image can produce a color+ * list. Evidence is limited to the run id, counters, per-model counts, a per-call+ * ledger (model, nullable image digest, response kind, and for documents the+ * digest of the delivered text), and SHA-256 digests of the delivered image bytes+ * and document text. Headers, auth values, prompts, raw image bytes, and document+ * text are never recorded.** Authentication. This fixture never holds the runner's ZAI key. It holds only* the SHA-256 digest of that key, hashes the bearer token it receives, and@@ -330,6 +340,97 @@ function extractPngBytes(messages) {return Buffer.from(payload, 'base64')}+/** The Host tool that returns the text of an attached document (issue #678). */+const ATTACHMENT_READ_TOOL = 'clerum__attachment_read'++/** Prefix of the document answer; the digest after it is computed from the delivered text. */+const DOCUMENT_ANSWER_PREFIX = 'DOCUMENT_FIXTURE_SHA256:'++/** Length of the digest prefix echoed in the answer. */+const DOCUMENT_ANSWER_DIGEST_CHARS = 16++/** The `attached_file` line the Host writes into the turn-context block. */+const ATTACHED_FILE_LINE = /^attached_file: id="([^"\n]+)"/m++/** The wrapper the Host puts around a tool result (`BasicSafety.wrapForLlm`). */+const ATTACHMENT_TOOL_OUTPUT = new RegExp(+ `^<tool_output name="${ATTACHMENT_READ_TOOL}" sanitized="(?:true|false)">\\n([\\s\\S]*)\\n</tool_output>$`+)++/** A document turn the fixture cannot read pixel-exactly gets a stable reason code. */+class FixtureDocumentError extends Error {+ constructor(code, message) {+ super(message)+ this.name = 'FixtureDocumentError'+ this.code = code+ }+}++function lastUserIndex(messages) {+ for (let index = messages.length - 1; index >= 0; index -= 1) {+ if (messages[index]?.role === 'user') return index+ }+ return -1+}++/**+ * Classifies the request as one of the two turns of an attachment read.+ *+ * - `null`: not a document turn (no `attached_file` line, or the read tool is not offered).+ * - `{ stage: 'read', attachmentId }`: the user message lists a readable file and no tool+ * result follows it yet, so the model has to ask for the text.+ * - `{ stage: 'answer', text }`: a `role:'tool'` message after the last user message carries+ * the wrapped tool result, so the model can answer from what was delivered.+ *+ * The fixture is stateless: both turns are recognised from the message list alone.+ */+function classifyDocumentTurn(body) {+ const tools = Array.isArray(body.tools) ? body.tools : []+ const offered = tools.some(tool => tool?.function?.name === ATTACHMENT_READ_TOOL)+ if (!offered) return null++ const userIndex = lastUserIndex(body.messages)+ if (userIndex < 0) return null+ const user = body.messages[userIndex]+ if (typeof user.content !== 'string') return null+ const attached = ATTACHED_FILE_LINE.exec(user.content)+ if (!attached) return null++ const toolMessages = body.messages+ .slice(userIndex + 1)+ .filter(message => message?.role === 'tool' && typeof message.content === 'string')+ if (toolMessages.length === 0) return { stage: 'read', attachmentId: attached[1] }+ if (toolMessages.length > 1)+ throw new FixtureDocumentError(+ 'document-tool-result-count',+ `exactly one attachment read is supported, received ${toolMessages.length}`+ )++ const wrapped = ATTACHMENT_TOOL_OUTPUT.exec(toolMessages[0].content)+ if (!wrapped)+ throw new FixtureDocumentError(+ 'document-tool-output-malformed',+ 'the tool result is not a clerum__attachment_read wrapper'+ )+ let result+ try {+ result = JSON.parse(wrapped[1])+ } catch {+ throw new FixtureDocumentError('document-tool-output-malformed', 'the tool result is not JSON')+ }+ if (+ !result ||+ typeof result !== 'object' ||+ result.kind !== 'text' ||+ typeof result.text !== 'string'+ )+ throw new FixtureDocumentError(+ 'document-tool-output-unreadable',+ 'the tool result does not carry the document text'+ )+ return { stage: 'answer', text: result.text }+}+const INTERNAL_HOST_SUFFIXES = ['.svc.cluster.local','.svc',@@ -457,6 +558,36 @@ function completionBody(completionId, model, content) {}}+/** An assistant turn that asks the Host to run `name` with `args` (non-stream only). */+function toolCallCompletionBody(completionId, model, name, args) {+ return {+ id: completionId,+ object: 'chat.completion',+ created: FIXTURE_CREATED,+ model,+ choices: [+ {+ index: 0,+ message: {+ role: 'assistant',+ content: null,+ tool_calls: [+ {+ id: `call-${completionId}`,+ type: 'function',+ function: { name, arguments: JSON.stringify(args) },+ },+ ],+ },+ finish_reason: 'tool_calls',+ logprobs: null,+ },+ ],+ usage: FIXTURE_USAGE,+ system_fingerprint: FIXTURE_SYSTEM_FINGERPRINT,+ }+}+function streamResponse(completionId, model, content) {const chunk = (delta, finishReason, withUsage) => ({id: completionId,@@ -541,6 +672,9 @@ export function createImageFixtureFetch(blockedEgress: 0,imageDecodeFailures: 0,unauthorizedAttempts: 0,+ documentReadRequests: 0,+ documentAnswers: 0,+ documentFailures: 0,},attempts: [],}@@ -555,8 +689,14 @@ export function createImageFixtureFetch(* request was refused before its body was parsed, and `imageSha256` is null* whenever the attempt carried no usable image.*/- const recordCall = (model, imageSha256, responseKind) => {- state.attempts.push({ model, imageSha256, responseKind })+ const recordCall = (model, imageSha256, responseKind, documentSha256) => {+ // Only document rows carry `documentSha256`, so the image rows keep the exact+ // shape they always had.+ state.attempts.push(+ documentSha256 === undefined+ ? { model, imageSha256, responseKind }+ : { model, imageSha256, responseKind, documentSha256 }+ )}const refuse = (model, reason, { status = 400, code = reason, imageSha256 = null } = {}) => {state.counters.rejectedAttempts += 1@@ -564,10 +704,15 @@ export function createImageFixtureFetch(publish()return fixtureErrorResponse(status, code, reason)}- const answer = (model, content, responseKind, { imageSha256 = null, stream = false } = {}) => {+ const answer = (+ model,+ content,+ responseKind,+ { imageSha256 = null, documentSha256, stream = false } = {}+ ) => {sequence += 1const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`- recordCall(model, imageSha256, responseKind)+ recordCall(model, imageSha256, responseKind, documentSha256)publish()return stream? streamResponse(completionId, model, content)@@ -631,6 +776,49 @@ export function createImageFixtureFetch(// request without pixels can ever receive a color list.if (model !== UNSUPPORTED_MODEL && model !== SUPPORTED_MODEL)return refuse(model, 'unsupported-model', { code: 'unsupported_model' })++ let documentTurn+ try {+ documentTurn = classifyDocumentTurn(body)+ } catch (error) {+ if (!(error instanceof FixtureDocumentError)) throw error+ state.counters.documentFailures += 1+ return refuse(model, error.code, { code: error.code })+ }+ if (documentTurn) {+ if (stream) {+ // The Host reads the provider without streaming; a stream here means the+ // wiring changed, and a tool call cannot be answered in this fixture's SSE.+ state.counters.documentFailures += 1+ return refuse(model, 'document-stream-unsupported', {+ code: 'document_stream_unsupported',+ })+ }+ if (documentTurn.stage === 'read') {+ // Ask for the text. The answer to this request carries no file content:+ // the only way the text reaches the next request is the Host's own tool.+ state.counters.documentReadRequests += 1+ sequence += 1+ const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`+ recordCall(model, null, 'document-read-requested', null)+ publish()+ return jsonResponse(+ toolCallCompletionBody(completionId, model, ATTACHMENT_READ_TOOL, {+ attachmentId: documentTurn.attachmentId,+ })+ )+ }+ // The digest in the answer is derived from the delivered text and from+ // nothing else: the prompt and the file name never contribute to it.+ const documentSha256 = createHash('sha256').update(documentTurn.text, 'utf8').digest('hex')+ state.counters.documentAnswers += 1+ return answer(+ model,+ `${DOCUMENT_ANSWER_PREFIX}${documentSha256.slice(0, DOCUMENT_ANSWER_DIGEST_CHARS)}`,+ 'document-answer',+ { documentSha256 }+ )+ }state.counters.textOnlyResponses += 1return answer(model, TEXT_ONLY_CONTENT, 'text-only', { stream })}diff --git a/tests/e2e/fixtures/image-capabilities/provider.test.mjs b/tests/e2e/fixtures/image-capabilities/provider.test.mjsindex da0934b1a..8b1fb4f15 100644--- a/tests/e2e/fixtures/image-capabilities/provider.test.mjs+++ b/tests/e2e/fixtures/image-capabilities/provider.test.mjs@@ -697,3 +697,226 @@ test('getEvidence returns a private copy and onEvidence publishes snapshots', asassert.equal(second.attempts.length, 2)assert.notEqual(h.getEvidence(), h.getEvidence())})++// ---------------------------------------------------------------------------+// Documents (issue #678): the Host lists an attached file in the turn-context+// block and offers `clerum__attachment_read`; the text reaches the provider only+// in the tool result of the second request.+// ---------------------------------------------------------------------------++const READ_TOOL = 'clerum__attachment_read'+const ATTACHMENT_ID = 'att-11111111-2222-4333-8444-555555555555'+const DOCUMENT_TEXT = 'ledger token 7f3a9c1e\nsecond line\n'++const readToolDefinition = {+ type: 'function',+ function: {+ name: READ_TOOL,+ description: 'Read an attached file.',+ parameters: { type: 'object' },+ },+}++function documentUserMessage(text = 'Summarize the attached file.') {+ return {+ role: 'user',+ content:+ '<turn-context>\ndate: 2026-09-29\n' ++ `attached_file: id="${ATTACHMENT_ID}" name="notes.txt" class=text bytes=34 reader=text\n` ++ "If the user's request refers to an attached file, read it with clerum__attachment_read before answering.\n" ++ `</turn-context>\n\n${text}`,+ }+}++function wrappedToolOutput(payload, { name = READ_TOOL } = {}) {+ return `<tool_output name="${name}" sanitized="false">\n${JSON.stringify(payload)}\n</tool_output>`+}++function documentReadResult(text = DOCUMENT_TEXT) {+ return {+ attachmentId: ATTACHMENT_ID,+ referenceId: 'ref-1',+ kind: 'text',+ byteRange: { offset: 0, length: Buffer.byteLength(text) },+ truncated: false,+ text,+ }+}++function documentAnswerMessages(toolContent) {+ return [+ documentUserMessage(),+ {+ role: 'assistant',+ content: null,+ tool_calls: [+ { id: 'call-1', type: 'function', function: { name: READ_TOOL, arguments: '{}' } },+ ],+ },+ { role: 'tool', tool_call_id: 'call-1', content: toolContent },+ ]+}++const sha256 = text => createHash('sha256').update(text, 'utf8').digest('hex')++test('document turn 1: asks for the text with a tool call and carries no file content', async () => {+ const h = harness()++ const response = await call(+ h,+ chatBody(VISUAL_MODEL, [documentUserMessage()], { tools: [readToolDefinition] })+ )+ assert.equal(response.status, 200)+ const body = await response.json()++ assert.equal(body.choices[0].finish_reason, 'tool_calls')+ const [toolCall] = body.choices[0].message.tool_calls+ assert.equal(toolCall.type, 'function')+ assert.equal(toolCall.function.name, READ_TOOL)+ assert.deepEqual(JSON.parse(toolCall.function.arguments), { attachmentId: ATTACHMENT_ID })++ const evidence = h.getEvidence()+ assert.equal(evidence.counters.documentReadRequests, 1)+ assert.equal(evidence.counters.documentAnswers, 0)+ assert.deepEqual(evidence.attempts, [+ {+ model: VISUAL_MODEL,+ imageSha256: null,+ responseKind: 'document-read-requested',+ documentSha256: null,+ },+ ])+})++test('document turn 2: the answer is a digest of the delivered text and the ledger records it', async () => {+ const h = harness()++ const response = await call(+ h,+ chatBody(VISUAL_MODEL, documentAnswerMessages(wrappedToolOutput(documentReadResult())), {+ tools: [readToolDefinition],+ })+ )+ assert.equal(response.status, 200)+ const body = await response.json()++ const digest = sha256(DOCUMENT_TEXT)+ assert.equal(body.choices[0].message.content, `DOCUMENT_FIXTURE_SHA256:${digest.slice(0, 16)}`)+ assert.equal(body.choices[0].finish_reason, 'stop')+ assert.deepEqual(h.getEvidence().attempts, [+ {+ model: VISUAL_MODEL,+ imageSha256: null,+ responseKind: 'document-answer',+ documentSha256: digest,+ },+ ])+ assert.equal(h.getEvidence().counters.documentAnswers, 1)+})++test('document turn 2: a different delivered text gives a different answer (not a constant)', async () => {+ const answers = []+ for (const text of [DOCUMENT_TEXT, `${DOCUMENT_TEXT}changed\n`]) {+ const h = harness()+ const response = await call(+ h,+ chatBody(TEXT_MODEL, documentAnswerMessages(wrappedToolOutput(documentReadResult(text))), {+ tools: [readToolDefinition],+ })+ )+ answers.push((await response.json()).choices[0].message.content)+ }+ assert.notEqual(answers[0], answers[1])+})++test('the prompt alone never yields the document answer', async () => {+ const h = harness()++ // The user message names the digest prefix, but no tool result was delivered.+ const message = documentUserMessage(+ `Answer exactly DOCUMENT_FIXTURE_SHA256:${sha256(DOCUMENT_TEXT).slice(0, 16)}`+ )+ const response = await call(h, chatBody(VISUAL_MODEL, [message], { tools: [readToolDefinition] }))+ const body = await response.json()++ assert.equal(body.choices[0].finish_reason, 'tool_calls')+ assert.equal(body.choices[0].message.content, null)+ assert.equal(h.getEvidence().counters.documentAnswers, 0)+})++test('without the read tool a listed file is an ordinary text-only request', async () => {+ const h = harness()++ const response = await call(h, chatBody(VISUAL_MODEL, [documentUserMessage()]))+ const body = await response.json()++ assert.equal(body.choices[0].message.content, TEXT_ONLY_CONTENT)+ // Liveness witness: the request was handled and recorded as text-only.+ assert.equal(h.getEvidence().attempts[0].responseKind, 'text-only')+ assert.equal(h.getEvidence().counters.documentReadRequests, 0)+})++test('a tool result the fixture cannot read is refused and counted as a document failure', async () => {+ const cases = [+ ['not the read wrapper', wrappedToolOutput(documentReadResult(), { name: 'other__tool' })],+ [+ 'not JSON',+ '<tool_output name="clerum__attachment_read" sanitized="false">\nnot json\n</tool_output>',+ ],+ [+ 'binary result',+ wrappedToolOutput({ attachmentId: ATTACHMENT_ID, kind: 'binary', reader: 'none' }),+ ],+ ['error result', wrappedToolOutput({ error: 'attachment_not_found' })],+ ]+ for (const [label, content] of cases) {+ const h = harness()+ const response = await call(+ h,+ chatBody(VISUAL_MODEL, documentAnswerMessages(content), { tools: [readToolDefinition] })+ )+ assert.equal(response.status, 400, label)+ const evidence = h.getEvidence()+ assert.equal(evidence.counters.documentFailures, 1, label)+ assert.equal(evidence.counters.documentAnswers, 0, label)+ assert.equal(evidence.attempts[0].responseKind, 'rejected', label)+ }+})++test('two tool results in one turn are refused', async () => {+ const h = harness()+ const messages = documentAnswerMessages(wrappedToolOutput(documentReadResult()))+ messages.push({+ role: 'tool',+ tool_call_id: 'call-2',+ content: wrappedToolOutput(documentReadResult()),+ })++ const response = await call(h, chatBody(VISUAL_MODEL, messages, { tools: [readToolDefinition] }))++ assert.equal(response.status, 400)+ assert.equal(h.getEvidence().counters.documentFailures, 1)+})++test('a streamed document turn is refused instead of answered', async () => {+ const h = harness()++ const response = await call(+ h,+ chatBody(VISUAL_MODEL, [documentUserMessage()], { tools: [readToolDefinition], stream: true })+ )++ assert.equal(response.status, 400)+ const evidence = h.getEvidence()+ assert.equal(evidence.counters.documentFailures, 1)+ assert.equal(evidence.counters.documentReadRequests, 0)+})++test('image rows keep their original shape (no documentSha256)', async () => {+ const { png } = palettePng()+ const h = harness()++ await call(h, chatBody(VISUAL_MODEL, imageMessages(png)))++ assert.equal('documentSha256' in h.getEvidence().attempts[0], false)+})diff --git a/tests/e2e/fixtures/image-capabilities/sdk.test.mjs b/tests/e2e/fixtures/image-capabilities/sdk.test.mjsindex 715ee7047..ff212de9d 100644--- a/tests/e2e/fixtures/image-capabilities/sdk.test.mjs+++ b/tests/e2e/fixtures/image-capabilities/sdk.test.mjs@@ -5,21 +5,28 @@ import test from 'node:test'import { buildImageFixture } from '../../../../desktop-app/test/e2e-playwright/helpers/qaRecorderImageFixture.ts'import { createImageFixtureFetch } from './provider.mjs'+// The pinned SDK reads `globalThis.fetch` once, when `openai/shims/web` first+// loads, so every test in this file shares one fixture and one key. Each test+// asserts only on the ledger rows it appended.+const IMAGE_CAPABILITIES_CREDENTIAL = randomBytes(32).toString('hex')+const fixture = createImageFixtureFetch(+ async () => {+ throw new Error('Unexpected network call')+ },+ {+ runId: 'image-capabilities-123456abcdef',+ credentialHash: createHash('sha256').update(IMAGE_CAPABILITIES_CREDENTIAL).digest('hex'),+ }+)+globalThis.fetch = fixture.fetch+const require = createRequire(new URL('../../../../mcp-host/package.json', import.meta.url))+require('openai/shims/web')++const appendedRows = before => fixture.getEvidence().attempts.slice(before)+// Uses the real compiled provider and pinned SDK. No outbound request is allowed.test('the production ZAI serializer and pinned SDK reach the pixel fixture', async () => {- const IMAGE_CAPABILITIES_CREDENTIAL = randomBytes(32).toString('hex')- const fixture = createImageFixtureFetch(- async () => {- throw new Error('Unexpected network call')- },- {- runId: 'image-capabilities-123456abcdef',- credentialHash: createHash('sha256').update(IMAGE_CAPABILITIES_CREDENTIAL).digest('hex'),- }- )- globalThis.fetch = fixture.fetch- const require = createRequire(new URL('../../../../mcp-host/package.json', import.meta.url))- require('openai/shims/web')+ const before = fixture.getEvidence().attempts.lengthconst { OpenAICompatibleProvider } = require('./dist/llm/openaiCompatible.js')const provider = new OpenAICompatibleProvider({@@ -45,7 +52,7 @@ test('the production ZAI serializer and pinned SDK reach the pixel fixture', asy[])assert.equal(result.content, image.orderedColors.join(', '))- assert.deepEqual(fixture.getEvidence().attempts, [+ assert.deepEqual(appendedRows(before), [{model: 'glm-5.3-flash',responseKind: 'tile-colors',@@ -53,3 +60,95 @@ test('the production ZAI serializer and pinned SDK reach the pixel fixture', asy},])})++// Issue #678: the same production serializer and pinned SDK, driven through the+// two turns of an attachment read. The turn-context block and the tool-result+// wrapper come from the compiled Host, so the fixture is proven against the+// format the Host really writes and not against a copy of it.+test('the production ZAI serializer and pinned SDK complete an attachment read against the fixture', async () => {+ const before = fixture.getEvidence().attempts.length+ const { OpenAICompatibleProvider } = require('./dist/llm/openaiCompatible.js')+ const { buildTurnContextBlock } = require('./dist/core/orchestration/turnContext.js')+ const { BasicSafety } = require('./dist/core/safety/safety.js')+ const provider = new OpenAICompatibleProvider(+ {+ id: 'zai',+ baseURL: 'https://api.z.ai/api/coding/paas/v4',+ defaultModel: 'glm-5.3-flash',+ },+ IMAGE_CAPABILITIES_CREDENTIAL,+ 'glm-5.3-flash'+ )++ const attachmentId = 'att-11111111-2222-4333-8444-555555555555'+ const documentText = `sdk token ${randomBytes(6).toString('hex')}\nsecond line\n`+ const block = buildTurnContextBlock({+ date: new Date('2026-09-29T00:00:00.000Z'),+ channel: { type: 'desktop' },+ attachedFiles: [+ {+ attachmentId,+ name: 'notes.txt',+ class: 'text',+ byteLength: Buffer.byteLength(documentText),+ reader: 'text',+ mismatch: false,+ declaredMediaType: 'text/plain',+ detectedMediaType: 'text/plain',+ },+ ],+ })+ const tools = [+ {+ name: 'clerum__attachment_read',+ description: 'Read an attached file.',+ parameters: { type: 'object', properties: { attachmentId: { type: 'string' } } },+ },+ ]+ const user = { role: 'user', content: `${block}\n\nSummarize the attached file.` }++ const first = await provider.completeSingleTurnWithTools([user], tools)+ // The wire says `tool_calls`; the Host's `mapFinishReason` reports it as `tool_use`.+ assert.equal(first.finish_reason, 'tool_use')+ assert.equal(first.tool_calls?.length, 1)+ assert.equal(first.tool_calls[0].name, 'clerum__attachment_read')+ assert.deepEqual(first.tool_calls[0].arguments, { attachmentId })++ const toolOutput = new BasicSafety().wrapForLlm(+ 'clerum__attachment_read',+ JSON.stringify({+ attachmentId,+ referenceId: 'ref-1',+ kind: 'text',+ byteRange: { offset: 0, length: Buffer.byteLength(documentText) },+ truncated: false,+ text: documentText,+ }),+ false+ )+ const second = await provider.completeSingleTurnWithTools(+ [+ user,+ { role: 'assistant', content: '', tool_calls: first.tool_calls },+ { role: 'tool', tool_call_id: first.tool_calls[0].id, content: toolOutput },+ ],+ tools+ )++ const digest = createHash('sha256').update(documentText, 'utf8').digest('hex')+ assert.equal(second.content, `DOCUMENT_FIXTURE_SHA256:${digest.slice(0, 16)}`)+ assert.deepEqual(appendedRows(before), [+ {+ model: 'glm-5.3-flash',+ imageSha256: null,+ responseKind: 'document-read-requested',+ documentSha256: null,+ },+ {+ model: 'glm-5.3-flash',+ imageSha256: null,+ responseKind: 'document-answer',+ documentSha256: digest,+ },+ ])+})diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex 8c88caac1..2e21df64b 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.100",+ "version": "1.0.101","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.100",+ "version": "1.0.101","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex b80adc934..0523a0d75 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.100",+ "version": "1.0.101","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",