Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): add a document-upload lane to the image-capabilities provider fixture

Alfredo Lopezbe37b70fe24a
diff --git a/Makefile b/Makefile
index ecddfb408..7ecb70530 100644
--- a/Makefile
+++ b/Makefile
@@ -308,6 +308,13 @@ minikube-run-image-capabilities: ## Run the visible image journey with an isolat
T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run
+.PHONY: minikube-run-document-upload
+minikube-run-document-upload: ## Run the visible Desktop document-upload journey (issue #678) with the isolated external-provider fixture and restore the Host
+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
+ IMAGE_CAPABILITIES_LANE=document-upload \
+ bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run
+
minikube-restore-image-capabilities: ## Resume restoration of a recorded image fixture run (IMAGE_CAPABILITIES_RUN_DIR=<dir printed by minikube-run-image-capabilities>)
@test -n "$(IMAGE_CAPABILITIES_RUN_DIR)" || { echo "IMAGE_CAPABILITIES_RUN_DIR is required: the run directory printed by 'make minikube-run-image-capabilities'"; exit 1; }
@T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
diff --git a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
index 47582627a..50e53a8c0 100644
--- a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
+++ b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
@@ -52,11 +52,25 @@ export const FIXTURE_RESPONSE_KIND = {
tileColors: 'tile-colors',
textOnly: 'text-only',
rejected: 'rejected',
+ // Issue #678: the two provider turns of an attachment read. The first asks for
+ // the `clerum__attachment_read` tool; the second answers from the text the tool
+ // delivered.
+ documentReadRequested: 'document-read-requested',
+ documentAnswer: 'document-answer',
} as const
/** The fixture's text-only answer. It names no image and no color. */
export const FIXTURE_TEXT_ONLY_CONTENT = 'IMAGE_FIXTURE_TEXT_OK'
+/**
+ * Prefix of the fixture's document answer. The answer is this prefix plus the
+ * first {@link FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS} hex characters of the
+ * sha256 of the text the Host delivered to the model, so it cannot be produced
+ * from the prompt alone.
+ */
+export const FIXTURE_DOCUMENT_ANSWER_PREFIX = 'DOCUMENT_FIXTURE_SHA256:'
+export const FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS = 16
+
const SHA256_HEX = /^[a-f0-9]{64}$/
const MINIKUBE_PROFILE_PATTERN = /^[a-z0-9][a-z0-9-]{0,62}$/
const LOOPBACK_HOSTS = new Set(['127.0.0.1', 'localhost', '::1'])
@@ -250,6 +264,12 @@ export interface FixtureProviderAttempt {
/** Digest of the exact PNG bytes that reached the wire, or null when none did. */
imageSha256: string | null
responseKind: string
+ /**
+ * Digest of the document text that reached the wire in a tool-result message.
+ * The fixture omits the field on image rows, so it is `undefined` there and
+ * `null` on the read-request row, which carries no document yet.
+ */
+ documentSha256?: string | null
}
export interface FixtureEvidenceCounters {
@@ -261,6 +281,9 @@ export interface FixtureEvidenceCounters {
textOnlyResponses: number
textModelImageRefusals: number
blockedEgress: number
+ documentReadRequests: number
+ documentAnswers: number
+ documentFailures: number
}
export interface ImageCapabilityEvidenceSnapshot {
@@ -284,6 +307,9 @@ const COUNTER_KEYS: readonly (keyof FixtureEvidenceCounters)[] = [
'textOnlyResponses',
'textModelImageRefusals',
'blockedEgress',
+ 'documentReadRequests',
+ 'documentAnswers',
+ 'documentFailures',
]
function requireCounters(value: unknown, source: string): FixtureEvidenceCounters {
@@ -324,6 +350,20 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu
)
}
+ // Present only on document rows; an image row must not grow the field.
+ if ('documentSha256' in value) {
+ const documentSha256 = value.documentSha256
+ if (
+ documentSha256 !== null &&
+ !(typeof documentSha256 === 'string' && SHA256_HEX.test(documentSha256))
+ ) {
+ throw new Error(
+ `${source}: attempts[${index}].documentSha256 must be a sha256 hex digest or null`
+ )
+ }
+ return { model, imageSha256, responseKind, documentSha256 }
+ }
+
return { model, imageSha256, responseKind }
}
@@ -467,7 +507,8 @@ export function appendedAttempts(
!current ||
current.model !== row.model ||
current.imageSha256 !== row.imageSha256 ||
- current.responseKind !== row.responseKind
+ current.responseKind !== row.responseKind ||
+ current.documentSha256 !== row.documentSha256
) {
throw new Error(
`[image-capabilities] ledger row ${index} changed between reads; the ledger must be append-only.`
diff --git a/desktop-app/test/e2e-playwright/playwright.document-upload.config.ts b/desktop-app/test/e2e-playwright/playwright.document-upload.config.ts
new file mode 100644
index 000000000..b37bee4e1
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/playwright.document-upload.config.ts
@@ -0,0 +1,54 @@
+import { defineConfig } from '@playwright/test'
+import path from 'node:path'
+import { requireImageCapabilitiesFixtureEnv } from './helpers/imageCapabilityEvidence'
+
+// E2E_GUARDIAN_IPC_FLOW: this file only configures the runner. The spec owns
+// visible IPC transitions and reads the external provider ledger as its wire oracle.
+
+/*
+ * Desktop App document-upload E2E lane (issue #678).
+ *
+ * It reuses the image-capabilities fixture stack (derived Host image, provider
+ * fixture, seeded catalog and login), so it has the same contract as
+ * playwright.image-capabilities.config.ts and for the same reasons:
+ *
+ * - It loads NO env file. Every binding is exported by the runner
+ * (`make minikube-run-document-upload`) and a missing or inconsistent one
+ * fails here, at config load, before Playwright launches anything.
+ * - It matches only the document-upload journey, so no recorder journey that
+ * talks to a real provider can run from this lane.
+ *
+ * The journey attaches a text document through the visible composer picker and
+ * asserts, through the provider fixture's ledger, that the document text reached
+ * the model through the `clerum__attachment_read` tool and not through the prompt.
+ */
+
+const repoRoot = path.resolve(__dirname, '../../..')
+
+if (process.env.NODE_ENV !== undefined && process.env.NODE_ENV !== 'test') {
+ throw new Error(
+ `NODE_ENV must be "test" (or unset) for the document-upload fixture lane; ` +
+ `received "${process.env.NODE_ENV}".`
+ )
+}
+process.env.NODE_ENV = 'test'
+
+// Fail at config load, with every missing binding listed at once.
+requireImageCapabilitiesFixtureEnv(process.env)
+
+const recorderRoot = process.env.QA_RECORDER_ROOT
+ ? path.resolve(process.env.QA_RECORDER_ROOT)
+ : path.join(repoRoot, '.local-notes', 'qa-recorder')
+
+export default defineConfig({
+ testDir: '.',
+ testMatch: /qa-recorder-document-upload\.spec\.ts/,
+ // One real Electron launch, one sign-in and one chat with two provider turns.
+ timeout: 300_000,
+ expect: { timeout: 30_000 },
+ retries: 0,
+ workers: 1,
+ reporter: [['list']],
+ outputDir: path.join(recorderRoot, 'runs', 'desktop-app-document-upload'),
+ preserveOutput: 'always',
+})
diff --git a/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts b/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts
new file mode 100644
index 000000000..4e7ab6b3c
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/qa-recorder-document-upload.spec.ts
@@ -0,0 +1,358 @@
+import { type ElectronApplication, type Page, type TestInfo, expect, test } from '@playwright/test'
+import { randomBytes } from 'node:crypto'
+import fs from 'node:fs'
+import { createRequire } from 'node:module'
+import path from 'node:path'
+import {
+ FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS,
+ FIXTURE_DOCUMENT_ANSWER_PREFIX,
+ FIXTURE_RESPONSE_KIND,
+ IMAGE_CAPABILITY_FIXTURE_MODELS,
+ appendedAttempts,
+ readImageCapabilityEvidence,
+ requireImageCapabilitiesFixtureEnv,
+ resolveImageCapabilitiesMode,
+ sha256Hex,
+} from './helpers/imageCapabilityEvidence'
+import {
+ DESKTOP_APP_ROOT,
+ EXTERNAL_REST_API_BASE_URL,
+ MAIN_ENTRY,
+ RPC_PROXY_BASE_URL,
+ assertAllowedTarget,
+ desktopCredentials,
+ finalizeRecording,
+ launchDesktopApp,
+ login,
+ screenshotAndLog,
+} from './qa-recorder-helpers'
+
+/*
+ * E2E_GUARDIAN_IPC_FLOW: Desktop chat, the model catalog, the agent list and the
+ * composer all travel over Electron IPC (main-process handlers in desktop-app/src),
+ * so the renderer has no HTTP request to await for any transition in this
+ * journey. The visible/business oracles are the composer's own DOM contracts,
+ * the persisted thread, and the provider-boundary ledger of the derived in-cluster
+ * fixture.
+ *
+ * Issue #678: any document attached from the composer travels inline to the Host
+ * as a `kind:'file'` attachment. The Host never puts its text in the prompt: it
+ * announces the file in the turn context and the model reads it through the
+ * `clerum__attachment_read` tool. The provider fixture asks for that tool call,
+ * then answers with a digest of the text it received in the tool result. So the
+ * answer on screen can only be produced if the exact bytes attached here crossed
+ * the composer, IPC, the Host admission, the read tool and the provider wire.
+ *
+ * Only the external provider peer is simulated. The run is selected by the
+ * fixture lane's own run id and never reaches a real provider.
+ */
+
+const MODE = resolveImageCapabilitiesMode()
+
+/**
+ * The prompt names neither the file content nor its digest: the answer cannot be
+ * derived from it.
+ */
+const DOCUMENT_PROMPT = 'Summarize the attached file.'
+
+function instanceBindingCheck(app: ElectronApplication, page: Page, testInfo: TestInfo) {
+ const expectedRun = path.resolve(testInfo.outputPath('electron-isolation'))
+ const expectedData = path.join(expectedRun, 'user-data')
+ const expectedConfig = path.join(expectedRun, 'runtime-config.json')
+ const expectedExecutable = createRequire(path.join(DESKTOP_APP_ROOT, 'package.json'))(
+ 'electron'
+ ) as string
+ const restOrigin = new URL(EXTERNAL_REST_API_BASE_URL).origin
+ const rpcOrigin = new URL(RPC_PROXY_BASE_URL).origin
+ const slug = `${restOrigin}_${rpcOrigin}`
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, '_')
+ .replace(/^_+|_+$/g, '')
+ .slice(0, 48)
+ const expectedEnvKey = `${slug}-${sha256Hex(Buffer.from(`${restOrigin}|rpc=${rpcOrigin}`)).slice(0, 12)}`
+ return async () => {
+ // Observe the running processes and IPC configuration, rather than trusting
+ // launch environment variables. This never changes selection or auth state.
+ const identity = await app.evaluate(({ app: electronApp }) => ({
+ pid: process.pid,
+ argv: process.argv,
+ executable: electronApp.getPath('exe'),
+ userData: electronApp.getPath('userData'),
+ }))
+ const runtime = await page.evaluate(async () => {
+ const state = await window.clerum.auth.getRuntimeConfigState()
+ return { storagePath: state.storagePath, envKey: state.envKey }
+ })
+ expect(identity.pid).toBe(app.process().pid)
+ expect(identity.argv.includes(MAIN_ENTRY)).toBe(true)
+ expect(fs.realpathSync(identity.executable)).toBe(fs.realpathSync(expectedExecutable))
+ expect(path.resolve(identity.userData)).toBe(expectedData)
+ expect(path.resolve(runtime.storagePath)).toBe(expectedConfig)
+ expect(runtime.envKey).toBe(expectedEnvKey)
+ }
+}
+
+/**
+ * Correlates the two rendered messages with one durable Host task and returns
+ * its id once the task result matches the expected answer. Read-only IPC reads
+ * after the visible send; they log in, select and send nothing.
+ */
+async function observeCompletedTask(
+ page: Page,
+ host: string,
+ expectedAnswer: RegExp
+): Promise<string> {
+ const visibleIds = await page
+ .locator('[data-chat-message-id]')
+ .evaluateAll(elements => elements.map(element => element.getAttribute('data-chat-message-id')))
+ expect(visibleIds).toHaveLength(2)
+ let observed: { taskId: string; result: string } | null = null
+ await expect
+ .poll(
+ async () => {
+ observed = await page.evaluate(
+ async ({ hostRef, ids }) => {
+ const chats = await window.clerum.chat.list(hostRef)
+ for (const chat of chats.slice(0, 20)) {
+ const messages = await window.clerum.chat.loadMessages(hostRef, chat.id)
+ const pair = ids.map(id => messages.find(message => message.id === id))
+ const taskId = pair[0]?.task_id
+ if (!taskId || pair[1]?.task_id !== taskId) continue
+ const result = await window.clerum.rpc.getTaskResult(hostRef, taskId)
+ return { taskId, result: JSON.stringify(result) }
+ }
+ return null
+ },
+ { hostRef: host, ids: visibleIds }
+ )
+ return observed?.result ?? ''
+ },
+ { timeout: 30_000 }
+ )
+ .toMatch(expectedAnswer)
+ expect(observed).not.toBeNull()
+ return observed!.taskId
+}
+
+function composer(page: Page) {
+ return page.getByRole('textbox', { name: 'Agent message composer' })
+}
+
+function sendButton(page: Page) {
+ return page.getByTestId('send-button')
+}
+
+function modelChip(page: Page) {
+ return page.getByTestId('selected-chat-model')
+}
+
+function fileChips(page: Page) {
+ return page.getByTestId('composer-file-chip')
+}
+
+/** Start a blank chat and prove the thread is empty before anything is sent. */
+async function startBlankChat(page: Page) {
+ await page.getByTestId('nav-new-chat').click()
+ await expect(composer(page)).toBeVisible({ timeout: 30_000 })
+ await expect(page.locator('[data-chat-message-id]')).toHaveCount(0, { timeout: 20_000 })
+}
+
+/**
+ * Land on the exact configured agent through the visible Switch chat agent
+ * menu, and trust the composer only once the switch control reports it.
+ */
+async function openConfiguredAgentChat(page: Page, hostRef: string) {
+ const escaped = hostRef.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+ await page.getByTestId('nav-chat').click()
+ const switchAgent = page.getByRole('button', { name: 'Switch chat agent' })
+ await expect(switchAgent).toBeVisible({ timeout: 30_000 })
+ await switchAgent.click()
+ const agentMenuItem = page.getByRole('menuitem', { name: new RegExp(`^${escaped}$`, 'i') })
+ await expect(agentMenuItem).toHaveCount(1)
+ await agentMenuItem.click()
+ await expect(switchAgent).toContainText(new RegExp(escaped, 'i'))
+}
+
+/** Selects the model through the visible popover and asserts the chip reports it. */
+async function selectSupportedModel(page: Page, model: string) {
+ await expect(modelChip(page)).toBeVisible({ timeout: 30_000 })
+ await modelChip(page).click()
+ const menu = page.getByRole('menu', { name: 'Select model' })
+ await expect(menu).toBeVisible({ timeout: 20_000 })
+ const row = page.getByTestId(`model-option-${model}`)
+ await expect(row).toHaveCount(1)
+ await row.click()
+ await expect(modelChip(page)).toHaveAttribute('data-model-id', model)
+}
+
+/**
+ * Reach the authenticated shell through the real form. While the form is still
+ * unsent, the authenticated surface must not be mounted, otherwise the journey
+ * could pass without a real sign-in.
+ */
+async function signIn(page: Page) {
+ await expect(page.locator('.boot-overlay')).toBeHidden({ timeout: 30_000 })
+
+ const emailInput = page.locator('#email-input')
+ const settingsMenu = page.getByTestId('nav-settings-menu')
+ await expect(emailInput.or(settingsMenu)).toBeVisible({ timeout: 30_000 })
+ if (await settingsMenu.isVisible()) {
+ if ((await settingsMenu.getAttribute('aria-expanded')) !== 'true') {
+ await settingsMenu.click()
+ }
+ await expect(settingsMenu).toHaveAttribute('aria-expanded', 'true')
+ await page.getByTestId('logout-btn').click()
+ }
+
+ await expect(emailInput).toBeVisible({ timeout: 20_000 })
+ await expect(composer(page)).toHaveCount(0)
+ await expect(sendButton(page)).toHaveCount(0)
+ await expect(page.getByTestId('nav-chat')).toHaveCount(0)
+
+ await login(page, desktopCredentials())
+ await expect(page.getByTestId('nav-chat')).toBeVisible({ timeout: 30_000 })
+}
+
+/** Attach a document the way a user does: Add context -> Upload Files -> native picker. */
+async function attachDocument(
+ page: Page,
+ document: { fileName: string; mimeType: string; buffer: Buffer }
+) {
+ const attachButton = page.getByRole('button', { name: 'Add context' })
+ await expect(attachButton).toBeEnabled()
+ await attachButton.click()
+ const uploadItem = page.getByRole('menuitem', { name: 'Upload Files' })
+ await expect(uploadItem).toBeEnabled({ timeout: 15_000 })
+
+ const [fileChooser] = await Promise.all([
+ page.waitForEvent('filechooser', { timeout: 15_000 }),
+ uploadItem.click(),
+ ])
+ await fileChooser.setFiles({
+ name: document.fileName,
+ mimeType: document.mimeType,
+ buffer: document.buffer,
+ })
+}
+
+/**
+ * The fixture's document answer, matched from the exported marker. The
+ * underscores are optional in the pattern because the bubble renders Markdown,
+ * which may treat intraword underscores as emphasis; the ledger below pins the
+ * exact digest regardless.
+ */
+function documentAnswerRegex(documentSha: string): RegExp {
+ const prefix = FIXTURE_DOCUMENT_ANSWER_PREFIX.replace(/_/g, '[_-]?')
+ return new RegExp(`${prefix}${documentSha.slice(0, FIXTURE_DOCUMENT_ANSWER_DIGEST_CHARS)}`, 'i')
+}
+
+test('document-upload fixture: an attached text file reaches the model through the read tool', async ({}, testInfo) => {
+ test.skip(
+ MODE !== 'fixture',
+ 'Set IMAGE_CAPABILITIES_RUN_ID=image-capabilities-<12 hex> (the runner does) to run the ' +
+ 'derived-provider fixture journey.'
+ )
+
+ const env = requireImageCapabilitiesFixtureEnv()
+ await assertAllowedTarget('EXTERNAL_REST_API_BASE_URL', env.externalRestApiBaseUrl)
+ await assertAllowedTarget('RPC_PROXY_BASE_URL', env.rpcProxyBaseUrl)
+
+ // Precondition read before anything is sent: the derived fixture must be live
+ // in the deployment that serves this host, for this run and this profile.
+ const baseline = readImageCapabilityEvidence(env)
+ expect(baseline.runId).toBe(env.runId)
+ expect(baseline.profile).toBe(env.profile)
+
+ // A random token the prompt never mentions: the digest of these exact bytes is
+ // the only thing that can appear in the answer.
+ const document = {
+ fileName: `notes-${randomBytes(3).toString('hex')}.txt`,
+ mimeType: 'text/plain',
+ buffer: Buffer.from(`document-upload ${randomBytes(6).toString('hex')}\nsecond line\n`, 'utf8'),
+ }
+ const documentSha = sha256Hex(document.buffer)
+ fs.mkdirSync(testInfo.outputPath(), { recursive: true })
+ fs.writeFileSync(testInfo.outputPath(document.fileName), document.buffer)
+
+ let app: ElectronApplication | undefined
+ let recordedPage: Page | undefined
+
+ try {
+ const launched = await launchDesktopApp(
+ testInfo,
+ `issue678-minikube${new URL(EXTERNAL_REST_API_BASE_URL).port}`
+ )
+ app = launched.app
+ recordedPage = launched.page
+ const page = launched.page
+
+ const assertBinding = instanceBindingCheck(app, page, testInfo)
+ await assertBinding()
+ await signIn(page)
+ await assertBinding()
+ await openConfiguredAgentChat(page, env.hostRef)
+
+ await test.step('the document is admitted as a ready file chip and the send is enabled', async () => {
+ await startBlankChat(page)
+ await selectSupportedModel(page, IMAGE_CAPABILITY_FIXTURE_MODELS.supported)
+
+ await attachDocument(page, document)
+ const chip = fileChips(page).filter({ hasText: document.fileName })
+ await expect(chip).toHaveCount(1, { timeout: 20_000 })
+ await expect(chip).toHaveAttribute('data-file-status', 'ready', { timeout: 20_000 })
+
+ await composer(page).fill(DOCUMENT_PROMPT)
+ await expect(sendButton(page)).toBeEnabled({ timeout: 20_000 })
+ })
+
+ await test.step('the answer carries the digest of the delivered text and the ledger shows both provider turns', async () => {
+ const before = readImageCapabilityEvidence(env)
+ await assertBinding()
+ await sendButton(page).click()
+ await expect(fileChips(page)).toHaveCount(0, { timeout: 30_000 })
+
+ await expect(page.getByTestId('message-list')).toBeVisible({ timeout: 20_000 })
+ await expect(page.locator('[data-chat-message-id]')).toHaveCount(2, { timeout: 120_000 })
+ const response = page.getByTestId('agent-response').locator('.message-block.markdown-content')
+ await expect(response).toHaveCount(1, { timeout: 120_000 })
+ await expect(response).toContainText(documentAnswerRegex(documentSha), { timeout: 120_000 })
+ await observeCompletedTask(page, env.hostRef, documentAnswerRegex(documentSha))
+
+ const after = readImageCapabilityEvidence(env)
+ const appended = appendedAttempts(before, after)
+ const readRows = appended.filter(
+ row => row.responseKind === FIXTURE_RESPONSE_KIND.documentReadRequested
+ )
+ const answerRows = appended.filter(
+ row => row.responseKind === FIXTURE_RESPONSE_KIND.documentAnswer
+ )
+
+ // Turn 1: the model asked for the read tool. The prompt alone produced no answer.
+ expect(readRows).toHaveLength(1)
+ expect(readRows[0]?.model).toBe(IMAGE_CAPABILITY_FIXTURE_MODELS.supported)
+ expect(readRows[0]?.documentSha256).toBeNull()
+ // Turn 2: the tool result reached the wire carrying exactly the attached bytes.
+ expect(answerRows).toHaveLength(1)
+ expect(answerRows[0]?.model).toBe(IMAGE_CAPABILITY_FIXTURE_MODELS.supported)
+ expect(answerRows[0]?.documentSha256).toBe(documentSha)
+
+ // A document is never an image: no pixels moved, and the fixture refused nothing.
+ expect(appended.filter(row => row.imageSha256 !== null)).toHaveLength(0)
+ expect(after.counters.imageAttempts - before.counters.imageAttempts).toBe(0)
+ expect(after.counters.documentReadRequests - before.counters.documentReadRequests).toBe(1)
+ expect(after.counters.documentAnswers - before.counters.documentAnswers).toBe(1)
+ expect(after.counters.documentFailures - before.counters.documentFailures).toBe(0)
+ })
+
+ await screenshotAndLog(page, testInfo, 'desktop-document-upload-fixture')
+ } catch (error) {
+ if (recordedPage && !recordedPage.isClosed()) {
+ await recordedPage
+ .screenshot({ path: testInfo.outputPath('failure.png') })
+ .catch(() => undefined)
+ }
+ throw error
+ } finally {
+ await finalizeRecording(app, recordedPage)
+ }
+})
diff --git a/docs/testing/desktop-headful-journeys.md b/docs/testing/desktop-headful-journeys.md
index 6deec7505..ac0e734ff 100644
--- a/docs/testing/desktop-headful-journeys.md
+++ b/docs/testing/desktop-headful-journeys.md
@@ -33,8 +33,9 @@ test under `.local-notes/qa-recorder/runs/desktop-app/` (git-ignored).
| Settings + chat smoke | `qa-recorder-settings-chat.spec.ts` | 1 — Settings tabs + one chat message | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |
| Image capabilities | `qa-recorder-image-capabilities.spec.ts` | 2 — capability gate on a text-only model, fixture-backed image answer | `QA_RECORDER_CONFIRM_CHAT` (required) | ✅ |
| Model selector | `qa-recorder-model-selector.spec.ts` | 1 — a multi-model catalog lists rows without capability tags | — | ✅ |
+| Document upload | `qa-recorder-document-upload.spec.ts` | 1 — attached text file read through the read tool (fixture lane only) | `IMAGE_CAPABILITIES_RUN_ID` (runner-set) | ✅ |
-**Totals: 13 specs, 22 tests.** Read-only journeys need no confirmation flag; every
+**Totals: 14 specs, 23 tests.** Read-only journeys need no confirmation flag; every
journey still calls the loopback health guard on both API URLs — except against a
non-loopback target, where `QA_RECORDER_ALLOW_REMOTE=1` skips the precheck, so a
wrong URL surfaces as an Electron timeout instead of a fast refusal. Mutating / paid
diff --git a/scripts/e2e/image-capabilities-fixture.mjs b/scripts/e2e/image-capabilities-fixture.mjs
index dd6e97f0f..a49505eec 100644
--- a/scripts/e2e/image-capabilities-fixture.mjs
+++ b/scripts/e2e/image-capabilities-fixture.mjs
@@ -22,6 +22,31 @@ export const fixtureConfigKeys = [
'CONTROL_API_REAL_PG_CONTEXT',
]
+/**
+ * The Playwright journeys this runner can drive against the swapped Host. The
+ * lane is chosen by `IMAGE_CAPABILITIES_LANE` and only from this closed set, so
+ * an environment value can never name an arbitrary config path.
+ */
+export const playwrightLanes = {
+ image: {
+ config: 'desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',
+ label: 'image-capabilities-playwright',
+ },
+ 'document-upload': {
+ config: 'desktop-app/test/e2e-playwright/playwright.document-upload.config.ts',
+ label: 'document-upload-playwright',
+ },
+}
+
+export function playwrightLaneConfig(env) {
+ const lane = env.IMAGE_CAPABILITIES_LANE ?? 'image'
+ if (!Object.hasOwn(playwrightLanes, lane))
+ throw new Error(
+ `IMAGE_CAPABILITIES_LANE must be one of ${Object.keys(playwrightLanes).join(', ')} (received ${JSON.stringify(lane)})`
+ )
+ return { lane, ...playwrightLanes[lane] }
+}
+
export function proveImages(manifest, observed, head, profile) {
if (!/^[a-f0-9]{40}$/.test(head) || manifest.profile !== profile)
throw new Error('Fixture image ownership mismatch')
@@ -157,6 +182,7 @@ async function main() {
)
throw new Error('Matching branch profile and context required')
command('bash', ['scripts/minikube/require-t2-mutation-lock.sh'])
+ const { lane, config: laneConfig, label: laneLabel } = playwrightLaneConfig(process.env)
const head = command('git', ['rev-parse', 'HEAD']).trim()
const branch = command('git', ['branch', '--show-current']).trim()
if (command('git', ['status', '--porcelain']).trim()) throw new Error('Clean checkout required')
@@ -715,7 +741,7 @@ async function main() {
QA_RECORDER_ROOT: evidence,
}
command('npm', ['run', 'verify:electron', '--prefix', 'desktop-app'])
- process.stdout.write(`Image capability fixture ready: ${state.runId}\n`)
+ process.stdout.write(`Image capability fixture ready: ${state.runId} (lane ${lane})\n`)
const output = command(
'node',
[
@@ -727,12 +753,12 @@ async function main() {
'--kill-grace-seconds',
'5',
'--label',
- 'image-capabilities-playwright',
+ laneLabel,
'--',
'node',
'desktop-app/node_modules/@playwright/test/cli.js',
'test',
- '--config=desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',
+ `--config=${laneConfig}`,
],
{ env: environment, timeout: 930_000 }
)
diff --git a/scripts/e2e/image-capabilities-fixture.test.mjs b/scripts/e2e/image-capabilities-fixture.test.mjs
index d702274d7..22c5e9c6b 100644
--- a/scripts/e2e/image-capabilities-fixture.test.mjs
+++ b/scripts/e2e/image-capabilities-fixture.test.mjs
@@ -6,6 +6,7 @@ import {
fixtureImage,
installSignalRestore,
modelInputs,
+ playwrightLaneConfig,
proveImages,
requireOwnedResource,
runAnnotation,
@@ -24,6 +25,26 @@ function manifest() {
}
}
+test('the Playwright lane defaults to the image journey and accepts only known lanes', () => {
+ assert.deepEqual(playwrightLaneConfig({}), {
+ lane: 'image',
+ config: 'desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',
+ label: 'image-capabilities-playwright',
+ })
+ assert.deepEqual(playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'document-upload' }), {
+ lane: 'document-upload',
+ config: 'desktop-app/test/e2e-playwright/playwright.document-upload.config.ts',
+ label: 'document-upload-playwright',
+ })
+ for (const lane of ['', 'documents', '../evil.config.ts', '__proto__', 'toString']) {
+ assert.throws(
+ () => playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane }),
+ /IMAGE_CAPABILITIES_LANE must be one of image, document-upload/,
+ lane
+ )
+ }
+})
+
test('requires both exact source revisions and the live derived-base identity', () => {
assert.deepEqual(proveImages(manifest(), ids, head, profile), { head, profile, images: ids })
for (const mutate of [
diff --git a/tests/e2e/fixtures/image-capabilities/README.md b/tests/e2e/fixtures/image-capabilities/README.md
index 45dc63463..f74ec51f1 100644
--- a/tests/e2e/fixtures/image-capabilities/README.md
+++ b/tests/e2e/fixtures/image-capabilities/README.md
@@ -18,6 +18,41 @@ compiled ZAI provider with the PNG generated by the browser journey.
The preload runs with Node's `--import`; the production `main.js` remains the
main program so its normal startup guard and crash handlers execute.
+## Documents (issue #678)
+
+The same fixture also answers the two provider turns of a document attachment.
+The Host never puts a file's text in the prompt: it announces the file in the
+turn context and the model reads it through the `clerum__attachment_read` tool.
+When that tool is offered and the last user message announces an
+`attached_file`, the fixture first replies with a tool call for it. After the
+tool result arrives, it replies `DOCUMENT_FIXTURE_SHA256:` plus the first 16 hex
+characters of the SHA-256 of the text in that result. The prompt alone can never
+produce that answer.
+
+The ledger records `document-read-requested` (no digest yet) and
+`document-answer` (with `documentSha256`, never the text). Malformed, non-text
+or repeated tool results and streamed document turns are refused and counted in
+`documentFailures`. Image rows keep their previous shape.
+
+`sdk.test.mjs` drives both turns through the compiled Host: the turn-context
+block, the safety wrapper and the ZAI provider are the production ones.
+
+The browser journey has its own lane, selected by the runner:
+
+```sh
+make minikube-build-image-capabilities-fixture
+make minikube-t2
+make minikube-run-document-upload
+```
+
+`make minikube-run-document-upload` is `minikube-run-image-capabilities` with
+`IMAGE_CAPABILITIES_LANE=document-upload`; the config is
+`desktop-app/test/e2e-playwright/playwright.document-upload.config.ts`. It
+attaches a text file through Add context -> Upload Files, waits for a `ready`
+file chip, sends, and requires the digest of the attached bytes in the answer,
+one `document-read-requested` row, one `document-answer` row carrying that
+digest, no image row and no `documentFailures`.
+
## Local execution
Use Node 24 and Bash 5, a clean development branch based on current `origin/dev`,
diff --git a/tests/e2e/fixtures/image-capabilities/provider.mjs b/tests/e2e/fixtures/image-capabilities/provider.mjs
index 7165f2776..c25139bde 100644
--- a/tests/e2e/fixtures/image-capabilities/provider.mjs
+++ b/tests/e2e/fixtures/image-capabilities/provider.mjs
@@ -25,15 +25,25 @@
* the six tile centers. A run that never delivered the image cannot pass by
* guessing, and a request that carries no image never receives a color list.
*
+ * Documents (issue #678). The Host never puts an attached document's text in the
+ * prompt: the first request lists the file by id and offers the
+ * `clerum__attachment_read` tool. So the one tool call this fixture ever makes
+ * is that read, answered with `finish_reason: 'tool_calls'`; the second request
+ * carries the tool result, and the answer is `DOCUMENT_FIXTURE_SHA256:` plus the
+ * first 16 hex characters of the SHA-256 of the delivered text. A run whose
+ * document never reached the provider cannot produce that digest. Document rows
+ * add `documentSha256` (the digest of the delivered text, never the text).
+ *
* Determinism and sanitation. Responses carry a synthetic completion id derived
- * from the run id, a fixed test usage block, `finish_reason: 'stop'`, and never a
- * tool call. Both accepted models answer a text-only request with
- * `IMAGE_FIXTURE_TEXT_OK`, so a legitimate title or summary call is never turned
- * into a false failure; only a delivered image can produce a color list. Evidence
- * is limited to the run id, counters, per-model counts, a per-call ledger
- * (model, nullable image digest, response kind), and SHA-256 digests of the
- * delivered image bytes. Headers, auth values, prompts, and raw image bytes are
- * never recorded.
+ * from the run id, a fixed test usage block, and `finish_reason: 'stop'` (or
+ * `'tool_calls'` for the document read). Both accepted models answer a text-only
+ * request with `IMAGE_FIXTURE_TEXT_OK`, so a legitimate title or summary call is
+ * never turned into a false failure; only a delivered image can produce a color
+ * list. Evidence is limited to the run id, counters, per-model counts, a per-call
+ * ledger (model, nullable image digest, response kind, and for documents the
+ * digest of the delivered text), and SHA-256 digests of the delivered image bytes
+ * and document text. Headers, auth values, prompts, raw image bytes, and document
+ * text are never recorded.
*
* Authentication. This fixture never holds the runner's ZAI key. It holds only
* the SHA-256 digest of that key, hashes the bearer token it receives, and
@@ -330,6 +340,97 @@ function extractPngBytes(messages) {
return Buffer.from(payload, 'base64')
}
+/** The Host tool that returns the text of an attached document (issue #678). */
+const ATTACHMENT_READ_TOOL = 'clerum__attachment_read'
+
+/** Prefix of the document answer; the digest after it is computed from the delivered text. */
+const DOCUMENT_ANSWER_PREFIX = 'DOCUMENT_FIXTURE_SHA256:'
+
+/** Length of the digest prefix echoed in the answer. */
+const DOCUMENT_ANSWER_DIGEST_CHARS = 16
+
+/** The `attached_file` line the Host writes into the turn-context block. */
+const ATTACHED_FILE_LINE = /^attached_file: id="([^"\n]+)"/m
+
+/** The wrapper the Host puts around a tool result (`BasicSafety.wrapForLlm`). */
+const ATTACHMENT_TOOL_OUTPUT = new RegExp(
+ `^<tool_output name="${ATTACHMENT_READ_TOOL}" sanitized="(?:true|false)">\\n([\\s\\S]*)\\n</tool_output>$`
+)
+
+/** A document turn the fixture cannot read pixel-exactly gets a stable reason code. */
+class FixtureDocumentError extends Error {
+ constructor(code, message) {
+ super(message)
+ this.name = 'FixtureDocumentError'
+ this.code = code
+ }
+}
+
+function lastUserIndex(messages) {
+ for (let index = messages.length - 1; index >= 0; index -= 1) {
+ if (messages[index]?.role === 'user') return index
+ }
+ return -1
+}
+
+/**
+ * Classifies the request as one of the two turns of an attachment read.
+ *
+ * - `null`: not a document turn (no `attached_file` line, or the read tool is not offered).
+ * - `{ stage: 'read', attachmentId }`: the user message lists a readable file and no tool
+ * result follows it yet, so the model has to ask for the text.
+ * - `{ stage: 'answer', text }`: a `role:'tool'` message after the last user message carries
+ * the wrapped tool result, so the model can answer from what was delivered.
+ *
+ * The fixture is stateless: both turns are recognised from the message list alone.
+ */
+function classifyDocumentTurn(body) {
+ const tools = Array.isArray(body.tools) ? body.tools : []
+ const offered = tools.some(tool => tool?.function?.name === ATTACHMENT_READ_TOOL)
+ if (!offered) return null
+
+ const userIndex = lastUserIndex(body.messages)
+ if (userIndex < 0) return null
+ const user = body.messages[userIndex]
+ if (typeof user.content !== 'string') return null
+ const attached = ATTACHED_FILE_LINE.exec(user.content)
+ if (!attached) return null
+
+ const toolMessages = body.messages
+ .slice(userIndex + 1)
+ .filter(message => message?.role === 'tool' && typeof message.content === 'string')
+ if (toolMessages.length === 0) return { stage: 'read', attachmentId: attached[1] }
+ if (toolMessages.length > 1)
+ throw new FixtureDocumentError(
+ 'document-tool-result-count',
+ `exactly one attachment read is supported, received ${toolMessages.length}`
+ )
+
+ const wrapped = ATTACHMENT_TOOL_OUTPUT.exec(toolMessages[0].content)
+ if (!wrapped)
+ throw new FixtureDocumentError(
+ 'document-tool-output-malformed',
+ 'the tool result is not a clerum__attachment_read wrapper'
+ )
+ let result
+ try {
+ result = JSON.parse(wrapped[1])
+ } catch {
+ throw new FixtureDocumentError('document-tool-output-malformed', 'the tool result is not JSON')
+ }
+ if (
+ !result ||
+ typeof result !== 'object' ||
+ result.kind !== 'text' ||
+ typeof result.text !== 'string'
+ )
+ throw new FixtureDocumentError(
+ 'document-tool-output-unreadable',
+ 'the tool result does not carry the document text'
+ )
+ return { stage: 'answer', text: result.text }
+}
+
const INTERNAL_HOST_SUFFIXES = [
'.svc.cluster.local',
'.svc',
@@ -457,6 +558,36 @@ function completionBody(completionId, model, content) {
}
}
+/** An assistant turn that asks the Host to run `name` with `args` (non-stream only). */
+function toolCallCompletionBody(completionId, model, name, args) {
+ return {
+ id: completionId,
+ object: 'chat.completion',
+ created: FIXTURE_CREATED,
+ model,
+ choices: [
+ {
+ index: 0,
+ message: {
+ role: 'assistant',
+ content: null,
+ tool_calls: [
+ {
+ id: `call-${completionId}`,
+ type: 'function',
+ function: { name, arguments: JSON.stringify(args) },
+ },
+ ],
+ },
+ finish_reason: 'tool_calls',
+ logprobs: null,
+ },
+ ],
+ usage: FIXTURE_USAGE,
+ system_fingerprint: FIXTURE_SYSTEM_FINGERPRINT,
+ }
+}
+
function streamResponse(completionId, model, content) {
const chunk = (delta, finishReason, withUsage) => ({
id: completionId,
@@ -541,6 +672,9 @@ export function createImageFixtureFetch(
blockedEgress: 0,
imageDecodeFailures: 0,
unauthorizedAttempts: 0,
+ documentReadRequests: 0,
+ documentAnswers: 0,
+ documentFailures: 0,
},
attempts: [],
}
@@ -555,8 +689,14 @@ export function createImageFixtureFetch(
* request was refused before its body was parsed, and `imageSha256` is null
* whenever the attempt carried no usable image.
*/
- const recordCall = (model, imageSha256, responseKind) => {
- state.attempts.push({ model, imageSha256, responseKind })
+ const recordCall = (model, imageSha256, responseKind, documentSha256) => {
+ // Only document rows carry `documentSha256`, so the image rows keep the exact
+ // shape they always had.
+ state.attempts.push(
+ documentSha256 === undefined
+ ? { model, imageSha256, responseKind }
+ : { model, imageSha256, responseKind, documentSha256 }
+ )
}
const refuse = (model, reason, { status = 400, code = reason, imageSha256 = null } = {}) => {
state.counters.rejectedAttempts += 1
@@ -564,10 +704,15 @@ export function createImageFixtureFetch(
publish()
return fixtureErrorResponse(status, code, reason)
}
- const answer = (model, content, responseKind, { imageSha256 = null, stream = false } = {}) => {
+ const answer = (
+ model,
+ content,
+ responseKind,
+ { imageSha256 = null, documentSha256, stream = false } = {}
+ ) => {
sequence += 1
const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`
- recordCall(model, imageSha256, responseKind)
+ recordCall(model, imageSha256, responseKind, documentSha256)
publish()
return stream
? streamResponse(completionId, model, content)
@@ -631,6 +776,49 @@ export function createImageFixtureFetch(
// request without pixels can ever receive a color list.
if (model !== UNSUPPORTED_MODEL && model !== SUPPORTED_MODEL)
return refuse(model, 'unsupported-model', { code: 'unsupported_model' })
+
+ let documentTurn
+ try {
+ documentTurn = classifyDocumentTurn(body)
+ } catch (error) {
+ if (!(error instanceof FixtureDocumentError)) throw error
+ state.counters.documentFailures += 1
+ return refuse(model, error.code, { code: error.code })
+ }
+ if (documentTurn) {
+ if (stream) {
+ // The Host reads the provider without streaming; a stream here means the
+ // wiring changed, and a tool call cannot be answered in this fixture's SSE.
+ state.counters.documentFailures += 1
+ return refuse(model, 'document-stream-unsupported', {
+ code: 'document_stream_unsupported',
+ })
+ }
+ if (documentTurn.stage === 'read') {
+ // Ask for the text. The answer to this request carries no file content:
+ // the only way the text reaches the next request is the Host's own tool.
+ state.counters.documentReadRequests += 1
+ sequence += 1
+ const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`
+ recordCall(model, null, 'document-read-requested', null)
+ publish()
+ return jsonResponse(
+ toolCallCompletionBody(completionId, model, ATTACHMENT_READ_TOOL, {
+ attachmentId: documentTurn.attachmentId,
+ })
+ )
+ }
+ // The digest in the answer is derived from the delivered text and from
+ // nothing else: the prompt and the file name never contribute to it.
+ const documentSha256 = createHash('sha256').update(documentTurn.text, 'utf8').digest('hex')
+ state.counters.documentAnswers += 1
+ return answer(
+ model,
+ `${DOCUMENT_ANSWER_PREFIX}${documentSha256.slice(0, DOCUMENT_ANSWER_DIGEST_CHARS)}`,
+ 'document-answer',
+ { documentSha256 }
+ )
+ }
state.counters.textOnlyResponses += 1
return answer(model, TEXT_ONLY_CONTENT, 'text-only', { stream })
}
diff --git a/tests/e2e/fixtures/image-capabilities/provider.test.mjs b/tests/e2e/fixtures/image-capabilities/provider.test.mjs
index da0934b1a..8b1fb4f15 100644
--- a/tests/e2e/fixtures/image-capabilities/provider.test.mjs
+++ b/tests/e2e/fixtures/image-capabilities/provider.test.mjs
@@ -697,3 +697,226 @@ test('getEvidence returns a private copy and onEvidence publishes snapshots', as
assert.equal(second.attempts.length, 2)
assert.notEqual(h.getEvidence(), h.getEvidence())
})
+
+// ---------------------------------------------------------------------------
+// Documents (issue #678): the Host lists an attached file in the turn-context
+// block and offers `clerum__attachment_read`; the text reaches the provider only
+// in the tool result of the second request.
+// ---------------------------------------------------------------------------
+
+const READ_TOOL = 'clerum__attachment_read'
+const ATTACHMENT_ID = 'att-11111111-2222-4333-8444-555555555555'
+const DOCUMENT_TEXT = 'ledger token 7f3a9c1e\nsecond line\n'
+
+const readToolDefinition = {
+ type: 'function',
+ function: {
+ name: READ_TOOL,
+ description: 'Read an attached file.',
+ parameters: { type: 'object' },
+ },
+}
+
+function documentUserMessage(text = 'Summarize the attached file.') {
+ return {
+ role: 'user',
+ content:
+ '<turn-context>\ndate: 2026-09-29\n' +
+ `attached_file: id="${ATTACHMENT_ID}" name="notes.txt" class=text bytes=34 reader=text\n` +
+ "If the user's request refers to an attached file, read it with clerum__attachment_read before answering.\n" +
+ `</turn-context>\n\n${text}`,
+ }
+}
+
+function wrappedToolOutput(payload, { name = READ_TOOL } = {}) {
+ return `<tool_output name="${name}" sanitized="false">\n${JSON.stringify(payload)}\n</tool_output>`
+}
+
+function documentReadResult(text = DOCUMENT_TEXT) {
+ return {
+ attachmentId: ATTACHMENT_ID,
+ referenceId: 'ref-1',
+ kind: 'text',
+ byteRange: { offset: 0, length: Buffer.byteLength(text) },
+ truncated: false,
+ text,
+ }
+}
+
+function documentAnswerMessages(toolContent) {
+ return [
+ documentUserMessage(),
+ {
+ role: 'assistant',
+ content: null,
+ tool_calls: [
+ { id: 'call-1', type: 'function', function: { name: READ_TOOL, arguments: '{}' } },
+ ],
+ },
+ { role: 'tool', tool_call_id: 'call-1', content: toolContent },
+ ]
+}
+
+const sha256 = text => createHash('sha256').update(text, 'utf8').digest('hex')
+
+test('document turn 1: asks for the text with a tool call and carries no file content', async () => {
+ const h = harness()
+
+ const response = await call(
+ h,
+ chatBody(VISUAL_MODEL, [documentUserMessage()], { tools: [readToolDefinition] })
+ )
+ assert.equal(response.status, 200)
+ const body = await response.json()
+
+ assert.equal(body.choices[0].finish_reason, 'tool_calls')
+ const [toolCall] = body.choices[0].message.tool_calls
+ assert.equal(toolCall.type, 'function')
+ assert.equal(toolCall.function.name, READ_TOOL)
+ assert.deepEqual(JSON.parse(toolCall.function.arguments), { attachmentId: ATTACHMENT_ID })
+
+ const evidence = h.getEvidence()
+ assert.equal(evidence.counters.documentReadRequests, 1)
+ assert.equal(evidence.counters.documentAnswers, 0)
+ assert.deepEqual(evidence.attempts, [
+ {
+ model: VISUAL_MODEL,
+ imageSha256: null,
+ responseKind: 'document-read-requested',
+ documentSha256: null,
+ },
+ ])
+})
+
+test('document turn 2: the answer is a digest of the delivered text and the ledger records it', async () => {
+ const h = harness()
+
+ const response = await call(
+ h,
+ chatBody(VISUAL_MODEL, documentAnswerMessages(wrappedToolOutput(documentReadResult())), {
+ tools: [readToolDefinition],
+ })
+ )
+ assert.equal(response.status, 200)
+ const body = await response.json()
+
+ const digest = sha256(DOCUMENT_TEXT)
+ assert.equal(body.choices[0].message.content, `DOCUMENT_FIXTURE_SHA256:${digest.slice(0, 16)}`)
+ assert.equal(body.choices[0].finish_reason, 'stop')
+ assert.deepEqual(h.getEvidence().attempts, [
+ {
+ model: VISUAL_MODEL,
+ imageSha256: null,
+ responseKind: 'document-answer',
+ documentSha256: digest,
+ },
+ ])
+ assert.equal(h.getEvidence().counters.documentAnswers, 1)
+})
+
+test('document turn 2: a different delivered text gives a different answer (not a constant)', async () => {
+ const answers = []
+ for (const text of [DOCUMENT_TEXT, `${DOCUMENT_TEXT}changed\n`]) {
+ const h = harness()
+ const response = await call(
+ h,
+ chatBody(TEXT_MODEL, documentAnswerMessages(wrappedToolOutput(documentReadResult(text))), {
+ tools: [readToolDefinition],
+ })
+ )
+ answers.push((await response.json()).choices[0].message.content)
+ }
+ assert.notEqual(answers[0], answers[1])
+})
+
+test('the prompt alone never yields the document answer', async () => {
+ const h = harness()
+
+ // The user message names the digest prefix, but no tool result was delivered.
+ const message = documentUserMessage(
+ `Answer exactly DOCUMENT_FIXTURE_SHA256:${sha256(DOCUMENT_TEXT).slice(0, 16)}`
+ )
+ const response = await call(h, chatBody(VISUAL_MODEL, [message], { tools: [readToolDefinition] }))
+ const body = await response.json()
+
+ assert.equal(body.choices[0].finish_reason, 'tool_calls')
+ assert.equal(body.choices[0].message.content, null)
+ assert.equal(h.getEvidence().counters.documentAnswers, 0)
+})
+
+test('without the read tool a listed file is an ordinary text-only request', async () => {
+ const h = harness()
+
+ const response = await call(h, chatBody(VISUAL_MODEL, [documentUserMessage()]))
+ const body = await response.json()
+
+ assert.equal(body.choices[0].message.content, TEXT_ONLY_CONTENT)
+ // Liveness witness: the request was handled and recorded as text-only.
+ assert.equal(h.getEvidence().attempts[0].responseKind, 'text-only')
+ assert.equal(h.getEvidence().counters.documentReadRequests, 0)
+})
+
+test('a tool result the fixture cannot read is refused and counted as a document failure', async () => {
+ const cases = [
+ ['not the read wrapper', wrappedToolOutput(documentReadResult(), { name: 'other__tool' })],
+ [
+ 'not JSON',
+ '<tool_output name="clerum__attachment_read" sanitized="false">\nnot json\n</tool_output>',
+ ],
+ [
+ 'binary result',
+ wrappedToolOutput({ attachmentId: ATTACHMENT_ID, kind: 'binary', reader: 'none' }),
+ ],
+ ['error result', wrappedToolOutput({ error: 'attachment_not_found' })],
+ ]
+ for (const [label, content] of cases) {
+ const h = harness()
+ const response = await call(
+ h,
+ chatBody(VISUAL_MODEL, documentAnswerMessages(content), { tools: [readToolDefinition] })
+ )
+ assert.equal(response.status, 400, label)
+ const evidence = h.getEvidence()
+ assert.equal(evidence.counters.documentFailures, 1, label)
+ assert.equal(evidence.counters.documentAnswers, 0, label)
+ assert.equal(evidence.attempts[0].responseKind, 'rejected', label)
+ }
+})
+
+test('two tool results in one turn are refused', async () => {
+ const h = harness()
+ const messages = documentAnswerMessages(wrappedToolOutput(documentReadResult()))
+ messages.push({
+ role: 'tool',
+ tool_call_id: 'call-2',
+ content: wrappedToolOutput(documentReadResult()),
+ })
+
+ const response = await call(h, chatBody(VISUAL_MODEL, messages, { tools: [readToolDefinition] }))
+
+ assert.equal(response.status, 400)
+ assert.equal(h.getEvidence().counters.documentFailures, 1)
+})
+
+test('a streamed document turn is refused instead of answered', async () => {
+ const h = harness()
+
+ const response = await call(
+ h,
+ chatBody(VISUAL_MODEL, [documentUserMessage()], { tools: [readToolDefinition], stream: true })
+ )
+
+ assert.equal(response.status, 400)
+ const evidence = h.getEvidence()
+ assert.equal(evidence.counters.documentFailures, 1)
+ assert.equal(evidence.counters.documentReadRequests, 0)
+})
+
+test('image rows keep their original shape (no documentSha256)', async () => {
+ const { png } = palettePng()
+ const h = harness()
+
+ await call(h, chatBody(VISUAL_MODEL, imageMessages(png)))
+
+ assert.equal('documentSha256' in h.getEvidence().attempts[0], false)
+})
diff --git a/tests/e2e/fixtures/image-capabilities/sdk.test.mjs b/tests/e2e/fixtures/image-capabilities/sdk.test.mjs
index 715ee7047..ff212de9d 100644
--- a/tests/e2e/fixtures/image-capabilities/sdk.test.mjs
+++ b/tests/e2e/fixtures/image-capabilities/sdk.test.mjs
@@ -5,21 +5,28 @@ import test from 'node:test'
import { buildImageFixture } from '../../../../desktop-app/test/e2e-playwright/helpers/qaRecorderImageFixture.ts'
import { createImageFixtureFetch } from './provider.mjs'
+// The pinned SDK reads `globalThis.fetch` once, when `openai/shims/web` first
+// loads, so every test in this file shares one fixture and one key. Each test
+// asserts only on the ledger rows it appended.
+const IMAGE_CAPABILITIES_CREDENTIAL = randomBytes(32).toString('hex')
+const fixture = createImageFixtureFetch(
+ async () => {
+ throw new Error('Unexpected network call')
+ },
+ {
+ runId: 'image-capabilities-123456abcdef',
+ credentialHash: createHash('sha256').update(IMAGE_CAPABILITIES_CREDENTIAL).digest('hex'),
+ }
+)
+globalThis.fetch = fixture.fetch
+const require = createRequire(new URL('../../../../mcp-host/package.json', import.meta.url))
+require('openai/shims/web')
+
+const appendedRows = before => fixture.getEvidence().attempts.slice(before)
+
// Uses the real compiled provider and pinned SDK. No outbound request is allowed.
test('the production ZAI serializer and pinned SDK reach the pixel fixture', async () => {
- const IMAGE_CAPABILITIES_CREDENTIAL = randomBytes(32).toString('hex')
- const fixture = createImageFixtureFetch(
- async () => {
- throw new Error('Unexpected network call')
- },
- {
- runId: 'image-capabilities-123456abcdef',
- credentialHash: createHash('sha256').update(IMAGE_CAPABILITIES_CREDENTIAL).digest('hex'),
- }
- )
- globalThis.fetch = fixture.fetch
- const require = createRequire(new URL('../../../../mcp-host/package.json', import.meta.url))
- require('openai/shims/web')
+ const before = fixture.getEvidence().attempts.length
const { OpenAICompatibleProvider } = require('./dist/llm/openaiCompatible.js')
const provider = new OpenAICompatibleProvider(
{
@@ -45,7 +52,7 @@ test('the production ZAI serializer and pinned SDK reach the pixel fixture', asy
[]
)
assert.equal(result.content, image.orderedColors.join(', '))
- assert.deepEqual(fixture.getEvidence().attempts, [
+ assert.deepEqual(appendedRows(before), [
{
model: 'glm-5.3-flash',
responseKind: 'tile-colors',
@@ -53,3 +60,95 @@ test('the production ZAI serializer and pinned SDK reach the pixel fixture', asy
},
])
})
+
+// Issue #678: the same production serializer and pinned SDK, driven through the
+// two turns of an attachment read. The turn-context block and the tool-result
+// wrapper come from the compiled Host, so the fixture is proven against the
+// format the Host really writes and not against a copy of it.
+test('the production ZAI serializer and pinned SDK complete an attachment read against the fixture', async () => {
+ const before = fixture.getEvidence().attempts.length
+ const { OpenAICompatibleProvider } = require('./dist/llm/openaiCompatible.js')
+ const { buildTurnContextBlock } = require('./dist/core/orchestration/turnContext.js')
+ const { BasicSafety } = require('./dist/core/safety/safety.js')
+ const provider = new OpenAICompatibleProvider(
+ {
+ id: 'zai',
+ baseURL: 'https://api.z.ai/api/coding/paas/v4',
+ defaultModel: 'glm-5.3-flash',
+ },
+ IMAGE_CAPABILITIES_CREDENTIAL,
+ 'glm-5.3-flash'
+ )
+
+ const attachmentId = 'att-11111111-2222-4333-8444-555555555555'
+ const documentText = `sdk token ${randomBytes(6).toString('hex')}\nsecond line\n`
+ const block = buildTurnContextBlock({
+ date: new Date('2026-09-29T00:00:00.000Z'),
+ channel: { type: 'desktop' },
+ attachedFiles: [
+ {
+ attachmentId,
+ name: 'notes.txt',
+ class: 'text',
+ byteLength: Buffer.byteLength(documentText),
+ reader: 'text',
+ mismatch: false,
+ declaredMediaType: 'text/plain',
+ detectedMediaType: 'text/plain',
+ },
+ ],
+ })
+ const tools = [
+ {
+ name: 'clerum__attachment_read',
+ description: 'Read an attached file.',
+ parameters: { type: 'object', properties: { attachmentId: { type: 'string' } } },
+ },
+ ]
+ const user = { role: 'user', content: `${block}\n\nSummarize the attached file.` }
+
+ const first = await provider.completeSingleTurnWithTools([user], tools)
+ // The wire says `tool_calls`; the Host's `mapFinishReason` reports it as `tool_use`.
+ assert.equal(first.finish_reason, 'tool_use')
+ assert.equal(first.tool_calls?.length, 1)
+ assert.equal(first.tool_calls[0].name, 'clerum__attachment_read')
+ assert.deepEqual(first.tool_calls[0].arguments, { attachmentId })
+
+ const toolOutput = new BasicSafety().wrapForLlm(
+ 'clerum__attachment_read',
+ JSON.stringify({
+ attachmentId,
+ referenceId: 'ref-1',
+ kind: 'text',
+ byteRange: { offset: 0, length: Buffer.byteLength(documentText) },
+ truncated: false,
+ text: documentText,
+ }),
+ false
+ )
+ const second = await provider.completeSingleTurnWithTools(
+ [
+ user,
+ { role: 'assistant', content: '', tool_calls: first.tool_calls },
+ { role: 'tool', tool_call_id: first.tool_calls[0].id, content: toolOutput },
+ ],
+ tools
+ )
+
+ const digest = createHash('sha256').update(documentText, 'utf8').digest('hex')
+ assert.equal(second.content, `DOCUMENT_FIXTURE_SHA256:${digest.slice(0, 16)}`)
+ assert.deepEqual(appendedRows(before), [
+ {
+ model: 'glm-5.3-flash',
+ imageSha256: null,
+ responseKind: 'document-read-requested',
+ documentSha256: null,
+ },
+ {
+ model: 'glm-5.3-flash',
+ imageSha256: null,
+ responseKind: 'document-answer',
+ documentSha256: digest,
+ },
+ ])
+})
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index 8c88caac1..2e21df64b 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.100",
+ "version": "1.0.101",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.100",
+ "version": "1.0.101",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index b80adc934..0523a0d75 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.100",
+ "version": "1.0.101",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",