Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(codex): cover the full Desktop image journey in Playwright
Alfredo Lopezc3a47d785205
diff --git a/desktop-app/test/codexImageChallenge.test.ts b/desktop-app/test/codexImageChallenge.test.tsnew file mode 100644index 000000000..6bb3e0e4d--- /dev/null+++ b/desktop-app/test/codexImageChallenge.test.ts@@ -0,0 +1,40 @@+import { describe, expect, it } from 'vitest'+import { createRequire } from 'node:module'+import { challengeImage, paddedChallengeImage } from './e2e-playwright/codexImageChallenge.js'++const MIB = 1024 * 1024+const { inspectVisualImage } = createRequire(import.meta.url)(+ '../../packages/llm-provider-attempt-contract/visualPayload.cjs'+) as {+ inspectVisualImage: (input: { mimeType: string; data: string }) => {+ ok: boolean+ message?: string+ }+}++describe('Codex image challenge fixtures', () => {+ it('keeps the answer out of the small PNG/JPEG containers', () => {+ for (const format of ['png', 'jpeg'] as const) {+ const image = challengeImage(format)+ expect(image.code).toMatch(/^[0-9A-F]{16}$/)+ expect(image.bytes.includes(Buffer.from(image.code))).toBe(false)+ expect(+ inspectVisualImage({+ mimeType: `image/${format}`,+ data: image.bytes.toString('base64'),+ }).ok+ ).toBe(true)+ }+ })++ it('pads a 5 MiB JPEG without putting the answer in the file bytes as text', () => {+ const image = paddedChallengeImage('jpeg', 5 * MIB)+ expect(image.bytes).toHaveLength(5 * MIB)+ expect(image.bytes.includes(Buffer.from(image.code))).toBe(false)+ const inspected = inspectVisualImage({+ mimeType: 'image/jpeg',+ data: image.bytes.toString('base64'),+ })+ expect(inspected.ok, inspected.message).toBe(true)+ })+})diff --git a/desktop-app/test/e2e-playwright/codex-image-input.spec.ts b/desktop-app/test/e2e-playwright/codex-image-input.spec.tsindex db129fe7e..23ba89ad8 100644--- a/desktop-app/test/e2e-playwright/codex-image-input.spec.ts+++ b/desktop-app/test/e2e-playwright/codex-image-input.spec.ts@@ -6,7 +6,7 @@* existing stored-session reset needs explicit opt-in before fixtures run.*/import { randomUUID } from 'node:crypto'-import { challengeImage } from './codexImageChallenge.js'+import { challengeImage, paddedChallengeImage } from './codexImageChallenge.js'import { expect, test } from './fixtures.js'const hostRef = process.env.E2E_HOST_REF ?? ''@@ -14,8 +14,15 @@ const hostLabel = process.env.E2E_CODEX_HOST_LABEL ?? ''const modelId = process.env.E2E_CODEX_IMAGE_MODEL ?? ''const modelLabel = process.env.E2E_CODEX_IMAGE_MODEL_LABEL ?? ''const mode = process.env.E2E_CODEX_IMAGE_MODE ?? ''+const MIB = 1024 * 1024const escapeRegex = (value: string): string => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')+type UploadFile = {+ name: string+ mimeType: 'image/png' | 'image/jpeg'+ buffer: Buffer+}+test.beforeAll(() => {expect(process.env.E2E_CODEX_IMAGE_INPUT,@@ -73,6 +80,84 @@ async function startOwnedChat(page: import('@playwright/test').Page): Promise<voawait expect(selector).toHaveAttribute('data-model', modelId, { timeout: 30_000 })}+async function uploadThroughChooser(+ page: import('@playwright/test').Page,+ files: UploadFile[]+): Promise<void> {+ await page.getByRole('button', { name: 'Add context', exact: true }).click()+ const upload = page.getByRole('menuitem', { name: 'Upload Files', exact: true })+ await expect(upload).toBeVisible()+ const chooserReady = page.waitForEvent('filechooser')+ await upload.click()+ const chooser = await chooserReady+ await chooser.setFiles(files)+}++async function assertPreview(+ page: import('@playwright/test').Page,+ filename: string+): Promise<void> {+ const chip = page.getByRole('button', { name: filename, exact: true })+ await expect(chip).toBeVisible()+ await chip.click()+ const preview = page.getByRole('dialog', { name: filename, exact: true })+ await expect(preview).toBeVisible()+ await expect(preview.getByRole('img', { name: filename, exact: true })).toBeVisible()+ await preview.getByRole('button', { name: 'Close image preview', exact: true }).click()+ await expect(preview).toHaveCount(0)+}++async function sendVisualTurn(+ page: import('@playwright/test').Page,+ input: { prompt: string; filenames: string[]; codes: string[] }+): Promise<void> {+ const composer = page.getByTestId('chat-input')+ await composer.fill(input.prompt)+ const send = page.getByTestId('send-button')+ await expect(send).toBeEnabled()+ await send.click()+ await expect(composer).toHaveValue('')+ const thread = page.getByTestId('message-list')+ await expect(thread.getByText(input.prompt, { exact: true })).toBeVisible()+ for (const filename of input.filenames) {+ await expect(+ thread.locator('.message-attachment-label').filter({ hasText: filename })+ ).toBeVisible()+ }+ const answer = page.getByTestId('agent-response')+ if (mode === 'enabled') {+ for (const code of input.codes) {+ await expect(answer).toContainText(new RegExp(`\\b${code}\\b`, 'i'), { timeout: 180_000 })+ }+ await expect(answer).not.toHaveClass(/chat-bubble--error/)+ await expect(thread.getByTestId('progress-stepper')).toHaveCount(0)+ } else {+ await expect(answer.locator('.error-bubble-message')).toContainText(+ 'Image input is not enabled for this Codex model',+ { timeout: 180_000 }+ )+ await expect(answer).toHaveClass(/chat-bubble--error/)+ for (const code of input.codes) {+ await expect(answer).not.toContainText(new RegExp(`\\b${code}\\b`, 'i'))+ }+ }+ await expect(answer).toHaveCount(1)+ await expect(send).toHaveAttribute('aria-label', 'Send message')+ await expect(thread.locator('.chat-message--in-flight')).toHaveCount(0)+ await expect(page.getByRole('status', { name: /^Running on fallback:/ })).toHaveCount(0)+ const selector = page.getByTestId('model-selector-up')+ await expect(selector).toHaveAttribute('data-host-ref', hostRef)+ await expect(selector).toHaveAttribute('data-provider', 'codex-subscription')+ await expect(selector).toHaveAttribute('data-model', modelId)+}++function visualPrompt(count: number): string {+ if (count === 1) {+ return 'Read the hexadecimal code in the attached image. Reply with that code only.'+ }+ return 'Read the hexadecimal code in each attached image, in the order the files were attached. Reply with those codes only, one per line.'+}+for (const format of ['png', 'jpeg'] as const) {test(`${format} visual input follows the explicitly provisioned capability lane`, async ({appPage,@@ -82,64 +167,159 @@ for (const format of ['png', 'jpeg'] as const) {// A generic response or discarded image cannot satisfy this 64-bit challenge.const image = challengeImage(format)const filename = `visual-${randomUUID()}.${format}`- const prompt = 'Read the hexadecimal code in the attached image. Reply with that code only.'await test.step('select the owned Codex Host and model through UI', async () => {await startOwnedChat(appPage)})await test.step('upload through the chooser and verify the rendered preview', async () => {- await appPage.getByRole('button', { name: 'Add context', exact: true }).click()- const upload = appPage.getByRole('menuitem', { name: 'Upload Files', exact: true })- await expect(upload).toBeVisible()- const chooserReady = appPage.waitForEvent('filechooser')- await upload.click()- const chooser = await chooserReady- await chooser.setFiles({ name: filename, mimeType: `image/${format}`, buffer: image.bytes })- const chip = appPage.getByRole('button', { name: filename, exact: true })- await expect(chip).toBeVisible()- await chip.click()- const preview = appPage.getByRole('dialog', { name: filename, exact: true })- await expect(preview).toBeVisible()- await expect(preview.getByRole('img', { name: filename, exact: true })).toBeVisible()- await preview.getByRole('button', { name: 'Close image preview', exact: true }).click()- await expect(preview).toHaveCount(0)+ await uploadThroughChooser(appPage, [+ { name: filename, mimeType: `image/${format}`, buffer: image.bytes },+ ])+ await assertPreview(appPage, filename)})await test.step('submit and assert the correlated terminal result', async () => {- const composer = appPage.getByTestId('chat-input')- await composer.fill(prompt)- const send = appPage.getByTestId('send-button')- await expect(send).toBeEnabled()- await send.click()- await expect(composer).toHaveValue('')- const thread = appPage.getByTestId('message-list')- await expect(thread.getByText(prompt, { exact: true })).toBeVisible()- await expect(- thread.locator('.message-attachment-label').filter({ hasText: filename })- ).toBeVisible()- const answer = appPage.getByTestId('agent-response')- const expectedCode = new RegExp(`\\b${image.code}\\b`, 'i')- if (mode === 'enabled') {- // One terminal wait, not separate 180-second waits for creation and text.- await expect(answer).toContainText(expectedCode, { timeout: 180_000 })- await expect(answer).not.toHaveClass(/chat-bubble--error/)- // Do not let tool-based attachment loading/OCR substitute for the- // direct model-input behavior this lane must prove.- await expect(thread.getByTestId('progress-stepper')).toHaveCount(0)- } else {- await expect(answer.locator('.error-bubble-message')).toContainText(- 'Image input is not enabled for this Codex model',- { timeout: 180_000 }- )- await expect(answer).toHaveClass(/chat-bubble--error/)- await expect(answer).not.toContainText(expectedCode)- }- await expect(answer).toHaveCount(1)- await expect(send).toHaveAttribute('aria-label', 'Send message')- await expect(thread.locator('.chat-message--in-flight')).toHaveCount(0)- await expect(appPage.getByRole('status', { name: /^Running on fallback:/ })).toHaveCount(0)- const selector = appPage.getByTestId('model-selector-up')- await expect(selector).toHaveAttribute('data-host-ref', hostRef)- await expect(selector).toHaveAttribute('data-provider', 'codex-subscription')- await expect(selector).toHaveAttribute('data-model', modelId)+ await sendVisualTurn(appPage, {+ prompt: visualPrompt(1),+ filenames: [filename],+ codes: [image.code],+ })})})}++test('5 MiB JPEG follows the same capability lane as a small image', async ({ appPage }) => {+ test.setTimeout(360_000)+ const image = paddedChallengeImage('jpeg', 5 * MIB)+ const filename = `visual-${randomUUID()}.jpeg`+ await startOwnedChat(appPage)+ await uploadThroughChooser(appPage, [+ { name: filename, mimeType: 'image/jpeg', buffer: image.bytes },+ ])+ await assertPreview(appPage, filename)+ await sendVisualTurn(appPage, {+ prompt: visualPrompt(1),+ filenames: [filename],+ codes: [image.code],+ })+})++test('10 MiB PNG + 5 MiB JPEG follow the 15 MiB combined budget', async ({ appPage }) => {+ test.setTimeout(360_000)+ const ten = paddedChallengeImage('png', 10 * MIB)+ const five = paddedChallengeImage('jpeg', 5 * MIB)+ const tenName = `visual-${randomUUID()}.png`+ const fiveName = `visual-${randomUUID()}.jpeg`+ await startOwnedChat(appPage)+ await uploadThroughChooser(appPage, [+ { name: tenName, mimeType: 'image/png', buffer: ten.bytes },+ { name: fiveName, mimeType: 'image/jpeg', buffer: five.bytes },+ ])+ await assertPreview(appPage, tenName)+ await assertPreview(appPage, fiveName)+ await sendVisualTurn(appPage, {+ prompt: visualPrompt(2),+ filenames: [tenName, fiveName],+ codes: [ten.code, five.code],+ })+})++test('three 5 MiB images follow the combined budget', async ({ appPage }) => {+ test.setTimeout(360_000)+ const first = paddedChallengeImage('png', 5 * MIB)+ const second = paddedChallengeImage('jpeg', 5 * MIB)+ const third = paddedChallengeImage('png', 5 * MIB)+ const files: UploadFile[] = [+ { name: `visual-${randomUUID()}.png`, mimeType: 'image/png', buffer: first.bytes },+ { name: `visual-${randomUUID()}.jpeg`, mimeType: 'image/jpeg', buffer: second.bytes },+ { name: `visual-${randomUUID()}.png`, mimeType: 'image/png', buffer: third.bytes },+ ]+ await startOwnedChat(appPage)+ await uploadThroughChooser(appPage, files)+ for (const file of files) await assertPreview(appPage, file.name)+ await sendVisualTurn(appPage, {+ prompt: visualPrompt(3),+ filenames: files.map(file => file.name),+ codes: [first.code, second.code, third.code],+ })+})++test('refuses a single image over the 10 MiB limit before send', async ({ appPage }) => {+ const image = paddedChallengeImage('png', 11 * MIB)+ const filename = `over-${randomUUID()}.png`+ await startOwnedChat(appPage)+ await uploadThroughChooser(appPage, [+ { name: filename, mimeType: 'image/png', buffer: image.bytes },+ ])+ await expect(appPage.getByRole('alert')).toContainText(+ `${filename} is too large. Max size is 10 MiB.`+ )+ await expect(appPage.getByRole('button', { name: filename, exact: true })).toHaveCount(0)+ await expect(appPage.getByTestId('agent-response')).toHaveCount(0)+})++test('refuses the file that would pass the 15 MiB total and keeps the one that fits', async ({+ appPage,+}) => {+ const fits = paddedChallengeImage('png', 10 * MIB)+ const over = paddedChallengeImage('jpeg', 6 * MIB)+ const fitsName = `fits-${randomUUID()}.png`+ const overName = `over-total-${randomUUID()}.jpeg`+ await startOwnedChat(appPage)+ await uploadThroughChooser(appPage, [+ { name: fitsName, mimeType: 'image/png', buffer: fits.bytes },+ { name: overName, mimeType: 'image/jpeg', buffer: over.bytes },+ ])+ await expect(appPage.getByRole('alert')).toContainText(+ `${overName} was not added. Attachments can total at most 15 MiB per message.`+ )+ await expect(appPage.getByRole('button', { name: fitsName, exact: true })).toBeVisible()+ await expect(appPage.getByRole('button', { name: overName, exact: true })).toHaveCount(0)+ await expect(appPage.getByTestId('agent-response')).toHaveCount(0)+})++test('explains a fourth image instead of dropping it silently', async ({ appPage }) => {+ const files = [0, 1, 2].map(index => {+ const image = challengeImage(index === 1 ? 'jpeg' : 'png')+ const format = index === 1 ? 'jpeg' : 'png'+ return {+ name: `visual-${randomUUID()}.${format}`,+ mimeType: `image/${format}` as const,+ buffer: image.bytes,+ }+ })+ const extra = challengeImage('png')+ const extraName = `visual-${randomUUID()}.png`+ await startOwnedChat(appPage)+ await uploadThroughChooser(appPage, files)+ for (const file of files) {+ await expect(appPage.getByRole('button', { name: file.name, exact: true })).toBeVisible()+ }+ await uploadThroughChooser(appPage, [+ { name: extraName, mimeType: 'image/png', buffer: extra.bytes },+ ])+ await expect(appPage.getByRole('alert')).toContainText(+ 'You can attach up to 3 images per message.'+ )+ await expect(appPage.getByRole('button', { name: extraName, exact: true })).toHaveCount(0)+ await expect(appPage.getByTestId('agent-response')).toHaveCount(0)+})++test('refuses an unsupported image type before send', async ({ appPage }) => {+ const filename = `visual-${randomUUID()}.gif`+ await startOwnedChat(appPage)+ await appPage.getByRole('button', { name: 'Add context', exact: true }).click()+ const upload = appPage.getByRole('menuitem', { name: 'Upload Files', exact: true })+ await expect(upload).toBeVisible()+ const chooserReady = appPage.waitForEvent('filechooser')+ await upload.click()+ const chooser = await chooserReady+ await chooser.setFiles({+ name: filename,+ mimeType: 'image/gif',+ buffer: Buffer.from('GIF89a'),+ })+ await expect(appPage.getByRole('alert')).toContainText(+ `${filename} is not supported. Use PNG or JPEG.`+ )+ await expect(appPage.getByRole('button', { name: filename, exact: true })).toHaveCount(0)+ await expect(appPage.getByTestId('agent-response')).toHaveCount(0)+})diff --git a/desktop-app/test/e2e-playwright/codexImageChallenge.ts b/desktop-app/test/e2e-playwright/codexImageChallenge.tsindex daa6e034c..6a6a51c85 100644--- a/desktop-app/test/e2e-playwright/codexImageChallenge.ts+++ b/desktop-app/test/e2e-playwright/codexImageChallenge.ts@@ -2,12 +2,18 @@ import { randomBytes } from 'node:crypto'import { createRequire } from 'node:module'import path from 'node:path'+const require = createRequire(path.resolve(__dirname, '../../../mcp-host/package.json'))+const fixtures = createRequire(__filename)(+ '../../../packages/llm-provider-attempt-contract/testImageFixtures.cjs'+) as {+ padJpegToSize: (jpeg: Buffer | string, targetBytes: number) => Buffer+ padPngToSize: (png: Buffer | string, targetBytes: number) => Buffer+}+/** A neutral image with an answer absent from the filename and user prompt. */export function challengeImage(format: 'png' | 'jpeg'): { code: string; bytes: Buffer } {// Reuse the Host renderer installed for T0; do not add another dependency.- const { createCanvas } = createRequire(path.resolve(__dirname, '../../../mcp-host/package.json'))(- '@napi-rs/canvas'- )+ const { createCanvas } = require('@napi-rs/canvas')const code = randomBytes(8).toString('hex').toUpperCase()const canvas = createCanvas(800, 120)const context = canvas.getContext('2d')@@ -21,3 +27,16 @@ export function challengeImage(format: 'png' | 'jpeg'): { code: string; bytes: Bbytes: format === 'png' ? canvas.toBuffer('image/png') : canvas.toBuffer('image/jpeg'),}}++/** Same pixel challenge, grown to an exact decoded size the contract still accepts. */+export function paddedChallengeImage(+ format: 'png' | 'jpeg',+ targetBytes: number+): { code: string; bytes: Buffer } {+ const image = challengeImage(format)+ const bytes =+ format === 'png'+ ? fixtures.padPngToSize(image.bytes, targetBytes)+ : fixtures.padJpegToSize(image.bytes, targetBytes)+ return { code: image.code, bytes }+}diff --git a/docs/testing/codex-subscription-validation.md b/docs/testing/codex-subscription-validation.mdindex 04e3faf05..0d1800391 100644--- a/docs/testing/codex-subscription-validation.md+++ b/docs/testing/codex-subscription-validation.md@@ -54,12 +54,17 @@ than silently skipping them.### Opt-in Desktop image lane`desktop-app/test/e2e-playwright/codex-image-input.spec.ts` covers direct PNG/JPEG-uploads through the visible composer. It creates a fresh 64-bit hexadecimal-challenge rendered only into image pixels. The filename and prompt do not carry-the answer. An enabled run requires that answer (hexadecimal case is ignored), a non-error response, no-tool steps substituting OCR/loading for direct image input, and no fallback badge.+uploads through the visible composer, including a 5 MiB JPEG, a 10+5 MiB pair,+three 5 MiB images, and composer refusals over 10 MiB, over 15 MiB total, a+fourth image, and a non-PNG/JPEG type. It creates a fresh 64-bit hexadecimal+challenge rendered only into image pixels. Large cases pad that same image so+the answer stays in pixels while the decoded size matches the hop budget. The+filename and prompt do not carry the answer. An enabled run requires that+answer (hexadecimal case is ignored), a non-error response, no tool steps+substituting OCR/loading for direct image input, and no fallback badge.The selector's stable data attributes verify the actual selected Host, providerand model. The disabled mode instead requires the explicit capability error.+Composer budget refusals never send.This is a real upstream lane, not a mock. Before running it, obtain separateauthorization for runtime, upstream access and the existing login fixture'sdiff --git a/packages/llm-provider-attempt-contract/index.test.cjs b/packages/llm-provider-attempt-contract/index.test.cjsindex f04b912cf..e6c549bea 100644--- a/packages/llm-provider-attempt-contract/index.test.cjs+++ b/packages/llm-provider-attempt-contract/index.test.cjs@@ -11,6 +11,8 @@ const {declaredHeaderPng,declaredHeaderPngOfSize,jpegOfSize,+ padJpegToSize,+ padPngToSize,realPng,realPngOfSize,} = require('./testImageFixtures.cjs')@@ -1101,6 +1103,21 @@ test('large real PNG: CRCs and inflatable scanlines at 3 MiB and at the 10 MiB c}})+test('shared pads keep a real fixture JPEG/PNG contract-valid at 5 MiB', () => {+ const jpeg = padJpegToSize(Buffer.from(IMAGE_DATA.jpeg, 'base64'), 5 * MIB)+ const png = padPngToSize(Buffer.from(IMAGE_DATA.png, 'base64'), 5 * MIB)+ assert.equal(jpeg.length, 5 * MIB)+ assert.equal(png.length, 5 * MIB)+ const jpegParsed = contract.parseCodexCompletionRequest(+ v2WithParts([imagePart(jpeg.toString('base64'), 'image/jpeg')])+ )+ const pngParsed = contract.parseCodexCompletionRequest(+ v2WithParts([imagePart(png.toString('base64'))])+ )+ assert.equal(jpegParsed.ok, true, jpegParsed.message)+ assert.equal(pngParsed.ok, true, pngParsed.message)+})+test('buildCodexProxyEnvelope: exact shape, no outer deadline, exact size boundary', () => {const ticket = 'header.payload.signature'.repeat(3)const parsed = contract.parseCodexCompletionRequest({diff --git a/packages/llm-provider-attempt-contract/testImageFixtures.cjs b/packages/llm-provider-attempt-contract/testImageFixtures.cjsindex 17fea0d35..a6ffbb252 100644--- a/packages/llm-provider-attempt-contract/testImageFixtures.cjs+++ b/packages/llm-provider-attempt-contract/testImageFixtures.cjs@@ -132,6 +132,42 @@ function padPngToSize(png, targetBytes) {])}+/**+ * Grow a real JPEG to an exact byte length by inserting COM segments after SOI.+ * The original SOF/SOS/scan/EOI stay intact, so a canvas image remains+ * decodable while the container still ends on EOI for the contract parser.+ */+function padJpegToSize(jpeg, targetBytes) {+ const bytes = Buffer.isBuffer(jpeg) ? jpeg : Buffer.from(String(jpeg), 'base64')+ if (bytes[0] !== 0xff || bytes[1] !== 0xd8) {+ throw new Error('JPEG SOI marker is missing')+ }+ const padding = targetBytes - bytes.length+ if (padding < 0) {+ throw new Error(`target ${targetBytes} is smaller than the JPEG ${bytes.length}`)+ }+ if (padding === 0) return Buffer.from(bytes)+ if (padding < 4) {+ throw new Error('target must leave room for a JPEG COM segment')+ }+ const comments = []+ let remaining = padding+ while (remaining > 0) {+ if (remaining < 4) {+ throw new Error('JPEG COM padding cannot finish on a short segment')+ }+ const chunk = Math.min(remaining, 4 + 65533)+ const payload = chunk - 4+ const com = Buffer.alloc(chunk)+ com[0] = 0xff+ com[1] = 0xfe+ com.writeUInt16BE(payload + 2, 2)+ comments.push(com)+ remaining -= chunk+ }+ return Buffer.concat([bytes.subarray(0, 2), ...comments, bytes.subarray(2)])+}+/*** Structurally framed JPEG of an exact byte length: SOI, SOF0 dimensions, SOS,* entropy pad, EOI. Marker framing only; scan data is not entropy-decodable.@@ -176,5 +212,6 @@ module.exports = {declaredHeaderPngOfSize,realPngOfSize,padPngToSize,+ padJpegToSize,jpegOfSize,}diff --git a/packages/llm-provider-attempt-contract/testImageFixtures.d.ts b/packages/llm-provider-attempt-contract/testImageFixtures.d.tsindex af521c68a..dfef9fee1 100644--- a/packages/llm-provider-attempt-contract/testImageFixtures.d.ts+++ b/packages/llm-provider-attempt-contract/testImageFixtures.d.ts@@ -14,4 +14,5 @@ export declare function realPngOfSize(seed: number): Bufferexport declare function padPngToSize(png: Buffer | string, targetBytes: number): Buffer+export declare function padJpegToSize(jpeg: Buffer | string, targetBytes: number): Bufferexport declare function jpegOfSize(targetBytes: number, width?: number, height?: number): Buffer