The open source coding agent.
TypeScript
test(app): add mock llm e2e fixture (#20375)
Kit Langtonc8ecd6402203
diff --git a/packages/app/e2e/fixtures.ts b/packages/app/e2e/fixtures.tsindex ca06858a458..7fc4cda057e 100644--- a/packages/app/e2e/fixtures.ts+++ b/packages/app/e2e/fixtures.ts@@ -1,5 +1,8 @@import { test as base, expect, type Page } from "@playwright/test"+import { ManagedRuntime } from "effect"import type { E2EWindow } from "../src/testing/terminal"+import type { Item, Reply, Usage } from "../../opencode/test/lib/llm-server"+import { TestLLMServer } from "../../opencode/test/lib/llm-server"import {healthPhase,cleanupSession,@@ -13,6 +16,24 @@ import {} from "./actions"import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils"+type LLMFixture = {+ url: string+ push: (...input: (Item | Reply)[]) => Promise<void>+ text: (value: string, opts?: { usage?: Usage }) => Promise<void>+ tool: (name: string, input: unknown) => Promise<void>+ toolHang: (name: string, input: unknown) => Promise<void>+ reason: (value: string, opts?: { text?: string; usage?: Usage }) => Promise<void>+ fail: (message?: unknown) => Promise<void>+ error: (status: number, body: unknown) => Promise<void>+ hang: () => Promise<void>+ hold: (value: string, wait: PromiseLike<unknown>) => Promise<void>+ hits: () => Promise<Array<{ url: URL; body: Record<string, unknown> }>>+ calls: () => Promise<number>+ wait: (count: number) => Promise<void>+ inputs: () => Promise<Record<string, unknown>[]>+ pending: () => Promise<number>+}+export const settingsKey = "settings.v3"const seedModel = (() => {@@ -26,6 +47,7 @@ const seedModel = (() => {})()type TestFixtures = {+ llm: LLMFixturesdk: ReturnType<typeof createSdk>gotoSession: (sessionID?: string) => Promise<void>withProject: <T>(@@ -36,7 +58,11 @@ type TestFixtures = {trackSession: (sessionID: string, directory?: string) => voidtrackDirectory: (directory: string) => void}) => Promise<T>,- options?: { extra?: string[] },+ options?: {+ extra?: string[]+ model?: { providerID: string; modelID: string }+ setup?: (directory: string) => Promise<void>+ },) => Promise<T>}@@ -46,6 +72,31 @@ type WorkerFixtures = {}export const test = base.extend<TestFixtures, WorkerFixtures>({+ llm: async ({}, use) => {+ const rt = ManagedRuntime.make(TestLLMServer.layer)+ try {+ const svc = await rt.runPromise(TestLLMServer.asEffect())+ await use({+ url: svc.url,+ push: (...input) => rt.runPromise(svc.push(...input)),+ text: (value, opts) => rt.runPromise(svc.text(value, opts)),+ tool: (name, input) => rt.runPromise(svc.tool(name, input)),+ toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)),+ reason: (value, opts) => rt.runPromise(svc.reason(value, opts)),+ fail: (message) => rt.runPromise(svc.fail(message)),+ error: (status, body) => rt.runPromise(svc.error(status, body)),+ hang: () => rt.runPromise(svc.hang),+ hold: (value, wait) => rt.runPromise(svc.hold(value, wait)),+ hits: () => rt.runPromise(svc.hits),+ calls: () => rt.runPromise(svc.calls),+ wait: (count) => rt.runPromise(svc.wait(count)),+ inputs: () => rt.runPromise(svc.inputs),+ pending: () => rt.runPromise(svc.pending),+ })+ } finally {+ await rt.dispose()+ }+ },page: async ({ page }, use) => {let boundary: string | undefinedsetHealthPhase(page, "test")@@ -99,7 +150,8 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({const root = await createTestProject()const sessions = new Map<string, string>()const dirs = new Set<string>()- await seedStorage(page, { directory: root, extra: options?.extra })+ await options?.setup?.(root)+ await seedStorage(page, { directory: root, extra: options?.extra, model: options?.model })const gotoSession = async (sessionID?: string) => {await page.goto(sessionPath(root, sessionID))@@ -133,7 +185,14 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({},})-async function seedStorage(page: Page, input: { directory: string; extra?: string[] }) {+async function seedStorage(+ page: Page,+ input: {+ directory: string+ extra?: string[]+ model?: { providerID: string; modelID: string }+ },+) {await seedProjects(page, input)await page.addInitScript((model: { providerID: string; modelID: string }) => {const win = window as E2EWindow@@ -158,7 +217,7 @@ async function seedStorage(page: Page, input: { directory: string; extra?: strinvariant: {},}),)- }, seedModel)+ }, input.model ?? seedModel)}export { expect }diff --git a/packages/app/e2e/prompt/prompt.spec.ts b/packages/app/e2e/prompt/prompt.spec.tsindex 0466d0988c8..1acf17f5bf4 100644--- a/packages/app/e2e/prompt/prompt.spec.ts+++ b/packages/app/e2e/prompt/prompt.spec.ts@@ -1,8 +1,44 @@+import fs from "node:fs/promises"+import path from "node:path"import { test, expect } from "../fixtures"import { promptSelector } from "../selectors"-import { cleanupSession, sessionIDFromUrl, withSession } from "../actions"+import { sessionIDFromUrl } from "../actions"+import { createSdk } from "../utils"-test("can send a prompt and receive a reply", async ({ page, sdk, gotoSession }) => {+async function config(dir: string, url: string) {+ await fs.writeFile(+ path.join(dir, "opencode.json"),+ JSON.stringify({+ $schema: "https://opencode.ai/config.json",+ enabled_providers: ["e2e-llm"],+ provider: {+ "e2e-llm": {+ name: "E2E LLM",+ npm: "@ai-sdk/openai-compatible",+ env: [],+ models: {+ "test-model": {+ name: "Test Model",+ tool_call: true,+ limit: { context: 128000, output: 32000 },+ },+ },+ options: {+ apiKey: "test-key",+ baseURL: url,+ },+ },+ },+ agent: {+ build: {+ model: "e2e-llm/test-model",+ },+ },+ }),+ )+}++test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => {test.setTimeout(120_000)const pageErrors: string[] = []@@ -11,42 +47,51 @@ test("can send a prompt and receive a reply", async ({ page, sdk, gotoSession })}page.on("pageerror", onPageError)- await gotoSession()-- const token = `E2E_OK_${Date.now()}`+ try {+ await withProject(+ async (project) => {+ const sdk = createSdk(project.directory)+ const token = `E2E_OK_${Date.now()}`- const prompt = page.locator(promptSelector)- await prompt.click()- await page.keyboard.type(`Reply with exactly: ${token}`)- await page.keyboard.press("Enter")+ await llm.text(token)+ await project.gotoSession()- await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })+ const prompt = page.locator(promptSelector)+ await prompt.click()+ await page.keyboard.type(`Reply with exactly: ${token}`)+ await page.keyboard.press("Enter")- const sessionID = (() => {- const id = sessionIDFromUrl(page.url())- if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)- return id- })()+ await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })- try {- await expect- .poll(- async () => {- const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])- return messages- .filter((m) => m.info.role === "assistant")- .flatMap((m) => m.parts)- .filter((p) => p.type === "text")- .map((p) => p.text)- .join("\n")- },- { timeout: 90_000 },- )+ const sessionID = (() => {+ const id = sessionIDFromUrl(page.url())+ if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)+ return id+ })()+ project.trackSession(sessionID)- .toContain(token)+ await expect+ .poll(+ async () => {+ const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])+ return messages+ .filter((m) => m.info.role === "assistant")+ .flatMap((m) => m.parts)+ .filter((p) => p.type === "text")+ .map((p) => p.text)+ .join("\n")+ },+ { timeout: 30_000 },+ )+ .toContain(token)+ },+ {+ model: { providerID: "e2e-llm", modelID: "test-model" },+ setup: (dir) => config(dir, llm.url),+ },+ )} finally {page.off("pageerror", onPageError)- await cleanupSession({ sdk, sessionID })}if (pageErrors.length > 0) {