The open source coding agent.
TypeScript
test(e2e): isolate prompt tests with per-worker backend (#20464)
Kit Langton38d22765920e
diff --git a/packages/app/e2e/actions.ts b/packages/app/e2e/actions.tsindex efd370d395c..dc023ddc0b5 100644--- a/packages/app/e2e/actions.ts+++ b/packages/app/e2e/actions.ts@@ -312,10 +312,11 @@ export async function openSettings(page: Page) {return dialog}-export async function seedProjects(page: Page, input: { directory: string; extra?: string[] }) {+export async function seedProjects(page: Page, input: { directory: string; extra?: string[]; serverUrl?: string }) {await page.addInitScript((args: { directory: string; serverUrl: string; extra: string[] }) => {const key = "opencode.global.dat:server"+ const defaultKey = "opencode.settings.dat:defaultServerUrl"const raw = localStorage.getItem(key)const parsed = (() => {if (!raw) return undefined@@ -331,6 +332,7 @@ export async function seedProjects(page: Page, input: { directory: string; extraconst lastProject = store.lastProject && typeof store.lastProject === "object" ? store.lastProject : {}const projects = store.projects && typeof store.projects === "object" ? store.projects : {}const nextProjects = { ...(projects as Record<string, unknown>) }+ const nextList = list.includes(args.serverUrl) ? list : [args.serverUrl, ...list]const add = (origin: string, directory: string) => {const current = nextProjects[origin]@@ -356,17 +358,18 @@ export async function seedProjects(page: Page, input: { directory: string; extralocalStorage.setItem(key,JSON.stringify({- list,+ list: nextList,projects: nextProjects,lastProject,}),)+ localStorage.setItem(defaultKey, args.serverUrl)},- { directory: input.directory, serverUrl, extra: input.extra ?? [] },+ { directory: input.directory, serverUrl: input.serverUrl ?? serverUrl, extra: input.extra ?? [] },)}-export async function createTestProject() {+export async function createTestProject(input?: { serverUrl?: string }) {const root = await fs.mkdtemp(path.join(os.tmpdir(), "opencode-e2e-project-"))const id = `e2e-${path.basename(root)}`@@ -381,7 +384,7 @@ export async function createTestProject() {stdio: "ignore",})- return resolveDirectory(root)+ return resolveDirectory(root, input?.serverUrl)}export async function cleanupTestProject(directory: string) {@@ -430,22 +433,22 @@ export async function waitSlug(page: Page, skip: string[] = []) {return next}-export async function resolveSlug(slug: string) {+export async function resolveSlug(slug: string, input?: { serverUrl?: string }) {const directory = base64Decode(slug)if (!directory) throw new Error(`Failed to decode workspace slug: ${slug}`)- const resolved = await resolveDirectory(directory)+ const resolved = await resolveDirectory(directory, input?.serverUrl)return { directory: resolved, slug: base64Encode(resolved), raw: slug }}-export async function waitDir(page: Page, directory: string) {- const target = await resolveDirectory(directory)+export async function waitDir(page: Page, directory: string, input?: { serverUrl?: string }) {+ const target = await resolveDirectory(directory, input?.serverUrl)await expect.poll(async () => {await assertHealthy(page, "waitDir")const slug = slugFromUrl(page.url())if (!slug) return ""- return resolveSlug(slug)+ return resolveSlug(slug, input).then((item) => item.directory).catch(() => "")},@@ -455,15 +458,15 @@ export async function waitDir(page: Page, directory: string) {return { directory: target, slug: base64Encode(target) }}-export async function waitSession(page: Page, input: { directory: string; sessionID?: string }) {- const target = await resolveDirectory(input.directory)+export async function waitSession(page: Page, input: { directory: string; sessionID?: string; serverUrl?: string }) {+ const target = await resolveDirectory(input.directory, input.serverUrl)await expect.poll(async () => {await assertHealthy(page, "waitSession")const slug = slugFromUrl(page.url())if (!slug) return false- const resolved = await resolveSlug(slug).catch(() => undefined)+ const resolved = await resolveSlug(slug, { serverUrl: input.serverUrl }).catch(() => undefined)if (!resolved || resolved.directory !== target) return falseconst current = sessionIDFromUrl(page.url())if (input.sessionID && current !== input.sessionID) return false@@ -473,7 +476,7 @@ export async function waitSession(page: Page, input: { directory: string; sessioif (input.sessionID && (!state || state.sessionID !== input.sessionID)) return falseif (!input.sessionID && state?.sessionID) return falseif (state?.dir) {- const dir = await resolveDirectory(state.dir).catch(() => state.dir ?? "")+ const dir = await resolveDirectory(state.dir, input.serverUrl).catch(() => state.dir ?? "")if (dir !== target) return false}@@ -489,9 +492,9 @@ export async function waitSession(page: Page, input: { directory: string; sessioreturn { directory: target, slug: base64Encode(target) }}-export async function waitSessionSaved(directory: string, sessionID: string, timeout = 30_000) {- const sdk = createSdk(directory)- const target = await resolveDirectory(directory)+export async function waitSessionSaved(directory: string, sessionID: string, timeout = 30_000, serverUrl?: string) {+ const sdk = createSdk(directory, serverUrl)+ const target = await resolveDirectory(directory, serverUrl)await expect.poll(@@ -501,7 +504,7 @@ export async function waitSessionSaved(directory: string, sessionID: string, tim.then((x) => x.data).catch(() => undefined)if (!data?.directory) return ""- return resolveDirectory(data.directory).catch(() => data.directory)+ return resolveDirectory(data.directory, serverUrl).catch(() => data.directory)},{ timeout },)@@ -666,8 +669,9 @@ export async function cleanupSession(input: {sessionID: stringdirectory?: stringsdk?: ReturnType<typeof createSdk>+ serverUrl?: string}) {- const sdk = input.sdk ?? (input.directory ? createSdk(input.directory) : undefined)+ const sdk = input.sdk ?? (input.directory ? createSdk(input.directory, input.serverUrl) : undefined)if (!sdk) throw new Error("cleanupSession requires sdk or directory")await waitSessionIdle(sdk, input.sessionID, 5_000).catch(() => undefined)const current = await status(sdk, input.sessionID).catch(() => undefined)@@ -1019,3 +1023,13 @@ export async function openWorkspaceMenu(page: Page, workspaceSlug: string) {await expect(menu).toBeVisible()return menu}++export async function assistantText(sdk: ReturnType<typeof createSdk>, sessionID: string) {+ const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])+ return messages+ .filter((m) => m.info.role === "assistant")+ .flatMap((m) => m.parts)+ .filter((p) => p.type === "text")+ .map((p) => p.text)+ .join("\n")+}diff --git a/packages/app/e2e/backend.ts b/packages/app/e2e/backend.tsnew file mode 100644index 00000000000..2acbe7179cc--- /dev/null+++ b/packages/app/e2e/backend.ts@@ -0,0 +1,119 @@+import { spawn } from "node:child_process"+import fs from "node:fs/promises"+import net from "node:net"+import os from "node:os"+import path from "node:path"+import { fileURLToPath } from "node:url"++type Handle = {+ url: string+ stop: () => Promise<void>+}++function freePort() {+ return new Promise<number>((resolve, reject) => {+ const server = net.createServer()+ server.once("error", reject)+ server.listen(0, () => {+ const address = server.address()+ if (!address || typeof address === "string") {+ server.close(() => reject(new Error("Failed to acquire a free port")))+ return+ }+ server.close((err) => {+ if (err) reject(err)+ else resolve(address.port)+ })+ })+ })+}++async function waitForHealth(url: string, probe = "/global/health") {+ const end = Date.now() + 120_000+ let last = ""+ while (Date.now() < end) {+ try {+ const res = await fetch(`${url}${probe}`)+ if (res.ok) return+ last = `status ${res.status}`+ } catch (err) {+ last = err instanceof Error ? err.message : String(err)+ }+ await new Promise((resolve) => setTimeout(resolve, 250))+ }+ throw new Error(`Timed out waiting for backend health at ${url}${probe}${last ? ` (${last})` : ""}`)+}++const LOG_CAP = 100++function cap(input: string[]) {+ if (input.length > LOG_CAP) input.splice(0, input.length - LOG_CAP)+}++function tail(input: string[]) {+ return input.slice(-40).join("")+}++export async function startBackend(label: string): Promise<Handle> {+ const port = await freePort()+ const sandbox = await fs.mkdtemp(path.join(os.tmpdir(), `opencode-e2e-${label}-`))+ const appDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")+ const repoDir = path.resolve(appDir, "../..")+ const opencodeDir = path.join(repoDir, "packages", "opencode")+ const env = {+ ...process.env,+ OPENCODE_DISABLE_SHARE: process.env.OPENCODE_DISABLE_SHARE ?? "true",+ OPENCODE_DISABLE_LSP_DOWNLOAD: "true",+ OPENCODE_DISABLE_DEFAULT_PLUGINS: "true",+ OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: "true",+ OPENCODE_TEST_HOME: path.join(sandbox, "home"),+ XDG_DATA_HOME: path.join(sandbox, "share"),+ XDG_CACHE_HOME: path.join(sandbox, "cache"),+ XDG_CONFIG_HOME: path.join(sandbox, "config"),+ XDG_STATE_HOME: path.join(sandbox, "state"),+ OPENCODE_CLIENT: "app",+ OPENCODE_STRICT_CONFIG_DEPS: "true",+ } satisfies Record<string, string | undefined>+ const out: string[] = []+ const err: string[] = []+ const proc = spawn(+ "bun",+ ["run", "--conditions=browser", "./src/index.ts", "serve", "--port", String(port), "--hostname", "127.0.0.1"],+ {+ cwd: opencodeDir,+ env,+ stdio: ["ignore", "pipe", "pipe"],+ },+ )+ proc.stdout?.on("data", (chunk) => { out.push(String(chunk)); cap(out) })+ proc.stderr?.on("data", (chunk) => { err.push(String(chunk)); cap(err) })++ const url = `http://127.0.0.1:${port}`+ try {+ await waitForHealth(url)+ } catch (error) {+ proc.kill("SIGTERM")+ await fs.rm(sandbox, { recursive: true, force: true }).catch(() => undefined)+ throw new Error(+ [+ `Failed to start isolated e2e backend for ${label}`,+ error instanceof Error ? error.message : String(error),+ tail(out),+ tail(err),+ ]+ .filter(Boolean)+ .join("\n"),+ )+ }++ return {+ url,+ async stop() {+ if (proc.exitCode === null) {+ proc.kill("SIGTERM")+ await new Promise((resolve) => proc.once("exit", () => resolve(undefined))).catch(() => undefined)+ }+ await fs.rm(sandbox, { recursive: true, force: true }).catch(() => undefined)+ },+ }+}diff --git a/packages/app/e2e/fixtures.ts b/packages/app/e2e/fixtures.tsindex 7fc4cda057e..c94c93992af 100644--- a/packages/app/e2e/fixtures.ts+++ b/packages/app/e2e/fixtures.ts@@ -3,6 +3,7 @@ import { ManagedRuntime } from "effect"import type { E2EWindow } from "../src/testing/terminal"import type { Item, Reply, Usage } from "../../opencode/test/lib/llm-server"import { TestLLMServer } from "../../opencode/test/lib/llm-server"+import { startBackend } from "./backend"import {healthPhase,cleanupSession,@@ -19,6 +20,20 @@ import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils"type LLMFixture = {url: stringpush: (...input: (Item | Reply)[]) => Promise<void>+ pushMatch: (+ match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,+ ...input: (Item | Reply)[]+ ) => Promise<void>+ textMatch: (+ match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,+ value: string,+ opts?: { usage?: Usage },+ ) => Promise<void>+ toolMatch: (+ match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,+ name: string,+ input: unknown,+ ) => Promise<void>text: (value: string, opts?: { usage?: Usage }) => Promise<void>tool: (name: string, input: unknown) => Promise<void>toolHang: (name: string, input: unknown) => Promise<void>@@ -46,32 +61,54 @@ const seedModel = (() => {}})()+type ProjectHandle = {+ directory: string+ slug: string+ gotoSession: (sessionID?: string) => Promise<void>+ trackSession: (sessionID: string, directory?: string) => void+ trackDirectory: (directory: string) => void+ sdk: ReturnType<typeof createSdk>+}++type ProjectOptions = {+ extra?: string[]+ model?: { providerID: string; modelID: string }+ setup?: (directory: string) => Promise<void>+ beforeGoto?: (project: { directory: string; sdk: ReturnType<typeof createSdk> }) => Promise<void>+}+type TestFixtures = {llm: LLMFixturesdk: ReturnType<typeof createSdk>gotoSession: (sessionID?: string) => Promise<void>- withProject: <T>(- callback: (project: {- directory: string- slug: string- gotoSession: (sessionID?: string) => Promise<void>- trackSession: (sessionID: string, directory?: string) => void- trackDirectory: (directory: string) => void- }) => Promise<T>,- options?: {- extra?: string[]- model?: { providerID: string; modelID: string }- setup?: (directory: string) => Promise<void>- },- ) => Promise<T>+ withProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>+ withBackendProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>}type WorkerFixtures = {+ backend: {+ url: string+ sdk: (directory?: string) => ReturnType<typeof createSdk>+ }directory: stringslug: string}export const test = base.extend<TestFixtures, WorkerFixtures>({+ backend: [+ async ({}, use, workerInfo) => {+ const handle = await startBackend(`w${workerInfo.workerIndex}`)+ try {+ await use({+ url: handle.url,+ sdk: (directory?: string) => createSdk(directory, handle.url),+ })+ } finally {+ await handle.stop()+ }+ },+ { scope: "worker" },+ ],llm: async ({}, use) => {const rt = ManagedRuntime.make(TestLLMServer.layer)try {@@ -79,6 +116,9 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({await use({url: svc.url,push: (...input) => rt.runPromise(svc.push(...input)),+ pushMatch: (match, ...input) => rt.runPromise(svc.pushMatch(match, ...input)),+ textMatch: (match, value, opts) => rt.runPromise(svc.textMatch(match, value, opts)),+ toolMatch: (match, name, input) => rt.runPromise(svc.toolMatch(match, name, input)),text: (value, opts) => rt.runPromise(svc.text(value, opts)),tool: (name, input) => rt.runPromise(svc.tool(name, input)),toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)),@@ -146,44 +186,70 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({await use(gotoSession)},withProject: async ({ page }, use) => {- await use(async (callback, options) => {- const root = await createTestProject()- const sessions = new Map<string, string>()- const dirs = new Set<string>()- await options?.setup?.(root)- await seedStorage(page, { directory: root, extra: options?.extra, model: options?.model })-- const gotoSession = async (sessionID?: string) => {- await page.goto(sessionPath(root, sessionID))- await waitSession(page, { directory: root, sessionID })- const current = sessionIDFromUrl(page.url())- if (current) trackSession(current)- }+ await use((callback, options) =>+ runProject(page, callback, options),+ )+ },+ withBackendProject: async ({ page, backend }, use) => {+ await use((callback, options) =>+ runProject(page, callback, { ...options, serverUrl: backend.url, sdk: backend.sdk }),+ )+ },+})- const trackSession = (sessionID: string, directory?: string) => {- sessions.set(sessionID, directory ?? root)- }+async function runProject<T>(+ page: Page,+ callback: (project: ProjectHandle) => Promise<T>,+ options?: ProjectOptions & {+ serverUrl?: string+ sdk?: (directory?: string) => ReturnType<typeof createSdk>+ },+) {+ const url = options?.serverUrl+ const root = await createTestProject(url ? { serverUrl: url } : undefined)+ const sdk = options?.sdk?.(root) ?? createSdk(root, url)+ const sessions = new Map<string, string>()+ const dirs = new Set<string>()+ await options?.setup?.(root)+ await seedStorage(page, {+ directory: root,+ extra: options?.extra,+ model: options?.model,+ serverUrl: url,+ })- const trackDirectory = (directory: string) => {- if (directory !== root) dirs.add(directory)- }+ const gotoSession = async (sessionID?: string) => {+ await page.goto(sessionPath(root, sessionID))+ await waitSession(page, { directory: root, sessionID, serverUrl: url })+ const current = sessionIDFromUrl(page.url())+ if (current) trackSession(current)+ }- try {- await gotoSession()- const slug = await waitSlug(page)- return await callback({ directory: root, slug, gotoSession, trackSession, trackDirectory })- } finally {- setHealthPhase(page, "cleanup")- await Promise.allSettled(- Array.from(sessions, ([sessionID, directory]) => cleanupSession({ sessionID, directory })),- )- await Promise.allSettled(Array.from(dirs, (directory) => cleanupTestProject(directory)))- await cleanupTestProject(root)- setHealthPhase(page, "test")- }- })- },-})+ const trackSession = (sessionID: string, directory?: string) => {+ sessions.set(sessionID, directory ?? root)+ }++ const trackDirectory = (directory: string) => {+ if (directory !== root) dirs.add(directory)+ }++ try {+ await options?.beforeGoto?.({ directory: root, sdk })+ await gotoSession()+ const slug = await waitSlug(page)+ return await callback({ directory: root, slug, gotoSession, trackSession, trackDirectory, sdk })+ } finally {+ setHealthPhase(page, "cleanup")+ await Promise.allSettled(+ Array.from(sessions, ([sessionID, directory]) =>+ cleanupSession({ sessionID, directory, serverUrl: url }),+ ),+ )+ await Promise.allSettled(Array.from(dirs, (directory) => cleanupTestProject(directory)))+ await cleanupTestProject(root)+ setHealthPhase(page, "test")+ }+}async function seedStorage(page: Page,@@ -191,6 +257,7 @@ async function seedStorage(directory: stringextra?: string[]model?: { providerID: string; modelID: string }+ serverUrl?: string},) {await seedProjects(page, input)diff --git a/packages/app/e2e/prompt/mock.ts b/packages/app/e2e/prompt/mock.tsnew file mode 100644index 00000000000..eb40a70cba0--- /dev/null+++ b/packages/app/e2e/prompt/mock.ts@@ -0,0 +1,46 @@+import { createSdk } from "../utils"++export const openaiModel = { providerID: "openai", modelID: "gpt-5.3-chat-latest" }++type Hit = { body: Record<string, unknown> }++export function bodyText(hit: Hit) {+ return JSON.stringify(hit.body)+}++export function titleMatch(hit: Hit) {+ return bodyText(hit).includes("Generate a title for this conversation")+}++export function promptMatch(token: string) {+ return (hit: Hit) => bodyText(hit).includes(token)+}++export async function withMockOpenAI<T>(input: { serverUrl: string; llmUrl: string; fn: () => Promise<T> }) {+ const sdk = createSdk(undefined, input.serverUrl)+ const prev = await sdk.global.config.get().then((res) => res.data ?? {})++ try {+ await sdk.global.config.update({+ config: {+ ...prev,+ model: `${openaiModel.providerID}/${openaiModel.modelID}`,+ enabled_providers: ["openai"],+ provider: {+ ...prev.provider,+ openai: {+ ...prev.provider?.openai,+ options: {+ ...prev.provider?.openai?.options,+ apiKey: "test-key",+ baseURL: input.llmUrl,+ },+ },+ },+ },+ })+ return await input.fn()+ } finally {+ await sdk.global.config.update({ config: prev })+ }+}diff --git a/packages/app/e2e/prompt/prompt-async.spec.ts b/packages/app/e2e/prompt/prompt-async.spec.tsindex 51fbc3e4ae3..99fa5f2d457 100644--- a/packages/app/e2e/prompt/prompt-async.spec.ts+++ b/packages/app/e2e/prompt/prompt-async.spec.ts@@ -1,47 +1,53 @@import { test, expect } from "../fixtures"import { promptSelector } from "../selectors"-import { cleanupSession, sessionIDFromUrl, withSession } from "../actions"+import { assistantText, sessionIDFromUrl, withSession } from "../actions"+import { openaiModel, promptMatch, withMockOpenAI } from "./mock"const text = (value: string | null) => (value ?? "").replace(/\u200B/g, "").trim()// Regression test for Issue #12453: the synchronous POST /message endpoint holds// the connection open while the agent works, causing "Failed to fetch" over// VPN/Tailscale. The fix switches to POST /prompt_async which returns immediately.-test("prompt succeeds when sync message endpoint is unreachable", async ({ page, sdk, gotoSession }) => {+test("prompt succeeds when sync message endpoint is unreachable", async ({+ page,+ llm,+ backend,+ withBackendProject,+}) => {test.setTimeout(120_000)// Simulate Tailscale/VPN killing the long-lived sync connectionawait page.route("**/session/*/message", (route) => route.abort("connectionfailed"))- await gotoSession()+ await withMockOpenAI({+ serverUrl: backend.url,+ llmUrl: llm.url,+ fn: async () => {+ const token = `E2E_ASYNC_${Date.now()}`+ await llm.textMatch(promptMatch(token), token)- const token = `E2E_ASYNC_${Date.now()}`- await page.locator(promptSelector).click()- await page.keyboard.type(`Reply with exactly: ${token}`)- await page.keyboard.press("Enter")+ await withBackendProject(+ async (project) => {+ await page.locator(promptSelector).click()+ await page.keyboard.type(`Reply with exactly: ${token}`)+ await page.keyboard.press("Enter")- await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })- const sessionID = sessionIDFromUrl(page.url())!+ await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })+ const sessionID = sessionIDFromUrl(page.url())!+ project.trackSession(sessionID)- try {- // Agent response arrives via SSE despite sync endpoint being dead- await expect- .poll(- async () => {- const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])- return messages- .filter((m) => m.info.role === "assistant")- .flatMap((m) => m.parts)- .filter((p) => p.type === "text")- .map((p) => p.text)- .join("\n")+ await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)++ await expect+ .poll(() => assistantText(project.sdk, sessionID), { timeout: 90_000 })+ .toContain(token)+ },+ {+ model: openaiModel,},- { timeout: 90_000 },)- .toContain(token)- } finally {- await cleanupSession({ sdk, sessionID })- }+ },+ })})test("failed prompt send restores the composer input", async ({ page, sdk, gotoSession }) => {diff --git a/packages/app/e2e/prompt/prompt.spec.ts b/packages/app/e2e/prompt/prompt.spec.tsindex 1acf17f5bf4..e4545e97a93 100644--- a/packages/app/e2e/prompt/prompt.spec.ts+++ b/packages/app/e2e/prompt/prompt.spec.ts@@ -1,44 +1,9 @@-import fs from "node:fs/promises"-import path from "node:path"import { test, expect } from "../fixtures"import { promptSelector } from "../selectors"-import { sessionIDFromUrl } from "../actions"-import { createSdk } from "../utils"+import { assistantText, sessionIDFromUrl } from "../actions"+import { openaiModel, promptMatch, titleMatch, withMockOpenAI } from "./mock"-async function config(dir: string, url: string) {- await fs.writeFile(- path.join(dir, "opencode.json"),- JSON.stringify({- $schema: "https://opencode.ai/config.json",- enabled_providers: ["e2e-llm"],- provider: {- "e2e-llm": {- name: "E2E LLM",- npm: "@ai-sdk/openai-compatible",- env: [],- models: {- "test-model": {- name: "Test Model",- tool_call: true,- limit: { context: 128000, output: 32000 },- },- },- options: {- apiKey: "test-key",- baseURL: url,- },- },- },- agent: {- build: {- model: "e2e-llm/test-model",- },- },- }),- )-}--test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => {+test("can send a prompt and receive a reply", async ({ page, llm, backend, withBackendProject }) => {test.setTimeout(120_000)const pageErrors: string[] = []@@ -48,48 +13,43 @@ test("can send a prompt and receive a reply", async ({ page, llm, withProject })page.on("pageerror", onPageError)try {- await withProject(- async (project) => {- const sdk = createSdk(project.directory)+ await withMockOpenAI({+ serverUrl: backend.url,+ llmUrl: llm.url,+ fn: async () => {const token = `E2E_OK_${Date.now()}`- await llm.text(token)- await project.gotoSession()+ await llm.textMatch(titleMatch, "E2E Title")+ await llm.textMatch(promptMatch(token), token)- const prompt = page.locator(promptSelector)- await prompt.click()- await page.keyboard.type(`Reply with exactly: ${token}`)- await page.keyboard.press("Enter")+ await withBackendProject(+ async (project) => {+ const prompt = page.locator(promptSelector)+ await prompt.click()+ await page.keyboard.type(`Reply with exactly: ${token}`)+ await page.keyboard.press("Enter")- await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })+ await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })- const sessionID = (() => {- const id = sessionIDFromUrl(page.url())- if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)- return id- })()- project.trackSession(sessionID)+ const sessionID = (() => {+ const id = sessionIDFromUrl(page.url())+ if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)+ return id+ })()+ project.trackSession(sessionID)- await expect- .poll(- async () => {- const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])- return messages- .filter((m) => m.info.role === "assistant")- .flatMap((m) => m.parts)- .filter((p) => p.type === "text")- .map((p) => p.text)- .join("\n")- },- { timeout: 30_000 },- )- .toContain(token)- },- {- model: { providerID: "e2e-llm", modelID: "test-model" },- setup: (dir) => config(dir, llm.url),+ await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)++ await expect+ .poll(() => assistantText(project.sdk, sessionID), { timeout: 30_000 })+ .toContain(token)+ },+ {+ model: openaiModel,+ },+ )},- )+ })} finally {page.off("pageerror", onPageError)}diff --git a/packages/app/e2e/utils.ts b/packages/app/e2e/utils.tsindex f07a8d3f111..17a87856646 100644--- a/packages/app/e2e/utils.ts+++ b/packages/app/e2e/utils.ts@@ -26,21 +26,21 @@ export const serverNamePattern = new RegExp(`(?:${serverNames.map(escape).join("export const modKey = process.platform === "darwin" ? "Meta" : "Control"export const terminalToggleKey = "Control+Backquote"-export function createSdk(directory?: string) {- return createOpencodeClient({ baseUrl: serverUrl, directory, throwOnError: true })+export function createSdk(directory?: string, baseUrl = serverUrl) {+ return createOpencodeClient({ baseUrl, directory, throwOnError: true })}-export async function resolveDirectory(directory: string) {- return createSdk(directory)+export async function resolveDirectory(directory: string, baseUrl = serverUrl) {+ return createSdk(directory, baseUrl).path.get().then((x) => x.data?.directory ?? directory)}-export async function getWorktree() {- const sdk = createSdk()+export async function getWorktree(baseUrl = serverUrl) {+ const sdk = createSdk(undefined, baseUrl)const result = await sdk.path.get()const data = result.data- if (!data?.worktree) throw new Error(`Failed to resolve a worktree from ${serverUrl}/path`)+ if (!data?.worktree) throw new Error(`Failed to resolve a worktree from ${baseUrl}/path`)return data.worktree}diff --git a/packages/opencode/test/lib/llm-server.ts b/packages/opencode/test/lib/llm-server.tsindex fb84f1175ae..1c624cd0d50 100644--- a/packages/opencode/test/lib/llm-server.ts+++ b/packages/opencode/test/lib/llm-server.ts@@ -20,6 +20,13 @@ type Hit = {body: Record<string, unknown>}+type Match = (hit: Hit) => boolean++type Queue = {+ item: Item+ match?: Match+}+type Wait = {count: numberready: Deferred.Deferred<void>@@ -420,7 +427,7 @@ const reset = Effect.fn("TestLLMServer.reset")(function* (item: Sse) {for (const part of item.tail) res.write(line(part))res.destroy(new Error("connection reset"))})- yield* Effect.never+ return yield* Effect.never})function fail(item: HttpError) {@@ -581,6 +588,9 @@ namespace TestLLMServer {export interface Service {readonly url: stringreadonly push: (...input: (Item | Reply)[]) => Effect.Effect<void>+ readonly pushMatch: (match: Match, ...input: (Item | Reply)[]) => Effect.Effect<void>+ readonly textMatch: (match: Match, value: string, opts?: { usage?: Usage }) => Effect.Effect<void>+ readonly toolMatch: (match: Match, name: string, input: unknown) => Effect.Effect<void>readonly text: (value: string, opts?: { usage?: Usage }) => Effect.Effect<void>readonly tool: (name: string, input: unknown) => Effect.Effect<void>readonly toolHang: (name: string, input: unknown) => Effect.Effect<void>@@ -605,11 +615,15 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServconst router = yield* HttpRouter.HttpRouterlet hits: Hit[] = []- let list: Item[] = []+ let list: Queue[] = []let waits: Wait[] = []const queue = (...input: (Item | Reply)[]) => {- list = [...list, ...input.map(item)]+ list = [...list, ...input.map((value) => ({ item: item(value) }))]+ }++ const queueMatch = (match: Match, ...input: (Item | Reply)[]) => {+ list = [...list, ...input.map((value) => ({ item: item(value), match }))]}const notify = Effect.fnUntraced(function* () {@@ -619,19 +633,21 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServyield* Effect.forEach(ready, (item) => Deferred.succeed(item.ready, void 0))})- const pull = () => {- const first = list[0]- if (!first) return- list = list.slice(1)- return first+ const pull = (hit: Hit) => {+ const index = list.findIndex((entry) => !entry.match || entry.match(hit))+ if (index === -1) return+ const first = list[index]+ list = [...list.slice(0, index), ...list.slice(index + 1)]+ return first.item}const handle = Effect.fn("TestLLMServer.handle")(function* (mode: "chat" | "responses") {const req = yield* HttpServerRequest.HttpServerRequest- const next = pull()- if (!next) return HttpServerResponse.text("unexpected request", { status: 500 })const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({})))- hits = [...hits, hit(req.originalUrl, body)]+ const current = hit(req.originalUrl, body)+ const next = pull(current)+ if (!next) return HttpServerResponse.text("unexpected request", { status: 500 })+ hits = [...hits, current]yield* notify()if (next.type !== "sse") return fail(next)if (mode === "responses") return send(responses(next, modelFrom(body)))@@ -655,6 +671,21 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServpush: Effect.fn("TestLLMServer.push")(function* (...input: (Item | Reply)[]) {queue(...input)}),+ pushMatch: Effect.fn("TestLLMServer.pushMatch")(function* (match: Match, ...input: (Item | Reply)[]) {+ queueMatch(match, ...input)+ }),+ textMatch: Effect.fn("TestLLMServer.textMatch")(function* (+ match: Match,+ value: string,+ opts?: { usage?: Usage },+ ) {+ const out = reply().text(value)+ if (opts?.usage) out.usage(opts.usage)+ queueMatch(match, out.stop().item())+ }),+ toolMatch: Effect.fn("TestLLMServer.toolMatch")(function* (match: Match, name: string, input: unknown) {+ queueMatch(match, reply().tool(name, input).item())+ }),text: Effect.fn("TestLLMServer.text")(function* (value: string, opts?: { usage?: Usage }) {const out = reply().text(value)if (opts?.usage) out.usage(opts.usage)