The open source coding agent.
TypeScript
test(app): block real llm calls in e2e prompts (#20579)
Kit Langtona09b08672906
diff --git a/packages/app/e2e/actions.ts b/packages/app/e2e/actions.tsindex dc023ddc0b5..df8e0768ed6 100644--- a/packages/app/e2e/actions.ts+++ b/packages/app/e2e/actions.ts@@ -1,5 +1,5 @@import { base64Decode, base64Encode } from "@opencode-ai/util/encode"-import { expect, type Locator, type Page } from "@playwright/test"+import { expect, type Locator, type Page, type Route } from "@playwright/test"import fs from "node:fs/promises"import os from "node:os"import path from "node:path"@@ -43,6 +43,27 @@ export async function defocus(page: Page) {.catch(() => undefined)}+export async function withNoReplyPrompt<T>(page: Page, fn: () => Promise<T>) {+ const url = "**/session/*/prompt_async"+ const route = async (input: Route) => {+ const body = input.request().postDataJSON()+ await input.continue({+ postData: JSON.stringify({ ...body, noReply: true }),+ headers: {+ ...input.request().headers(),+ "content-type": "application/json",+ },+ })+ }++ await page.route(url, route)+ try {+ return await fn()+ } finally {+ await page.unroute(url, route)+ }+}+async function terminalID(term: Locator) {const id = await term.getAttribute(terminalAttr)if (id) return iddiff --git a/packages/app/e2e/projects/projects-switch.spec.ts b/packages/app/e2e/projects/projects-switch.spec.tsindex b46c1b407e3..f87a47cf091 100644--- a/packages/app/e2e/projects/projects-switch.spec.ts+++ b/packages/app/e2e/projects/projects-switch.spec.ts@@ -10,6 +10,7 @@ import {waitSession,waitSessionSaved,waitSlug,+ withNoReplyPrompt,} from "../actions"import { projectSwitchSelector, promptSelector, workspaceItemSelector, workspaceNewSessionSelector } from "../selectors"import { dirSlug, resolveDirectory } from "../utils"@@ -81,8 +82,10 @@ test("switching back to a project opens the latest workspace session", async ({// Create a session by sending a promptconst prompt = page.locator(promptSelector)await expect(prompt).toBeVisible()- await prompt.fill("test")- await page.keyboard.press("Enter")+ await withNoReplyPrompt(page, async () => {+ await prompt.fill("test")+ await page.keyboard.press("Enter")+ })// Wait for the URL to update with the new session IDawait expect.poll(() => sessionIDFromUrl(page.url()) ?? "", { timeout: 15_000 }).not.toBe("")diff --git a/packages/app/e2e/projects/workspace-new-session.spec.ts b/packages/app/e2e/projects/workspace-new-session.spec.tsindex 3a7a6bbc221..835c8c99eda 100644--- a/packages/app/e2e/projects/workspace-new-session.spec.ts+++ b/packages/app/e2e/projects/workspace-new-session.spec.ts@@ -9,6 +9,7 @@ import {waitSession,waitSessionSaved,waitSlug,+ withNoReplyPrompt,} from "../actions"import { promptSelector, workspaceItemSelector, workspaceNewSessionSelector } from "../selectors"import { createSdk } from "../utils"@@ -58,8 +59,10 @@ async function createSessionFromWorkspace(const prompt = page.locator(promptSelector)await expect(prompt).toBeVisible()- await prompt.fill(text)- await page.keyboard.press("Enter")+ await withNoReplyPrompt(page, async () => {+ await prompt.fill(text)+ await page.keyboard.press("Enter")+ })await expect.poll(() => sessionIDFromUrl(page.url()) ?? "", { timeout: 15_000 }).not.toBe("")const sessionID = sessionIDFromUrl(page.url())diff --git a/packages/app/e2e/session/session-model-persistence.spec.ts b/packages/app/e2e/session/session-model-persistence.spec.tsindex 36cbb0fbf19..66bc451bcfc 100644--- a/packages/app/e2e/session/session-model-persistence.spec.ts+++ b/packages/app/e2e/session/session-model-persistence.spec.ts@@ -8,11 +8,11 @@ import {waitSession,waitSessionIdle,waitSlug,+ withNoReplyPrompt,} from "../actions"import {promptAgentSelector,promptModelSelector,- promptSelector,promptVariantSelector,workspaceItemSelector,workspaceNewSessionSelector,@@ -231,11 +231,14 @@ async function goto(page: Page, directory: string, sessionID?: string) {}async function submit(page: Page, value: string) {- const prompt = page.locator(promptSelector)+ const prompt = page.locator('[data-component="prompt-input"]')await expect(prompt).toBeVisible()- await prompt.click()- await prompt.fill(value)- await prompt.press("Enter")++ await withNoReplyPrompt(page, async () => {+ await prompt.click()+ await prompt.fill(value)+ await prompt.press("Enter")+ })await expect.poll(() => sessionIDFromUrl(page.url()) ?? "", { timeout: 30_000 }).not.toBe("")const id = sessionIDFromUrl(page.url())diff --git a/packages/app/test/e2e/no-real-llm.test.ts b/packages/app/test/e2e/no-real-llm.test.tsnew file mode 100644index 00000000000..c801df56f8f--- /dev/null+++ b/packages/app/test/e2e/no-real-llm.test.ts@@ -0,0 +1,27 @@+import { describe, expect, test } from "bun:test"+import path from "node:path"+import { fileURLToPath } from "node:url"++const dir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../e2e")++function hasPrompt(src: string) {+ if (!src.includes("withProject(")) return false+ if (src.includes("withNoReplyPrompt(")) return false+ if (src.includes("session.promptAsync({") && !src.includes("noReply: true")) return true+ if (!src.includes("promptSelector")) return false+ return src.includes('keyboard.press("Enter")') || src.includes('prompt.press("Enter")')+}++describe("e2e llm guard", () => {+ test("withProject specs do not submit prompt replies", async () => {+ const bad: string[] = []++ for await (const file of new Bun.Glob("**/*.spec.ts").scan({ cwd: dir, absolute: true })) {+ const src = await Bun.file(file).text()+ if (!hasPrompt(src)) continue+ bad.push(path.relative(dir, file))+ }++ expect(bad).toEqual([])+ })+})