Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

ci(platform): run AutoPilot journeys against a scripted model server (#15431)

Nicholas Tindledb0b2131141c
diff --git a/.github/scripts/test_e2e_seed_cache_contract.py b/.github/scripts/test_e2e_seed_cache_contract.py
index 2319452b9f..7e18145b51 100644
--- a/.github/scripts/test_e2e_seed_cache_contract.py
+++ b/.github/scripts/test_e2e_seed_cache_contract.py
@@ -41,6 +41,18 @@ class E2EMatrixContractTests(unittest.TestCase):
)[0]
self.assertIn("matrix.suite == 'smoke'", body)
+ def test_only_journeys_answer_copilot_from_scripts(self):
+ workflow = (
+ Path(__file__).resolve().parents[1] / "workflows/platform-fullstack-ci.yml"
+ ).read_text(encoding="utf-8")
+ self.assertEqual(_matrix_leg(workflow, "smoke")["scripted_model"], "false")
+ self.assertEqual(_matrix_leg(workflow, "journeys")["scripted_model"], "true")
+ prepare = workflow.split("- name: Set up Platform - Prepare Docker build\n", 1)[
+ 1
+ ].split("\n - name:", 1)[0]
+ self.assertIn("-f docker-compose.journeys.yml", prepare)
+ self.assertIn("matrix.scripted_model", prepare)
+
def test_journeys_check_is_advisory(self):
checker = (
Path(__file__).resolve().parents[1]
@@ -121,7 +133,7 @@ class E2ECoverageContractTests(unittest.TestCase):
).read_text(encoding="utf-8")
self.assertIn("--min-tests ${{ matrix.min_tests }}", workflow)
self.assertEqual(_matrix_leg(workflow, "smoke")["min_tests"], "28")
- self.assertEqual(_matrix_leg(workflow, "journeys")["min_tests"], "5")
+ self.assertEqual(_matrix_leg(workflow, "journeys")["min_tests"], "9")
# Smoke stays strict: no test.fail() known failures.
self.assertEqual(_matrix_leg(workflow, "smoke")["known_failures"], "false")
diff --git a/.github/workflows/platform-fullstack-ci.yml b/.github/workflows/platform-fullstack-ci.yml
index a464b453d8..6f17ac5705 100644
--- a/.github/workflows/platform-fullstack-ci.yml
+++ b/.github/workflows/platform-fullstack-ci.yml
@@ -101,6 +101,7 @@ jobs:
../../.github/scripts/test_validate_cobertura.py \
../../.github/scripts/test_e2e_seed_cache_contract.py \
../../.github/scripts/test_hosted_journeys_env.py \
+ ../scripted_model/tests \
--junitxml="$RUNNER_TEMP/fullstack-validator-results/playwright-validator-tests.xml"
- name: Set up Backend - Generate Prisma client
@@ -191,6 +192,7 @@ jobs:
embeddings: true
seed: true
hosted: false
+ scripted_model: false
known_failures: false
# Holistic *-journey.spec.ts flows on the hosted configuration:
# hosted feature flags, credits on. Advisory until proven stable
@@ -201,11 +203,15 @@ jobs:
# selfhost-journey asserts nothing hosted-only exists; it needs a
# self-hosted stack (TODO in src/playwright/journeys/README.md).
playwright_args: --project hosted
- min_tests: 5
+ min_tests: 9
embeddings: false
# Journeys sign up their own users and need no seeded data.
seed: false
hosted: true
+ # AutoPilot answers from scripts (autogpt_platform/scripted_model)
+ # instead of CHAT_TEST_MODE's canned reply, so journeys run the
+ # real copilot pipeline: tools, persistence, streaming, cost.
+ scripted_model: true
# Journeys may pin a known bug with test.fail(); it must still fail.
known_failures: true
permissions:
@@ -253,7 +259,11 @@ jobs:
if [[ "${{ matrix.embeddings }}" == "true" ]]; then
echo "OPENAI_INTERNAL_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> ../backend/.env
fi
- echo "CHAT_TEST_MODE=true" >> ../backend/.env
+ # The journeys leg's AutoPilot talks to the scripted model instead
+ # (docker-compose.journeys.yml).
+ if [[ "${{ matrix.scripted_model }}" != "true" ]]; then
+ echo "CHAT_TEST_MODE=true" >> ../backend/.env
+ fi
echo "SCHEDULER_STARTUP_EMBEDDING_BACKFILL=false" >> ../backend/.env
env:
# Used by E2E test data script to generate embeddings for approved store agents
@@ -287,7 +297,11 @@ jobs:
# Resolve extends and generate a flat compose file that bake can understand
export NEXT_PUBLIC_SOURCEMAPS NEXT_PUBLIC_PW_TEST
- docker compose -f docker-compose.yml config > docker-compose.resolved.yml
+ compose_files=(-f docker-compose.yml)
+ if [[ "${{ matrix.scripted_model }}" == "true" ]]; then
+ compose_files+=(-f docker-compose.journeys.yml)
+ fi
+ docker compose "${compose_files[@]}" config > docker-compose.resolved.yml
# Ensure NEXT_PUBLIC_SOURCEMAPS is in resolved compose
# (docker compose config on some versions drops this arg)
diff --git a/autogpt_platform/docker-compose.journeys.yml b/autogpt_platform/docker-compose.journeys.yml
new file mode 100644
index 0000000000..e16955b764
--- /dev/null
+++ b/autogpt_platform/docker-compose.journeys.yml
@@ -0,0 +1,82 @@
+# The journeys configuration: AutoPilot answers from scripts instead of a real
+# model, so a Playwright journey drives the real copilot pipeline (engine,
+# tools, persistence, streaming, cost accounting) with deterministic model
+# turns. Only CI's journeys leg and local test stacks add this file; a plain
+# `docker compose up` never reads it, so self-hosted installs are unaffected.
+#
+# docker compose -f docker-compose.yml -f docker-compose.journeys.yml up -d
+#
+# Scripts live in frontend/src/playwright/journeys/model-scripts. See
+# scripted_model/README.md.
+
+x-scripted-model-env: &scripted-model-env
+ # OpenRouter mode pointed at the fake. The SDK engine's CLI gets this URL
+ # without /v1 as ANTHROPIC_BASE_URL (backend/copilot/sdk/env.py); the
+ # baseline engine and side calls (session titles) use it as their
+ # OpenAI-compatible endpoint. The key is never checked.
+ CHAT_TEST_MODE: "false"
+ CHAT_USE_LOCAL: "false"
+ CHAT_USE_OPENROUTER: "true"
+ CHAT_USE_CLAUDE_CODE_SUBSCRIPTION: "false"
+ CHAT_BASE_URL: http://scripted_model:4010/api/v1
+ CHAT_API_KEY: "scripted-model-accepts-any-key" # pragma: allowlist secret
+ # Memory (graphiti-memory is a hosted flag) would otherwise send that key to
+ # openrouter.ai and api.openai.com. Kept inside the stack, its calls fail
+ # fast and the turn goes on without memory, as it does on any failure.
+ GRAPHITI_LLM_BASE_URL: http://scripted_model:4010/api/v1
+ GRAPHITI_EMBEDDER_BASE_URL: http://scripted_model:4010/api/v1
+ GRAPHITI_EMBEDDER_API_KEY: "scripted-model-accepts-any-key" # pragma: allowlist secret
+ # Journeys provoke failures on purpose; keep them out of Sentry even when a
+ # local .env carries a real DSN.
+ SENTRY_DSN: ""
+
+services:
+ scripted_model:
+ build:
+ context: ./scripted_model
+ volumes:
+ - ./frontend/src/playwright/journeys/model-scripts:/scripts:ro
+ # Journeys read the journal from the host.
+ ports:
+ - "127.0.0.1:4010:4010"
+ healthcheck:
+ test:
+ [
+ "CMD",
+ "python",
+ "-c",
+ "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4010/healthz', timeout=2)",
+ ]
+ interval: 5s
+ timeout: 3s
+ retries: 10
+ networks:
+ - app-network
+
+ copilot_executor:
+ environment: *scripted-model-env
+ depends_on:
+ scripted_model:
+ condition: service_healthy
+
+ rest_server:
+ environment: *scripted-model-env
+
+ executor:
+ environment: *scripted-model-env
+
+ scheduler_server:
+ environment: *scripted-model-env
+
+ database_manager:
+ environment: *scripted-model-env
+
+ notification_server:
+ environment: *scripted-model-env
+
+ websocket_server:
+ environment: *scripted-model-env
+
+ frontend:
+ environment:
+ DISABLE_SENTRY: "true"
diff --git a/autogpt_platform/frontend/src/playwright/journeys/README.md b/autogpt_platform/frontend/src/playwright/journeys/README.md
index 50e662ed44..8405179a76 100644
--- a/autogpt_platform/frontend/src/playwright/journeys/README.md
+++ b/autogpt_platform/frontend/src/playwright/journeys/README.md
@@ -52,14 +52,15 @@ product does what the user expects.
## Run locally
Start a full stack in the journeys configuration, then point Playwright at it.
-Mirror CI's backend settings:
+Mirror CI's backend settings; `docker-compose.journeys.yml` adds the scripted
+model AutoPilot chats with (see below):
```bash
cd autogpt_platform
make init-env
python ../.github/scripts/hosted_journeys_env.py >> backend/.env
-echo "CHAT_TEST_MODE=true" >> backend/.env
-NEXT_PUBLIC_PW_TEST=true docker compose up -d --build
+NEXT_PUBLIC_PW_TEST=true docker compose -f docker-compose.yml \
+ -f docker-compose.journeys.yml up -d --build
cd frontend
PLAYWRIGHT_JOURNEY_COST_PER_NODE=1 PLAYWRIGHT_JOURNEY_TIER=PRO \
pnpm exec playwright test --config playwright.journeys.config.ts --project hosted
@@ -73,8 +74,41 @@ reads the hosted flag values the fixtures hand it.
per node execution: `1` with `EXECUTION_COST_COUNT_THRESHOLD=1`, otherwise
leave it unset for `0`.
-The tier grant uses `DATABASE_URL` from `frontend/.env`; if your stack's
-Postgres isn't on `localhost:5432`, set `PLAYWRIGHT_JOURNEY_DATABASE_URL`.
+The tier grant and the copilot cost check use `DATABASE_URL` from
+`frontend/.env`; if your stack's Postgres isn't on `localhost:5432`, set
+`PLAYWRIGHT_JOURNEY_DATABASE_URL`.
+
+### Chatting with AutoPilot: the scripted model
+
+Journeys that chat run the real copilot pipeline (engine, tool calls,
+persistence, streaming and reconnects, cost accounting) against
+`autogpt_platform/scripted_model`, a fake model provider that plays each
+model turn from a script. `docker-compose.journeys.yml` starts it and points
+AutoPilot at it, in place of `CHAT_TEST_MODE`'s canned reply.
+
+- **Write the script** in `model-scripts/<name>.json`: the text, tool calls
+ (with their exact arguments) and token usage of each model turn, in order.
+ The format is in `scripted_model/README.md`.
+- **Pick it** with `scriptedRun("<name>")` from `scripted-model.ts`, and put
+ its `marker` in the chat message.
+- **Import the script** in the spec to know what to expect, as
+ `copilot-build-agent-journey.spec.ts` does.
+- **Check the run** with the helpers in `scripted-model.ts`:
+ `expectEachTurnOnce` (every turn asked for once, every expected tool result
+ seen, every reply delivered in full) and `expectChargedWhatWasBilled` (the
+ copilot's cost ledger charged exactly what the model billed). They read the
+ scripted model's journal at `localhost:4010`.
+
+- **Chat helpers** are in `copilot-chat.ts`. `recordChatStreams` records the
+ body of every chat stream as the browser received it (Chromium keeps no
+ streamed body for Playwright), and `cutChatStreams` fails the open ones the
+ way a dropped connection does. `context.setOffline` can't do that: Chromium
+ lets a stream that is already open finish.
+
+A request the script can't answer (no marker, past the last turn, a tool the
+platform didn't offer) fails the turn with the reason, which shows in the
+scripted model's container log and journal. These journeys skip against a
+deployed app, whose AutoPilot uses real models.
### Projects: `hosted` and `selfhost`
@@ -146,15 +180,15 @@ allows. Flag values live in `hosted-flags.json`, read by both sides:
Differences from hosted that are still open:
-| Area | Journeys leg | Hosted |
-| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ |
-| Payments | `enable-platform-payment` **on**; journey users get a paid tier without Stripe, so checkout, trials and Stripe webhooks go untested | On, with Stripe: NO_TIER users get a 402 until they subscribe or start a trial |
-| Node execution fee | 1 credit per node (`EXECUTION_COST_COUNT_THRESHOLD=1`), so a run's cost is visible | 1 credit per 100 node executions |
-| Copilot | `CHAT_TEST_MODE` dummy reply; no real turn pipeline, tools or cost | Real models |
-| Flag source | Fixed values from `hosted-flags.json`; JSON flags (model routing, tier limits) at code defaults | PostHog/LaunchDarkly with targeting |
-| `BEHAVE_AS` | `LOCAL` on both sides; the frontend's is build time | `CLOUD` |
-| Pods | One `executor`, one `copilot_executor` | Many. Scaling needs the fixed host ports 8002/8008 removed first |
-| Frontend runtime | Next.js standalone server in Docker | Vercel (body size limits, edge routing) |
-| External services | No Stripe, OAuth providers, Postmark or E2B; ClamAV, Redis, RabbitMQ and FalkorDB are real | Real |
-| LLM keys | None; blocks that need platform credentials fail | Platform keys |
-| Origin | `http://localhost:3000` | HTTPS |
+| Area | Journeys leg | Hosted |
+| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------ |
+| Payments | `enable-platform-payment` **on**; journey users get a paid tier without Stripe, so checkout, trials and Stripe webhooks go untested | On, with Stripe: NO_TIER users get a 402 until they subscribe or start a trial |
+| Node execution fee | 1 credit per node (`EXECUTION_COST_COUNT_THRESHOLD=1`), so a run's cost is visible | 1 credit per 100 node executions |
+| Copilot | The scripted model: the real engine (Claude Agent SDK), tools, persistence, streaming and cost, with scripted model turns. Memory (Graphiti) calls fail, so turns run without memory | Real models |
+| Flag source | Fixed values from `hosted-flags.json`; JSON flags (model routing, tier limits) at code defaults | PostHog/LaunchDarkly with targeting |
+| `BEHAVE_AS` | `LOCAL` on both sides; the frontend's is build time | `CLOUD` |
+| Pods | One `executor`, one `copilot_executor` | Many. Scaling needs the fixed host ports 8002/8008 removed first |
+| Frontend runtime | Next.js standalone server in Docker | Vercel (body size limits, edge routing) |
+| External services | No Stripe, OAuth providers, Postmark or E2B; ClamAV, Redis, RabbitMQ and FalkorDB are real | Real |
+| LLM keys | None; blocks that need platform credentials fail | Platform keys |
+| Origin | `http://localhost:3000` | HTTPS |
diff --git a/autogpt_platform/frontend/src/playwright/journeys/copilot-build-agent-journey.spec.ts b/autogpt_platform/frontend/src/playwright/journeys/copilot-build-agent-journey.spec.ts
new file mode 100644
index 0000000000..481574dac9
--- /dev/null
+++ b/autogpt_platform/frontend/src/playwright/journeys/copilot-build-agent-journey.spec.ts
@@ -0,0 +1,115 @@
+import type { Page } from "@playwright/test";
+import {
+ currentSessionID,
+ finishedChatStream,
+ recordChatStreams,
+ savedAssistantTexts,
+ savedMessages,
+ startChat,
+} from "./copilot-chat";
+import { expect, test } from "./fixtures";
+import buildDoublerScript from "./model-scripts/build-doubler-agent.json";
+import {
+ expectChargedWhatWasBilled,
+ expectEachTurnOnce,
+ type ModelScript,
+ readJournal,
+ requireScriptedModel,
+ scriptedRun,
+ streamedTexts,
+} from "./scripted-model";
+
+// The model reads the building guide, saves the agent with create_agent and
+// says it's done; AutoPilot runs both tools for real.
+const SCRIPT: ModelScript = buildDoublerScript;
+const AGENT_NAME = "Journey Doubler";
+
+test.describe("copilot build agent journey", () => {
+ test.beforeAll(requireScriptedModel);
+
+ test("AutoPilot builds an agent: the streamed answer is what's saved, the agent is in the Library, and the chat costs what the model billed", async ({
+ page,
+ journeyUser,
+ }) => {
+ const run = scriptedRun("build-doubler-agent");
+ const modelTexts = SCRIPT.turns.map((turn) => turn.text ?? "");
+ expect(JSON.stringify(SCRIPT), "the script builds the agent").toContain(
+ AGENT_NAME,
+ );
+
+ await recordChatStreams(page);
+ await startChat(
+ page,
+ `Build me an agent that doubles a number. ${run.marker}`,
+ );
+ const streamed = streamedTexts(await finishedChatStream(page, "POST"));
+ expect(streamed, "the stream should carry what the model said").toEqual(
+ modelTexts,
+ );
+ const sessionID = await currentSessionID(page);
+
+ await expect
+ .poll(() => savedAssistantTexts(page, sessionID), {
+ message: "the saved chat should hold exactly what was streamed",
+ })
+ .toEqual(streamed);
+ await page.reload();
+ const transcript = page.getByRole("log");
+ for (const text of streamed) {
+ await expect(
+ transcript.getByText(text, { exact: true }),
+ "the reloaded chat should show what was streamed, once",
+ ).toHaveCount(1);
+ }
+
+ const saved = await savedAgent(page, sessionID);
+ expect(saved.agent_name).toBe(AGENT_NAME);
+ await page.goto("/library");
+ await expect(
+ page
+ .getByTestId("library-agent-card")
+ .filter({ hasText: AGENT_NAME })
+ .and(page.locator(`[data-agent-id="${saved.library_agent_id}"]`)),
+ "the Library should show the agent the chat saved",
+ ).toBeVisible();
+ expect(
+ await libraryAgentsNamed(page, AGENT_NAME),
+ "the chat should save the agent once",
+ ).toHaveLength(1);
+
+ const journal = await readJournal(run.nonce);
+ expectEachTurnOnce(journal, SCRIPT);
+ await expectChargedWhatWasBilled(userID(journeyUser), journal);
+ });
+});
+
+function userID(user: { id?: string }) {
+ if (!user.id) throw new Error("the journey user should have an id");
+ return user.id;
+}
+
+async function savedAgent(page: Page, sessionID: string) {
+ const results = (await savedMessages(page, sessionID))
+ .filter((message) => message.role === "tool" && message.content)
+ .map((message) => parseJSON(message.content as string))
+ .filter((content) => content?.type === "agent_builder_saved");
+ expect(results, "the chat should record one saved agent").toHaveLength(1);
+ return results[0] as { agent_name: string; library_agent_id: string };
+}
+
+async function libraryAgentsNamed(page: Page, name: string) {
+ const response = await page.request.get("/api/proxy/api/library/agents", {
+ params: { search_term: name },
+ });
+ expect(response.ok(), `GET library agents: ${response.status()}`).toBe(true);
+ const body = (await response.json()) as { agents: { name: string }[] };
+ return body.agents.filter((agent) => agent.name === name);
+}
+
+function parseJSON(text: string) {
+ try {
+ return JSON.parse(text) as Record<string, unknown>;
+ } catch {
+ return null;
+ }
+}
diff --git a/autogpt_platform/frontend/src/playwright/journeys/copilot-chat.ts b/autogpt_platform/frontend/src/playwright/journeys/copilot-chat.ts
new file mode 100644
index 0000000000..145975f30f
--- /dev/null
+++ b/autogpt_platform/frontend/src/playwright/journeys/copilot-chat.ts
@@ -0,0 +1,176 @@
+import {
+ expect,
+ type Page,
+ type Request as PlaywrightRequest,
+} from "@playwright/test";
+
+/** Opens AutoPilot and sends the first message of a new chat. */
+export async function startChat(page: Page, message: string) {
+ await page.goto("/home");
+ const input = page.getByRole("textbox", { name: "Chat message input" });
+ await expect(input).toBeVisible();
+ await input.fill(message);
+ await input.press("Enter");
+}
+
+interface RecordedStream {
+ method: string;
+ text: string;
+ done: boolean;
+}
+
+/**
+ * Records the body of every chat stream the page reads, as the browser
+ * received it, and lets a journey cut the open ones the way a dropped
+ * connection would (cutChatStreams). Chromium neither keeps a streamed
+ * response's body for Playwright nor fails an open stream when the context
+ * goes offline, so the page's fetch hands the app a stream the journey
+ * controls. Call before the page loads.
+ */
+export async function recordChatStreams(page: Page) {
+ await page.context().addInitScript(() => {
+ const streams: RecordedStream[] = [];
+ const openStreams = new Set<() => void>();
+ Object.defineProperty(window, "__journeyChatStreams", { value: streams });
+ Object.defineProperty(window, "__journeyCutChatStreams", {
+ value: () => {
+ const cuts = [...openStreams];
+ cuts.forEach((cut) => cut());
+ return cuts.length;
+ },
+ });
+ const pageFetch = window.fetch.bind(window);
+ window.fetch = async (input, init) => {
+ const response = await pageFetch(input, init);
+ const url = input instanceof Request ? input.url : String(input);
+ const method = (
+ init?.method ?? (input instanceof Request ? input.method : "GET")
+ ).toUpperCase();
+ const { pathname } = new URL(url, window.location.href);
+ if (!/\/api\/chat\/sessions\/[^/]+\/stream$/.test(pathname)) {
+ return response;
+ }
+ if (!response.body) return response;
+ const record: RecordedStream = { method, text: "", done: false };
+ streams.push(record);
+ const upstream = response.body.getReader();
+ const decoder = new TextDecoder();
+ let cutOff = false;
+ let pageSide: ReadableStreamDefaultController<Uint8Array> | null = null;
+ function end() {
+ record.done = true;
+ openStreams.delete(drop);
+ }
+ function drop() {
+ cutOff = true;
+ end();
+ void upstream.cancel();
+ // What the app's reader sees when the connection drops.
+ pageSide?.error(new TypeError("network error"));
+ }
+ const body = new ReadableStream<Uint8Array>({
+ start(controller) {
+ pageSide = controller;
+ openStreams.add(drop);
+ },
+ async pull(controller) {
+ try {
+ const { done, value } = await upstream.read();
+ if (cutOff) return;
+ if (done) {
+ end();
+ controller.close();
+ return;
+ }
+ record.text += decoder.decode(value, { stream: true });
+ controller.enqueue(value);
+ } catch (error) {
+ if (cutOff) return;
+ end();
+ controller.error(error);
+ }
+ },
+ cancel(reason) {
+ end();
+ return upstream.cancel(reason);
+ },
+ });
+ return new Response(body, {
+ status: response.status,
+ statusText: response.statusText,
+ headers: response.headers,
+ });
+ };
+ });
+}
+
+/** Fails every open chat stream as a dropped connection would. */
+export async function cutChatStreams(page: Page) {
+ const cut = await page.evaluate(() =>
+ (
+ window as unknown as { __journeyCutChatStreams: () => number }
+ ).__journeyCutChatStreams(),
+ );
+ expect(cut, "a chat stream should be open to cut").toBeGreaterThan(0);
+}
+
+/** The body of the first chat stream sent with `method`, once it has ended. */
+export async function finishedChatStream(page: Page, method: "GET" | "POST") {
+ const handle = await page.waitForFunction(
+ (wanted) => {
+ const streams = (
+ window as unknown as { __journeyChatStreams?: RecordedStream[] }
+ ).__journeyChatStreams;
+ const stream = streams?.find((s) => s.method === wanted && s.done);
+ return stream ? stream.text : null;
+ },
+ method,
+ { timeout: 60_000 },
+ );
+ return (await handle.jsonValue()) as string;
+}
+
+/** The start of `text`, to see an answer begin streaming. */
+export function firstWords(text: string, count = 5) {
+ return text.split(" ").slice(0, count).join(" ");
+}
+
+/** The request that sends a message and streams the turn back (POST), or
+ * reconnects to a running turn (GET). It goes to the backend directly. */
+export function isChatStream(method: "GET" | "POST") {
+ return (request: PlaywrightRequest) =>
+ request.method() === method &&
+ /\/api\/chat\/sessions\/[^/]+\/stream$/.test(
+ new URL(request.url()).pathname,
+ );
+}
+
+export async function currentSessionID(page: Page) {
+ await expect
+ .poll(() => new URL(page.url()).searchParams.get("sessionId"), {
+ message: "the chat should get a session",
+ })
+ .toBeTruthy();
+ return new URL(page.url()).searchParams.get("sessionId") as string;
+}
+
+export interface SavedMessage {
+ role: string;
+ content: string | null;
+}
+
+export async function savedMessages(page: Page, sessionID: string) {
+ const response = await page.request.get(
+ `/api/proxy/api/chat/sessions/${sessionID}`,
+ );
+ expect(response.ok(), `GET chat session: ${response.status()}`).toBe(true);
+ const body = (await response.json()) as { messages: SavedMessage[] };
+ return body.messages;
+}
+
+/** The assistant's text, one entry per saved message that has any. */
+export async function savedAssistantTexts(page: Page, sessionID: string) {
+ return (await savedMessages(page, sessionID))
+ .filter((message) => message.role === "assistant" && message.content)
+ .map((message) => message.content);
+}
diff --git a/autogpt_platform/frontend/src/playwright/journeys/copilot-follow-up-journey.spec.ts b/autogpt_platform/frontend/src/playwright/journeys/copilot-follow-up-journey.spec.ts
new file mode 100644
index 0000000000..1157333239
--- /dev/null
+++ b/autogpt_platform/frontend/src/playwright/journeys/copilot-follow-up-journey.spec.ts
@@ -0,0 +1,81 @@
+import type { Page } from "@playwright/test";
+import {
+ currentSessionID,
+ firstWords,
+ savedMessages,
+ startChat,
+} from "./copilot-chat";
+import { expect, test } from "./fixtures";
+import followUpScript from "./model-scripts/answer-then-follow-up.json";
+import {
+ expectChargedWhatWasBilled,
+ expectEachTurnOnce,
+ type ModelScript,
+ readJournal,
+ requireScriptedModel,
+ scriptedRun,
+} from "./scripted-model";
+
+// A slow first answer, then the answer to the follow-up the script expects
+// the model to be shown.
+const SCRIPT: ModelScript = followUpScript;
+const [ANSWER, FOLLOW_UP_ANSWER] = SCRIPT.turns.map((turn) => turn.text ?? "");
+const FOLLOW_UP = SCRIPT.turns[1].expect_user_text ?? "";
+
+test.describe("copilot follow-up journey", () => {
+ test.beforeAll(requireScriptedModel);
+
+ test("a follow-up queued while AutoPilot answers runs once, after the answer", async ({
+ page,
+ journeyUser,
+ }) => {
+ const run = scriptedRun("answer-then-follow-up");
+ const firstMessage = `Tell me about today. ${run.marker}`;
+ await startChat(page, firstMessage);
+ const transcript = page.getByRole("log");
+ await expect(
+ transcript.getByText(firstWords(ANSWER)),
+ "the answer should start streaming",
+ ).toBeVisible();
+
+ await page
+ .getByRole("textbox", { name: "Chat message input" })
+ .fill(FOLLOW_UP);
+ await page.getByRole("button", { name: "Queue message" }).click();
+
+ await expect(
+ transcript.getByText(FOLLOW_UP_ANSWER, { exact: true }),
+ "the follow-up should be answered, once",
+ ).toHaveCount(1);
+ const sessionID = await currentSessionID(page);
+ await expect
+ .poll(() => savedConversation(page, sessionID), {
+ message:
+ "the chat should save the follow-up once, after the first answer, with its own answer",
+ })
+ .toEqual([
+ ["user", firstMessage],
+ ["assistant", ANSWER],
+ ["user", FOLLOW_UP],
+ ["assistant", FOLLOW_UP_ANSWER],
+ ]);
+ await page.reload();
+ for (const text of [ANSWER, FOLLOW_UP, FOLLOW_UP_ANSWER]) {
+ await expect(
+ page.getByRole("log").getByText(text, { exact: true }),
+ "the reloaded chat should show each message once",
+ ).toHaveCount(1);
+ }
+
+ const journal = await readJournal(run.nonce);
+ expectEachTurnOnce(journal, SCRIPT);
+ if (!journeyUser.id) throw new Error("the journey user should have an id");
+ await expectChargedWhatWasBilled(journeyUser.id, journal);
+ });
+});
+
+async function savedConversation(page: Page, sessionID: string) {
+ return (await savedMessages(page, sessionID))
+ .filter((message) => ["user", "assistant"].includes(message.role))
+ .map((message) => [message.role, message.content]);
+}
diff --git a/autogpt_platform/frontend/src/playwright/journeys/copilot-reconnect-journey.spec.ts b/autogpt_platform/frontend/src/playwright/journeys/copilot-reconnect-journey.spec.ts
new file mode 100644
index 0000000000..8162c1fb39
--- /dev/null
+++ b/autogpt_platform/frontend/src/playwright/journeys/copilot-reconnect-journey.spec.ts
@@ -0,0 +1,116 @@
+import type { BrowserContext, Page } from "@playwright/test";
+import {
+ currentSessionID,
+ cutChatStreams,
+ firstWords,
+ isChatStream,
+ recordChatStreams,
+ savedAssistantTexts,
+ startChat,
+} from "./copilot-chat";
+import { expect, test } from "./fixtures";
+import type { JourneyUser } from "./journey-user";
+import slowAnswerScript from "./model-scripts/slow-answer.json";
+import {
+ expectChargedWhatWasBilled,
+ expectEachTurnOnce,
+ type ModelScript,
+ readJournal,
+ requireScriptedModel,
+ scriptedRun,
+} from "./scripted-model";
+
+// One answer, streamed a word at a time over a few seconds.
+const SCRIPT: ModelScript = slowAnswerScript;
+const ANSWER = SCRIPT.turns[0].text ?? "";
+
+test.describe("copilot reconnect journey", () => {
+ test.beforeAll(requireScriptedModel);
+
+ test("an answer cut off by a dropped connection arrives whole, once and as saved", async ({
+ page,
+ context,
+ journeyUser,
+ }) => {
+ await loseConnectionMidAnswer({
+ page,
+ context,
+ journeyUser,
+ offline: false,
+ });
+ });
+
+ test("an answer cut off while the browser is offline arrives whole, once and as saved", async ({
+ page,
+ context,
+ journeyUser,
+ }) => {
+ // While offline, the pending follow-ups peek in useCopilotPendingChips.ts
+ // (`void getV2GetPendingMessages(sessionId).then(...)`, no catch) rejects
+ // with "Failed to fetch" and nothing handles it, so the page reports an
+ // uncaught error. Everything the user sees is right. Remove this marker
+ // when that request handles a network failure.
+ test.fail(true, "unhandled 'Failed to fetch' from useCopilotPendingChips");
+ await loseConnectionMidAnswer({
+ page,
+ context,
+ journeyUser,
+ offline: true,
+ });
+ });
+});
+
+interface Args {
+ page: Page;
+ context: BrowserContext;
+ journeyUser: JourneyUser;
+ offline: boolean;
+}
+
+async function loseConnectionMidAnswer({
+ page,
+ context,
+ journeyUser,
+ offline,
+}: Args) {
+ const run = scriptedRun("slow-answer");
+ await recordChatStreams(page);
+ await startChat(page, `Answer me slowly. ${run.marker}`);
+ const transcript = page.getByRole("log");
+ await expect(
+ transcript.getByText(firstWords(ANSWER)),
+ "the answer should start streaming",
+ ).toBeVisible();
+
+ // The connection drops mid-answer while the model is still writing it.
+ const reconnected = page.waitForRequest(isChatStream("GET"));
+ if (offline) await context.setOffline(true);
+ await cutChatStreams(page);
+ await expect(
+ page.getByText(/Reconnecting/).first(),
+ "the chat should say it lost the connection",
+ ).toBeVisible();
+ if (offline) await context.setOffline(false);
+ await reconnected;
+
+ await expect(
+ transcript.getByText(ANSWER, { exact: true }),
+ "the whole answer should show, once, without a reload",
+ ).toHaveCount(1);
+ const sessionID = await currentSessionID(page);
+ await expect
+ .poll(() => savedAssistantTexts(page, sessionID), {
+ message: "the chat should save the answer once",
+ })
+ .toEqual([ANSWER]);
+ await page.reload();
+ await expect(
+ page.getByRole("log").getByText(ANSWER, { exact: true }),
+ "the reloaded chat should show the answer once",
+ ).toHaveCount(1);
+
+ const journal = await readJournal(run.nonce);
+ expectEachTurnOnce(journal, SCRIPT);
+ if (!journeyUser.id) throw new Error("the journey user should have an id");
+ await expectChargedWhatWasBilled(journeyUser.id, journal);
+}
diff --git a/autogpt_platform/frontend/src/playwright/journeys/journey-database.ts b/autogpt_platform/frontend/src/playwright/journeys/journey-database.ts
new file mode 100644
index 0000000000..cf51c7457a
--- /dev/null
+++ b/autogpt_platform/frontend/src/playwright/journeys/journey-database.ts
@@ -0,0 +1,28 @@
+import { Client } from "pg";
+
+/**
+ * Runs `query` against the stack's database, for what a journey can't do or
+ * read through the app: granting a plan, reading the cost ledger.
+ *
+ * Uses PLAYWRIGHT_JOURNEY_DATABASE_URL, else DATABASE_URL from frontend/.env.
+ * Unset on a preview, which journeys can't reach into.
+ */
+export async function withJourneyDatabase<T>(
+ purpose: string,
+ query: (client: Client) => Promise<T>,
+) {
+ const connectionString =
+ process.env.PLAYWRIGHT_JOURNEY_DATABASE_URL ?? process.env.DATABASE_URL;
+ if (!connectionString) {
+ throw new Error(
+ `${purpose} needs PLAYWRIGHT_JOURNEY_DATABASE_URL or DATABASE_URL`,
+ );
+ }
+ const client = new Client({ connectionString });
+ await client.connect();
+ try {
+ return await query(client);
+ } finally {
+ await client.end();
+ }
+}
diff --git a/autogpt_platform/frontend/src/playwright/journeys/model-scripts/answer-then-follow-up.json b/autogpt_platform/frontend/src/playwright/journeys/model-scripts/answer-then-follow-up.json
new file mode 100644
index 0000000000..e591603e2e
--- /dev/null
+++ b/autogpt_platform/frontend/src/playwright/journeys/model-scripts/answer-then-follow-up.json
@@ -0,0 +1,24 @@
+{
+ "description": "A first answer slow enough to queue a follow-up while it streams, then the answer to that follow-up. copilot-follow-up-journey.spec.ts follows it.",
+ "title": "Follow-up",
+ "turns": [
+ {
+ "text": "Here is the first answer, which takes a few seconds to arrive so that there is time to queue a follow-up question while it streams.",
+ "usage": {
+ "input_tokens": 2600,
+ "output_tokens": 30,
+ "cache_read_input_tokens": 24000
+ },
+ "stream": { "chunk_words": 1, "delay_ms": 200 }
+ },
+ {
+ "expect_user_text": "What about tomorrow?",
+ "text": "And here is the answer to your follow-up about tomorrow.",
+ "usage": {
+ "input_tokens": 2700,
+ "output_tokens": 14,
+ "cache_read_input_tokens": 24300
+ }
+ }
+ ]
+}
diff --git a/autogpt_platform/frontend/src/playwright/journeys/model-scripts/build-doubler-agent.json b/autogpt_platform/frontend/src/playwright/journeys/model-scripts/build-doubler-agent.json
new file mode 100644
index 0000000000..ca0ff4d646
--- /dev/null
+++ b/autogpt_platform/frontend/src/playwright/journeys/model-scripts/build-doubler-agent.json
@@ -0,0 +1,104 @@
+{
+ "description": "AutoPilot reads the agent-building guide, then builds and saves an agent that doubles a number. copilot-build-agent-journey.spec.ts follows it.",
+ "title": "Doubler agent",
+ "turns": [
+ {
+ "text": "I'll read the agent-building guide first, so the agent I build is valid.",
+ "tool_calls": [
+ {
+ "name": "run_capability",
+ "input": { "id": "skill:agent_building_guide", "input": {} }
+ }
+ ],
+ "usage": {
+ "input_tokens": 3200,
+ "output_tokens": 45,
+ "cache_read_input_tokens": 24000,
+ "cache_creation_input_tokens": 1800
+ }
+ },
+ {
+ "expect_tool_results": [
+ { "tool": "run_capability", "contains": "Agent Generation Guide" }
+ ],
+ "text": "Now I'll build an agent that takes a number and returns double that number.",
+ "tool_calls": [
+ {
+ "name": "run_capability",
+ "input": {
+ "id": "tool:create_agent",
+ "input": {
+ "library_check_ack": true,
+ "agent_json": {
+ "name": "Journey Doubler",
+ "description": "Takes a number and returns double that number.",
+ "nodes": [
+ {
+ "id": "5d1a9f3e-6b1c-4c8e-9a51-2f0e7c3b1a01",
+ "block_id": "c0a8e994-ebf1-4a9c-a4d8-89d09c86741b",
+ "input_default": {
+ "name": "number",
+ "title": "Number",
+ "value": null
+ },
+ "metadata": { "position": { "x": 0, "y": 0 } }
+ },
+ {
+ "id": "5d1a9f3e-6b1c-4c8e-9a51-2f0e7c3b1a02",
+ "block_id": "b1ab9b19-67a6-406d-abf5-2dba76d00c79",
+ "input_default": {
+ "operation": "Multiply",
+ "b": 2,
+ "round_result": false
+ },
+ "metadata": { "position": { "x": 500, "y": 0 } }
+ },
+ {
+ "id": "5d1a9f3e-6b1c-4c8e-9a51-2f0e7c3b1a03",
+ "block_id": "363ae599-353e-4804-937e-b2ee3cef3da4",
+ "input_default": { "name": "doubled", "title": "Doubled" },
+ "metadata": { "position": { "x": 1000, "y": 0 } }
+ }
+ ],
+ "links": [
+ {
+ "id": "5d1a9f3e-6b1c-4c8e-9a51-2f0e7c3b1a04",
+ "source_id": "5d1a9f3e-6b1c-4c8e-9a51-2f0e7c3b1a01",
+ "source_name": "result",
+ "sink_id": "5d1a9f3e-6b1c-4c8e-9a51-2f0e7c3b1a02",
+ "sink_name": "a",
+ "is_static": false
+ },
+ {
+ "id": "5d1a9f3e-6b1c-4c8e-9a51-2f0e7c3b1a05",
+ "source_id": "5d1a9f3e-6b1c-4c8e-9a51-2f0e7c3b1a02",
+ "source_name": "result",
+ "sink_id": "5d1a9f3e-6b1c-4c8e-9a51-2f0e7c3b1a03",
+ "sink_name": "value",
+ "is_static": false
+ }
+ ]
+ }
+ }
+ }
+ }
+ ],
+ "usage": {
+ "input_tokens": 9000,
+ "output_tokens": 380,
+ "cache_read_input_tokens": 25800
+ }
+ },
+ {
+ "expect_tool_results": [
+ { "tool": "run_capability", "contains": "agent_builder_saved" }
+ ],
+ "text": "Done. Journey Doubler is saved in your Library: give it a number and it returns double that number.",
+ "usage": {
+ "input_tokens": 400,
+ "output_tokens": 40,
+ "cache_read_input_tokens": 34800
+ }
+ }
+ ]
+}
diff --git a/autogpt_platform/frontend/src/playwright/journeys/model-scripts/slow-answer.json b/autogpt_platform/frontend/src/playwright/journeys/model-scripts/slow-answer.json
new file mode 100644
index 0000000000..fd6c678137
--- /dev/null
+++ b/autogpt_platform/frontend/src/playwright/journeys/model-scripts/slow-answer.json
@@ -0,0 +1,15 @@
+{
+ "description": "One answer streamed a word at a time over about five seconds, so a journey can lose the connection mid-reply. copilot-reconnect-journey.spec.ts follows it.",
+ "title": "Slow answer",
+ "turns": [
+ {
+ "text": "Here is a long answer that arrives slowly, one word at a time, so that you can drop the connection halfway through and check that every word still reaches the chat exactly once after the connection comes back.",
+ "usage": {
+ "input_tokens": 2600,
+ "output_tokens": 48,
+ "cache_read_input_tokens": 24000
+ },
+ "stream": { "chunk_words": 1, "delay_ms": 120 }
+ }
+ ]
+}
diff --git a/autogpt_platform/frontend/src/playwright/journeys/scripted-model.ts b/autogpt_platform/frontend/src/playwright/journeys/scripted-model.ts
new file mode 100644
index 0000000000..27fc59e5b1
--- /dev/null
+++ b/autogpt_platform/frontend/src/playwright/journeys/scripted-model.ts
@@ -0,0 +1,180 @@
+import { randomBytes } from "crypto";
+import { expect, test } from "@playwright/test";
+import { withJourneyDatabase } from "./journey-database";
+
+/**
+ * Journeys that chat with AutoPilot run against the scripted model
+ * (autogpt_platform/scripted_model): the real copilot pipeline, with model
+ * turns played from model-scripts/<name>.json. The stack must be started with
+ * docker-compose.journeys.yml, which CI's journeys leg does.
+ */
+const SCRIPTED_MODEL_URL =
+ process.env.PLAYWRIGHT_SCRIPTED_MODEL_URL ?? "http://localhost:4010";
+
+export interface ModelScript {
+ title?: string;
+ turns: {
+ text?: string;
+ tool_calls?: { name: string; input: Record<string, unknown> }[];
+ expect_user_text?: string;
+ }[];
+}
+
+export interface JournalEntry {
+ kind: "turn" | "side_call" | "error" | "unexpected";
+ model?: string;
+ turn?: number | null;
+ error?: string;
+ cost_usd?: number | null;
+ checks?: { what: string; ok: boolean; detail: string }[];
+ completed?: boolean | null;
+}
+
+/** A chat message that makes the model follow `script`, and its run's id. */
+export function scriptedRun(script: string) {
+ const nonce = randomBytes(6).toString("hex");
+ return { nonce, marker: `[[script:${script}#${nonce}]]` };
+}
+
+/**
+ * Skips the journey against a deployed app, which talks to real models.
+ * On a local stack, fails unless the scripted model is up.
+ */
+export async function requireScriptedModel() {
+ const { hostname } = new URL(
+ process.env.PLAYWRIGHT_BASE_URL ?? "http://localhost:3000",
+ );
+ test.skip(
+ !["localhost", "127.0.0.1", "[::1]"].includes(hostname),
+ "scripted model journeys need a local stack with docker-compose.journeys.yml",
+ );
+ const healthy = await fetch(`${SCRIPTED_MODEL_URL}/healthz`)
+ .then((response) => response.ok)
+ .catch(() => false);
+ if (!healthy) {
+ throw new Error(
+ `the scripted model isn't answering at ${SCRIPTED_MODEL_URL}. Start ` +
+ "the stack with docker-compose.journeys.yml (see scripted_model/README.md).",
+ );
+ }
+}
+
+/** Every model request a run made, oldest first. */
+export async function readJournal(nonce: string) {
+ const response = await fetch(
+ `${SCRIPTED_MODEL_URL}/__scripted__/journal?nonce=${nonce}`,
+ );
+ expect(response.ok, `GET journal: ${response.status}`).toBe(true);
+ const body = (await response.json()) as { entries: JournalEntry[] };
+ return body.entries;
+}
+
+/**
+ * Checks the platform asked for every scripted turn exactly once and each
+ * reply reached it whole: no refusal, no retried or repeated turn, and every
+ * tool result the script expected.
+ */
+export function expectEachTurnOnce(
+ journal: JournalEntry[],
+ script: ModelScript,
+) {
+ const turns = journal.filter((entry) => entry.kind === "turn");
+ const problems = journal.filter(
+ (entry) => entry.kind === "error" || entry.kind === "unexpected",
+ );
+ expect(problems, "the model should answer every request").toEqual([]);
+ expect(
+ turns.map((entry) => entry.turn),
+ "the platform should ask for each scripted turn exactly once",
+ ).toEqual(script.turns.map((_, index) => index + 1));
+ expect(
+ turns.flatMap((entry) => entry.checks ?? []).filter((check) => !check.ok),
+ "every tool result and user message should be what the script expected",
+ ).toEqual([]);
+ expect(
+ turns.filter((entry) => entry.completed !== true),
+ "every reply should reach the platform in full",
+ ).toEqual([]);
+}
+
+/**
+ * Checks the copilot's cost ledger charged the user exactly what the model
+ * billed for its turns: every turn once, none missed, none twice. Each ledger
+ * row may be off by rounding, as the backend's billing audit allows.
+ *
+ * Side calls (the chat's title) are left out. The platform shares one OpenAI
+ * client across the copilot executor's worker event loops
+ * (`_get_aux_client` in backend/copilot/service.py), so a title request that
+ * lands on another loop's pooled connection is sent twice: the provider
+ * answers and bills both, the platform records one. Include them once that's
+ * fixed.
+ */
+export async function expectChargedWhatWasBilled(
+ userID: string,
+ journal: JournalEntry[],
+) {
+ const billed = journal
+ .filter((entry) => entry.kind === "turn")
+ .reduce(
+ (sum, entry) => sum + Math.round((entry.cost_usd ?? 0) * 1_000_000),
+ 0,
+ );
+ expect(billed, "the model should bill the run").toBeGreaterThan(0);
+ await expect(async () => {
+ const rows = (await copilotCostRows(userID)).filter(
+ (row) => row.blockName !== "copilot:title",
+ );
+ const charged = rows.reduce((sum, row) => sum + row.costMicrodollars, 0);
+ expect(
+ Math.abs(charged - billed),
+ `charged ${charged} microdollars for turns billed ${billed}: ` +
+ JSON.stringify(rows),
+ ).toBeLessThanOrEqual(2 * Math.max(rows.length, 1));
+ }, "the chat should be charged exactly what the model billed").toPass({
+ timeout: 20_000,
+ });
+}
+
+async function copilotCostRows(userID: string) {
+ return withJourneyDatabase("reading the copilot cost ledger", async (db) => {
+ const result = await db.query<{
+ blockName: string;
+ costMicrodollars: string | null;
+ chatSessionId: string | null;
+ }>(
+ `SELECT "blockName", "costMicrodollars"::text AS "costMicrodollars", "chatSessionId"
+ FROM platform."PlatformCostLog"
+ WHERE "userId" = $1 AND "metadata"->>'source' = 'copilot'
+ ORDER BY "createdAt"`,
+ [userID],
+ );
+ return result.rows.map((row) => ({
+ ...row,
+ costMicrodollars: Number(row.costMicrodollars ?? 0),
+ }));
+ });
+}
+
+/**
+ * The text a chat stream carried, one entry per text block, as the browser
+ * received it.
+ */
+export function streamedTexts(body: string) {
+ const texts = new Map<string, string>();
+ for (const line of body.split("\n")) {
+ if (!line.startsWith("data:")) continue;
+ const event = parseEvent(line.slice("data:".length).trim());
+ if (event?.type === "text-delta" && event.id) {
+ texts.set(event.id, (texts.get(event.id) ?? "") + (event.delta ?? ""));
+ }
+ }
+ return [...texts.values()];
+}
+
+function parseEvent(data: string) {
+ try {
+ return JSON.parse(data) as { type?: string; id?: string; delta?: string };
+ } catch {
+ return null;
+ }
+}
diff --git a/autogpt_platform/frontend/src/playwright/journeys/subscription-tier.ts b/autogpt_platform/frontend/src/playwright/journeys/subscription-tier.ts
index 8f729788bd..d1a3234dcc 100644
--- a/autogpt_platform/frontend/src/playwright/journeys/subscription-tier.ts
+++ b/autogpt_platform/frontend/src/playwright/journeys/subscription-tier.ts
@@ -1,4 +1,4 @@
-import { Client } from "pg";
+import { withJourneyDatabase } from "./journey-database";
/**
* Gives a journey user a paid plan without Stripe, so the hosted paywall lets
@@ -6,23 +6,13 @@ import { Client } from "pg";
* models an admin grant, which the Stripe reconciliation sweep never revokes.
*
* Opt in with PLAYWRIGHT_JOURNEY_TIER (e.g. PRO). It needs the stack's
- * database: PLAYWRIGHT_JOURNEY_DATABASE_URL, else DATABASE_URL from
- * frontend/.env. Unset on a preview, where an existing paid account
- * (PLAYWRIGHT_JOURNEY_EMAIL) is the way past the paywall.
+ * database (see withJourneyDatabase). Unset on a preview, where an existing
+ * paid account (PLAYWRIGHT_JOURNEY_EMAIL) is the way past the paywall.
*/
export async function grantSubscriptionTier(userID: string) {
const tier = process.env.PLAYWRIGHT_JOURNEY_TIER;
if (!tier) return;
- const connectionString =
- process.env.PLAYWRIGHT_JOURNEY_DATABASE_URL ?? process.env.DATABASE_URL;
- if (!connectionString) {
- throw new Error(
- "PLAYWRIGHT_JOURNEY_TIER needs PLAYWRIGHT_JOURNEY_DATABASE_URL or DATABASE_URL",
- );
- }
- const client = new Client({ connectionString });
- await client.connect();
- try {
+ await withJourneyDatabase("PLAYWRIGHT_JOURNEY_TIER", async (client) => {
const result = await client.query(
`UPDATE platform."User" SET "subscriptionTier" = $1::platform."SubscriptionTier" WHERE id = $2`,
[tier, userID],
@@ -30,7 +20,5 @@ export async function grantSubscriptionTier(userID: string) {
if (result.rowCount !== 1) {
throw new Error(`no platform user ${userID} to grant ${tier} to`);
}
- } finally {
- await client.end();
- }
+ });
}
diff --git a/autogpt_platform/scripted_model/.dockerignore b/autogpt_platform/scripted_model/.dockerignore
new file mode 100644
index 0000000000..f84a9af397
--- /dev/null
+++ b/autogpt_platform/scripted_model/.dockerignore
@@ -0,0 +1,3 @@
+__pycache__
+*.pyc
+tests
diff --git a/autogpt_platform/scripted_model/Dockerfile b/autogpt_platform/scripted_model/Dockerfile
new file mode 100644
index 0000000000..4f1ff493fd
--- /dev/null
+++ b/autogpt_platform/scripted_model/Dockerfile
@@ -0,0 +1,15 @@
+# Build context: autogpt_platform/scripted_model
+# Standard library only, so the image builds in seconds with nothing to install.
+FROM python:3.13-slim
+
+RUN useradd --system --no-create-home scripted-model
+WORKDIR /app
+COPY scripted_model ./scripted_model
+
+USER scripted-model
+ENV PYTHONUNBUFFERED=1 \
+ SCRIPTED_MODEL_PORT=4010 \
+ SCRIPTED_MODEL_SCRIPTS_DIR=/scripts
+
+EXPOSE 4010
+CMD ["python", "-m", "scripted_model"]
diff --git a/autogpt_platform/scripted_model/README.md b/autogpt_platform/scripted_model/README.md
new file mode 100644
index 0000000000..f6cfe2affe
--- /dev/null
+++ b/autogpt_platform/scripted_model/README.md
@@ -0,0 +1,132 @@
+# Scripted model
+
+A fake model provider for tests. A Playwright journey drives the real copilot
+pipeline (engine, tool routing and execution, persistence, streaming and
+resume, cost accounting) while this server plays the model, turn by turn, from
+a script. `CHAT_TEST_MODE`'s canned reply skips all of that.
+
+It only runs where `docker-compose.journeys.yml` is added: CI's journeys leg
+and local test stacks. Self-hosted and hosted deployments never start it.
+
+## How the platform reaches it
+
+`docker-compose.journeys.yml` puts the backend in OpenRouter mode with
+`CHAT_BASE_URL=http://scripted_model:4010/api/v1`, so both engines and every
+side call come here:
+
+| Caller | Request | Dialect |
+| --- | --- | --- |
+| SDK engine (the Claude Agent SDK's CLI, the default) | `POST /api/v1/messages` | Anthropic Messages, streamed |
+| Baseline engine (`FORCE_FLAG_COPILOT_SDK=false`) | `POST /api/v1/chat/completions` | OpenAI chat completions, streamed |
+| Side calls (session title) | `POST /api/v1/chat/completions` | non-streamed, no tools |
+
+The CLI also sends `HEAD /api/api/hello` to check it can reach the server,
+and may ask `POST /api/v1/messages/count_tokens`; both are answered.
+
+## Scripts
+
+A journey picks a script with a marker in its chat message:
+`[[script:<name>]]`, or `[[script:<name>#<nonce>]]` to find its own requests
+in the journal. Scripts are JSON files in
+`frontend/src/playwright/journeys/model-scripts/`, mounted at `/scripts` and
+read on every request (edits apply without a restart). A journey imports its
+script to know what to expect.
+
+```json
+{
+ "description": "What the journey checks.",
+ "title": "Answer to side calls, e.g. the chat's title",
+ "turns": [
+ {
+ "text": "Let me check your library first.",
+ "tool_calls": [{ "name": "find_library_agent", "input": { "query": "doubler" } }],
+ "usage": { "input_tokens": 3000, "output_tokens": 40,
+ "cache_read_input_tokens": 20000, "cache_creation_input_tokens": 0 },
+ "stream": { "chunk_words": 3, "delay_ms": 15 }
+ },
+ {
+ "expect_tool_results": [{ "tool": "find_library_agent", "contains": "No agents" }],
+ "text": "You don't have one yet. Want me to build it?",
+ "usage": { "input_tokens": 3400, "output_tokens": 20 }
+ },
+ {
+ "expect_user_text": "Yes, build it",
+ "text": "On it.",
+ "usage": { "input_tokens": 3500, "output_tokens": 5 }
+ }
+ ]
+}
+```
+
+- **Which turn answers a request:** turn `n` answers the request whose
+ conversation already holds `n` assistant messages. Requests carry the whole
+ conversation, so the server keeps no state: concurrent runs of one script,
+ a retried turn and a resumed session all land on the right turn. A
+ follow-up the user queued during a turn reaches the model after that turn's
+ answer, so it gets the next turn.
+- **Tools:** name them as the baseline engine does (`find_library_agent`).
+ The SDK engine offers them as `mcp__copilot__find_library_agent`; the server
+ calls whichever the request offers. Most platform tools (`create_agent`,
+ `edit_agent` and others) are deferred: on both engines the model reaches
+ them through `run_capability` with `{"id": "tool:<name>", "input": {...}}`,
+ and `create_agent` needs the building guide read first
+ (`{"id": "skill:agent_building_guide"}`). See
+ `model-scripts/build-doubler-agent.json`.
+- **Checks** are logged, and a journey asserts on them through the journal:
+ `expect_tool_results` against the tool results the platform sends back,
+ `expect_user_text` against what the user said since the last answer.
+- **Streaming:** text goes out `chunk_words` words at a time, `delay_ms`
+ apart. Slow it down to give a journey time to cut the connection.
+
+A request the script can't answer fails the turn with a 400 that says why:
+no marker, an unknown script, a turn past the script's end (an extra tool
+round or retry), or a tool the platform didn't offer. A journey fails loudly
+instead of waiting on a reply that never comes. A request that offers no tools
+is a side call and gets the script's `title`.
+
+## Cost
+
+The server keeps the provider's invoice, priced in `invoice.py` from
+Anthropic's list prices (the same as `backend/copilot/anthropic_rates.json`):
+
+- chat completions report it as `usage.cost`, as OpenRouter does, which is
+ what the baseline engine charges;
+- the SDK engine charges the CLI's own estimate, priced from the same list.
+ Message ids are `msg_…`, not OpenRouter's `gen-…`, so the platform doesn't
+ try to look the cost up on openrouter.ai.
+
+Each journal entry records `usage` and `cost_usd`, so a journey can check that
+the platform recorded exactly what the provider billed.
+
+## Journal
+
+`GET http://localhost:4010/__scripted__/journal?nonce=<nonce>` (or
+`?script=<name>`) lists every request: dialect, model, which turn answered it,
+the tool results it carried, expectation checks, usage, cost, and whether the
+whole reply reached the platform (`completed`). Refused requests are there
+with their reason. The container log has one line per request.
+
+## Run it
+
+With the stack, add the override file:
+
+```bash
+cd autogpt_platform
+docker compose -f docker-compose.yml -f docker-compose.journeys.yml up -d
+```
+
+On its own (standard library only, Python 3.10+):
+
+```bash
+cd autogpt_platform/scripted_model
+SCRIPTED_MODEL_SCRIPTS_DIR=../frontend/src/playwright/journeys/model-scripts \
+ SCRIPTED_MODEL_PORT=4010 python -m scripted_model
+```
+
+Tests, with the backend's environment (they use the real `anthropic` and
+`openai` clients):
+
+```bash
+cd autogpt_platform/backend
+poetry run pytest ../scripted_model/tests
+```
diff --git a/autogpt_platform/scripted_model/conftest.py b/autogpt_platform/scripted_model/conftest.py
new file mode 100644
index 0000000000..6f3eb846ef
--- /dev/null
+++ b/autogpt_platform/scripted_model/conftest.py
@@ -0,0 +1,34 @@
+import json
+import threading
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+
+from scripted_model.server import ScriptedModelServer
+
+
+class RunningServer:
+ def __init__(self, server: ScriptedModelServer, scripts_dir: Path) -> None:
+ self.server = server
+ self.scripts_dir = scripts_dir
+ host, port = server.server_address[:2]
+ self.url = f"http://{host}:{port}"
+
+ def add_script(self, name: str, script: dict) -> None:
+ (self.scripts_dir / f"{name}.json").write_text(json.dumps(script))
+
+ def journal(self, **query: str) -> list[dict]:
+ return self.server.journal.find(**query)
+
+
+def scripted_model(tmp_path: Path) -> Iterator[RunningServer]:
+ server = ScriptedModelServer(("127.0.0.1", 0), tmp_path)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ yield RunningServer(server, tmp_path)
+ finally:
+ server.shutdown()
+ server.server_close()
diff --git a/autogpt_platform/scripted_model/scripted_model/__init__.py b/autogpt_platform/scripted_model/scripted_model/__init__.py
new file mode 100644
index 0000000000..2c971c1639
--- /dev/null
+++ b/autogpt_platform/scripted_model/scripted_model/__init__.py
@@ -0,0 +1 @@
+"""A fake model server that answers the copilot from scripts."""
diff --git a/autogpt_platform/scripted_model/scripted_model/__main__.py b/autogpt_platform/scripted_model/scripted_model/__main__.py
new file mode 100644
index 0000000000..7a2e306960
--- /dev/null
+++ b/autogpt_platform/scripted_model/scripted_model/__main__.py
@@ -0,0 +1,24 @@
+"""``python -m scripted_model``: serve scripts from SCRIPTED_MODEL_SCRIPTS_DIR."""
+
+import logging
+import os
+from pathlib import Path
+
+from .server import ScriptedModelServer, logger
+
+
+def main() -> None:
+ logging.basicConfig(
+ level=os.environ.get("SCRIPTED_MODEL_LOG_LEVEL", "INFO"),
+ format="%(asctime)s scripted-model %(levelname)s %(message)s",
+ )
+ port = int(os.environ.get("SCRIPTED_MODEL_PORT", "4010"))
+ scripts_dir = Path(os.environ.get("SCRIPTED_MODEL_SCRIPTS_DIR", "/scripts"))
+ server = ScriptedModelServer(("0.0.0.0", port), scripts_dir)
+ known = sorted(p.stem for p in scripts_dir.glob("*.json"))
+ logger.info("serving on :%s with scripts from %s: %s", port, scripts_dir, known)
+ server.serve_forever()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/autogpt_platform/scripted_model/scripted_model/chat_completions.py b/autogpt_platform/scripted_model/scripted_model/chat_completions.py
new file mode 100644
index 0000000000..2a166ce0dd
--- /dev/null
+++ b/autogpt_platform/scripted_model/scripted_model/chat_completions.py
@@ -0,0 +1,162 @@
+"""OpenAI chat completions, the dialect of the baseline engine and side calls.
+
+Usage is reported the way OpenRouter reports it: ``prompt_tokens`` counts the
+cache buckets too, which ``prompt_tokens_details`` breaks out.
+"""
+
+import json
+import time
+import uuid
+from collections.abc import Iterator
+from typing import Any
+
+from .invoice import requested_cache_ttl
+from .planner import Conversation, Reply, ToolResult, bare_name
+from .streaming import Event, json_chunks, text_chunks
+
+
+def conversation(body: dict[str, Any]) -> Conversation:
+ messages = body.get("messages") or []
+ last_assistant = max(
+ (i for i, m in enumerate(messages) if m.get("role") == "assistant"), default=-1
+ )
+ answered = messages[last_assistant] if last_assistant >= 0 else {}
+ tool_names = {
+ call.get("id"): bare_name(str(call.get("function", {}).get("name")))
+ for call in answered.get("tool_calls") or []
+ }
+ return Conversation(
+ model=str(body.get("model", "")),
+ stream=bool(body.get("stream")),
+ user_text="\n".join(
+ _text(m.get("content")) for m in messages if m.get("role") == "user"
+ ),
+ assistant_messages=sum(1 for m in messages if m.get("role") == "assistant"),
+ offered_tools=tuple(
+ t["function"]["name"]
+ for t in body.get("tools") or []
+ if isinstance(t.get("function"), dict) and "name" in t["function"]
+ ),
+ tool_results=tuple(
+ ToolResult(
+ tool=tool_names.get(m.get("tool_call_id"), "<unknown>"),
+ content=_text(m.get("content")),
+ # Chat completions has no error flag on a tool message.
+ is_error=False,
+ )
+ for m in messages[last_assistant + 1 :]
+ if m.get("role") == "tool"
+ ),
+ cache_ttl=requested_cache_ttl(body),
+ latest_user_text="\n".join(
+ _text(m.get("content"))
+ for m in messages[last_assistant + 1 :]
+ if m.get("role") == "user"
+ ),
+ )
+
+
+def events(reply: Reply, model: str) -> Iterator[Event]:
+ """The reply as chat-completion chunks, ending in a usage-only chunk."""
+ base = {
+ "id": _id("chatcmpl-"),
+ "object": "chat.completion.chunk",
+ "created": int(time.time()),
+ "model": model,
+ }
+
+ def chunk(
+ delta: dict[str, Any], finish: str | None = None, delay_ms: int = 0
+ ) -> Event:
+ choice = {"index": 0, "delta": delta, "finish_reason": finish}
+ return Event(None, {**base, "choices": [choice]}, delay_ms)
+
+ yield chunk({"role": "assistant", "content": ""})
+ for piece in text_chunks(reply.text, reply.chunk_words):
+ yield chunk({"content": piece}, delay_ms=reply.delay_ms)
+ for index, (name, arguments) in enumerate(reply.tool_calls):
+ call = {"index": index, "id": _id("call_"), "type": "function"}
+ yield chunk(
+ {"tool_calls": [{**call, "function": {"name": name, "arguments": ""}}]}
+ )
+ for piece in json_chunks(arguments):
+ delta = {"tool_calls": [{"index": index, "function": {"arguments": piece}}]}
+ yield chunk(delta, delay_ms=reply.delay_ms)
+ yield chunk({}, finish=_finish_reason(reply))
+ yield Event(None, {**base, "choices": [], "usage": _usage(reply)}, delay_ms=0)
+ yield Event(None, "[DONE]", delay_ms=0)
+
+
+def completion(reply: Reply, model: str) -> dict[str, Any]:
+ """The reply as one non-streamed chat completion."""
+ message: dict[str, Any] = {"role": "assistant", "content": reply.text or None}
+ if reply.tool_calls:
+ message["tool_calls"] = [
+ {
+ "id": _id("call_"),
+ "type": "function",
+ "function": {"name": name, "arguments": json.dumps(arguments)},
+ }
+ for name, arguments in reply.tool_calls
+ ]
+ return {
+ "id": _id("chatcmpl-"),
+ "object": "chat.completion",
+ "created": int(time.time()),
+ "model": model,
+ "choices": [
+ {"index": 0, "message": message, "finish_reason": _finish_reason(reply)}
+ ],
+ "usage": _usage(reply),
+ }
+
+
+def error(message_text: str) -> dict[str, Any]:
+ return {
+ "error": {
+ "message": f"scripted-model: {message_text}",
+ "type": "invalid_request_error",
+ "code": "scripted_model",
+ }
+ }
+
+
+def _usage(reply: Reply) -> dict[str, Any]:
+ usage = reply.usage
+ prompt = (
+ usage.input_tokens
+ + usage.cache_read_input_tokens
+ + usage.cache_creation_input_tokens
+ )
+ body: dict[str, Any] = {
+ "prompt_tokens": prompt,
+ "completion_tokens": usage.output_tokens,
+ "total_tokens": prompt + usage.output_tokens,
+ "prompt_tokens_details": {
+ "cached_tokens": usage.cache_read_input_tokens,
+ "cache_write_tokens": usage.cache_creation_input_tokens,
+ },
+ }
+ if reply.cost_usd is not None:
+ # OpenRouter's extension: what the call was billed, in US dollars.
+ body["cost"] = reply.cost_usd
+ return body
+
+
+def _finish_reason(reply: Reply) -> str:
+ return "tool_calls" if reply.tool_calls else "stop"
+
+
+def _text(content: Any) -> str:
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ return "\n".join(
+ str(part.get("text", "")) if isinstance(part, dict) else str(part)
+ for part in content
+ )
+ return ""
+
+
+def _id(prefix: str) -> str:
+ return prefix + uuid.uuid4().hex[:24]
diff --git a/autogpt_platform/scripted_model/scripted_model/invoice.py b/autogpt_platform/scripted_model/scripted_model/invoice.py
new file mode 100644
index 0000000000..b6300eb108
--- /dev/null
+++ b/autogpt_platform/scripted_model/scripted_model/invoice.py
@@ -0,0 +1,67 @@
+"""What each model call costs: the fake provider's own invoice.
+
+The baseline engine is charged the ``usage.cost`` an OpenRouter-style
+response carries, so the fake reports one. The SDK engine is charged the
+CLI's estimate from its own copy of Anthropic's list prices, which these
+match (backend/copilot/anthropic_rates.json). A journey checks what the
+platform recorded against the sum of these, read from the journal.
+"""
+
+from dataclasses import dataclass
+from typing import Any
+
+from .scripts import Usage
+
+
+@dataclass(frozen=True)
+class Price:
+ """US dollars per million tokens."""
+
+ input: float
+ output: float
+ cache_read: float
+ cache_write_5m: float
+ cache_write_1h: float
+
+
+# Keyed by model name without the vendor prefix (``anthropic/``).
+PRICES: dict[str, Price] = {
+ "claude-sonnet-5-5": Price(2.0, 10.0, 0.2, 2.5, 4.0),
+ "claude-opus-5-5": Price(4.0, 20.0, 0.2, 5.0, 8.0),
+ "claude-haiku-4-5": Price(1.0, 5.0, 0.1, 1.25, 2.0),
+}
+
+
+def cost_usd(model: str, usage: Usage, cache_ttl: str | None) -> float | None:
+ """The call's price, or None for a model this table doesn't list."""
+ price = PRICES.get(model.rsplit("/", 1)[-1])
+ if price is None:
+ return None
+ write = price.cache_write_1h if cache_ttl == "1h" else price.cache_write_5m
+ return (
+ usage.input_tokens * price.input
+ + usage.output_tokens * price.output
+ + usage.cache_read_input_tokens * price.cache_read
+ + usage.cache_creation_input_tokens * write
+ ) / 1_000_000
+
+
+def requested_cache_ttl(body: Any) -> str | None:
+ """The prompt-cache lifetime a request asked for: "1h" if any marker asks
+ for an hour, "5m" for any other marker, None when nothing is cached."""
+ ttls = set(_cache_ttls(body))
+ if not ttls:
+ return None
+ return "1h" if "1h" in ttls else "5m"
+
+
+def _cache_ttls(node: Any):
+ if isinstance(node, dict):
+ control = node.get("cache_control")
+ if isinstance(control, dict):
+ yield str(control.get("ttl") or "5m")
+ for value in node.values():
+ yield from _cache_ttls(value)
+ elif isinstance(node, list):
+ for value in node:
+ yield from _cache_ttls(value)
diff --git a/autogpt_platform/scripted_model/scripted_model/journal.py b/autogpt_platform/scripted_model/scripted_model/journal.py
new file mode 100644
index 0000000000..fd63d2994e
--- /dev/null
+++ b/autogpt_platform/scripted_model/scripted_model/journal.py
@@ -0,0 +1,112 @@
+"""What the model was asked and what it answered, for a journey to inspect.
+
+``GET /__scripted__/journal?nonce=<nonce>`` returns one run's entries. A
+journey reads it to wait for a reply to finish streaming, or to print what a
+tool really returned when a check failed.
+"""
+
+import threading
+import time
+from collections import deque
+from dataclasses import asdict, dataclass
+from typing import Any
+
+from .planner import Conversation, Plan
+from .scripts import find_marker
+
+# Enough for a CI run's journeys; the oldest entries go first.
+MAX_ENTRIES = 1000
+# A tool result can be a whole agent graph; the start says what happened.
+MAX_RESULT_CHARS = 2000
+
+
+@dataclass(frozen=True)
+class Request:
+ dialect: str
+ path: str
+
+
+class Journal:
+ def __init__(self) -> None:
+ self._entries: deque[dict[str, Any]] = deque(maxlen=MAX_ENTRIES)
+ self._lock = threading.Lock()
+ self._next_id = 1
+
+ def record_plan(
+ self, request: Request, conversation: Conversation, planned: Plan
+ ) -> dict[str, Any]:
+ return self._add(
+ request,
+ conversation,
+ kind=planned.kind,
+ turn=None if planned.turn_index is None else planned.turn_index + 1,
+ turns=planned.turn_count,
+ text=planned.reply.text,
+ tool_calls=[name for name, _ in planned.reply.tool_calls],
+ usage=asdict(planned.reply.usage),
+ cost_usd=planned.reply.cost_usd,
+ checks=[
+ {"what": c.what, "ok": c.ok, "detail": c.detail} for c in planned.checks
+ ],
+ completed=None,
+ )
+
+ def record_error(
+ self, request: Request, conversation: Conversation, reason: str
+ ) -> dict[str, Any]:
+ return self._add(request, conversation, kind="error", error=reason)
+
+ def record_unexpected(self, method: str, path: str) -> dict[str, Any]:
+ with self._lock:
+ entry = {"id": self._take_id(), "at": time.time(), "kind": "unexpected"}
+ entry.update(method=method, path=path)
+ self._entries.append(entry)
+ return entry
+
+ def finish(self, entry: dict[str, Any], completed: bool) -> None:
+ """Mark whether the whole reply reached the platform."""
+ with self._lock:
+ entry["completed"] = completed
+ entry["finished_at"] = time.time()
+
+ def find(
+ self, script: str | None = None, nonce: str | None = None
+ ) -> list[dict[str, Any]]:
+ with self._lock:
+ return [
+ dict(entry)
+ for entry in self._entries
+ if (script is None or entry.get("script") == script)
+ and (nonce is None or entry.get("nonce") == nonce)
+ ]
+
+ def _add(
+ self, request: Request, conversation: Conversation, **fields: Any
+ ) -> dict[str, Any]:
+ marker = find_marker(conversation.user_text)
+ with self._lock:
+ entry: dict[str, Any] = {
+ "id": self._take_id(),
+ "at": time.time(),
+ "dialect": request.dialect,
+ "path": request.path,
+ "model": conversation.model,
+ "script": marker.script if marker else None,
+ "nonce": marker.nonce if marker else None,
+ "tool_results": [
+ {
+ "tool": r.tool,
+ "is_error": r.is_error,
+ "content": r.content[:MAX_RESULT_CHARS],
+ }
+ for r in conversation.tool_results
+ ],
+ }
+ entry.update(fields)
+ self._entries.append(entry)
+ return entry
+
+ def _take_id(self) -> int:
+ entry_id = self._next_id
+ self._next_id += 1
+ return entry_id
diff --git a/autogpt_platform/scripted_model/scripted_model/messages.py b/autogpt_platform/scripted_model/scripted_model/messages.py
new file mode 100644
index 0000000000..3b646ed145
--- /dev/null
+++ b/autogpt_platform/scripted_model/scripted_model/messages.py
@@ -0,0 +1,185 @@
+"""Anthropic Messages API, the dialect the Claude Agent SDK's CLI speaks."""
+
+import uuid
+from collections.abc import Iterator
+from typing import Any
+
+from .invoice import requested_cache_ttl
+from .planner import Conversation, Reply, ToolResult, bare_name
+from .streaming import Event, json_chunks, text_chunks
+
+
+def conversation(body: dict[str, Any]) -> Conversation:
+ messages = body.get("messages") or []
+ last_assistant = max(
+ (i for i, m in enumerate(messages) if m.get("role") == "assistant"), default=-1
+ )
+ answered = messages[last_assistant] if last_assistant >= 0 else {}
+ tool_names = {
+ block.get("id"): bare_name(str(block.get("name")))
+ for block in _blocks(answered)
+ if block.get("type") == "tool_use"
+ }
+ return Conversation(
+ model=str(body.get("model", "")),
+ stream=bool(body.get("stream")),
+ user_text="\n".join(
+ _text(m.get("content")) for m in messages if m.get("role") == "user"
+ ),
+ assistant_messages=sum(1 for m in messages if m.get("role") == "assistant"),
+ offered_tools=tuple(t["name"] for t in body.get("tools") or [] if "name" in t),
+ tool_results=tuple(
+ ToolResult(
+ tool=tool_names.get(block.get("tool_use_id"), "<unknown>"),
+ content=_text(block.get("content")),
+ is_error=bool(block.get("is_error")),
+ )
+ for message in messages[last_assistant + 1 :]
+ for block in _blocks(message)
+ if block.get("type") == "tool_result"
+ ),
+ cache_ttl=requested_cache_ttl(body),
+ latest_user_text="\n".join(
+ _text(m.get("content"))
+ for m in messages[last_assistant + 1 :]
+ if m.get("role") == "user"
+ ),
+ )
+
+
+def events(reply: Reply, model: str) -> Iterator[Event]:
+ """The reply as a Messages stream: text, then one block per tool call."""
+ yield Event("message_start", _message_start(reply, model), delay_ms=0)
+ index = 0
+ if reply.text:
+ yield _block_start(index, {"type": "text", "text": ""})
+ for piece in text_chunks(reply.text, reply.chunk_words):
+ delta = {"type": "text_delta", "text": piece}
+ yield Event(
+ "content_block_delta",
+ {"type": "content_block_delta", "index": index, "delta": delta},
+ reply.delay_ms,
+ )
+ yield _block_stop(index)
+ index += 1
+ for name, arguments in reply.tool_calls:
+ block = {"type": "tool_use", "id": _id("toolu_"), "name": name, "input": {}}
+ yield _block_start(index, block)
+ for piece in json_chunks(arguments):
+ delta = {"type": "input_json_delta", "partial_json": piece}
+ yield Event(
+ "content_block_delta",
+ {"type": "content_block_delta", "index": index, "delta": delta},
+ reply.delay_ms,
+ )
+ yield _block_stop(index)
+ index += 1
+ yield Event(
+ "message_delta",
+ {
+ "type": "message_delta",
+ "delta": {"stop_reason": _stop_reason(reply), "stop_sequence": None},
+ "usage": {"output_tokens": reply.usage.output_tokens},
+ },
+ delay_ms=0,
+ )
+ yield Event("message_stop", {"type": "message_stop"}, delay_ms=0)
+
+
+def message(reply: Reply, model: str) -> dict[str, Any]:
+ """The reply as one non-streamed Messages response."""
+ content: list[dict[str, Any]] = (
+ [{"type": "text", "text": reply.text}] if reply.text else []
+ )
+ content += [
+ {"type": "tool_use", "id": _id("toolu_"), "name": name, "input": arguments}
+ for name, arguments in reply.tool_calls
+ ]
+ return {
+ **_message_start(reply, model)["message"],
+ "content": content,
+ "stop_reason": _stop_reason(reply),
+ "usage": _usage(reply),
+ }
+
+
+def error(message_text: str) -> dict[str, Any]:
+ return {
+ "type": "error",
+ "error": {
+ "type": "invalid_request_error",
+ "message": f"scripted-model: {message_text}",
+ },
+ }
+
+
+def _message_start(reply: Reply, model: str) -> dict[str, Any]:
+ return {
+ "type": "message_start",
+ "message": {
+ "id": _id("msg_"),
+ "type": "message",
+ "role": "assistant",
+ "model": model,
+ "content": [],
+ "stop_reason": None,
+ "stop_sequence": None,
+ "usage": {**_usage(reply), "output_tokens": 1},
+ },
+ }
+
+
+def _usage(reply: Reply) -> dict[str, Any]:
+ usage = reply.usage
+ written = usage.cache_creation_input_tokens
+ return {
+ "input_tokens": usage.input_tokens,
+ "output_tokens": usage.output_tokens,
+ "cache_read_input_tokens": usage.cache_read_input_tokens,
+ "cache_creation_input_tokens": written,
+ # The CLI prices a cache write by the lifetime it bought.
+ "cache_creation": {
+ "ephemeral_5m_input_tokens": 0 if reply.cache_ttl == "1h" else written,
+ "ephemeral_1h_input_tokens": written if reply.cache_ttl == "1h" else 0,
+ },
+ }
+
+
+def _block_start(index: int, block: dict[str, Any]) -> Event:
+ return Event(
+ "content_block_start",
+ {"type": "content_block_start", "index": index, "content_block": block},
+ delay_ms=0,
+ )
+
+
+def _block_stop(index: int) -> Event:
+ return Event(
+ "content_block_stop", {"type": "content_block_stop", "index": index}, delay_ms=0
+ )
+
+
+def _stop_reason(reply: Reply) -> str:
+ return "tool_use" if reply.tool_calls else "end_turn"
+
+
+def _blocks(message: dict[str, Any]) -> list[dict[str, Any]]:
+ content = message.get("content")
+ return (
+ [b for b in content if isinstance(b, dict)] if isinstance(content, list) else []
+ )
+
+
+def _text(content: Any) -> str:
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ return "\n".join(
+ str(block.get("text", "")) if isinstance(block, dict) else str(block)
+ for block in content
+ )
+ return ""
+
+
+def _id(prefix: str) -> str:
+ return prefix + uuid.uuid4().hex[:24]
diff --git a/autogpt_platform/scripted_model/scripted_model/planner.py b/autogpt_platform/scripted_model/scripted_model/planner.py
new file mode 100644
index 0000000000..945f7b350f
--- /dev/null
+++ b/autogpt_platform/scripted_model/scripted_model/planner.py
@@ -0,0 +1,198 @@
+"""Decide what to answer a model request, whichever wire format it came in."""
+
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+
+from .invoice import cost_usd
+from .scripts import (
+ Expectation,
+ Marker,
+ Script,
+ ScriptError,
+ Turn,
+ Usage,
+ find_marker,
+ load_script,
+)
+
+# What a side call (a request offering no tools: a session title, a summary)
+# costs when no script names it.
+SIDE_CALL_USAGE = Usage(input_tokens=200, output_tokens=10)
+
+
+@dataclass(frozen=True)
+class ToolResult:
+ tool: str
+ content: str
+ is_error: bool
+
+
+@dataclass(frozen=True)
+class Conversation:
+ """The parts of a model request a script cares about."""
+
+ model: str
+ stream: bool
+ # Every user message's text, in order; where the marker is looked for.
+ user_text: str
+ assistant_messages: int
+ # Tool names the request offers the model, as the model must call them.
+ offered_tools: tuple[str, ...]
+ # Results the platform sent back since the last assistant message.
+ tool_results: tuple[ToolResult, ...]
+ # What the user said since the last assistant message.
+ latest_user_text: str = ""
+ # The prompt-cache lifetime the request asked for, if any.
+ cache_ttl: str | None = None
+
+
+@dataclass(frozen=True)
+class Check:
+ what: str
+ ok: bool
+ detail: str
+
+
+@dataclass(frozen=True)
+class Reply:
+ text: str
+ # (name as offered to the model, input)
+ tool_calls: tuple[tuple[str, dict], ...]
+ usage: Usage
+ chunk_words: int = 3
+ delay_ms: int = 15
+ # Which bucket cache writes are reported in, and what the call costs.
+ cache_ttl: str | None = None
+ cost_usd: float | None = None
+
+
+@dataclass(frozen=True)
+class Plan:
+ kind: str # "turn" or "side_call"
+ reply: Reply
+ marker: Marker | None = None
+ turn_index: int | None = None
+ turn_count: int | None = None
+ checks: tuple[Check, ...] = field(default_factory=tuple)
+
+
+def plan(conversation: Conversation, scripts_dir: Path) -> Plan:
+ """The priced reply for this request, or ScriptError when the script can't
+ give one."""
+ planned = _plan(conversation, scripts_dir)
+ reply = planned.reply
+ priced = replace(
+ reply,
+ cache_ttl=conversation.cache_ttl,
+ cost_usd=cost_usd(conversation.model, reply.usage, conversation.cache_ttl),
+ )
+ return replace(planned, reply=priced)
+
+
+def _plan(conversation: Conversation, scripts_dir: Path) -> Plan:
+ """The reply for this request, or ScriptError when the script can't give one.
+
+ A request offering no tools is a side call the platform makes around a
+ turn (a session title, say): it gets the script's title, or a fixed
+ answer. A request offering tools is a model turn and must belong to a
+ script.
+ """
+ marker = find_marker(conversation.user_text)
+ script = load_script(scripts_dir, marker.script) if marker else None
+ if not conversation.offered_tools:
+ title = script.title if script else "Scripted reply"
+ return Plan(
+ kind="side_call",
+ marker=marker,
+ reply=Reply(text=title, tool_calls=(), usage=SIDE_CALL_USAGE, delay_ms=0),
+ )
+ if script is None:
+ raise ScriptError(
+ "this conversation has no [[script:<name>]] marker, so the scripted "
+ "model has nothing to answer with. Put the marker in the first message."
+ )
+ index = conversation.assistant_messages
+ if index >= len(script.turns):
+ raise ScriptError(
+ f"script {script.name!r} has {len(script.turns)} turn(s), but the "
+ f"platform asked for turn {index + 1}: the conversation went further "
+ "than the script expected (an extra tool round, retry or message)."
+ )
+ turn = script.turns[index]
+ return Plan(
+ kind="turn",
+ marker=marker,
+ turn_index=index,
+ turn_count=len(script.turns),
+ reply=_reply(script, index, turn, conversation.offered_tools),
+ checks=_checks(turn, conversation),
+ )
+
+
+def _reply(script: Script, index: int, turn: Turn, offered: tuple[str, ...]) -> Reply:
+ return Reply(
+ text=turn.text,
+ tool_calls=tuple(
+ (_offered_name(script, index, call.name, offered), call.input)
+ for call in turn.tool_calls
+ ),
+ usage=turn.usage,
+ chunk_words=turn.chunk_words,
+ delay_ms=turn.delay_ms,
+ )
+
+
+def _offered_name(
+ script: Script, index: int, name: str, offered: tuple[str, ...]
+) -> str:
+ """The tool as the platform offers it: scripts name ``create_agent``, the
+ SDK engine offers it as ``mcp__<server>__create_agent``."""
+ for candidate in offered:
+ if candidate == name or candidate.endswith(f"__{name}"):
+ return candidate
+ raise ScriptError(
+ f"script {script.name!r} turn {index + 1} calls tool {name!r}, which the "
+ f"platform did not offer. Offered: {sorted(bare_name(t) for t in offered)}"
+ )
+
+
+def _checks(turn: Turn, conversation: Conversation) -> tuple[Check, ...]:
+ checks = [_check(e, conversation.tool_results) for e in turn.expect_tool_results]
+ if turn.expect_user_text is not None:
+ said = conversation.latest_user_text
+ checks.append(
+ Check(
+ "user message",
+ turn.expect_user_text in said,
+ (
+ "matched"
+ if turn.expect_user_text in said
+ else f"the user said: {said[-500:]!r}"
+ ),
+ )
+ )
+ return tuple(checks)
+
+
+def _check(expectation: Expectation, results: tuple[ToolResult, ...]) -> Check:
+ what = f"{expectation.tool} result"
+ matches = [r for r in results if r.tool == expectation.tool]
+ if not matches:
+ seen = [r.tool for r in results]
+ return Check(what, False, f"no result from {expectation.tool!r}; got {seen}")
+ for result in matches:
+ if result.is_error != expectation.is_error:
+ continue
+ if expectation.contains is None or expectation.contains in result.content:
+ return Check(what, True, "matched")
+ first = matches[0]
+ return Check(
+ what,
+ False,
+ f"{expectation.tool!r} returned is_error={first.is_error}: {first.content[:500]}",
+ )
+
+
+def bare_name(tool: str) -> str:
+ """``mcp__copilot__create_agent`` -> ``create_agent``."""
+ return tool.rsplit("__", 1)[-1]
diff --git a/autogpt_platform/scripted_model/scripted_model/scripts.py b/autogpt_platform/scripted_model/scripted_model/scripts.py
new file mode 100644
index 0000000000..4c40264b46
--- /dev/null
+++ b/autogpt_platform/scripted_model/scripted_model/scripts.py
@@ -0,0 +1,203 @@
+"""Scripts: what the fake model answers, one model turn at a time.
+
+A journey picks a script with a marker in its chat message,
+``[[script:<name>]]``, or ``[[script:<name>#<nonce>]]`` to find its own
+requests in the journal later. The script is ``<scripts dir>/<name>.json``.
+It is read on every request, so an edit applies without a restart.
+
+Turn ``n`` answers the model request whose conversation already holds ``n``
+assistant messages. Every request carries the whole conversation, so this
+needs no state: concurrent runs of one script, a platform retry and a resumed
+session all land on the right turn.
+"""
+
+import json
+import re
+from collections.abc import Set
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+MARKER = re.compile(r"\[\[script:([a-z0-9][a-z0-9-]*)(?:#([A-Za-z0-9_.-]+))?\]\]")
+
+DEFAULT_TITLE = "Scripted chat"
+
+
+class ScriptError(Exception):
+ """A model request the script can't answer. The message says why."""
+
+
+@dataclass(frozen=True)
+class Usage:
+ input_tokens: int
+ output_tokens: int
+ cache_read_input_tokens: int = 0
+ cache_creation_input_tokens: int = 0
+
+
+@dataclass(frozen=True)
+class ToolCall:
+ name: str
+ input: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class Expectation:
+ """A tool result the turn expects to find in the request. Only logged."""
+
+ tool: str
+ contains: str | None = None
+ is_error: bool = False
+
+
+@dataclass(frozen=True)
+class Turn:
+ text: str
+ tool_calls: tuple[ToolCall, ...]
+ usage: Usage
+ expect_tool_results: tuple[Expectation, ...]
+ # Text the user's newest message must contain, e.g. a queued follow-up.
+ expect_user_text: str | None
+ # How the text streams: this many words per delta, this long between them.
+ chunk_words: int
+ delay_ms: int
+
+
+@dataclass(frozen=True)
+class Script:
+ name: str
+ title: str
+ turns: tuple[Turn, ...]
+
+
+@dataclass(frozen=True)
+class Marker:
+ script: str
+ nonce: str | None
+
+
+def find_marker(text: str) -> Marker | None:
+ match = MARKER.search(text)
+ if match is None:
+ return None
+ return Marker(script=match.group(1), nonce=match.group(2))
+
+
+def load_script(scripts_dir: Path, name: str) -> Script:
+ path = scripts_dir / f"{name}.json"
+ try:
+ raw = json.loads(path.read_text(encoding="utf-8"))
+ except FileNotFoundError:
+ known = sorted(p.stem for p in scripts_dir.glob("*.json"))
+ raise ScriptError(f"no script named {name!r}; known scripts: {known}")
+ except json.JSONDecodeError as error:
+ raise ScriptError(f"script {name!r} is not valid JSON: {error}")
+ return parse_script(name, raw)
+
+
+def parse_script(name: str, raw: Any) -> Script:
+ data = _object(
+ raw, f"script {name!r}", required={"turns"}, optional={"title", "description"}
+ )
+ turns = data["turns"]
+ if not isinstance(turns, list) or not turns:
+ raise ScriptError(f"script {name!r}: 'turns' must be a non-empty list")
+ return Script(
+ name=name,
+ title=str(data.get("title") or DEFAULT_TITLE),
+ turns=tuple(_turn(f"script {name!r} turn {i}", t) for i, t in enumerate(turns)),
+ )
+
+
+def _turn(where: str, raw: Any) -> Turn:
+ data = _object(
+ raw,
+ where,
+ required={"usage"},
+ optional={
+ "text",
+ "tool_calls",
+ "expect_tool_results",
+ "expect_user_text",
+ "stream",
+ },
+ )
+ text = data.get("text", "")
+ calls = [
+ _tool_call(f"{where} tool call {i}", c)
+ for i, c in enumerate(data.get("tool_calls", []))
+ ]
+ if not isinstance(text, str) or not (text or calls):
+ raise ScriptError(f"{where}: needs 'text', 'tool_calls' or both")
+ stream = _object(
+ data.get("stream", {}), f"{where} stream", optional={"chunk_words", "delay_ms"}
+ )
+ return Turn(
+ text=text,
+ tool_calls=tuple(calls),
+ usage=_usage(f"{where} usage", data["usage"]),
+ expect_tool_results=tuple(
+ _expectation(f"{where} expectation {i}", e)
+ for i, e in enumerate(data.get("expect_tool_results", []))
+ ),
+ expect_user_text=(
+ None
+ if data.get("expect_user_text") is None
+ else str(data["expect_user_text"])
+ ),
+ chunk_words=_count(
+ stream.get("chunk_words", 3), f"{where} chunk_words", minimum=1
+ ),
+ delay_ms=_count(stream.get("delay_ms", 15), f"{where} delay_ms"),
+ )
+
+
+def _tool_call(where: str, raw: Any) -> ToolCall:
+ data = _object(raw, where, required={"name", "input"})
+ if not isinstance(data["input"], dict):
+ raise ScriptError(f"{where}: 'input' must be a JSON object")
+ return ToolCall(name=str(data["name"]), input=data["input"])
+
+
+def _usage(where: str, raw: Any) -> Usage:
+ data = _object(
+ raw,
+ where,
+ required={"input_tokens", "output_tokens"},
+ optional={"cache_read_input_tokens", "cache_creation_input_tokens"},
+ )
+ return Usage(
+ **{key: _count(value, f"{where} {key}") for key, value in data.items()}
+ )
+
+
+def _expectation(where: str, raw: Any) -> Expectation:
+ data = _object(raw, where, required={"tool"}, optional={"contains", "is_error"})
+ return Expectation(
+ tool=str(data["tool"]),
+ contains=None if data.get("contains") is None else str(data["contains"]),
+ is_error=bool(data.get("is_error", False)),
+ )
+
+
+def _object(
+ raw: Any,
+ where: str,
+ required: Set[str] = frozenset(),
+ optional: Set[str] = frozenset(),
+) -> dict[str, Any]:
+ """``raw`` as a JSON object with exactly the allowed keys: a typo fails
+ loudly instead of being ignored."""
+ if not isinstance(raw, dict):
+ raise ScriptError(f"{where}: must be a JSON object")
+ if missing := required - raw.keys():
+ raise ScriptError(f"{where}: missing {sorted(missing)}")
+ if unknown := raw.keys() - required - optional:
+ raise ScriptError(f"{where}: unknown keys {sorted(unknown)}")
+ return raw
+
+
+def _count(value: Any, where: str, minimum: int = 0) -> int:
+ if not isinstance(value, int) or isinstance(value, bool) or value < minimum:
+ raise ScriptError(f"{where}: must be an integer >= {minimum}")
+ return value
diff --git a/autogpt_platform/scripted_model/scripted_model/server.py b/autogpt_platform/scripted_model/scripted_model/server.py
new file mode 100644
index 0000000000..586ecbf2e9
--- /dev/null
+++ b/autogpt_platform/scripted_model/scripted_model/server.py
@@ -0,0 +1,240 @@
+"""The HTTP server: routes, streamed responses and the journal."""
+
+import json
+import logging
+import time
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from http import HTTPStatus
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from typing import Any, cast
+from urllib.parse import parse_qs, urlparse
+
+from . import chat_completions, messages
+from .journal import Journal, Request
+from .planner import Conversation, Plan, Reply, plan
+from .scripts import ScriptError
+from .streaming import Event, sse
+
+logger = logging.getLogger("scripted_model")
+
+JOURNAL_PATH = "/__scripted__/journal"
+
+
+@dataclass(frozen=True)
+class Dialect:
+ name: str
+ conversation: Callable[[dict[str, Any]], Conversation]
+ events: Callable[[Reply, str], Iterator[Event]]
+ whole: Callable[[Reply, str], dict[str, Any]]
+ error: Callable[[str], dict[str, Any]]
+
+
+ANTHROPIC = Dialect(
+ "anthropic",
+ messages.conversation,
+ messages.events,
+ messages.message,
+ messages.error,
+)
+OPENAI = Dialect(
+ "openai",
+ chat_completions.conversation,
+ chat_completions.events,
+ chat_completions.completion,
+ chat_completions.error,
+)
+
+
+class ScriptedModelServer(ThreadingHTTPServer):
+ daemon_threads = True
+
+ def __init__(self, address: tuple[str, int], scripts_dir: Path) -> None:
+ super().__init__(address, Handler)
+ self.scripts_dir = scripts_dir
+ self.journal = Journal()
+
+
+class Handler(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+
+ @property
+ def model_server(self) -> ScriptedModelServer:
+ return cast(ScriptedModelServer, self.server)
+
+ def do_HEAD(self) -> None:
+ # The CLI checks it can reach its base URL before a session starts.
+ if urlparse(self.path).path.endswith("/api/hello"):
+ self._send_json(HTTPStatus.OK, {}, body=False)
+ else:
+ self._unexpected()
+
+ def do_GET(self) -> None:
+ url = urlparse(self.path)
+ if url.path == "/healthz":
+ self._send_json(HTTPStatus.OK, {"ok": True})
+ elif url.path == JOURNAL_PATH:
+ query = {key: values[-1] for key, values in parse_qs(url.query).items()}
+ entries = self.model_server.journal.find(
+ script=query.get("script"), nonce=query.get("nonce")
+ )
+ self._send_json(HTTPStatus.OK, {"entries": entries})
+ elif url.path.endswith("/api/hello"):
+ self._send_json(HTTPStatus.OK, {})
+ else:
+ self._unexpected()
+
+ def do_POST(self) -> None:
+ path = urlparse(self.path).path
+ if path.endswith("/messages/count_tokens"):
+ body = self._read_json()
+ tokens = max(1, len(json.dumps(body)) // 4) if body is not None else 1
+ self._send_json(HTTPStatus.OK, {"input_tokens": tokens})
+ elif path.endswith("/messages"):
+ self._answer(ANTHROPIC)
+ elif path.endswith("/chat/completions"):
+ self._answer(OPENAI)
+ else:
+ self._unexpected()
+
+ def _answer(self, dialect: Dialect) -> None:
+ body = self._read_json()
+ if not isinstance(body, dict):
+ self._send_json(
+ HTTPStatus.BAD_REQUEST, dialect.error("body is not a JSON object")
+ )
+ return
+ try:
+ conversation = dialect.conversation(body)
+ except (AttributeError, KeyError, TypeError) as error:
+ # A body this dialect can't read: answer it rather than leave the
+ # client waiting on a handler that died.
+ logger.error("unreadable %s request: %r", dialect.name, error)
+ self._send_json(
+ HTTPStatus.BAD_REQUEST, dialect.error(f"malformed request: {error!r}")
+ )
+ return
+ try:
+ planned = plan(conversation, self.model_server.scripts_dir)
+ except ScriptError as error:
+ self._refuse(dialect, conversation, str(error))
+ return
+ entry = self.model_server.journal.record_plan(
+ self._request(dialect), conversation, planned
+ )
+ _log_plan(dialect.name, conversation, planned)
+ if conversation.stream:
+ self._stream(dialect.events(planned.reply, conversation.model), entry)
+ else:
+ self._send_json(
+ HTTPStatus.OK, dialect.whole(planned.reply, conversation.model)
+ )
+ self.model_server.journal.finish(entry, completed=True)
+
+ def _refuse(
+ self, dialect: Dialect, conversation: Conversation, reason: str
+ ) -> None:
+ """A request the script can't answer fails the turn with the reason,
+ rather than leaving the chat waiting on a reply that never comes."""
+ logger.error("refused %s request: %s", dialect.name, reason)
+ self.model_server.journal.record_error(
+ self._request(dialect), conversation, reason
+ )
+ self._send_json(HTTPStatus.BAD_REQUEST, dialect.error(reason))
+
+ def _request(self, dialect: Dialect) -> Request:
+ return Request(dialect.name, self.path)
+
+ def _stream(self, events: Iterator[Event], entry: dict[str, Any]) -> None:
+ self.send_response(HTTPStatus.OK)
+ self.send_header("Content-Type", "text/event-stream")
+ self.send_header("Cache-Control", "no-cache")
+ self.send_header("Transfer-Encoding", "chunked")
+ self.end_headers()
+ try:
+ for event in events:
+ if event.delay_ms:
+ time.sleep(event.delay_ms / 1000)
+ self._write_chunk(sse(event))
+ self._write_chunk(b"")
+ except ConnectionError:
+ logger.warning(
+ "client went away mid-stream (journal entry %s)", entry["id"]
+ )
+ self.close_connection = True
+ self.model_server.journal.finish(entry, completed=False)
+ return
+ self.model_server.journal.finish(entry, completed=True)
+
+ def _write_chunk(self, data: bytes) -> None:
+ self.wfile.write(f"{len(data):X}\r\n".encode() + data + b"\r\n")
+ self.wfile.flush()
+
+ def _read_json(self) -> Any:
+ try:
+ return json.loads(self._read_body() or b"null")
+ except json.JSONDecodeError:
+ return None
+
+ def _read_body(self) -> bytes:
+ if self.headers.get("Transfer-Encoding", "").lower() == "chunked":
+ return self._read_chunked()
+ return self.rfile.read(int(self.headers.get("Content-Length") or 0))
+
+ def _read_chunked(self) -> bytes:
+ parts = []
+ while size := int(self.rfile.readline().split(b";")[0].strip() or b"0", 16):
+ parts.append(self.rfile.read(size))
+ self.rfile.readline()
+ self.rfile.readline()
+ return b"".join(parts)
+
+ def _unexpected(self) -> None:
+ if self.command in ("POST", "PUT", "PATCH"):
+ # Unread, the body would be taken for the next request's start.
+ self._read_body()
+ logger.warning("unexpected request: %s %s", self.command, self.path)
+ self.model_server.journal.record_unexpected(self.command, self.path)
+ self._send_json(
+ HTTPStatus.NOT_FOUND,
+ {
+ "error": {
+ "message": f"scripted-model: nothing serves {self.command} {self.path}"
+ }
+ },
+ )
+
+ def _send_json(self, status: HTTPStatus, payload: Any, body: bool = True) -> None:
+ data = json.dumps(payload).encode()
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ if body:
+ self.wfile.write(data)
+
+ def log_message(self, format: str, *args: Any) -> None:
+ logger.debug(format, *args)
+
+
+def _log_plan(dialect: str, conversation: Conversation, planned: Plan) -> None:
+ script = planned.marker.script if planned.marker else "-"
+ if planned.kind == "side_call":
+ logger.info(
+ "%s side call (%s) for script %s", dialect, conversation.model, script
+ )
+ return
+ calls = ", ".join(name for name, _ in planned.reply.tool_calls) or "none"
+ logger.info(
+ "%s turn %s/%s of %s (%s): tool calls: %s",
+ dialect,
+ (planned.turn_index or 0) + 1,
+ planned.turn_count,
+ script,
+ conversation.model,
+ calls,
+ )
+ for check in planned.checks:
+ level = logging.INFO if check.ok else logging.ERROR
+ logger.log(level, "expected %s: %s", check.what, check.detail)
diff --git a/autogpt_platform/scripted_model/scripted_model/streaming.py b/autogpt_platform/scripted_model/scripted_model/streaming.py
new file mode 100644
index 0000000000..7520ce44e5
--- /dev/null
+++ b/autogpt_platform/scripted_model/scripted_model/streaming.py
@@ -0,0 +1,40 @@
+"""Cutting a reply into the deltas a streamed response sends."""
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+
+_WORD = re.compile(r"\s*\S+")
+
+
+@dataclass(frozen=True)
+class Event:
+ """One server-sent event, sent after waiting ``delay_ms``."""
+
+ name: str | None
+ data: Any
+ delay_ms: int
+
+
+def text_chunks(text: str, words: int) -> list[str]:
+ """``text`` in pieces of ``words`` words that join back to exactly ``text``."""
+ tokens = _WORD.findall(text)
+ if not tokens:
+ return [text] if text else []
+ tokens[-1] += text[sum(len(t) for t in tokens) :]
+ return ["".join(tokens[i : i + words]) for i in range(0, len(tokens), words)]
+
+
+def json_chunks(value: Any, pieces: int = 3) -> list[str]:
+ """``value`` as JSON text, cut into a few partial strings as a model streams
+ tool arguments."""
+ text = json.dumps(value)
+ size = max(1, -(-len(text) // pieces))
+ return [text[i : i + size] for i in range(0, len(text), size)]
+
+
+def sse(event: Event) -> bytes:
+ prefix = f"event: {event.name}\n" if event.name else ""
+ data = event.data if isinstance(event.data, str) else json.dumps(event.data)
+ return f"{prefix}data: {data}\n\n".encode()
diff --git a/autogpt_platform/scripted_model/tests/test_scripted_model.py b/autogpt_platform/scripted_model/tests/test_scripted_model.py
new file mode 100644
index 0000000000..2a0c40c9b4
--- /dev/null
+++ b/autogpt_platform/scripted_model/tests/test_scripted_model.py
@@ -0,0 +1,407 @@
+"""The scripted model, spoken to through the clients the platform itself uses.
+
+The Anthropic client parses a Messages stream as the Claude Agent SDK's CLI
+does, and the OpenAI client is the baseline engine's own. A script that round
+trips through both is one the copilot can follow.
+"""
+
+import json
+import re
+import threading
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+import anthropic
+import openai
+import pytest
+
+from scripted_model.scripts import parse_script
+
+JOURNEY_SCRIPTS = (
+ Path(__file__).resolve().parents[2]
+ / "frontend/src/playwright/journeys/model-scripts"
+)
+SONNET = "anthropic/claude-sonnet-5-5"
+
+LOOKUP = {
+ "title": "Haiku lookup",
+ "turns": [
+ {
+ "text": "Let me check your library first.",
+ "tool_calls": [{"name": "find_library_agent", "input": {"query": "haiku"}}],
+ "usage": {
+ "input_tokens": 3000,
+ "output_tokens": 40,
+ "cache_read_input_tokens": 20000,
+ "cache_creation_input_tokens": 1000,
+ },
+ },
+ {
+ "expect_tool_results": [
+ {"tool": "find_library_agent", "contains": "No agents"}
+ ],
+ "text": "You have none yet.",
+ "usage": {"input_tokens": 3100, "output_tokens": 8},
+ },
+ ],
+}
+SDK_TOOLS = [
+ {
+ "name": "mcp__copilot__find_library_agent",
+ "description": "Search the user's library.",
+ "input_schema": {"type": "object"},
+ }
+]
+CHAT_TOOLS = [
+ {
+ "type": "function",
+ "function": {"name": "find_library_agent", "parameters": {"type": "object"}},
+ }
+]
+
+
+def test_messages_follow_the_script_through_a_tool_round(scripted_model):
+ scripted_model.add_script("lookup", LOOKUP)
+ client = _anthropic(scripted_model)
+ # The CLI marks its cached prefix; an hour-long one prices writes higher.
+ system = [
+ {
+ "type": "text",
+ "text": "You are AutoPilot.",
+ "cache_control": {"type": "ephemeral", "ttl": "1h"},
+ }
+ ]
+ messages = [
+ {"role": "user", "content": "Any haiku agent? [[script:lookup#sdk-run]]"}
+ ]
+
+ first = _stream(client, messages, system)
+ call = first.content[1]
+ assert first.content[0].text == "Let me check your library first."
+ assert (call.name, call.input) == (
+ "mcp__copilot__find_library_agent",
+ {"query": "haiku"},
+ )
+ assert first.stop_reason == "tool_use"
+ assert (
+ first.usage.input_tokens,
+ first.usage.output_tokens,
+ first.usage.cache_read_input_tokens,
+ first.usage.cache_creation.ephemeral_1h_input_tokens,
+ ) == (3000, 40, 20000, 1000)
+
+ messages += [
+ {"role": "assistant", "content": [b.model_dump() for b in first.content]},
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": call.id,
+ "content": "No agents match 'haiku'.",
+ }
+ ],
+ },
+ ]
+ second = _stream(client, messages, system)
+ assert [b.text for b in second.content] == ["You have none yet."]
+ assert second.stop_reason == "end_turn"
+
+ first_entry, second_entry = scripted_model.journal(nonce="sdk-run")
+ assert first_entry["cost_usd"] == pytest.approx(
+ (3000 * 2 + 40 * 10 + 20000 * 0.2 + 1000 * 4.0) / 1_000_000
+ )
+ assert second_entry["checks"] == [
+ {"what": "find_library_agent result", "ok": True, "detail": "matched"}
+ ]
+ assert first_entry["completed"] and second_entry["completed"]
+
+
+def test_chat_completions_follow_the_script_and_bill_like_openrouter(
+ scripted_model,
+):
+ scripted_model.add_script("lookup", LOOKUP)
+ client = _openai(scripted_model)
+ messages: list[dict] = [
+ {"role": "user", "content": "Any haiku agent? [[script:lookup#chat-run]]"}
+ ]
+
+ text, calls, usage = _stream_chat(client, messages)
+ assert text == "Let me check your library first."
+ assert [(c["name"], json.loads(c["arguments"])) for c in calls] == [
+ ("find_library_agent", {"query": "haiku"})
+ ]
+ assert (usage.prompt_tokens, usage.completion_tokens) == (24000, 40)
+ # No cache marker asked for an hour, so the write is a five-minute one.
+ assert usage.model_extra["cost"] == pytest.approx(
+ (3000 * 2 + 40 * 10 + 20000 * 0.2 + 1000 * 2.5) / 1_000_000
+ )
+
+ messages += [
+ {
+ "role": "assistant",
+ "content": text,
+ "tool_calls": [
+ {
+ "id": calls[0]["id"],
+ "type": "function",
+ "function": {
+ "name": calls[0]["name"],
+ "arguments": calls[0]["arguments"],
+ },
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": calls[0]["id"],
+ "content": "No agents match 'haiku'.",
+ },
+ ]
+ text, calls, _ = _stream_chat(client, messages)
+ assert (text, calls) == ("You have none yet.", [])
+ assert scripted_model.journal(nonce="chat-run")[-1]["checks"][0]["ok"]
+
+
+def test_a_failed_expectation_is_recorded_with_what_the_tool_said(scripted_model):
+ scripted_model.add_script("lookup", LOOKUP)
+ messages = [
+ {"role": "user", "content": "[[script:lookup#failed-check]]"},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "tool_use", "id": "toolu_1", "name": "x__find_library_agent"}
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_1",
+ "content": "Search is down.",
+ "is_error": True,
+ }
+ ],
+ },
+ ]
+ _stream(_anthropic(scripted_model), messages)
+ (entry,) = scripted_model.journal(nonce="failed-check")
+ assert entry["checks"] == [
+ {
+ "what": "find_library_agent result",
+ "ok": False,
+ "detail": "'find_library_agent' returned is_error=True: Search is down.",
+ }
+ ]
+
+
+def test_a_follow_up_message_gets_the_next_turn(scripted_model):
+ """A message the user queued while a turn ran arrives after that turn's
+ answer, and the script's next turn answers it."""
+ scripted_model.add_script(
+ "follow-up",
+ {
+ "turns": [
+ {
+ "text": "First answer.",
+ "usage": {"input_tokens": 1, "output_tokens": 1},
+ },
+ {
+ "expect_user_text": "one more thing",
+ "text": "Second answer.",
+ "usage": {"input_tokens": 1, "output_tokens": 1},
+ },
+ ]
+ },
+ )
+ client = _anthropic(scripted_model)
+ for nonce, follow_up in (("asked", "And one more thing?"), ("forgot", "Thanks!")):
+ messages = [
+ {"role": "user", "content": f"[[script:follow-up#{nonce}]]"},
+ {"role": "assistant", "content": "First answer."},
+ {"role": "user", "content": follow_up},
+ ]
+ assert [b.text for b in _stream(client, messages).content] == ["Second answer."]
+
+ (asked,) = scripted_model.journal(nonce="asked")
+ (forgot,) = scripted_model.journal(nonce="forgot")
+ assert asked["checks"] == [
+ {"what": "user message", "ok": True, "detail": "matched"}
+ ]
+ assert forgot["checks"] == [
+ {"what": "user message", "ok": False, "detail": "the user said: 'Thanks!'"}
+ ]
+
+
+def test_a_request_the_script_cannot_answer_fails_with_the_reason(scripted_model):
+ scripted_model.add_script("lookup", LOOKUP)
+ client = _anthropic(scripted_model)
+ one_round = [
+ {"role": "user", "content": "[[script:lookup]]"},
+ {"role": "assistant", "content": "Looking."},
+ {"role": "user", "content": "And?"},
+ {"role": "assistant", "content": "None."},
+ {"role": "user", "content": "Once more?"},
+ ]
+ cases = [
+ ("hello", SDK_TOOLS, "no [[script:<name>]] marker"),
+ (
+ "[[script:nope]]",
+ SDK_TOOLS,
+ "no script named 'nope'; known scripts: ['lookup']",
+ ),
+ (
+ "[[script:lookup]]",
+ [{**SDK_TOOLS[0], "name": "mcp__copilot__other"}],
+ "did not offer",
+ ),
+ (one_round, SDK_TOOLS, "has 2 turn(s), but the platform asked for turn 3"),
+ ]
+ for content, tools, reason in cases:
+ messages = (
+ content
+ if isinstance(content, list)
+ else [{"role": "user", "content": content}]
+ )
+ with pytest.raises(anthropic.BadRequestError, match=re.escape(reason)):
+ client.messages.create(
+ model=SONNET, max_tokens=64, messages=messages, tools=tools
+ )
+
+ malformed = urllib.request.Request(
+ f"{scripted_model.url}/api/v1/messages",
+ data=json.dumps({"messages": ["not an object"]}).encode(),
+ headers={"Content-Type": "application/json"},
+ )
+ with pytest.raises(urllib.error.HTTPError) as refused:
+ urllib.request.urlopen(malformed, timeout=5)
+ assert refused.value.code == 400
+ assert "malformed request" in refused.value.read().decode()
+
+ with pytest.raises(openai.BadRequestError, match="no script named 'nope'"):
+ _openai(scripted_model).chat.completions.create(
+ model=SONNET,
+ messages=[{"role": "user", "content": "[[script:nope]]"}],
+ tools=CHAT_TOOLS,
+ )
+ assert [e["kind"] for e in scripted_model.journal()] == ["error"] * 5
+
+
+def test_a_side_call_gets_the_script_title(scripted_model):
+ scripted_model.add_script("lookup", LOOKUP)
+ client = _openai(scripted_model)
+
+ titled = client.chat.completions.create(
+ model="anthropic/claude-haiku-4-5",
+ messages=[{"role": "user", "content": "Title: [[script:lookup#side]]"}],
+ )
+ untitled = client.chat.completions.create(
+ model="anthropic/claude-haiku-4-5",
+ messages=[{"role": "user", "content": "Title this chat."}],
+ )
+
+ assert titled.choices[0].message.content == "Haiku lookup"
+ assert untitled.choices[0].message.content == "Scripted reply"
+ assert titled.usage.model_extra["cost"] == pytest.approx((200 * 1 + 10 * 5) / 1e6)
+ assert scripted_model.journal(nonce="side")[0]["kind"] == "side_call"
+
+
+def test_concurrent_runs_of_one_script_each_get_their_own_turns(scripted_model):
+ """Every request carries its whole conversation, so runs sharing a script
+ land on their own turns however their requests interleave."""
+ scripted_model.add_script("lookup", LOOKUP)
+ runs = 8
+ # Every run's first turn happens before any run's second turn.
+ between_turns = threading.Barrier(runs)
+ failures: list[BaseException] = []
+
+ def run(nonce: str) -> None:
+ try:
+ client = _anthropic(scripted_model)
+ messages = [{"role": "user", "content": f"[[script:lookup#{nonce}]]"}]
+ first = _stream(client, messages)
+ between_turns.wait(timeout=10)
+ call = first.content[1]
+ messages += [
+ {
+ "role": "assistant",
+ "content": [b.model_dump() for b in first.content],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": call.id,
+ "content": f"No agents for {nonce}.",
+ }
+ ],
+ },
+ ]
+ _stream(client, messages)
+ except BaseException as error:
+ failures.append(error)
+
+ threads = [threading.Thread(target=run, args=(f"run-{i}",)) for i in range(runs)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(timeout=30)
+
+ assert failures == []
+ for i in range(runs):
+ entries = scripted_model.journal(nonce=f"run-{i}")
+ assert [e["turn"] for e in entries] == [1, 2]
+ assert entries[1]["tool_results"][0]["content"] == f"No agents for run-{i}."
+
+
+ "path", sorted(JOURNEY_SCRIPTS.glob("*.json")), ids=lambda path: path.stem
+)
+def test_every_journey_script_is_valid(path: Path):
+ parse_script(path.stem, json.loads(path.read_text(encoding="utf-8")))
+
+
+def _anthropic(server) -> anthropic.Anthropic:
+ # Like the CLI: CHAT_BASE_URL without its /v1.
+ return anthropic.Anthropic(
+ base_url=f"{server.url}/api", api_key="any", max_retries=0
+ )
+
+
+def _openai(server) -> openai.OpenAI:
+ return openai.OpenAI(base_url=f"{server.url}/api/v1", api_key="any", max_retries=0)
+
+
+def _stream(client: anthropic.Anthropic, messages: list[dict], system=None):
+ extra = {"system": system} if system else {}
+ with client.messages.stream(
+ model=SONNET, max_tokens=1024, messages=messages, tools=SDK_TOOLS, **extra
+ ) as stream:
+ return stream.get_final_message()
+
+
+def _stream_chat(client: openai.OpenAI, messages: list[dict]):
+ stream = client.chat.completions.create(
+ model=SONNET,
+ messages=messages,
+ tools=CHAT_TOOLS,
+ stream=True,
+ stream_options={"include_usage": True},
+ )
+ text, calls, usage = "", {}, None
+ for chunk in stream:
+ usage = chunk.usage or usage
+ for choice in chunk.choices:
+ text += choice.delta.content or ""
+ for call in choice.delta.tool_calls or []:
+ entry = calls.setdefault(
+ call.index, {"id": "", "name": "", "arguments": ""}
+ )
+ entry["id"] = call.id or entry["id"]
+ if call.function:
+ entry["name"] += call.function.name or ""
+ entry["arguments"] += call.function.arguments or ""
+ return text, [calls[i] for i in sorted(calls)], usage