Skip to content

Email for professionals and teams

Stop Workers AI streams from repeating text (#120)

rome-xi6e73e28cae65
diff --git a/src/lib/agent/model.ts b/src/lib/agent/model.ts
index 8709b15..b4e271c 100644
--- a/src/lib/agent/model.ts
+++ b/src/lib/agent/model.ts
@@ -3,11 +3,57 @@ import { createWorkersAI } from "workers-ai-provider";
import { getAgentProviderConfig } from "./provider";
import { agentTimeContext } from "./time";
+// workers-ai-provider 4.0.0 maps both `response` and `choices[0].delta` from the same SSE event, so models that send
+// both (e.g. Llama 3.3 70B) repeat their text and tool calls. Drop the native copy until cloudflare/ai#615 is released.
+function dropDuplicateNativeFields(line: string) {
+ const match = /^(data: ?)(\{.*\})(\r?)$/s.exec(line);
+ if (!match) return line;
+ try {
+ const event = JSON.parse(match[2]);
+ const delta = event?.choices?.[0]?.delta;
+ if (!delta) return line;
+ delete event.response;
+ if (delta.tool_calls) delete event.tool_calls;
+ return `${match[1]}${JSON.stringify(event)}${match[3]}`;
+ } catch {
+ return line;
+ }
+}
+
+function dedupeWorkersAIStream(stream: ReadableStream) {
+ const decoder = new TextDecoder();
+ const encoder = new TextEncoder();
+ let pending = "";
+ return stream.pipeThrough(new TransformStream<Uint8Array | string, Uint8Array>({
+ transform(chunk, controller) {
+ const lines = (pending + (typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true }))).split("\n");
+ pending = lines.pop() ?? "";
+ for (const line of lines) controller.enqueue(encoder.encode(`${dropDuplicateNativeFields(line)}\n`));
+ },
+ flush(controller) {
+ const rest = pending + decoder.decode();
+ if (rest) controller.enqueue(encoder.encode(dropDuplicateNativeFields(rest)));
+ },
+ }));
+}
+
+function withDedupedStreams(binding: Ai): Ai {
+ return new Proxy(binding, {
+ get(target, prop, receiver) {
+ if (prop !== "run") return Reflect.get(target, prop, receiver);
+ return async (...args: unknown[]) => {
+ const result = await (target.run as (...runArgs: unknown[]) => Promise<unknown>).apply(target, args);
+ return result instanceof ReadableStream ? dedupeWorkersAIStream(result) : result;
+ };
+ },
+ });
+}
+
export async function getAgentModel(env: CloudflareEnv, requestedModel?: string | null) {
const config = await getAgentProviderConfig(env);
const modelId = requestedModel && config.models.includes(requestedModel) ? requestedModel : config.model;
const details = { modelId, provider: config.provider === "cloudflare" ? "cloudflare" : config.preset, rates: config.rates[modelId] };
- if (config.provider === "cloudflare") return env.AI ? { model: createWorkersAI({ binding: env.AI })(modelId), ...details } : null;
+ if (config.provider === "cloudflare") return env.AI ? { model: createWorkersAI({ binding: withDedupedStreams(env.AI) })(modelId), ...details } : null;
if (config.baseUrl && config.apiKey && modelId) {
const provider = createOpenAICompatible({ name: "mailflare", baseURL: config.baseUrl, apiKey: config.apiKey });
return { model: provider.chatModel(modelId), ...details };
diff --git a/tests/workers-ai-stream.test.mjs b/tests/workers-ai-stream.test.mjs
new file mode 100644
index 0000000..abc5e85
--- /dev/null
+++ b/tests/workers-ai-stream.test.mjs
@@ -0,0 +1,164 @@
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { dirname, join } from "node:path";
+import test, { after } from "node:test";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { build } from "esbuild";
+import { streamText } from "ai";
+import { z } from "zod";
+
+const root = join(dirname(fileURLToPath(import.meta.url)), "..");
+const bundleDirectory = mkdtempSync(join(root, "node_modules", "mailflare-workers-ai-stream-bundle-"));
+after(() => rmSync(bundleDirectory, { recursive: true, force: true }));
+
+await build({
+ stdin: {
+ contents: `
+ export { getAgentModel } from "./src/lib/agent/model.ts";
+ `,
+ resolveDir: root,
+ sourcefile: "workers-ai-stream-test-entry.ts",
+ },
+ outfile: join(bundleDirectory, "entry.mjs"),
+ bundle: true,
+ platform: "node",
+ format: "esm",
+ target: "node24",
+ tsconfig: join(root, "tsconfig.json"),
+ packages: "external",
+ alias: {
+ "next/headers": "next/headers.js",
+ "cloudflare:workers": "./server/runtime/cloudflare-workers.ts",
+ },
+ logLevel: "silent",
+});
+
+const { getAgentModel } = await import(pathToFileURL(join(bundleDirectory, "entry.mjs")).href);
+
+function createMockDb() {
+ return {
+ prepare() {
+ return {
+ bind() {
+ return {
+ all: async () => ({ results: [] }),
+ raw: async () => [],
+ };
+ },
+ all: async () => ({ results: [] }),
+ raw: async () => [],
+ };
+ },
+ };
+}
+
+function createSseStream(events) {
+ return new ReadableStream({
+ start(controller) {
+ const encoder = new TextEncoder();
+ for (const event of events) {
+ controller.enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`));
+ }
+ controller.enqueue(encoder.encode("data: [DONE]\n\n"));
+ controller.close();
+ },
+ });
+}
+
+test("Workers AI dual-format SSE stream produces deduplicated text", async () => {
+ const binding = {
+ run: async () => createSseStream([
+ { response: "OK", choices: [{ index: 0, delta: { content: "OK" } }] },
+ { response: ".", choices: [{ index: 0, delta: { content: "." }, finish_reason: "stop" }] },
+ ]),
+ };
+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });
+ assert.ok(agent?.model);
+ const result = streamText({ model: agent.model, prompt: "say OK" });
+ assert.equal(await result.text, "OK.");
+});
+
+test("Workers AI native-only stream emits text once", async () => {
+ const binding = {
+ run: async () => createSseStream([
+ { response: "OK" },
+ { response: ".", finish_reason: "stop" },
+ ]),
+ };
+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });
+ assert.ok(agent?.model);
+ const result = streamText({ model: agent.model, prompt: "say OK" });
+ assert.equal(await result.text, "OK.");
+});
+
+test("Workers AI OpenAI-only stream emits text once", async () => {
+ const binding = {
+ run: async () => createSseStream([
+ { choices: [{ index: 0, delta: { content: "OK" } }] },
+ { choices: [{ index: 0, delta: { content: "." }, finish_reason: "stop" }] },
+ ]),
+ };
+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });
+ assert.ok(agent?.model);
+ const result = streamText({ model: agent.model, prompt: "say OK" });
+ assert.equal(await result.text, "OK.");
+});
+
+test("Workers AI stream preserves legitimately repeated prose across events", async () => {
+ const binding = {
+ run: async () => createSseStream([
+ { response: "ha", choices: [{ index: 0, delta: { content: "ha" } }] },
+ { response: "ha", choices: [{ index: 0, delta: { content: "ha" }, finish_reason: "stop" }] },
+ ]),
+ };
+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });
+ assert.ok(agent?.model);
+ const result = streamText({ model: agent.model, prompt: "laugh" });
+ assert.equal(await result.text, "haha");
+});
+
+test("Workers AI dual-format stream emits exactly one tool call with parsed arguments", async () => {
+ const toolCall = [{ index: 0, id: "call_lookup", type: "function", function: { name: "lookup", arguments: JSON.stringify({ city: "Hanoi" }) } }];
+ const binding = {
+ run: async () => createSseStream([
+ { tool_calls: toolCall, choices: [{ index: 0, delta: { tool_calls: toolCall } }] },
+ ]),
+ };
+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });
+ assert.ok(agent?.model);
+ const result = streamText({
+ model: agent.model,
+ tools: {
+ lookup: {
+ description: "lookup info",
+ parameters: z.object({ city: z.string() }),
+ execute: async (args) => args,
+ },
+ },
+ prompt: "lookup Hanoi",
+ });
+ const calls = await result.toolCalls;
+ assert.equal(calls.length, 1);
+ assert.equal(calls[0].toolName, "lookup");
+ assert.deepEqual(calls[0].input, { city: "Hanoi" });
+});
+
+test("Workers AI stream handles events split across chunk boundaries", async () => {
+ const payload = `data: ${JSON.stringify({ response: "OK", choices: [{ index: 0, delta: { content: "OK" } }] })}\n\ndata: [DONE]\n\n`;
+ const encoder = new TextEncoder();
+ const bytes = encoder.encode(payload);
+ const splitAt = 12;
+ const binding = {
+ run: async () => new ReadableStream({
+ start(controller) {
+ controller.enqueue(bytes.subarray(0, splitAt));
+ controller.enqueue(bytes.subarray(splitAt));
+ controller.close();
+ },
+ }),
+ };
+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });
+ assert.ok(agent?.model);
+ const result = streamText({ model: agent.model, prompt: "say OK" });
+ assert.equal(await result.text, "OK");
+});