Email for professionals and teams
Stop Workers AI streams from repeating text (#120)
rome-xi6e73e28cae65
diff --git a/src/lib/agent/model.ts b/src/lib/agent/model.tsindex 8709b15..b4e271c 100644--- a/src/lib/agent/model.ts+++ b/src/lib/agent/model.ts@@ -3,11 +3,57 @@ import { createWorkersAI } from "workers-ai-provider";import { getAgentProviderConfig } from "./provider";import { agentTimeContext } from "./time";+// workers-ai-provider 4.0.0 maps both `response` and `choices[0].delta` from the same SSE event, so models that send+// both (e.g. Llama 3.3 70B) repeat their text and tool calls. Drop the native copy until cloudflare/ai#615 is released.+function dropDuplicateNativeFields(line: string) {+ const match = /^(data: ?)(\{.*\})(\r?)$/s.exec(line);+ if (!match) return line;+ try {+ const event = JSON.parse(match[2]);+ const delta = event?.choices?.[0]?.delta;+ if (!delta) return line;+ delete event.response;+ if (delta.tool_calls) delete event.tool_calls;+ return `${match[1]}${JSON.stringify(event)}${match[3]}`;+ } catch {+ return line;+ }+}++function dedupeWorkersAIStream(stream: ReadableStream) {+ const decoder = new TextDecoder();+ const encoder = new TextEncoder();+ let pending = "";+ return stream.pipeThrough(new TransformStream<Uint8Array | string, Uint8Array>({+ transform(chunk, controller) {+ const lines = (pending + (typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true }))).split("\n");+ pending = lines.pop() ?? "";+ for (const line of lines) controller.enqueue(encoder.encode(`${dropDuplicateNativeFields(line)}\n`));+ },+ flush(controller) {+ const rest = pending + decoder.decode();+ if (rest) controller.enqueue(encoder.encode(dropDuplicateNativeFields(rest)));+ },+ }));+}++function withDedupedStreams(binding: Ai): Ai {+ return new Proxy(binding, {+ get(target, prop, receiver) {+ if (prop !== "run") return Reflect.get(target, prop, receiver);+ return async (...args: unknown[]) => {+ const result = await (target.run as (...runArgs: unknown[]) => Promise<unknown>).apply(target, args);+ return result instanceof ReadableStream ? dedupeWorkersAIStream(result) : result;+ };+ },+ });+}+export async function getAgentModel(env: CloudflareEnv, requestedModel?: string | null) {const config = await getAgentProviderConfig(env);const modelId = requestedModel && config.models.includes(requestedModel) ? requestedModel : config.model;const details = { modelId, provider: config.provider === "cloudflare" ? "cloudflare" : config.preset, rates: config.rates[modelId] };- if (config.provider === "cloudflare") return env.AI ? { model: createWorkersAI({ binding: env.AI })(modelId), ...details } : null;+ if (config.provider === "cloudflare") return env.AI ? { model: createWorkersAI({ binding: withDedupedStreams(env.AI) })(modelId), ...details } : null;if (config.baseUrl && config.apiKey && modelId) {const provider = createOpenAICompatible({ name: "mailflare", baseURL: config.baseUrl, apiKey: config.apiKey });return { model: provider.chatModel(modelId), ...details };diff --git a/tests/workers-ai-stream.test.mjs b/tests/workers-ai-stream.test.mjsnew file mode 100644index 0000000..abc5e85--- /dev/null+++ b/tests/workers-ai-stream.test.mjs@@ -0,0 +1,164 @@+import assert from "node:assert/strict";+import { mkdtempSync, rmSync } from "node:fs";+import { dirname, join } from "node:path";+import test, { after } from "node:test";+import { fileURLToPath, pathToFileURL } from "node:url";+import { build } from "esbuild";+import { streamText } from "ai";+import { z } from "zod";++const root = join(dirname(fileURLToPath(import.meta.url)), "..");+const bundleDirectory = mkdtempSync(join(root, "node_modules", "mailflare-workers-ai-stream-bundle-"));+after(() => rmSync(bundleDirectory, { recursive: true, force: true }));++await build({+ stdin: {+ contents: `+ export { getAgentModel } from "./src/lib/agent/model.ts";+ `,+ resolveDir: root,+ sourcefile: "workers-ai-stream-test-entry.ts",+ },+ outfile: join(bundleDirectory, "entry.mjs"),+ bundle: true,+ platform: "node",+ format: "esm",+ target: "node24",+ tsconfig: join(root, "tsconfig.json"),+ packages: "external",+ alias: {+ "next/headers": "next/headers.js",+ "cloudflare:workers": "./server/runtime/cloudflare-workers.ts",+ },+ logLevel: "silent",+});++const { getAgentModel } = await import(pathToFileURL(join(bundleDirectory, "entry.mjs")).href);++function createMockDb() {+ return {+ prepare() {+ return {+ bind() {+ return {+ all: async () => ({ results: [] }),+ raw: async () => [],+ };+ },+ all: async () => ({ results: [] }),+ raw: async () => [],+ };+ },+ };+}++function createSseStream(events) {+ return new ReadableStream({+ start(controller) {+ const encoder = new TextEncoder();+ for (const event of events) {+ controller.enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`));+ }+ controller.enqueue(encoder.encode("data: [DONE]\n\n"));+ controller.close();+ },+ });+}++test("Workers AI dual-format SSE stream produces deduplicated text", async () => {+ const binding = {+ run: async () => createSseStream([+ { response: "OK", choices: [{ index: 0, delta: { content: "OK" } }] },+ { response: ".", choices: [{ index: 0, delta: { content: "." }, finish_reason: "stop" }] },+ ]),+ };+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });+ assert.ok(agent?.model);+ const result = streamText({ model: agent.model, prompt: "say OK" });+ assert.equal(await result.text, "OK.");+});++test("Workers AI native-only stream emits text once", async () => {+ const binding = {+ run: async () => createSseStream([+ { response: "OK" },+ { response: ".", finish_reason: "stop" },+ ]),+ };+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });+ assert.ok(agent?.model);+ const result = streamText({ model: agent.model, prompt: "say OK" });+ assert.equal(await result.text, "OK.");+});++test("Workers AI OpenAI-only stream emits text once", async () => {+ const binding = {+ run: async () => createSseStream([+ { choices: [{ index: 0, delta: { content: "OK" } }] },+ { choices: [{ index: 0, delta: { content: "." }, finish_reason: "stop" }] },+ ]),+ };+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });+ assert.ok(agent?.model);+ const result = streamText({ model: agent.model, prompt: "say OK" });+ assert.equal(await result.text, "OK.");+});++test("Workers AI stream preserves legitimately repeated prose across events", async () => {+ const binding = {+ run: async () => createSseStream([+ { response: "ha", choices: [{ index: 0, delta: { content: "ha" } }] },+ { response: "ha", choices: [{ index: 0, delta: { content: "ha" }, finish_reason: "stop" }] },+ ]),+ };+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });+ assert.ok(agent?.model);+ const result = streamText({ model: agent.model, prompt: "laugh" });+ assert.equal(await result.text, "haha");+});++test("Workers AI dual-format stream emits exactly one tool call with parsed arguments", async () => {+ const toolCall = [{ index: 0, id: "call_lookup", type: "function", function: { name: "lookup", arguments: JSON.stringify({ city: "Hanoi" }) } }];+ const binding = {+ run: async () => createSseStream([+ { tool_calls: toolCall, choices: [{ index: 0, delta: { tool_calls: toolCall } }] },+ ]),+ };+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });+ assert.ok(agent?.model);+ const result = streamText({+ model: agent.model,+ tools: {+ lookup: {+ description: "lookup info",+ parameters: z.object({ city: z.string() }),+ execute: async (args) => args,+ },+ },+ prompt: "lookup Hanoi",+ });+ const calls = await result.toolCalls;+ assert.equal(calls.length, 1);+ assert.equal(calls[0].toolName, "lookup");+ assert.deepEqual(calls[0].input, { city: "Hanoi" });+});++test("Workers AI stream handles events split across chunk boundaries", async () => {+ const payload = `data: ${JSON.stringify({ response: "OK", choices: [{ index: 0, delta: { content: "OK" } }] })}\n\ndata: [DONE]\n\n`;+ const encoder = new TextEncoder();+ const bytes = encoder.encode(payload);+ const splitAt = 12;+ const binding = {+ run: async () => new ReadableStream({+ start(controller) {+ controller.enqueue(bytes.subarray(0, splitAt));+ controller.enqueue(bytes.subarray(splitAt));+ controller.close();+ },+ }),+ };+ const agent = await getAgentModel({ DB: createMockDb(), AI: binding });+ assert.ok(agent?.model);+ const result = streamText({ model: agent.model, prompt: "say OK" });+ assert.equal(await result.text, "OK");+});