Skip to content

A personal agent with a browser, terminal, files, and work that keeps going built with CopilotKit and AG-UI.

TypeScript

fix: make file-producing task retries idempotent (#45)

Charan Rathore01236edc434f
diff --git a/apps/server/src/engine/model.ts b/apps/server/src/engine/model.ts
index 5688760..f0e1828 100644
--- a/apps/server/src/engine/model.ts
+++ b/apps/server/src/engine/model.ts
@@ -150,8 +150,15 @@ export async function executeModelTask(
}),
async (args) =>
cached("fill_pdf", args, async () => {
- const file = await service.files.fill(owner, args.fileId, args.fields);
- task = await ctx.checkpoint({ artifactIds: [...task.artifactIds, file.id] });
+ const file = await service.files.fill(
+ owner,
+ args.fileId,
+ args.fields,
+ `model:${task.id}:fill_pdf`,
+ );
+ task = await ctx.checkpoint({
+ artifactIds: [...new Set([...task.artifactIds, file.id])],
+ });
return { id: file.id, name: file.name, fields: file.fields };
}),
),
diff --git a/apps/server/src/engine/service.ts b/apps/server/src/engine/service.ts
index c7b94c9..fcbe6a7 100644
--- a/apps/server/src/engine/service.ts
+++ b/apps/server/src/engine/service.ts
@@ -986,7 +986,7 @@ export class AgentService {
let filledId = typeof task.state.filledId === "string" ? task.state.filledId : undefined;
if (!filledId) {
await ctx.guard();
- const filled = await this.files.fill(owner, source.fileId, fields);
+ const filled = await this.files.fill(owner, source.fileId, fields, `document:${task.id}`);
filledId = filled.id;
task = await ctx.checkpoint({
state: { ...task.state, source, filledId },
diff --git a/apps/server/src/files.ts b/apps/server/src/files.ts
index 239985e..af41eb5 100644
--- a/apps/server/src/files.ts
+++ b/apps/server/src/files.ts
@@ -1,5 +1,5 @@
-import { randomUUID } from "node:crypto";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { createHash, randomUUID } from "node:crypto";
+import { link, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { join, resolve, sep } from "node:path";
import type { Artifact } from "../../../packages/domain/src/index.ts";
import { fillPdf, inspectPdf } from "../../../packages/integrations/src/pdf.ts";
@@ -23,6 +23,21 @@ function assertFileId(id: string): void {
if (!FILE_ID.test(id)) throw new AppError("File not found", 404);
}
+// Keyed operations need the same ID on every retry, so the ID is derived from the
+// owner and operation key instead of random. It is shaped as a version 5 UUID
+// (first 128 bits of a SHA-256 hash, version and variant bits set) so it is
+// accepted anywhere a random file UUID is.
+export function deterministicFileId(owner: string, operationKey: string): string {
+ const bytes = createHash("sha256")
+ .update(JSON.stringify([owner, operationKey]))
+ .digest()
+ .subarray(0, 16);
+ bytes[6] = (bytes[6] & 0x0f) | 0x50;
+ bytes[8] = (bytes[8] & 0x3f) | 0x80;
+ const hex = bytes.toString("hex");
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+}
+
export class Files {
constructor(
private readonly db: Store,
@@ -35,11 +50,36 @@ export class Files {
bytes: Uint8Array,
source: string,
parentId?: string,
+ operationKey?: string,
): Promise<Artifact> {
+ const id = operationKey === undefined ? randomUUID() : deterministicFileId(owner, operationKey);
+ if (operationKey !== undefined) {
+ const existing = await this.db.get<Artifact>(owner, "files", id);
+ if (existing) return this.signed(owner, existing);
+ }
if (bytes.length > 10 * 1024 * 1024) throw new AppError("PDFs must be 10 MB or smaller", 413);
- const metadata = await inspectPdf(bytes);
+ let metadata = await inspectPdf(bytes);
if (metadata.pageCount > 500) throw new AppError("PDFs must have 500 pages or fewer", 422);
- const id = randomUUID();
+ const directory = join(this.config.dataDir, "files");
+ await mkdir(directory, { recursive: true, mode: 0o700 });
+ const path = this.pathFor(id);
+ if (operationKey === undefined) {
+ await writeFile(path, bytes, { mode: 0o600, flag: "wx" });
+ } else {
+ const temporary = `${path}.${randomUUID()}.tmp`;
+ try {
+ await writeFile(temporary, bytes, { mode: 0o600, flag: "wx" });
+ try {
+ await link(temporary, path);
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+ bytes = await readFile(path);
+ metadata = await inspectPdf(bytes);
+ }
+ } finally {
+ await rm(temporary, { force: true });
+ }
+ }
const safeName = sanitizeFileName(name);
const artifact: Artifact = {
id,
@@ -53,11 +93,12 @@ export class Files {
source,
parentId,
};
- const directory = join(this.config.dataDir, "files");
- await mkdir(directory, { recursive: true, mode: 0o700 });
- await writeFile(join(directory, `${id}.pdf`), bytes, { mode: 0o600, flag: "wx" });
- await this.db.put(owner, "files", artifact);
- return this.signed(owner, artifact);
+ if (operationKey === undefined) {
+ await this.db.put(owner, "files", artifact);
+ return this.signed(owner, artifact);
+ }
+ const persisted = await this.db.insertIfAbsent(owner, "files", artifact);
+ return this.signed(owner, persisted ?? (await this.get(owner, id)));
}
signed(owner: string, file: Artifact): Artifact {
return { ...file, url: this.auth.sign(owner, `/api/files/${file.id}/content`) };
@@ -82,7 +123,12 @@ export class Files {
await this.get(owner, id);
return readFile(this.pathFor(id));
}
- async fill(owner: string, id: string, values: Record<string, string | boolean>) {
+ async fill(
+ owner: string,
+ id: string,
+ values: Record<string, string | boolean>,
+ operationKey?: string,
+ ) {
const file = await this.get(owner, id);
const bytes = await this.bytes(owner, id);
const output = await fillPdf(bytes, values);
@@ -92,6 +138,16 @@ export class Files {
output,
`Filled from ${file.name}`,
id,
+ operationKey === undefined
+ ? undefined
+ : JSON.stringify([
+ "fill",
+ operationKey,
+ id,
+ Object.keys(values)
+ .sort()
+ .map((name) => [name, values[name]]),
+ ]),
);
}
}
diff --git a/apps/server/src/workspace.ts b/apps/server/src/workspace.ts
index 5609120..d13140f 100644
--- a/apps/server/src/workspace.ts
+++ b/apps/server/src/workspace.ts
@@ -474,6 +474,8 @@ export class WorkspaceService {
decodeURIComponent(filename),
await this.google(owner, connection.id).getAttachment(messageId, attachmentId),
`Gmail · ${message.subject}`,
+ undefined,
+ JSON.stringify(["attachment", connection.id, reference]),
);
await this.db.put(owner, "imports", {
id: reference,
diff --git a/tests/api.test.ts b/tests/api.test.ts
index d26bc14..6ea9420 100644
--- a/tests/api.test.ts
+++ b/tests/api.test.ts
@@ -6,11 +6,14 @@ import { after, before, test } from "node:test";
import { EventType } from "@ag-ui/core";
import { lastValueFrom, toArray } from "rxjs";
import { createApp } from "../apps/server/src/app.ts";
+import { Auth } from "../apps/server/src/auth.ts";
import type { Config } from "../apps/server/src/config.ts";
import { createStore, type Store } from "../apps/server/src/db.ts";
import { ConversationAgent } from "../apps/server/src/engine/conversation.ts";
import type { AgentService } from "../apps/server/src/engine/service.ts";
+import { deterministicFileId, Files } from "../apps/server/src/files.ts";
import type { ActionProposal, Artifact, Workspace } from "../packages/domain/src/index.ts";
+import { createSamplePdf } from "../packages/integrations/src/pdf.ts";
let db: Store,
app: Awaited<ReturnType<typeof createApp>>["app"],
@@ -58,6 +61,127 @@ test("API protects private data and rejects unrelated web origins", async () =>
403,
);
});
+test("file operations recover a published PDF when metadata persistence fails", async (t) => {
+ const owner = "file-recovery";
+ const bytes = await createSamplePdf();
+ const insert = db.insertIfAbsent.bind(db);
+ let publishedId = "";
+ t.mock.method(db, "insertIfAbsent", async (...args: Parameters<Store["insertIfAbsent"]>) => {
+ if (args[0] === owner && args[1] === "files" && !publishedId) {
+ publishedId = args[2].id;
+ throw new Error("metadata unavailable");
+ }
+ return insert(...args);
+ });
+ await assert.rejects(
+ agent.files.import(owner, "sample.pdf", bytes, "fixture", undefined, "import-once"),
+ /metadata unavailable/,
+ );
+ assert.ok(publishedId);
+ assert.equal((await agent.files.list(owner)).length, 0);
+ const restarted = new Files(db, config, new Auth(db, config, "sample-signing-key"));
+ const recovered = await restarted.import(
+ owner,
+ "sample.pdf",
+ bytes,
+ "fixture",
+ undefined,
+ "import-once",
+ );
+ assert.equal(recovered.id, publishedId);
+ assert.deepEqual(new Uint8Array(await restarted.bytes(owner, recovered.id)), bytes);
+ assert.equal((await restarted.list(owner)).length, 1);
+});
+
+test("file operation identity isolates owners, inputs and independent requests", async () => {
+ const owner = "file-identities";
+ const bytes = await createSamplePdf();
+ const source = await agent.files.import(owner, "sample.pdf", bytes, "fixture");
+ const fields = { participant_name: "Sample Student", permission_granted: true };
+ const [first, concurrent] = await Promise.all([
+ agent.files.fill(owner, source.id, fields, "task-fill"),
+ agent.files.fill(owner, source.id, fields, "task-fill"),
+ ]);
+ assert.equal(first.id, concurrent.id);
+ assert.equal(first.createdAt, concurrent.createdAt);
+ assert.equal((await agent.files.list(owner)).filter((file) => file.parentId).length, 1);
+ const reordered = await agent.files.fill(
+ owner,
+ source.id,
+ {
+ permission_granted: true,
+ participant_name: "Sample Student",
+ },
+ "task-fill",
+ );
+ assert.equal(reordered.id, first.id);
+ assert.notEqual((await agent.files.fill(owner, source.id, fields, "another-task")).id, first.id);
+ assert.notEqual(
+ (await agent.files.fill(owner, source.id, { participant_name: "Another Student" }, "task-fill"))
+ .id,
+ first.id,
+ );
+ const manual = await agent.files.fill(owner, source.id, fields);
+ assert.notEqual((await agent.files.fill(owner, source.id, fields)).id, manual.id);
+ const otherOwner = await agent.files.import(
+ "other-file-owner",
+ "sample.pdf",
+ bytes,
+ "fixture",
+ undefined,
+ "owner-scoped",
+ );
+ assert.notEqual(
+ (await agent.files.import(owner, "sample.pdf", bytes, "fixture", undefined, "owner-scoped")).id,
+ otherOwner.id,
+ );
+});
+
+test("keyed file IDs are UUIDs that work through lookup, content reads and attachments", async () => {
+ const owner = "local-user";
+ const sample = await createSamplePdf();
+ const source = await agent.files.import(owner, "form.pdf", sample, "test");
+ const fields = { participant_name: "Sample Student", permission_granted: true };
+ const filled = await agent.files.fill(owner, source.id, fields, "task-1:fill_pdf");
+ const imported = await agent.files.import(
+ owner,
+ "a.pdf",
+ sample,
+ "test",
+ undefined,
+ "attachment:1",
+ );
+ assert.equal((await agent.files.fill(owner, source.id, fields, "task-1:fill_pdf")).id, filled.id);
+ for (const id of [filled.id, imported.id]) {
+ assert.match(id, /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/);
+ assert.equal((await agent.files.get(owner, id)).id, id);
+ const content = await app.request(`/api/files/${id}/content`, { headers: headers() });
+ assert.equal(content.status, 200);
+ assert.equal(
+ Buffer.from(await content.arrayBuffer())
+ .subarray(0, 5)
+ .toString(),
+ "%PDF-",
+ );
+ const proposal = await app.request("/api/actions", {
+ method: "POST",
+ headers: headers(),
+ body: JSON.stringify({
+ kind: "email.send",
+ data: {
+ subject: "Attached",
+ body: "See attached.",
+ attachmentIds: [id],
+ },
+ }),
+ });
+ assert.equal(proposal.status, 201);
+ }
+ assert.equal(deterministicFileId(owner, "x"), deterministicFileId(owner, "x"));
+ assert.notEqual(deterministicFileId(owner, "x"), deterministicFileId("other", "x"));
+});
+
test("sample workspace serves a real PDF and filling creates a new version", async () => {
const response = await app.request("/api/workspace", { headers: headers() });
assert.equal(response.status, 200);
diff --git a/tests/model-worker.test.ts b/tests/model-worker.test.ts
index cbaa4b8..dd5419f 100644
--- a/tests/model-worker.test.ts
+++ b/tests/model-worker.test.ts
@@ -6,13 +6,96 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "node:test";
import { createApp } from "../apps/server/src/app.ts";
-import { createStore } from "../apps/server/src/db.ts";
+import { createStore, type Store } from "../apps/server/src/db.ts";
import { createDemoModel, demoModel } from "../apps/server/src/demo/model.ts";
+import type { AgentTask } from "../packages/domain/src/agent.ts";
import type { ActionProposal } from "../packages/domain/src/index.ts";
import { browserFixture } from "./helpers/browser.ts";
import { fixture as computerFixture } from "./helpers/computer.ts";
import { modelFixture } from "./helpers/model.ts";
+for (const checkpoint of ["artifact", "cache"] as const) {
+ test(`model fill reuses its PDF after a failed ${checkpoint} checkpoint`, async (t) => {
+ const directory = await mkdtemp(join(tmpdir(), "openmuse-model-fill-"));
+ const db = await createStore();
+ const calls: { name: string; arguments: object }[] = [];
+ const { requests } = await modelFixture(t, (index) => calls[index]);
+ const server = await createApp(db, {
+ mode: "sample",
+ port: 8787,
+ host: "127.0.0.1",
+ publicUrl: "http://localhost:8787",
+ dataDir: directory,
+ agentBackend: "model",
+ intelligenceApiKey: "test-project-key-never-sent",
+ model: "openai/fixture",
+ googleRedirectUri: "http://localhost:8787/api/google/callback",
+ allowedOrigins: [],
+ });
+ try {
+ const owner = "model-fill-replay";
+ await server.workspace.ensureSample(owner, server.actions);
+ const source = (await server.files.list(owner))[0];
+ const fill = {
+ name: "fill_pdf",
+ arguments: { fileId: source.id, fields: { participant_name: "Sample Student" } },
+ };
+ calls.push(fill, {
+ name: "ask_user",
+ arguments: { question: "Retry the interrupted operation?" },
+ });
+ const task = await server.agent.createTask(owner, { prompt: "Fill the sample form" });
+ const compareAndSwap = db.compareAndSwap.bind(db);
+ let interrupted = false;
+ t.mock.method(db, "compareAndSwap", async (...args: Parameters<Store["compareAndSwap"]>) => {
+ const patch = args[4] as Partial<AgentTask>;
+ if (
+ args[0] === owner &&
+ args[1] === "tasks" &&
+ args[2] === task.id &&
+ (checkpoint === "artifact" ? patch.artifactIds?.length : patch.state?.operations) &&
+ !interrupted
+ ) {
+ interrupted = true;
+ return null;
+ }
+ return compareAndSwap(...args);
+ });
+ await server.agent.worker.tick();
+ assert.ok(interrupted);
+ const waiting = await server.agent.getTask(owner, task.id);
+ assert.equal(waiting.status, "waiting_input", waiting.error ?? waiting.question);
+ const outputs = (await server.files.list(owner)).filter(
+ (file) => file.parentId === source.id,
+ );
+ assert.equal(outputs.length, 1);
+ assert.deepEqual(waiting.artifactIds, checkpoint === "artifact" ? [] : [outputs[0].id]);
+ assert.equal(waiting.state.operations, undefined);
+ requests.length = 0;
+ calls.splice(0, calls.length, fill, {
+ name: "finish_task",
+ arguments: { summary: "Saved the sample form." },
+ });
+ await server.agent.answer(owner, task.id, "Retry");
+ await server.agent.worker.tick();
+ const completed = await server.agent.getTask(owner, task.id);
+ assert.equal(completed.status, "succeeded", completed.error ?? completed.question);
+ assert.deepEqual(
+ completed.artifactIds.filter((id) => id === outputs[0].id),
+ [outputs[0].id],
+ );
+ assert.equal(
+ (await server.files.list(owner)).filter((file) => file.parentId === source.id).length,
+ 1,
+ );
+ } finally {
+ await server.agent.stop();
+ await db.close();
+ await rm(directory, { recursive: true, force: true });
+ }
+ });
+}
+
test("CopilotKit model worker executes server tools and persists the confirmed outcome", async (t) => {
const directory = await mkdtemp(join(tmpdir(), "openmuse-model-"));
const db = await createStore();
diff --git a/tests/workflows.test.ts b/tests/workflows.test.ts
index d37214c..99d523f 100644
--- a/tests/workflows.test.ts
+++ b/tests/workflows.test.ts
@@ -6,7 +6,8 @@ import { after, before, test } from "node:test";
import { createApp } from "../apps/server/src/app.ts";
import { createStore, type Store } from "../apps/server/src/db.ts";
import type { AgentNotification, AgentTask, Idea, Monitor } from "../packages/domain/src/agent.ts";
-import type { ActionProposal } from "../packages/domain/src/index.ts";
+import type { ActionProposal, Artifact } from "../packages/domain/src/index.ts";
+import { createSamplePdf } from "../packages/integrations/src/pdf.ts";
let db: Store, server: Awaited<ReturnType<typeof createApp>>, directory: string;
const owner = "workflow-user";
@@ -98,6 +99,100 @@ test("document job runs without a client, waits for review, and resumes from its
);
});
+test("document retry reuses the filled PDF after its task checkpoint is lost", async (t) => {
+ const replayOwner = "interrupted-document";
+ await server.workspace.ensureSample(replayOwner, server.actions);
+ const workspace = await server.workspace.snapshot(replayOwner);
+ const mail = workspace.mail.find((message) => message.attachments.length);
+ assert.ok(mail);
+ const task = await server.agent.createTask(replayOwner, {
+ prompt: "Fill the sample form",
+ kind: "document",
+ input: { messageId: mail.id, fields: { participant_name: "Sample Student" } },
+ });
+ const compareAndSwap = db.compareAndSwap.bind(db);
+ let interrupted = false;
+ t.mock.method(db, "compareAndSwap", async (...args: Parameters<Store["compareAndSwap"]>) => {
+ const [recordOwner, kind, id, , patch] = args;
+ if (
+ recordOwner === replayOwner &&
+ kind === "tasks" &&
+ id === task.id &&
+ (patch.state as AgentTask["state"] | undefined)?.filledId &&
+ !interrupted
+ ) {
+ interrupted = true;
+ return null;
+ }
+ return compareAndSwap(...args);
+ });
+ await server.agent.worker.tick();
+ assert.ok(interrupted);
+ const retry = await server.agent.getTask(replayOwner, task.id);
+ assert.equal(retry.status, "queued");
+ assert.equal(retry.state.filledId, undefined);
+ const outputs = (await db.list<Artifact>(replayOwner, "files")).filter((file) => file.parentId);
+ assert.equal(outputs.length, 1);
+
+ await server.agent.worker.tick();
+ const resumed = await server.agent.getTask(replayOwner, task.id);
+ assert.equal(resumed.status, "waiting_approval", resumed.error ?? resumed.question);
+ assert.equal(resumed.state.filledId, outputs[0].id);
+ assert.equal(
+ (await db.list<Artifact>(replayOwner, "files")).filter((file) => file.parentId).length,
+ 1,
+ );
+});
+
+test("attachment import recovers after losing its mapping and isolates reconnections", async (t) => {
+ const attachmentOwner = "attachment-recovery";
+ await server.workspace.ensureSample(attachmentOwner, server.actions);
+ const workspace = await server.workspace.snapshot(attachmentOwner);
+ const mail = workspace.mail[0];
+ const reference = `${mail.id}:sample-attachment:sample.pdf`;
+ await db.put(attachmentOwner, "mail", {
+ ...mail,
+ attachments: [reference],
+ connectionId: "sample-google",
+ });
+ const google = server.workspace.google(attachmentOwner);
+ const bytes = await createSamplePdf();
+ t.mock.method(google, "getAttachment", async () => bytes);
+ t.mock.method(server.workspace, "google", () => google);
+ const put = db.put.bind(db);
+ let interrupted = false;
+ t.mock.method(db, "put", async (...args: Parameters<Store["put"]>) => {
+ if (args[0] === attachmentOwner && args[1] === "imports" && !interrupted) {
+ interrupted = true;
+ throw new Error("mapping unavailable");
+ }
+ return put(...args);
+ });
+ const before = await server.files.list(attachmentOwner);
+ await assert.rejects(
+ server.workspace.importAttachment(attachmentOwner, reference),
+ /mapping unavailable/,
+ );
+ const published = (await server.files.list(attachmentOwner)).find(
+ (file) => !before.some((old) => old.id === file.id),
+ );
+ assert.ok(published);
+ const recovered = await server.workspace.importAttachment(attachmentOwner, reference);
+ assert.equal(recovered.id, published.id);
+ assert.equal((await server.files.list(attachmentOwner)).length, before.length + 1);
+
+ await db.put(attachmentOwner, "settings", { id: "google", connectionId: "new-connection" });
+ await db.put(attachmentOwner, "mail", {
+ ...mail,
+ attachments: [reference],
+ connectionId: "new-connection",
+ });
+ assert.notEqual(
+ (await server.workspace.importAttachment(attachmentOwner, reference)).id,
+ recovered.id,
+ );
+});
+
test("ideas ignore sent replies while retaining unfinished incoming requests", async () => {
const ideaOwner = "sent-reply-ideas";
await server.workspace.ensureSample(ideaOwner, server.actions);