The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
// Opt-in provider qualification, separate from the cheap default suite:
// PAPERCLIP_RECOVERY_LIVE=1 DAYTONA_API_KEY=... PAPERCLIP_DAYTONA_SDK_PATH=...
// PAPERCLIP_RECOVERY_IMAGE=ghcr.io/...@sha256:... vitest run --config vitest.config.ts scripts/workspace-recovery-provider.test.mjs
import { randomUUID, createHash } from "node:crypto";
import { spawn } from "node:child_process";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { eq } from "drizzle-orm";
import { expect, it, vi } from "vitest";
import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, issueRecoveryActions,
issues, workspaceOperations } from "@paperclipai/db";
import { NOOP_PLUGIN_TRACER } from "@paperclipai/plugin-sdk";
import { startEmbeddedPostgresTestDatabase } from "../src/__tests__/helpers/embedded-postgres.js";
import { readSystemRecovery } from "../src/services/system-recovery-state.js";
import { recoverRetainedWorkspace } from "../src/services/workspace-recovery.js";
import { classifyEnvironmentCapabilities, environmentRuntimeService } from "../src/services/environment-runtime.js";
import { attachPreservedWorkspaceSnapshots } from "../src/services/workspace-recovery-snapshot.js";
import { performSyncOut } from "../../packages/plugins/sandbox-providers/daytona/src/file-sync.js";
const boundary = vi.hoisted(() => ({ target: null }));
vi.mock("../src/services/environment-execution-target.js", () => ({
resolveEnvironmentExecutionTarget: async () => boundary.target,
}));
vi.mock("../../packages/plugins/sandbox-providers/daytona/src/plugin.js", () => ({
getPluginTracer: () => NOOP_PLUGIN_TRACER,
}));
const live = process.env.PAPERCLIP_RECOVERY_LIVE === "1";
const worker = process.env.PAPERCLIP_RECOVERY_WORKER;
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const quote = value => `'${value.replaceAll("'", "'\\''")}'`;
async function client() {
const modulePath = process.env.PAPERCLIP_DAYTONA_SDK_PATH;
if (!modulePath || !process.env.DAYTONA_API_KEY) throw new Error("Explicit Daytona qualification settings are required");
const { Daytona } = await import(/* @vite-ignore */ pathToFileURL(modulePath).href);
return new Daytona({ apiKey: process.env.DAYTONA_API_KEY, requestTimeoutMs: 120_000 });
}
it.skipIf(!live || !worker)("provider controller", async () => {
const f = JSON.parse(await readFile(process.env.PAPERCLIP_RECOVERY_FIXTURE, "utf8"));
process.env.PAPERCLIP_STORAGE_LOCAL_DIR = path.join(f.directory, "storage");
const db = createDb(f.connectionString);
try {
const sandbox = await (await client()).get(f.providerLeaseId);
const download = sandbox.fs.downloadFiles.bind(sandbox.fs);
sandbox.fs.downloadFiles = async (...args) => {
const result = await download(...args);
if (worker === "interrupt") {
// Interrupt after the real provider transfer and before host extraction.
// The controller joins its work and stops the export-only sandbox before
// exiting; an unjoined hard crash is separately fenced by ownership tests.
await writeFile(path.join(f.directory, "interrupted"), "archive received; extraction interrupted");
throw new Error("qualification_export_interrupted");
}
return result;
};
boundary.target = { kind: "remote", transport: "sandbox", providerKey: "daytona", leaseId: f.leaseId,
remoteCwd: f.remoteCwd, timeoutMs: 120_000,
runner: {
syncIn: async () => { throw new Error("Recovery must not upload into the retained source"); },
execute: async input => {
const startedAt = new Date().toISOString();
const result = await sandbox.process.executeCommand([input.command, ...(input.args ?? [])].map(quote).join(" "),
input.cwd, input.env, 120);
return { exitCode: result.exitCode, stdout: result.result, stderr: "", timedOut: false,
signal: null, pid: null, startedAt };
},
syncOut: operations => performSyncOut({ sandbox, operations, remoteDir: f.remoteCwd, timeoutSeconds: 120 }),
} };
const driver = {
driver: "sandbox",
acquireRunLease: async () => { throw new Error("Recovery must not acquire another sandbox"); },
releaseRunLease: async () => { throw new Error("Recovery must not destroy its retained source"); },
resolveCapabilities: async () => classifyEnvironmentCapabilities({ verifiedMethods: [] }),
resumeRunLease: async () => {
await sandbox.start(120); await sandbox.refreshData();
if (sandbox.state !== "started") throw new Error("Provider did not confirm start");
return { providerLeaseId: sandbox.id, metadata: { remoteCwd: f.remoteCwd } };
},
retryPendingSandboxTeardown: async () => {
await sandbox.stop(120); await sandbox.refreshData();
if (sandbox.state !== "stopped") throw new Error("Provider did not confirm stop");
return { providerLeaseId: sandbox.id, state: "stopped" };
},
};
const environmentRuntime = environmentRuntimeService(db, { drivers: [driver] });
await recoverRetainedWorkspace({ db, ...f, environmentRuntime });
const [action] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId));
const state = readSystemRecovery(action.evidence);
expect(state?.state).toBe(worker === "interrupt" ? "scheduled" : "complete");
await writeFile(path.join(f.directory, `${worker}.json`), JSON.stringify(state));
} finally { await db.$client.end(); }
}, 300_000);
it.skipIf(!live || Boolean(worker))("preserves provider files across an interrupted export and controller restart without restarting the old agent", async () => {
const image = process.env.PAPERCLIP_RECOVERY_IMAGE;
const snapshotId = process.env.PAPERCLIP_RECOVERY_SNAPSHOT_ID;
if (!image?.includes("@sha256:") && !/^[0-9a-f-]{36}$/i.test(snapshotId ?? ""))
throw new Error("An immutable qualification image or snapshot ID is required");
const directory = await mkdtemp(path.join(os.tmpdir(), "paperclip-provider-recovery-"));
const priorHome = process.env.PAPERCLIP_HOME;
process.env.PAPERCLIP_HOME = path.join(directory, "home");
const priorStorage = process.env.PAPERCLIP_STORAGE_LOCAL_DIR;
process.env.PAPERCLIP_STORAGE_LOCAL_DIR = path.join(directory, "storage");
const database = await startEmbeddedPostgresTestDatabase("provider-recovery-");
const db = createDb(database.connectionString);
const daytona = await client();
let sandbox;
const evidence = { image, snapshotId, interrupted: false, restored: false, oldAgentStarts: null, externalOperations: null,
prematureCleanup: false, controllerProcesses: [], snapshotCount: 0 };
try {
if (snapshotId) {
const snapshot = await daytona.snapshot.get(snapshotId);
expect(snapshot.id).toBe(snapshotId);
expect(snapshot.state).toBe("active");
}
sandbox = await daytona.create({ ...(snapshotId ? { snapshot: snapshotId } : { image, resources: { cpu: 2, memory: 4, disk: 10 } }),
autoStopInterval: 10, autoArchiveInterval: 60, labels: { purpose: "paperclip-recovery-qualification" } }, { timeout: 300 });
const remoteCwd = "/tmp/paperclip-recovery-fixture";
await sandbox.process.executeCommand(`mkdir -p ${remoteCwd}`, undefined, undefined, 30);
await sandbox.process.createSession("old-agent");
await sandbox.process.executeSessionCommand("old-agent", {
command: `sh -c ${quote(`echo start >> ${remoteCwd}/agent-starts; echo published >> ${remoteCwd}/external-operations; printf 'saved work\\n' > ${remoteCwd}/saved.txt; while :; do sleep 1; done`)}`,
runAsync: true,
});
await expect.poll(async () => (await sandbox.process.executeCommand(`cat ${remoteCwd}/saved.txt`, undefined, undefined, 30)).result)
.toContain("saved work");
const companyId = randomUUID(), issueId = randomUUID(), runId = randomUUID(), actionId = randomUUID(), agentId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Disposable provider qualification", issuePrefix: "LIVE" });
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
await db.insert(issues).values({ id: issueId, companyId, title: "Disposable preservation fixture", status: "blocked", assigneeAgentId: agentId });
await db.insert(heartbeatRuns).values({ id: runId, companyId, issueId, agentId, status: "failed", runtimeMode: "legacy",
contextSnapshot: { issueId }, finishedAt: new Date() });
const [environment] = await db.insert(environments).values({ name: runId, driver: "sandbox" }).returning();
const [lease] = await db.insert(environmentLeases).values({ companyId, issueId, heartbeatRunId: runId,
environmentId: environment.id, provider: "daytona", providerLeaseId: sandbox.id,
leasePolicy: "retain_on_failure", status: "released", releasedAt: new Date(),
metadata: { remoteCwd, driver: "sandbox", pluginId: randomUUID(), sandboxProviderPlugin: true,
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", runId } } }).returning();
const recovery = { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [lease.id] };
await db.update(heartbeatRuns).set({ resultJson: { workspaceRestoreRecovery: recovery, workspaceRestoreFailure: "restore_failed" } })
.where(eq(heartbeatRuns.id, runId));
await db.insert(issueRecoveryActions).values({ id: actionId, companyId, sourceIssueId: issueId,
kind: "active_run_watchdog", ownerType: "system", fingerprint: runId, nextAction: "",
cause: "legacy_execution_requires_reconciliation", evidence: { runId, workspaceRestoreRecovery: recovery } });
const fixture = path.join(directory, "fixture.json");
await writeFile(fixture, JSON.stringify({ connectionString: database.connectionString, companyId, issueId, runId, actionId,
leaseId: lease.id, providerLeaseId: sandbox.id, remoteCwd, directory }), { mode: 0o600 });
async function runController(phase) {
const child = spawn(process.execPath, [path.join(root, "node_modules/vitest/vitest.mjs"), "run", "--config", "vitest.config.ts",
"scripts/workspace-recovery-provider.test.mjs", "-t", "provider controller"], {
cwd: path.join(root, "server"), env: { ...process.env, PAPERCLIP_RECOVERY_WORKER: phase, PAPERCLIP_RECOVERY_FIXTURE: fixture },
stdio: ["ignore", "pipe", "pipe"],
});
evidence.controllerProcesses.push(child.pid);
// Keep provider/auth diagnostics off the terminal. Only the bounded,
// scrubbed verdict is published by the parent.
let output = "";
child.stdout.on("data", data => { output += data; });
child.stderr.on("data", data => { output += data; });
const code = await new Promise(resolve => child.on("exit", resolve));
if (code !== 0) {
await writeFile(path.join(directory, `${phase}.log`), output, { mode: 0o600 });
throw new Error(`Provider controller ${phase} failed; protected log: ${directory}/${phase}.log`);
}
}
await runController("interrupt");
evidence.interrupted = Boolean(await readFile(path.join(directory, "interrupted"), "utf8"));
const failed = JSON.parse(await readFile(path.join(directory, "interrupt.json"), "utf8"));
expect(failed.attempts).toBe(1);
// Honor the production persisted backoff; the probe never replenishes it.
if (failed.nextAttemptAt > Date.now()) await new Promise(resolve => setTimeout(resolve, failed.nextAttemptAt - Date.now() + 10));
await runController("resume");
const finished = JSON.parse(await readFile(path.join(directory, "resume.json"), "utf8"));
expect(finished.attempts).toBe(2);
const fresh = path.join(directory, "fresh"); await mkdir(fresh);
const attached = await attachPreservedWorkspaceSnapshots({ db, companyId, issueId, sourceRunIds: [runId], cwd: fresh });
const files = path.join(attached[0].path, "files");
expect(await readFile(path.join(files, "saved.txt"), "utf8")).toBe("saved work\n");
evidence.oldAgentStarts = (await readFile(path.join(files, "agent-starts"), "utf8")).trim().split("\n").length;
evidence.externalOperations = (await readFile(path.join(files, "external-operations"), "utf8")).trim().split("\n").length;
expect(evidence.oldAgentStarts).toBe(1); expect(evidence.externalOperations).toBe(1);
const operations = await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, runId));
evidence.snapshotCount = operations.filter(op => op.phase === "workspace_recovery_snapshot" && op.status === "succeeded").length;
expect(evidence.snapshotCount).toBe(1);
await sandbox.refreshData(); expect(sandbox.state).toBe("stopped");
const [retained] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id));
expect(retained.providerLeaseId).toBe(sandbox.id); expect(retained.metadata.workspaceRecoveryCleanup).toBeUndefined();
evidence.restored = true;
evidence.savedFileSha256 = createHash("sha256").update(await readFile(path.join(files, "saved.txt"))).digest("hex");
} finally {
// This test owns the disposable source. Production recovery never destroys it.
if (sandbox) await sandbox.delete(120);
if (process.env.PAPERCLIP_RECOVERY_EVIDENCE) await writeFile(process.env.PAPERCLIP_RECOVERY_EVIDENCE, JSON.stringify(evidence, null, 2));
await db.$client.end(); await database.cleanup();
if (priorHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = priorHome;
if (priorStorage === undefined) delete process.env.PAPERCLIP_STORAGE_LOCAL_DIR; else process.env.PAPERCLIP_STORAGE_LOCAL_DIR = priorStorage;
if (evidence.restored) await rm(directory, { recursive: true, force: true });
}
}, 900_000);