Skip to content

The open-source app everyone uses to manage agents at work

Recognize confirmed container loss when resuming Daytona leases

Devin Foley1f3ff75d334d
diff --git a/packages/plugins/sandbox-providers/daytona/README.md b/packages/plugins/sandbox-providers/daytona/README.md
index 618f0df429..a7e0198f8c 100644
--- a/packages/plugins/sandbox-providers/daytona/README.md
+++ b/packages/plugins/sandbox-providers/daytona/README.md
@@ -28,6 +28,7 @@ Notes:
- The current published Daytona SDK package is `@daytonaio/sdk`.
- The driver supports both `snapshot`-based and `image`-based sandbox creation. If both are set, validation rejects the config as ambiguous.
- Reusable leases map to Daytona stop/start semantics. Non-reusable leases are deleted on release. A provider-resolved `target` does not change the identity of an existing sandbox. Release closes the same scoped lease that a later sentinel-verified resume reopens.
+- A sandbox record can survive the loss of its underlying container. Resume treats it as expired only when a fresh provider read confirms the exact missing-container error for that sandbox and marks it unrecoverable. Unknown errors and failed confirmation reads preserve the lease. The host still requires a verified native-runner backup before replacement.
## Local development
diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
index 30066a5aaa..066436c998 100644
--- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts
@@ -1264,6 +1264,132 @@ describe("Daytona sandbox provider plugin", () => {
});
});
+ describe("missing-container resume", () => {
+ const sandboxId = "00000000-0000-4000-8000-000000000001";
+ const missing = `not found: failed to inspect sandbox container ${sandboxId}: Error response from daemon: No such container: ${sandboxId}`;
+ const params = {
+ driverKey: "daytona", companyId: "company-1", environmentId: "env-1", providerLeaseId: sandboxId,
+ config: { apiKey: "host-key", timeoutMs: 300000, livenessTimeoutMs: 100, reuseLease: true },
+ leaseMetadata: { workspaceSentinel: { path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json", token: "sentinel-token" } },
+ };
+ const resume = () => plugin.definition.onEnvironmentResumeLease!(params);
+ function missingSandbox() {
+ return { ...createMockSandbox({ id: sandboxId, state: "error", recoverable: false }), errorReason: missing };
+ }
+ function allowSentinel(sandbox: ReturnType<typeof missingSandbox>) {
+ sandbox.process.executeCommand.mockResolvedValueOnce({ exitCode: 0,
+ result: JSON.stringify({ token: "sentinel-token" }), artifacts: { stdout: JSON.stringify({ token: "sentinel-token" }) },
+ });
+ }
+
+ it("expires a provider record with a freshly confirmed missing container without replacing it", async () => {
+ const sandbox = missingSandbox();
+ mockGet.mockResolvedValue(sandbox);
+ for (let attempt = 0; attempt < 2; attempt++) {
+ await expect(resume()).resolves.toEqual({ providerLeaseId: null, metadata: { expired: true } });
+ }
+ expect(mockGet).toHaveBeenCalledTimes(2);
+ expect(sandbox.refreshData).toHaveBeenCalledTimes(2);
+ expect(sandbox.start).not.toHaveBeenCalled();
+ expect(sandbox.recover).not.toHaveBeenCalled();
+ expect(sandbox.delete).not.toHaveBeenCalled();
+ expect(mockCreate).not.toHaveBeenCalled();
+ });
+
+ it.each(["sandbox-opaque", "sandbox.with+[literal](characters)"])("matches the opaque sandbox ID literally: %s", async (id) => {
+ const sandbox = { ...missingSandbox(), id, errorReason: missing.replaceAll(sandboxId, id) };
+ mockGet.mockResolvedValue(sandbox);
+ await expect(plugin.definition.onEnvironmentResumeLease!({ ...params, providerLeaseId: id }))
+ .resolves.toEqual({ providerLeaseId: null, metadata: { expired: true } });
+ expect(sandbox.refreshData).toHaveBeenCalledOnce();
+ expect(sandbox.delete).not.toHaveBeenCalled();
+ expect(mockCreate).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ "not found: provider temporarily unavailable",
+ missing.replace("No such container:", "No such volume:"),
+ missing.replace(/000000000001$/, "000000000002"),
+ missing.replaceAll(sandboxId, "00000000-0000-4000-8000-000000000002"),
+ ])("preserves an unexplained unrecoverable error: %s", async (errorReason) => {
+ const sandbox = { ...missingSandbox(), errorReason };
+ mockGet.mockResolvedValue(sandbox);
+ await expect(resume()).rejects.toThrow("unrecoverable error state");
+ expect(sandbox.delete).not.toHaveBeenCalled();
+ expect(mockCreate).not.toHaveBeenCalled();
+ });
+
+ it("uses provider recovery when the sandbox is recoverable", async () => {
+ const sandbox = { ...missingSandbox(), recoverable: true };
+ allowSentinel(sandbox);
+ mockGet.mockResolvedValue(sandbox);
+ await expect(resume()).resolves.toMatchObject({ providerLeaseId: sandboxId });
+ expect(sandbox.recover).toHaveBeenCalled();
+ expect(sandbox.delete).not.toHaveBeenCalled();
+ });
+
+ it("reuses a sandbox whose fresh state disproves the cached loss", async () => {
+ const sandbox = missingSandbox();
+ sandbox.refreshData.mockImplementation(async () => { sandbox.state = "started"; });
+ allowSentinel(sandbox);
+ mockGet.mockResolvedValue(sandbox);
+ await expect(resume()).resolves.toMatchObject({ providerLeaseId: sandboxId });
+ expect(sandbox.start).not.toHaveBeenCalled();
+ expect(sandbox.delete).not.toHaveBeenCalled();
+ expect(mockCreate).not.toHaveBeenCalled();
+ });
+
+ it.each([new MockDaytonaTimeoutError("timed out"), new Error("provider 503")])("preserves the lease if confirmation fails: %s", async (error) => {
+ const sandbox = missingSandbox();
+ sandbox.refreshData.mockRejectedValue(error);
+ mockGet.mockResolvedValue(sandbox);
+ await expect(resume()).rejects.toThrow(error.message);
+ expect(sandbox.delete).not.toHaveBeenCalled();
+ expect(mockCreate).not.toHaveBeenCalled();
+ });
+
+ it("preserves an unknown error returned by the fresh provider read", async () => {
+ const sandbox = missingSandbox();
+ sandbox.refreshData.mockImplementation(async () => { sandbox.errorReason = "provider unavailable"; });
+ mockGet.mockResolvedValue(sandbox);
+ await expect(resume()).rejects.toThrow("unrecoverable error state");
+ expect(sandbox.delete).not.toHaveBeenCalled();
+ expect(mockCreate).not.toHaveBeenCalled();
+ });
+
+ it("rejects a refreshed handle belonging to a different sandbox", async () => {
+ const sandbox = missingSandbox();
+ sandbox.refreshData.mockImplementation(async () => { sandbox.id = "another-sandbox"; });
+ mockGet.mockResolvedValue(sandbox);
+ await expect(resume()).rejects.toThrow("handle mismatch");
+ expect(sandbox.delete).not.toHaveBeenCalled();
+ expect(mockCreate).not.toHaveBeenCalled();
+ });
+
+ it("accepts a typed not-found result while confirming the missing container", async () => {
+ const sandbox = missingSandbox();
+ sandbox.refreshData.mockRejectedValue(new MockDaytonaNotFoundError("missing"));
+ mockGet.mockResolvedValue(sandbox);
+ await expect(resume()).resolves.toEqual({ providerLeaseId: null, metadata: { expired: true } });
+ expect(sandbox.delete).not.toHaveBeenCalled();
+ expect(mockCreate).not.toHaveBeenCalled();
+ });
+
+ it("bounds a stalled confirmation and preserves the lease", async () => {
+ vi.useFakeTimers();
+ try {
+ const sandbox = missingSandbox();
+ sandbox.refreshData.mockImplementation(() => new Promise(() => {}));
+ mockGet.mockResolvedValue(sandbox);
+ const result = resume().then(() => null, error => error as Error);
+ await vi.advanceTimersByTimeAsync(101);
+ expect((await result)?.message).toContain("sandbox.refreshData");
+ expect(sandbox.delete).not.toHaveBeenCalled();
+ expect(mockCreate).not.toHaveBeenCalled();
+ } finally { vi.useRealTimers(); }
+ });
+ });
+
it("expires a reusable lease when the workspace sentinel does not match", async () => {
process.env.DAYTONA_API_KEY = "host-key";
const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" });
diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts
index 88ae8f3fdb..a053af7e7c 100644
--- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts
+++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts
@@ -502,6 +502,13 @@ async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number): P
await withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds));
}
+function hasMissingSandboxContainer(sandbox: Sandbox): boolean {
+ if (sandbox.state !== "error" || sandbox.recoverable !== false || typeof sandbox.errorReason !== "string") return false;
+ if (typeof sandbox.id !== "string" || !sandbox.id) return false;
+ return sandbox.errorReason ===
+ `not found: failed to inspect sandbox container ${sandbox.id}: Error response from daemon: No such container: ${sandbox.id}`;
+}
+
async function resolveSandboxWorkingDirectory(sandbox: Sandbox): Promise<string> {
const root = (await sandbox.getWorkDir())?.trim()
|| (await sandbox.getUserHomeDir())?.trim()
@@ -2228,6 +2235,26 @@ const plugin = definePlugin({
return { providerLeaseId: null, metadata: { expired: true } };
}
+ // Daytona can retain the API record after losing its container. Confirm
+ // the exact provider report again before allowing the host's existing
+ // backup-guarded replacement path. Unknown errors preserve the lease.
+ if (hasMissingSandboxContainer(sandbox)) {
+ try {
+ await withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData());
+ assertHandleMatchesLease(sandbox, params.providerLeaseId);
+ } catch (error) {
+ evictSandboxHandle(scope);
+ if (error instanceof DaytonaNotFoundError) {
+ return { providerLeaseId: null, metadata: { expired: true } };
+ }
+ throw error;
+ }
+ if (hasMissingSandboxContainer(sandbox)) {
+ evictSandboxHandle(scope);
+ return { providerLeaseId: null, metadata: { expired: true } };
+ }
+ }
+
// A stopped sandbox loses its session shell, so the stored session id is
// stale after a real restart. Clear the id only when the sandbox is not
// already running, and clear it before the restart. A stopped sandbox has