The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Permit idle sleep with consumed retries and passive worktrees (#15907)
Devin Foleyd49cbf4c4fc0
diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.mdindex e006680d1a..11b51cb692 100644--- a/doc/idle-sleep-safety.md+++ b/doc/idle-sleep-safety.md@@ -35,6 +35,32 @@ signed bootstrap writes. Only the task-drain read skips request tracking.Its authentication finishes under a separate work token before entering thereport, so it counts concurrent writes without blocking on its own scan.+## Retry history++Retry promotion keeps `scheduled_retry_at` on a run for inspection. That date+is consumed history when the run is terminal, has a finish time at or after the+retry date, and has no accounting work pending. The idle report uses the same+proof for failed ephemeral leases, coalesced wakes, and acknowledged question+responses. A live retry child still blocks through the instance-wide run check.+Unfinished runs, retry dates after the finish time, and pending accounting stay+awake. The inspection does not clear dates or change retry scheduling.++## Retained local workspaces++Local directories for primary projects and Git worktrees do not require an+awake process by themselves. The report permits their retained rows only in+known modes and states, with no runtime configuration, service state, commands,+pending reopen, or cleanup already eligible. Saved environment references alone+are passive. Shared workspaces with a project workspace remain blockers because+they can inherit runtime configuration. Unfamiliar providers, strategies,+configuration and malformed metadata also remain blockers.++This does not delete files, alter retention, or clear workspace records. Future+cleanup eligibility is an earliest allowed time; eligible cleanup can wait for+the next wake. Due cleanup, unfinished operations, runtime leases and runtime+services still keep the instance awake through their separate checks. The host+must keep workspace storage attached across sleep and preserve its backup policy.+## Accepted work and cleanupHTTP admission is installed before body parsing, authentication, webhooks anddiff --git a/server/src/__tests__/idle-sleep-safety.test.ts b/server/src/__tests__/idle-sleep-safety.test.tsindex 4b7d731972..51bbe35191 100644--- a/server/src/__tests__/idle-sleep-safety.test.ts+++ b/server/src/__tests__/idle-sleep-safety.test.ts@@ -6,7 +6,7 @@ import {agentInstructionWorkingCopies, agents, agentApiKeys, agentWakeupRequests, companies, companySecretProposals, createDb,adapterAuthSessions, environments, environmentLeases, executionWorkspaces,heartbeatRuns, issues, issueWatchdogs, issueThreadInteractions, issueQuestionResponseDeliveries,- projects, routines, toolApplications, toolConnections, type Db,+ projects, projectWorkspaces, routines, toolApplications, toolConnections, workspaceOperations, executionWorkspaceRuntimeLeases, type Db,} from "@paperclipai/db";import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";@@ -185,6 +185,37 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${suppreturn { companyId, agentId, runId };}+ it.each(["succeeded", "failed", "cancelled", "timed_out", "interrupted"])(+ "permits a finished %s run with a consumed retry date without changing history", async status => {+ const { runId } = await seed();+ const scheduledRetryAt = new Date(now - 1000);+ await db.update(heartbeatRuns).set({ status, finishedAt: new Date(now), scheduledRetryAt }).where(eq(heartbeatRuns.id, runId));+ expect(await read()).toEqual(none);+ expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))+ .toMatchObject([{ status, scheduledRetryAt }]);+ },+ );++ it.each([+ { status: "scheduled_retry", finishedAt: new Date(now) },+ { status: "queued", finishedAt: new Date(now) },+ { status: "running", finishedAt: new Date(now) },+ { status: "failed", finishedAt: null },+ { status: "failed", finishedAt: new Date(now), costAccountingPending: true },+ { status: "failed", finishedAt: new Date(now - 2000) },+ ])("retains retry work without terminal consumption proof: %j", async change => {+ const { runId } = await seed();+ await db.update(heartbeatRuns).set({ scheduledRetryAt: new Date(now - 1000), ...change }).where(eq(heartbeatRuns.id, runId));+ expect(await read()).toEqual(present);+ });++ it("keeps a live retry child awake after its parent's schedule is consumed", async () => {+ const { companyId, agentId, runId } = await seed();+ await db.update(heartbeatRuns).set({ finishedAt: new Date(now), scheduledRetryAt: new Date(now - 1000) }).where(eq(heartbeatRuns.id, runId));+ await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "on_demand", status: "scheduled_retry", retryOfRunId: runId, scheduledRetryAt: new Date(now + 1000) });+ expect(await read()).toEqual(present);+ });+async function seedQuestionDelivery(change: Partial<typeof issueQuestionResponseDeliveries.$inferInsert> = {}) {const seeded = await seed();await db.update(heartbeatRuns).set({ finishedAt: new Date(now) }).where(eq(heartbeatRuns.id, seeded.runId));@@ -521,6 +552,14 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp.toMatchObject([{ status: "failed", cleanupStatus: "success" }]);});+ it("permits consumed retry history only with exact failed-lease termination proof", async () => {+ const { lease, runId } = await seedFinishedFailedLease();+ await db.update(heartbeatRuns).set({ scheduledRetryAt: new Date(now - 1000) }).where(eq(heartbeatRuns.id, runId));+ expect(await read()).toEqual(none);+ await db.update(environmentLeases).set({ metadata: {} }).where(eq(environmentLeases.id, lease.id));+ expect(await read()).toEqual(present);+ });+it.each(["cleanup_failed", "cleanup_unknown", "unreleased", "retained_policy", "reusable_policy","active", "pending_cleanup", "missing_receipt", "wrong_company", "wrong_run", "wrong_lease",@@ -670,12 +709,92 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${suppexpect(await read()).toEqual(present);});- it("reports retained execution workspaces that may need delayed cleanup", async () => {+ async function seedRetainedWorkspace(change: Partial<typeof executionWorkspaces.$inferInsert> = {}) {const { companyId } = await seed();const [project] = await db.insert(projects).values({ companyId, name: "Fixture project" }).returning();- await db.insert(executionWorkspaces).values({- companyId, projectId: project!.id, name: "Fixture workspace", mode: "isolated",- strategyType: "git_worktree", status: "closed", cleanupEligibleAt: new Date(now + 86_400_000),+ const [workspace] = await db.insert(executionWorkspaces).values({+ companyId, projectId: project!.id, name: "Fixture workspace", mode: "isolated_workspace",+ strategyType: "git_worktree", status: "active", providerType: "git_worktree",+ cwd: "/tmp/fixture-retained-worktree", providerRef: "/tmp/fixture-retained-worktree",+ ...change,+ }).returning();+ return workspace!;+ }++ it.each(["local_fs", "git_worktree"].flatMap(providerType =>+ ["active", "idle", "in_review", "archived"].map(status => ({ providerType, status }))))(+ "permits a passive local directory without removing retained data: %j", async ({ providerType, status }) => {+ const metadata = { config: {+ environmentId: randomUUID(), provisionCommand: null, runtimeProvisionCommand: null,+ teardownCommand: null, cleanupCommand: null, workspaceRuntime: null,+ desiredState: null, serviceStates: null,+ }, retainedFixture: true };+ const strategyType = providerType === "local_fs" ? "project_primary" : "git_worktree";+ const workspace = await seedRetainedWorkspace({ providerType, strategyType, status, metadata, cleanupEligibleAt: new Date(Date.UTC(2099, 0, 1)) });+ expect(await read()).toEqual(none);+ expect(await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspace.id)))+ .toMatchObject([{ status, metadata, cwd: workspace.cwd, cleanupEligibleAt: workspace.cleanupEligibleAt }]);+ },+ );++ it.each([null, {}, { config: null }, { config: {} }, { config: { desiredState: "stopped", serviceStates: {} } }])(+ "permits passive local configuration: %j", async metadata => {+ await seedRetainedWorkspace({ metadata });+ expect(await read()).toEqual(none);+ },+ );++ it.each([+ { providerType: "cloud_sandbox" }, { providerType: "future_provider" },+ { strategyType: "adapter_managed" }, { strategyType: "future_strategy" },+ { mode: "future_mode" },+ { providerRef: "another-resource" }, { status: "cleanup_failed" }, { status: "future_state" },+ { cleanupEligibleAt: new Date(now) }, { metadata: { reopenPendingConsumption: true } },+ { metadata: { reopenPendingConsumption: "unknown" } },+ { metadata: { config: [] } }, { metadata: { config: "malformed" } },+ { metadata: { config: { workspaceRuntime: {} } } },+ { metadata: { config: { workspaceRuntime: { services: [] } } } },+ { metadata: { config: { desiredState: "running" } } },+ { metadata: { config: { desiredState: "manual" } } },+ { metadata: { config: { serviceStates: { web: "stopped" } } } },+ { metadata: { config: { provisionCommand: "echo fixture" } } },+ { metadata: { config: { runtimeProvisionCommand: "echo fixture" } } },+ { metadata: { config: { teardownCommand: "echo fixture" } } },+ { metadata: { config: { cleanupCommand: "echo fixture" } } },+ { metadata: { config: { futureWorker: true } } },+ ])("retains a directory with work or unfamiliar state: %j", async change => {+ await seedRetainedWorkspace(change);+ expect(await read()).toEqual(present);+ });++ it("fails closed for malformed directory metadata", async () => {+ const workspace = await seedRetainedWorkspace();+ await db.execute(sql`UPDATE execution_workspaces SET metadata = '[]'::jsonb WHERE id = ${workspace.id}`);+ expect(await read()).toEqual(present);+ });++ it("retains a shared directory that can inherit project runtime configuration", async () => {+ const workspace = await seedRetainedWorkspace();+ const [projectWorkspace] = await db.insert(projectWorkspaces).values({+ companyId: workspace.companyId, projectId: workspace.projectId, name: "Shared fixture",+ metadata: { runtimeConfig: { workspaceRuntime: { services: [] } } },+ }).returning();+ await db.update(executionWorkspaces).set({ mode: "shared_workspace", projectWorkspaceId: projectWorkspace!.id })+ .where(eq(executionWorkspaces.id, workspace.id));+ expect(await read()).toEqual(present);+ });++ it("retains a local directory with an unfinished operation", async () => {+ const workspace = await seedRetainedWorkspace();+ await db.insert(workspaceOperations).values({ companyId: workspace.companyId, executionWorkspaceId: workspace.id, phase: "cleanup", status: "running" });+ expect(await read()).toEqual(present);+ });++ it("retains a local directory with a runtime owner", async () => {+ const workspace = await seedRetainedWorkspace();+ await db.insert(executionWorkspaceRuntimeLeases).values({+ companyId: workspace.companyId, executionWorkspaceId: workspace.id,+ ownerKey: "fixture-owner", lastAction: "use", expiresAt: new Date(Date.UTC(2099, 0, 1)),});expect(await read()).toEqual(present);});diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.tsindex 4d1a5f6196..cc0ad42665 100644--- a/server/src/services/idle-sleep-safety.ts+++ b/server/src/services/idle-sleep-safety.ts@@ -22,6 +22,16 @@ export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number })pluginIds: string[];}>;+// Retry promotion keeps its due date as history on the run. Only a terminal+// run finished at or after that date has consumed the schedule. Newer retry+// dates, unfinished runs and accounting work still require the instance.+function finishedRun(alias: string): string {+ return `${alias}.status IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')+ AND ${alias}.finished_at IS NOT NULL+ AND (${alias}.scheduled_retry_at IS NULL OR ${alias}.scheduled_retry_at <= ${alias}.finished_at)+ AND ${alias}.cost_accounting_pending = false`;+}+// A failed turn may have successfully torn down its ephemeral allocation. A// cleanup label alone is not provider termination proof. Require the exact// host-owned receipt and finished run, and retain all cleanup/export intents.@@ -43,9 +53,7 @@ function finishedFailedLease(alias: string): string {AND ${receipt}->>'state' IN ('stopped', 'destroyed')AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = ${alias}.heartbeat_run_idAND r.company_id = ${alias}.company_id- AND r.status IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')- AND r.finished_at IS NOT NULL AND r.scheduled_retry_at IS NULL- AND r.cost_accounting_pending = false)`;+ AND ${finishedRun("r")})`;}// These are deliberately conservative. Completed agent runs and workspace@@ -60,7 +68,9 @@ const WORK_CHECKS = [COALESCE((runtime_config->'heartbeat'->>'intervalSec')::numeric, 0) > 0)`,`SELECT 1 FROM heartbeat_runs WHEREstatus NOT IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')- OR scheduled_retry_at IS NOT NULL OR cost_accounting_pending`,+ OR (scheduled_retry_at IS NOT NULL+ AND (finished_at IS NULL OR scheduled_retry_at > finished_at))+ OR cost_accounting_pending`,`SELECT 1 FROM agent_instruction_working_copies WHEREstate NOT IN ('saved', 'unchanged', 'resolved', 'unavailable', 'conflict')OR process_stopped_at IS NULL OR next_attempt_at IS NOT NULL@@ -78,14 +88,41 @@ const WORK_CHECKS = [OR (w.status = 'coalesced' AND (w.finished_at IS NULL OR NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = w.run_idAND r.company_id = w.company_id AND r.agent_id = w.agent_id- AND r.status IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')- AND r.finished_at IS NOT NULL AND r.scheduled_retry_at IS NULL- AND r.cost_accounting_pending = false)))`,+ AND ${finishedRun("r")})))`,`SELECT 1 FROM issues WHERE status NOT IN ('done', 'cancelled')`,// A completed issue can still need its first watchdog review, including a// retry after immediate evaluation failed before creating a review/run.`SELECT 1 FROM issue_watchdogs WHERE status <> 'disabled'`,`SELECT 1 FROM workspace_operations WHERE status NOT IN ('succeeded', 'failed', 'cancelled')`,+ // A local directory has no independent worker. Keep unfamiliar providers,+ // runtime/command configuration, reopen fences and due cleanup awake. Runs,+ // operations, runtime services, retained leases and backup work are checked+ // separately; inspecting the row never deletes or shortens retained data.+ `SELECT 1 FROM execution_workspaces w WHERE (+ w.provider_type IN ('local_fs', 'git_worktree')+ AND w.strategy_type IN ('project_primary', 'git_worktree')+ AND w.mode IN ('inherit', 'shared_workspace', 'isolated_workspace', 'operator_branch', 'reuse_existing', 'agent_default')+ AND w.status IN ('active', 'idle', 'in_review', 'archived')+ AND (w.metadata IS NULL OR jsonb_typeof(w.metadata) = 'object')+ -- A shared workspace can inherit runtime configuration from its project.+ AND (w.mode <> 'shared_workspace' OR w.project_workspace_id IS NULL)+ AND (w.provider_ref IS NULL OR w.provider_ref = w.cwd)+ AND (w.cleanup_eligible_at IS NULL OR w.cleanup_eligible_at > now())+ AND COALESCE(w.metadata->>'reopenPendingConsumption', 'false') = 'false'+ AND (w.metadata->'config' IS NULL OR w.metadata->'config' = 'null'::jsonb+ OR (jsonb_typeof(w.metadata->'config') = 'object'+ AND CASE WHEN jsonb_typeof(w.metadata->'config') = 'object' THEN+ (w.metadata->'config') - ARRAY['environmentId', 'provisionCommand', 'runtimeProvisionCommand',+ 'teardownCommand', 'cleanupCommand', 'workspaceRuntime', 'desiredState', 'serviceStates'] = '{}'::jsonb+ ELSE false END+ AND COALESCE(w.metadata->'config'->'workspaceRuntime', 'null'::jsonb) = 'null'::jsonb+ AND COALESCE(w.metadata->'config'->'serviceStates', 'null'::jsonb) IN ('null'::jsonb, '{}'::jsonb)+ AND COALESCE(w.metadata->'config'->>'desiredState', 'stopped') = 'stopped'+ AND COALESCE(w.metadata->'config'->>'provisionCommand', '') = ''+ AND COALESCE(w.metadata->'config'->>'runtimeProvisionCommand', '') = ''+ AND COALESCE(w.metadata->'config'->>'teardownCommand', '') = ''+ AND COALESCE(w.metadata->'config'->>'cleanupCommand', '') = ''))+ ) IS NOT TRUE`,`SELECT 1 FROM environment_leases l WHERE status NOT IN ('released', 'expired')AND (${finishedFailedLease("l")}) IS NOT TRUE`,`SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`,@@ -135,9 +172,7 @@ const WORK_CHECKS = [OR (d.target_run_id IS NOT NULL AND NOT EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = d.target_run_idAND r.company_id = d.company_id- AND r.status IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')- AND r.finished_at IS NOT NULL AND r.scheduled_retry_at IS NULL- AND r.cost_accounting_pending = false))+ AND ${finishedRun("r")}))OR (d.status = 'fallback_queued' AND d.target_run_id IS NULL AND NOT EXISTS (SELECT 1 FROM agent_wakeup_requests w WHERE w.company_id = d.company_idAND w.idempotency_key = 'question-response:' || d.interaction_id::text@@ -147,9 +182,7 @@ const WORK_CHECKS = [OR (w.status = 'coalesced' AND EXISTS (SELECT 1 FROM heartbeat_runs r WHERE r.id = w.run_idAND r.company_id = w.company_id AND r.agent_id = w.agent_id- AND r.status IN ('succeeded', 'failed', 'cancelled', 'timed_out', 'interrupted')- AND r.finished_at IS NOT NULL AND r.scheduled_retry_at IS NULL- AND r.cost_accounting_pending = false)))))`,+ AND ${finishedRun("r")})))))`,// These less common work sources fail closed on any retained state. Their// terminal-state exceptions can be added with tests for the owning service....[@@ -161,7 +194,7 @@ const WORK_CHECKS = ["native_run_finalizations", "company_transfer_runs", "decisions", "decision_effect_executions","decision_archive_notification_outbox", "browser_use_sessions", "browser_use_runs","browser_use_browsers", "environment_custom_image_setup_sessions", "feedback_exports",- "company_secret_proposals", "execution_workspaces",+ "company_secret_proposals","mcp_oauth_grants", "mcp_mutation_receipts", "mcp_event_subscriptions","mcp_event_deliveries", "mcp_attachment_uploads", "dot_agent_bindings","dot_runner_assignments", "dot_runner_operations", "dot_mailbox_items",