The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(runtime): keep the copyback lock transaction active (#16340)
Dottaaa04c5c2a704
diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.mdindex a9a6e6365f..86438c3ca4 100644--- a/doc/native-workspace-finalization-recovery.md+++ b/doc/native-workspace-finalization-recovery.md@@ -35,6 +35,14 @@ leave tar/Git children alive, so incomplete copyback requires operator stopverification even on the same host. PID reuse is checked against its recordedstart time rather than trusted by PID alone.+When PostgreSQL configures an idle-transaction timeout, the owner probes its+dedicated lock transaction at one third of that interval, capped at+30 seconds. The probe stops and joins before the transaction is released. This+keeps a live copyback from losing its lock only because filesystem work has no+database statements. It does not disable the database timeout or reconnect a+failed lock session. Publication still requires the physical connection and+the exact durable owner token. A disabled timeout adds no periodic probe.+## Automatic cloud recoveryNew Railway copyback receipts record the exact project, environment, service anddiff --git a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts b/server/src/services/native-runtime/native-workspace-finalization-ownership.tsindex 5168b9bf85..9d39705f5b 100644--- a/server/src/services/native-runtime/native-workspace-finalization-ownership.ts+++ b/server/src/services/native-runtime/native-workspace-finalization-ownership.ts@@ -7,6 +7,7 @@ import os from "node:os";import { and, eq, inArray, sql } from "drizzle-orm";import { heartbeatRuns, issues, issueRecoveryActions, nativeRunFinalizations, workspaceOperations, withDedicatedDbConnection, type Db } from "@paperclipai/db";import { currentNativeWorkspaceRecoveryResult } from "./native-workspace-recovery-binding.js";+import { withWorkspaceLockKeepalive } from "./native-workspace-lock-keepalive.js";import {currentNativeControllerIdentity,evaluateNativeControllerTakeover,@@ -71,149 +72,152 @@ export async function withNativeWorkspaceFinalizationOwnership<T>(let actionPromise: Promise<T> | undefined;try {return await withDedicatedDbConnection(input.db, (dedicated) => dedicated.transaction(async (lock) => {- const rows = await lock.execute(sql`select pg_try_advisory_xact_lock(hashtextextended(${`native-workspace-finalization:${input.companyId}:${input.runId}`}, 0)) as acquired`);+ const rows = await lock.execute(sql`select pg_try_advisory_xact_lock(hashtextextended(${`native-workspace-finalization:${input.companyId}:${input.runId}`}, 0)) as acquired,+ (select setting::bigint from pg_settings where name = 'idle_in_transaction_session_timeout') as idle_timeout_ms`);if (rows[0]?.acquired !== true) return { acquired: false } as const;- // Reads and durable progress use the ordinary pool, not the lock transaction.- const [run] = await input.db.select({ profile: heartbeatRuns.runnerProfileJson, issueId: heartbeatRuns.nativeIssueId, agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(scope).limit(1);- if (!run) throw new Error("native_workspace_finalization_binding_missing");- const rawPrior = run.profile?.[OWNER_KEY];- const prior = readOwner(rawPrior);- let automaticResultId: string | undefined;- const currentRecoveryResult = async (db: Db) => {- if (!run.issueId) return null;- // Use the arbiter's lock order and release row locks before copyback I/O.- const [coordinator] = await db.select().from(nativeRunFinalizations).where(and(- eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, input.runId),- eq(nativeRunFinalizations.issueId, run.issueId),- )).for("update");- const [issue] = await db.select().from(issues).where(and(- eq(issues.companyId, input.companyId), eq(issues.id, run.issueId),- )).for("update");- const [currentRun] = await db.select().from(heartbeatRuns).where(scope).for("update");- return coordinator && issue && currentRun ? currentNativeWorkspaceRecoveryResult(db, {- companyId: input.companyId, coordinator, issue, run: currentRun,- }) : null;- };- let platformProof: WorkspaceWorkerStopProof | null = null;- if (rawPrior != null) {- const requireStopVerification = async () => {- const fingerprint = nativeSha256({ runId: input.runId, owner: rawPrior });- const nextAction = "Verify the previous controller and its workspace-copyback processes have stopped, then release only this exact workspace owner receipt using doc/native-workspace-finalization-recovery.md. Resume workspace finalization without another provider turn.";- const recovery = issueRecoveryActionService(input.db);- const existing = run.issueId ? await recovery.getActiveForIssue(input.companyId, run.issueId) : null;- // Rechecking the same physical owner is not a new recovery attempt.- if (existing?.cause === NATIVE_WORKSPACE_OWNER_UNVERIFIED && existing.fingerprint === fingerprint) {- if (!existing.evidence?.workspaceOwnerStop) {- await input.db.update(issueRecoveryActions).set({ nextAction, updatedAt: new Date(),- evidence: sql`jsonb_set(coalesce(${issueRecoveryActions.evidence}, '{}'::jsonb), '{workspaceOwnerRecovery}', '{"state":"operator_required"}'::jsonb)`,- }).where(and(- eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.id, existing.id),- eq(issueRecoveryActions.fingerprint, fingerprint), eq(issueRecoveryActions.cause, NATIVE_WORKSPACE_OWNER_UNVERIFIED),- inArray(issueRecoveryActions.status, ["active", "escalated"]),- sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`,- sql`(${issueRecoveryActions.nextAction} is distinct from ${nextAction} or ${issueRecoveryActions.evidence}#>>'{workspaceOwnerRecovery,state}' is distinct from 'operator_required')`,- ));+ return withWorkspaceLockKeepalive(lock as unknown as Db, Number(rows[0].idle_timeout_ms), async () => {+ // Reads and durable progress use the ordinary pool, not the lock transaction.+ const [run] = await input.db.select({ profile: heartbeatRuns.runnerProfileJson, issueId: heartbeatRuns.nativeIssueId, agentId: heartbeatRuns.agentId }).from(heartbeatRuns).where(scope).limit(1);+ if (!run) throw new Error("native_workspace_finalization_binding_missing");+ const rawPrior = run.profile?.[OWNER_KEY];+ const prior = readOwner(rawPrior);+ let automaticResultId: string | undefined;+ const currentRecoveryResult = async (db: Db) => {+ if (!run.issueId) return null;+ // Use the arbiter's lock order and release row locks before copyback I/O.+ const [coordinator] = await db.select().from(nativeRunFinalizations).where(and(+ eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, input.runId),+ eq(nativeRunFinalizations.issueId, run.issueId),+ )).for("update");+ const [issue] = await db.select().from(issues).where(and(+ eq(issues.companyId, input.companyId), eq(issues.id, run.issueId),+ )).for("update");+ const [currentRun] = await db.select().from(heartbeatRuns).where(scope).for("update");+ return coordinator && issue && currentRun ? currentNativeWorkspaceRecoveryResult(db, {+ companyId: input.companyId, coordinator, issue, run: currentRun,+ }) : null;+ };+ let platformProof: WorkspaceWorkerStopProof | null = null;+ if (rawPrior != null) {+ const requireStopVerification = async () => {+ const fingerprint = nativeSha256({ runId: input.runId, owner: rawPrior });+ const nextAction = "Verify the previous controller and its workspace-copyback processes have stopped, then release only this exact workspace owner receipt using doc/native-workspace-finalization-recovery.md. Resume workspace finalization without another provider turn.";+ const recovery = issueRecoveryActionService(input.db);+ const existing = run.issueId ? await recovery.getActiveForIssue(input.companyId, run.issueId) : null;+ // Rechecking the same physical owner is not a new recovery attempt.+ if (existing?.cause === NATIVE_WORKSPACE_OWNER_UNVERIFIED && existing.fingerprint === fingerprint) {+ if (!existing.evidence?.workspaceOwnerStop) {+ await input.db.update(issueRecoveryActions).set({ nextAction, updatedAt: new Date(),+ evidence: sql`jsonb_set(coalesce(${issueRecoveryActions.evidence}, '{}'::jsonb), '{workspaceOwnerRecovery}', '{"state":"operator_required"}'::jsonb)`,+ }).where(and(+ eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.id, existing.id),+ eq(issueRecoveryActions.fingerprint, fingerprint), eq(issueRecoveryActions.cause, NATIVE_WORKSPACE_OWNER_UNVERIFIED),+ inArray(issueRecoveryActions.status, ["active", "escalated"]),+ sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`,+ sql`(${issueRecoveryActions.nextAction} is distinct from ${nextAction} or ${issueRecoveryActions.evidence}#>>'{workspaceOwnerRecovery,state}' is distinct from 'operator_required')`,+ ));+ }+ return { acquired: false } as const;}+ if (run.issueId) await recovery.upsertSourceScoped({+ companyId: input.companyId, sourceIssueId: run.issueId,+ kind: "active_run_watchdog", ownerType: "board", returnOwnerAgentId: run.agentId,+ cause: NATIVE_WORKSPACE_OWNER_UNVERIFIED,+ fingerprint,+ evidence: { runId: input.runId, owner: rawPrior, workspaceOwnerRecovery: { state: "operator_required" } },+ nextAction,+ wakePolicy: null, maxAttempts: 1, supersedeOnIdentityChange: true,+ });return { acquired: false } as const;+ };+ // The harness checks the exact owned deployment, including orphaned children.+ // A live deployment, unsigned answer or older receipt stays fenced.+ if (prior?.worker) platformProof = await requestWorkspaceWorkerStopProof({ runId: input.runId, ownerToken: prior.token, worker: prior.worker });+ if (!prior) return requireStopVerification();+ if (!platformProof && prior.hostname !== owner.hostname) return requireStopVerification();+ const knownJoinedHere = prior.controllerBootId === identity.bootId+ && prior.pid === identity.pid && joinedOwnerTokens.has(prior.token)+ && !activeOwnerTokens.has(prior.token);+ if (!platformProof && !knownJoinedHere) {+ const takeover = await evaluateNativeControllerTakeover({+ owner: { leaseOwner: prior.token, leaseExpiresAt: new Date(0), controllerPid: prior.pid,+ controllerProcessStartedAt: new Date(prior.processStartedAt) }, now: new Date(),+ });+ if (!takeover.allowed) return { acquired: false } as const;+ // A dead parent may have orphaned tar/Git children. Only a durable+ // completed-copyback barrier proves they joined; otherwise board proof+ // is required even on the same host. Both callers reuse this barrier.+ const [completedCopyback] = await input.db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(+ eq(workspaceOperations.companyId, input.companyId),+ eq(workspaceOperations.heartbeatRunId, input.runId),+ eq(workspaceOperations.issueId, run.issueId!),+ eq(workspaceOperations.phase, "workspace_finalize"),+ eq(workspaceOperations.status, "succeeded"),+ )).limit(1);+ if (!completedCopyback) return requireStopVerification();}- if (run.issueId) await recovery.upsertSourceScoped({- companyId: input.companyId, sourceIssueId: run.issueId,- kind: "active_run_watchdog", ownerType: "board", returnOwnerAgentId: run.agentId,- cause: NATIVE_WORKSPACE_OWNER_UNVERIFIED,- fingerprint,- evidence: { runId: input.runId, owner: rawPrior, workspaceOwnerRecovery: { state: "operator_required" } },- nextAction,- wakePolicy: null, maxAttempts: 1, supersedeOnIdentityChange: true,- });- return { acquired: false } as const;- };- // The harness checks the exact owned deployment, including orphaned children.- // A live deployment, unsigned answer or older receipt stays fenced.- if (prior?.worker) platformProof = await requestWorkspaceWorkerStopProof({ runId: input.runId, ownerToken: prior.token, worker: prior.worker });- if (!prior) return requireStopVerification();- if (!platformProof && prior.hostname !== owner.hostname) return requireStopVerification();- const knownJoinedHere = prior.controllerBootId === identity.bootId- && prior.pid === identity.pid && joinedOwnerTokens.has(prior.token)- && !activeOwnerTokens.has(prior.token);- if (!platformProof && !knownJoinedHere) {- const takeover = await evaluateNativeControllerTakeover({- owner: { leaseOwner: prior.token, leaseExpiresAt: new Date(0), controllerPid: prior.pid,- controllerProcessStartedAt: new Date(prior.processStartedAt) }, now: new Date(),- });- if (!takeover.allowed) return { acquired: false } as const;- // A dead parent may have orphaned tar/Git children. Only a durable- // completed-copyback barrier proves they joined; otherwise board proof- // is required even on the same host. Both callers reuse this barrier.- const [completedCopyback] = await input.db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(- eq(workspaceOperations.companyId, input.companyId),- eq(workspaceOperations.heartbeatRunId, input.runId),- eq(workspaceOperations.issueId, run.issueId!),- eq(workspaceOperations.phase, "workspace_finalize"),- eq(workspaceOperations.status, "succeeded"),- )).limit(1);- if (!completedCopyback) return requireStopVerification();}- }- // Claim before the potentially ambiguous write so finally can remove- // only our token even if the server committed but its response was lost.- const claimOwner = async (db: Db) => {- activeOwnerTokens.add(owner.token);- claimed = true;- const [claimedRun] = await db.update(heartbeatRuns).set({- runnerProfileJson: sql`jsonb_set(coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb), array[${OWNER_KEY}], ${JSON.stringify(owner)}::jsonb) || ${JSON.stringify(platformProof ? { nativeWorkspaceOwnerStopProof: platformProof } : {})}::jsonb`,- }).where(and(scope, prior- ? sql`${heartbeatRuns.runnerProfileJson}->${OWNER_KEY} = ${JSON.stringify(rawPrior)}::jsonb`- : sql`${heartbeatRuns.runnerProfileJson}->>${OWNER_KEY} is null`- )).returning({ id: heartbeatRuns.id });- return claimedRun;- };- // A stop receipt proves the old worker exited, not that its task is still- // current. Revalidate after the provider read and atomically with the CAS.- const claimedRun = platformProof ? await input.db.transaction(async tx => {- const result = await currentRecoveryResult(tx as unknown as Db);- if (!result) return null;- const claimedRun = await claimOwner(tx as unknown as Db);- if (claimedRun) automaticResultId = result.id;- return claimedRun;- }) : await claimOwner(input.db);- if (!claimedRun) return { acquired: false } as const;- if (platformProof && run.issueId) await logActivity(input.db, {- companyId: input.companyId, actorType: "system", actorId: "workspace-recovery",- action: "issue.workspace_owner_stop_verified", entityType: "issue", entityId: run.issueId,- runId: input.runId, details: { proof: platformProof },- });- if (prior) joinedOwnerTokens.delete(prior.token);- const ownership: NativeWorkspaceFinalizationOwnership = {- token: owner.token,- assertHeld: async () => {- try {- // A failed transaction is never transparently reconnected by the DB client.- await lock.execute(sql`select 1`);- const [current] = await input.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(owned).limit(1);- if (!current) throw new Error("owner_replaced");- if (automaticResultId) {- const result = await input.db.transaction(tx => currentRecoveryResult(tx as unknown as Db));- if (result?.id !== automaticResultId) throw new Error("task_replaced");+ // Claim before the potentially ambiguous write so finally can remove+ // only our token even if the server committed but its response was lost.+ const claimOwner = async (db: Db) => {+ activeOwnerTokens.add(owner.token);+ claimed = true;+ const [claimedRun] = await db.update(heartbeatRuns).set({+ runnerProfileJson: sql`jsonb_set(coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb), array[${OWNER_KEY}], ${JSON.stringify(owner)}::jsonb) || ${JSON.stringify(platformProof ? { nativeWorkspaceOwnerStopProof: platformProof } : {})}::jsonb`,+ }).where(and(scope, prior+ ? sql`${heartbeatRuns.runnerProfileJson}->${OWNER_KEY} = ${JSON.stringify(rawPrior)}::jsonb`+ : sql`${heartbeatRuns.runnerProfileJson}->>${OWNER_KEY} is null`+ )).returning({ id: heartbeatRuns.id });+ return claimedRun;+ };+ // A stop receipt proves the old worker exited, not that its task is still+ // current. Revalidate after the provider read and atomically with the CAS.+ const claimedRun = platformProof ? await input.db.transaction(async tx => {+ const result = await currentRecoveryResult(tx as unknown as Db);+ if (!result) return null;+ const claimedRun = await claimOwner(tx as unknown as Db);+ if (claimedRun) automaticResultId = result.id;+ return claimedRun;+ }) : await claimOwner(input.db);+ if (!claimedRun) return { acquired: false } as const;+ if (platformProof && run.issueId) await logActivity(input.db, {+ companyId: input.companyId, actorType: "system", actorId: "workspace-recovery",+ action: "issue.workspace_owner_stop_verified", entityType: "issue", entityId: run.issueId,+ runId: input.runId, details: { proof: platformProof },+ });+ if (prior) joinedOwnerTokens.delete(prior.token);+ const ownership: NativeWorkspaceFinalizationOwnership = {+ token: owner.token,+ assertHeld: async () => {+ try {+ // A failed transaction is never transparently reconnected by the DB client.+ await lock.execute(sql`select 1`);+ const [current] = await input.db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(owned).limit(1);+ if (!current) throw new Error("owner_replaced");+ if (automaticResultId) {+ const result = await input.db.transaction(tx => currentRecoveryResult(tx as unknown as Db));+ if (result?.id !== automaticResultId) throw new Error("task_replaced");+ }+ } catch (cause) {+ throw new NativeWorkspaceFinalizationOwnershipLostError({ cause });}- } catch (cause) {- throw new NativeWorkspaceFinalizationOwnershipLostError({ cause });- }- },- };- await ownership.assertHeld();- if (platformProof && run.issueId) await input.db.update(issueRecoveryActions).set({- evidence: sql`jsonb_set(coalesce(${issueRecoveryActions.evidence}, '{}'::jsonb), '{workspaceOwnerRecovery}', '{"state":"resuming"}'::jsonb)`,- updatedAt: new Date(),- }).where(and(- eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.sourceIssueId, run.issueId),- eq(issueRecoveryActions.cause, NATIVE_WORKSPACE_OWNER_UNVERIFIED),- eq(issueRecoveryActions.fingerprint, nativeSha256({ runId: input.runId, owner: rawPrior })),- inArray(issueRecoveryActions.status, ["active", "escalated"]),- ));- actionPromise = action(ownership);- const value = await actionPromise;- await ownership.assertHeld();- return { acquired: true, value } as const;+ },+ };+ await ownership.assertHeld();+ if (platformProof && run.issueId) await input.db.update(issueRecoveryActions).set({+ evidence: sql`jsonb_set(coalesce(${issueRecoveryActions.evidence}, '{}'::jsonb), '{workspaceOwnerRecovery}', '{"state":"resuming"}'::jsonb)`,+ updatedAt: new Date(),+ }).where(and(+ eq(issueRecoveryActions.companyId, input.companyId), eq(issueRecoveryActions.sourceIssueId, run.issueId),+ eq(issueRecoveryActions.cause, NATIVE_WORKSPACE_OWNER_UNVERIFIED),+ eq(issueRecoveryActions.fingerprint, nativeSha256({ runId: input.runId, owner: rawPrior })),+ inArray(issueRecoveryActions.status, ["active", "escalated"]),+ ));+ actionPromise = action(ownership);+ const value = await actionPromise;+ await ownership.assertHeld();+ return { acquired: true, value } as const;+ });}));} finally {// postgres.js may reject the transaction when the socket dies before thediff --git a/server/src/services/native-runtime/native-workspace-finalizer.test.ts b/server/src/services/native-runtime/native-workspace-finalizer.test.tsindex 30737efb30..bdc1ec177e 100644--- a/server/src/services/native-runtime/native-workspace-finalizer.test.ts+++ b/server/src/services/native-runtime/native-workspace-finalizer.test.ts@@ -892,4 +892,21 @@ describe("native workspace finalization recovery", () => {}),]);});+ it("preserves a live copyback owner across the configured idle transaction timeout", async () => {+ const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Idle copyback lock" });+ const rows = await db.execute(sql`select current_user as name`);+ const role = String(rows[0].name);+ await db.execute(sql`alter role ${sql.identifier(role)} set idle_in_transaction_session_timeout = '1s'`);+ try {+ const result = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async ownership => {+ await new Promise(resolve => setTimeout(resolve, 3200));+ await ownership.assertHeld();+ return "copied";+ });+ expect(result).toEqual({ acquired: true, value: "copied" });+ } finally {+ await db.execute(sql`alter role ${sql.identifier(role)} reset idle_in_transaction_session_timeout`);+ }+ });+});diff --git a/server/src/services/native-runtime/native-workspace-lock-keepalive.test.ts b/server/src/services/native-runtime/native-workspace-lock-keepalive.test.tsnew file mode 100644index 0000000000..9369ca8e6f--- /dev/null+++ b/server/src/services/native-runtime/native-workspace-lock-keepalive.test.ts@@ -0,0 +1,71 @@+import { setImmediate } from "node:timers/promises";+import * as timers from "node:timers/promises";+import { expect, it, vi } from "vitest";+import type { Db } from "@paperclipai/db";+import { withWorkspaceLockKeepalive } from "./native-workspace-lock-keepalive.js";++vi.mock("node:timers/promises", async importOriginal => {+ const actual = await importOriginal<typeof import("node:timers/promises")>();+ return { ...actual, setTimeout: vi.fn(actual.setTimeout) };+});++it.each([0, -1, NaN, Infinity])("does not probe when the idle timeout is %s", async timeout => {+ const execute = vi.fn();+ const value = {};+ let finish!: (value: object) => void;+ const pending = new Promise<object>(resolve => { finish = resolve; });+ vi.useFakeTimers();+ const sleep = vi.mocked(timers.setTimeout);+ const originalSleep = sleep.getMockImplementation()!;+ sleep.mockClear().mockImplementation((_delay, result, options) => new Promise((resolve, reject) => {+ const timer = setTimeout(() => resolve(result), 30_000);+ options?.signal?.addEventListener("abort", () => {+ clearTimeout(timer);+ reject(new Error("aborted"));+ }, { once: true });+ }));+ const result = withWorkspaceLockKeepalive({ execute } as unknown as Db, timeout, () => pending);+ try {+ await vi.advanceTimersByTimeAsync(60_001);+ expect(execute).not.toHaveBeenCalled();+ expect(sleep).not.toHaveBeenCalled();+ } finally {+ finish(value);+ await expect(result).resolves.toBe(value);+ sleep.mockImplementation(originalSleep);+ vi.useRealTimers();+ }+});++it("joins an in-flight probe before releasing the lock scope", async () => {+ let entered!: () => void, finish!: () => void;+ const started = new Promise<void>(resolve => { entered = resolve; });+ const pending = new Promise<void>(resolve => { finish = resolve; });+ const execute = vi.fn(async () => { entered(); await pending; });+ let settled = false;+ const result = withWorkspaceLockKeepalive({ execute } as unknown as Db, 3, async () => {+ await started;+ return 42;+ }).then(value => { settled = true; return value; });+ try {+ await started;+ await setImmediate();+ expect(settled).toBe(false);+ expect(execute).toHaveBeenCalledOnce();+ } finally { finish(); }+ await expect(result).resolves.toBe(42);+ await setImmediate();+ expect(execute).toHaveBeenCalledOnce();+});++it("preserves the work error when its keepalive query fails", async () => {+ let entered!: () => void;+ const started = new Promise<void>(resolve => { entered = resolve; });+ const execute = vi.fn(async () => { entered(); throw new Error("connection lost"); });+ const original = new Error("copyback failed");+ const work = vi.fn(async () => { await started; throw original; });+ await expect(withWorkspaceLockKeepalive({ execute } as unknown as Db, 3, work)).rejects.toBe(original);+ expect(work).toHaveBeenCalledOnce();+ await setImmediate();+ expect(execute).toHaveBeenCalledOnce();+});diff --git a/server/src/services/native-runtime/native-workspace-lock-keepalive.ts b/server/src/services/native-runtime/native-workspace-lock-keepalive.tsnew file mode 100644index 0000000000..87cb09c5fa--- /dev/null+++ b/server/src/services/native-runtime/native-workspace-lock-keepalive.ts@@ -0,0 +1,25 @@+import { setTimeout } from "node:timers/promises";+import { sql } from "drizzle-orm";+import type { Db } from "@paperclipai/db";++export async function withWorkspaceLockKeepalive<T>(+ lock: Pick<Db, "execute">,+ idleTimeoutMs: number,+ work: () => Promise<T>,+): Promise<T> {+ if (!Number.isFinite(idleTimeoutMs) || idleTimeoutMs <= 0) return work();+ const stop = new AbortController();+ const interval = Math.max(1, Math.min(30_000, Math.floor(idleTimeoutMs / 3)));+ const keeper = (async () => {+ while (!stop.signal.aborted) {+ await setTimeout(interval, undefined, { signal: stop.signal, ref: false });+ await lock.execute(sql`select 1`);+ }+ })().catch(() => undefined);+ try {+ return await work();+ } finally {+ stop.abort();+ await keeper;+ }+}