The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(server): gate periodic heartbeat recovery to one chain in flight (#11699)
Dottae82d86bfebe3
diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.mdindex 8b9c004282..428c32b3fe 100644--- a/doc/native-workspace-finalization-recovery.md+++ b/doc/native-workspace-finalization-recovery.md@@ -1,5 +1,15 @@# Native workspace finalization ownership and recovery+Periodic heartbeat recovery runs one pass at a time in each server process.+A tick that arrives during a slow pass reports its age and starts no duplicate.+A later tick can start a fresh pass after the current work settles. Shutdown+and idle sleep still track the actual recovery promise through cleanup. Direct+run operations and scheduled work delivery keep their existing paths. A pass+that remains active for three scheduler intervals produces an error log with its+age. A waiting timeout does not admit another pass: the actual work must settle+before ownership is released. A stuck operation requires investigation and+recovery at the operation or provider boundary.+A native provider can fail before it submits a result while its workspace stillcontains unpublished edits. A prepared workspace reference is not an exportreceipt. Terminal provider failure pins the exact bound sandbox and records adiff --git a/server/src/index.ts b/server/src/index.tsindex 584e9e6769..e83d603e2a 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -38,6 +38,7 @@ import { chatCompletionDeliveryService } from "./services/chat-completion-delive// instrumentationReady before opening DB connections or constructing the// HTTP server, so trace coverage does not depend on incidental timing.import { instrumentationReady, shutdownInstrumentation } from "./instrumentation.js";+import { createTrackedSingleFlight } from "./lib/single-flight.js";import { sentryReady, shutdownSentry, captureException } from "./sentry.js";import { setSentryRunLinkResolver } from "./sentry-links.js";import { createSentryRunLinkResolver } from "./services/sentry-run-links.js";@@ -1209,6 +1210,16 @@ async function startServerWithDatabaseTeardown(});heartbeatSchedulerInFlight.add(tracked);};+ const runPeriodicRecovery = createTrackedSingleFlight({+ trackWork: trackHeartbeatSchedulerWork,+ onSkipped: (inFlightForMs) => {+ if (inFlightForMs >= 3 * config.heartbeatSchedulerIntervalMs) {+ logger.error({ inFlightForMs }, "periodic heartbeat recovery remains in flight; duplicate pass suppressed");+ } else {+ logger.warn({ inFlightForMs }, "skipped periodic heartbeat recovery tick; previous chain still in flight");+ }+ },+ });const waitForHeartbeatSchedulerIdle = async () => {while (heartbeatSchedulerInFlight.size > 0) {await Promise.allSettled([...heartbeatSchedulerInFlight]);@@ -1887,7 +1898,7 @@ async function startServerWithDatabaseTeardown(if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) {// Periodically reap orphaned runs (5-min staleness threshold) and make sure// persisted queued work is still being driven forward.- trackHeartbeatSchedulerWork(heartbeat+ runPeriodicRecovery(() => heartbeat.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000 }).then(() => heartbeat.promoteDueScheduledRetries()).then(async (promotion) => {diff --git a/server/src/lib/single-flight.test.ts b/server/src/lib/single-flight.test.tsnew file mode 100644index 0000000000..06dabc9248--- /dev/null+++ b/server/src/lib/single-flight.test.ts@@ -0,0 +1,163 @@+import { describe, expect, it, vi } from "vitest";+import { createSingleFlight, createTrackedSingleFlight } from "./single-flight.js";+import { beginIdleTrackedWork, idleWorkSnapshot } from "../services/task-admission.js";++function deferred() {+ let resolve!: () => void;+ let reject!: (err: unknown) => void;+ const promise = new Promise<void>((res, rej) => {+ resolve = res;+ reject = rej;+ });+ return { promise, resolve, reject };+}++describe("createSingleFlight", () => {+ it("runs work when idle and reports busy until it settles", async () => {+ const gate = createSingleFlight();+ const gap = deferred();++ const first = gate.run(() => gap.promise);+ expect(first.started).toBe(true);+ expect(gate.busy).toBe(true);++ gap.resolve();+ await (first as { settled: Promise<void> }).settled;+ expect(gate.busy).toBe(false);+ });++ it("skips a tick while the previous invocation is still in flight", async () => {+ const clock = { nowMs: 1_000 };+ const gate = createSingleFlight(() => clock.nowMs);+ const gap = deferred();+ const work = vi.fn(() => gap.promise);++ const first = gate.run(work);+ clock.nowMs = 31_000;+ const second = gate.run(work);++ expect(second).toEqual({ started: false, inFlightForMs: 30_000 });+ expect(work).toHaveBeenCalledTimes(1);++ gap.resolve();+ await (first as { settled: Promise<void> }).settled;+ });++ it("admits new work after the previous invocation resolves", async () => {+ const gate = createSingleFlight();+ const gap = deferred();++ const first = gate.run(() => gap.promise);+ gap.resolve();+ await (first as { settled: Promise<void> }).settled;++ const second = gate.run(async () => {});+ expect(second.started).toBe(true);+ await (second as { settled: Promise<void> }).settled;+ });++ it("releases the gate when work rejects and never rejects the settled promise", async () => {+ const gate = createSingleFlight();+ const gap = deferred();++ const first = gate.run(() => gap.promise);+ gap.reject(new Error("chain failed"));+ await expect((first as { settled: Promise<void> }).settled).resolves.toBeUndefined();+ expect(gate.busy).toBe(false);++ const second = gate.run(async () => {});+ expect(second.started).toBe(true);+ await (second as { settled: Promise<void> }).settled;+ });++ it("propagates a synchronous throw and leaves the gate free", async () => {+ const gate = createSingleFlight();++ expect(() =>+ gate.run(() => {+ throw new Error("sync failure");+ }),+ ).toThrow("sync failure");+ expect(gate.busy).toBe(false);++ const second = gate.run(async () => {});+ expect(second.started).toBe(true);+ await (second as { settled: Promise<void> }).settled;+ });++ it("keeps the owned pass active after a caller stops waiting", async () => {+ const gate = createSingleFlight(), work = deferred(), observer = deferred();+ const first = gate.run(() => work.promise);+ if (!first.started) throw new Error("initial pass was not admitted");+ const stopped = Promise.race([first.settled, observer.promise]);+ observer.reject(new Error("observer stopped"));+ await expect(stopped).rejects.toThrow("observer stopped");+ const duplicate = vi.fn(async () => undefined);+ expect(gate.run(duplicate).started).toBe(false);+ expect(duplicate).not.toHaveBeenCalled();+ expect(gate.busy).toBe(true);+ work.resolve();+ await first.settled;+ expect(gate.busy).toBe(false);+ });++ it("keeps cleanup joined after the main operation completes", async () => {+ const gate = createSingleFlight(), operation = deferred(), cleanup = deferred();+ const first = gate.run(() => operation.promise.finally(() => cleanup.promise));+ if (!first.started) throw new Error("initial pass was not admitted");+ operation.resolve();+ await Promise.resolve();+ expect(gate.busy).toBe(true);+ const duplicate = vi.fn(async () => undefined);+ expect(gate.run(duplicate).started).toBe(false);+ expect(duplicate).not.toHaveBeenCalled();+ cleanup.resolve();+ await first.settled;+ expect(gate.busy).toBe(false);+ const next = gate.run(duplicate);+ expect(next.started).toBe(true);+ if (next.started) await next.settled;+ expect(duplicate).toHaveBeenCalledTimes(1);+ });++ it("tracks one periodic pass through scheduler shutdown and idle sleep", async () => {+ vi.useFakeTimers();+ const before = idleWorkSnapshot().active;+ const operation = deferred(), cleanup = deferred();+ const tracked: Promise<void>[] = [];+ const trackWork = vi.fn((work: Promise<void>) => {+ const finish = beginIdleTrackedWork();+ tracked.push(work.finally(finish));+ });+ const onSkipped = vi.fn();+ const tick = createTrackedSingleFlight({ trackWork, onSkipped });+ const heartbeat = vi.fn(() => operation.promise.finally(() => cleanup.promise));+ const timer = setInterval(() => tick(heartbeat), 30_000);+ try {+ await vi.advanceTimersByTimeAsync(60_000);+ expect(heartbeat).toHaveBeenCalledOnce();+ expect(trackWork).toHaveBeenCalledOnce();+ expect(onSkipped).toHaveBeenCalledExactlyOnceWith(30_000);+ clearInterval(timer);+ let shutdownComplete = false;+ const shutdown = Promise.all(tracked).then(() => { shutdownComplete = true; });+ await Promise.resolve();+ expect(shutdownComplete).toBe(false);+ expect(idleWorkSnapshot().active).toBe(before + 1);+ operation.resolve();+ await Promise.resolve();+ expect(shutdownComplete).toBe(false);+ expect(idleWorkSnapshot().active).toBe(before + 1);+ cleanup.resolve();+ await shutdown;+ expect(shutdownComplete).toBe(true);+ expect(idleWorkSnapshot().active).toBe(before);+ } finally {+ clearInterval(timer);+ operation.resolve();+ cleanup.resolve();+ await Promise.all(tracked);+ vi.useRealTimers();+ }+ });+});diff --git a/server/src/lib/single-flight.ts b/server/src/lib/single-flight.tsnew file mode 100644index 0000000000..68dc1b035f--- /dev/null+++ b/server/src/lib/single-flight.ts@@ -0,0 +1,51 @@+export interface SingleFlightSkip {+ started: false;+ inFlightForMs: number;+}++export interface SingleFlightStart {+ started: true;+ settled: Promise<void>;+}++export type SingleFlightResult = SingleFlightSkip | SingleFlightStart;++export function createSingleFlight(now: () => number = Date.now) {+ let inFlight: Promise<void> | null = null;+ let startedAtMs = 0;++ return {+ get busy(): boolean {+ return inFlight !== null;+ },+ run(work: () => Promise<unknown>): SingleFlightResult {+ if (inFlight) {+ return { started: false, inFlightForMs: now() - startedAtMs };+ }+ startedAtMs = now();+ const settled = Promise.resolve(work())+ .then(+ () => undefined,+ () => undefined,+ )+ .finally(() => {+ inFlight = null;+ });+ inFlight = settled;+ return { started: true, settled };+ },+ };+}++export function createTrackedSingleFlight(options: {+ trackWork: (work: Promise<void>) => void;+ onSkipped: (inFlightForMs: number) => void;+ now?: () => number;+}) {+ const gate = createSingleFlight(options.now);+ return (work: () => Promise<unknown>) => {+ const tick = gate.run(work);+ if (tick.started) options.trackWork(tick.settled);+ else options.onSkipped(tick.inFlightForMs);+ };+}