Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: schedule restored loops from their saved last run (#11754)

[._.]/ Adam Eivyfcc64bc95ae6
diff --git a/server/services/loops.js b/server/services/loops.js
index e519cbec0a..b5eddb9435 100644
--- a/server/services/loops.js
+++ b/server/services/loops.js
@@ -26,6 +26,12 @@ const DEFAULT_TIMEOUT_MS = 300_000;
const activeLoops = new Map();
+// Overdue loops restored at boot catch up once, spaced apart so they do not
+// all start together.
+const CATCHUP_BASE_DELAY_MS = 1000;
+const CATCHUP_STAGGER_MS = 10_000;
+const MAX_TIMER_DELAY_MS = 2 ** 31 - 1;
+
function formatInterval(ms) {
if (ms >= 86_400_000) return `${ms / 86_400_000}d`;
if (ms >= 3_600_000) return `${ms / 3_600_000}h`;
@@ -313,17 +319,35 @@ export async function createLoop({ prompt, interval, name, cwd, providerId, time
return loop;
}
-function startLoopTimer(loop, runImmediately = false) {
+// setTimeout/setInterval handles share one id space in Node, so a single
+// clear covers whichever of the pending first-fire timeout or the steady
+// interval is stored on the active entry.
+function clearLoopTimer(active) {
+ clearTimeout(active.timer);
+ clearInterval(active.timer);
+}
+
+// firstDelayMs (restore path) delays the first fire past one full interval's
+// worth of waiting: the first tick runs once, then the steady interval takes over.
+function startLoopTimer(loop, runImmediately = false, firstDelayMs = null) {
const runWithLogging = () => runIteration(loop.id);
- const timer = setInterval(runWithLogging, loop.intervalMs);
- activeLoops.set(loop.id, {
- timer,
+ const active = {
+ timer: null,
runId: null,
running: false,
iterationCount: loop.iterationCount || 0,
lastResult: null,
history: []
- });
+ };
+ if (firstDelayMs === null) {
+ active.timer = setInterval(runWithLogging, loop.intervalMs);
+ } else {
+ active.timer = setTimeout(() => {
+ active.timer = setInterval(runWithLogging, loop.intervalMs);
+ runWithLogging();
+ }, Math.min(Math.max(firstDelayMs, 0), MAX_TIMER_DELAY_MS));
+ }
+ activeLoops.set(loop.id, active);
if (runImmediately) {
setTimeout(runWithLogging, 100);
@@ -334,7 +358,7 @@ export async function stopLoop(id) {
const active = activeLoops.get(id);
if (!active) throw new ServerError(`Loop ${id} is not running`, { status: 409, code: 'INVALID_STATE' });
- clearInterval(active.timer);
+ clearLoopTimer(active);
activeLoops.delete(id);
await withLoopsTail(async () => {
@@ -376,7 +400,7 @@ export async function resumeLoop(id) {
export async function deleteLoop(id) {
if (activeLoops.has(id)) {
const active = activeLoops.get(id);
- clearInterval(active.timer);
+ clearLoopTimer(active);
activeLoops.delete(id);
}
@@ -446,7 +470,7 @@ export async function updateLoop(id, updates) {
if (activeLoops.has(id) && updates.interval) {
const active = activeLoops.get(id);
- clearInterval(active.timer);
+ clearLoopTimer(active);
active.timer = setInterval(() => runIteration(id), updatedRecord.intervalMs);
}
@@ -488,11 +512,20 @@ export async function restoreLoops() {
await ensureDir(LOOPS_OUTPUT_DIR);
const loops = await loadLoops();
let restored = 0;
+ let overdue = 0;
+ const now = Date.now();
for (const loop of loops) {
- if (loop.status === 'running') {
- startLoopTimer(loop, false);
- restored++;
+ if (loop.status !== 'running') continue;
+ // Honor the persisted lastRun so a restart does not push the next
+ // iteration out by a full interval. An overdue loop gets one staggered
+ // catch-up run, never one per missed interval.
+ let firstDelay = null;
+ if (Number.isFinite(loop.lastRun)) {
+ const remaining = loop.lastRun + loop.intervalMs - now;
+ firstDelay = remaining > 0 ? remaining : CATCHUP_BASE_DELAY_MS + overdue++ * CATCHUP_STAGGER_MS;
}
+ startLoopTimer(loop, false, firstDelay);
+ restored++;
}
if (restored > 0) {
console.log(`🔄 Restored ${restored} active loop(s)`);
diff --git a/server/services/loops.test.js b/server/services/loops.test.js
index 9a8cbc5ead..f72b654454 100644
--- a/server/services/loops.test.js
+++ b/server/services/loops.test.js
@@ -46,6 +46,7 @@ import { runPromptThroughProvider, resolveProviderAndModel } from './promptRunne
import { getActiveProvider, listSelectableProviders } from './providers.js';
import {
createLoop,
+ restoreLoops,
stopLoop,
triggerLoop,
updateLoop,
@@ -588,6 +589,66 @@ describe('loops.js', () => {
});
});
+ describe('restoreLoops honors the persisted lastRun', () => {
+ const HOUR = 3_600_000;
+ const rec = (id, lastRun, intervalMs = 24 * HOUR) => ({
+ id, name: id, prompt: 'p', intervalMs, status: 'running', lastRun, iterationCount: 1
+ });
+ const stubDisk = (loops) => {
+ tryReadFile.mockImplementation(async (path) =>
+ String(path).endsWith('loops.json') ? JSON.stringify(loops) : null
+ );
+ createdLoopIds.push(...loops.map(l => l.id));
+ };
+
+ it('fires at lastRun + interval, then every interval', async () => {
+ stubDisk([rec('l1', Date.now() - 23 * HOUR)]);
+ await restoreLoops();
+
+ await vi.advanceTimersByTimeAsync(HOUR - 1000);
+ expect(mockRunPrompt).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(mockRunPrompt).toHaveBeenCalledTimes(1);
+ await vi.advanceTimersByTimeAsync(24 * HOUR);
+ expect(mockRunPrompt).toHaveBeenCalledTimes(2);
+ });
+
+ it('runs one staggered catch-up per overdue loop, then resumes the interval', async () => {
+ stubDisk([rec('o1', Date.now() - 100 * HOUR, HOUR), rec('o2', Date.now() - 100 * HOUR, HOUR)]);
+ await restoreLoops();
+
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(mockRunPrompt).toHaveBeenCalledTimes(1);
+ await vi.advanceTimersByTimeAsync(9_000);
+ expect(mockRunPrompt).toHaveBeenCalledTimes(1);
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(mockRunPrompt).toHaveBeenCalledTimes(2);
+
+ // No further catch-ups for the missed intervals; next runs are an interval later.
+ await vi.advanceTimersByTimeAsync(HOUR - 12_000);
+ expect(mockRunPrompt).toHaveBeenCalledTimes(2);
+ await vi.advanceTimersByTimeAsync(12_000);
+ expect(mockRunPrompt).toHaveBeenCalledTimes(4);
+ });
+
+ it('stopLoop before the first fire cancels it', async () => {
+ stubDisk([rec('s1', Date.now() - 100 * HOUR, HOUR)]);
+ await restoreLoops();
+ await stopLoop('s1');
+ await vi.advanceTimersByTimeAsync(2 * HOUR);
+ expect(mockRunPrompt).not.toHaveBeenCalled();
+ });
+
+ it('keeps the full interval for a loop that never completed an iteration', async () => {
+ stubDisk([rec('n1', null, HOUR)]);
+ await restoreLoops();
+ await vi.advanceTimersByTimeAsync(HOUR - 1000);
+ expect(mockRunPrompt).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(mockRunPrompt).toHaveBeenCalledTimes(1);
+ });
+ });
+
// ===========================================================================
// Early rejection recovery (#9649) — if loadLoops rejects or setup fails
// before the main run, active.running must be reset so subsequent ticks