Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: schedule restored loops from their saved last run (#11754)
[._.]/ Adam Eivyfcc64bc95ae6
diff --git a/server/services/loops.js b/server/services/loops.jsindex e519cbec0a..b5eddb9435 100644--- a/server/services/loops.js+++ b/server/services/loops.js@@ -26,6 +26,12 @@ const DEFAULT_TIMEOUT_MS = 300_000;const activeLoops = new Map();+// Overdue loops restored at boot catch up once, spaced apart so they do not+// all start together.+const CATCHUP_BASE_DELAY_MS = 1000;+const CATCHUP_STAGGER_MS = 10_000;+const MAX_TIMER_DELAY_MS = 2 ** 31 - 1;+function formatInterval(ms) {if (ms >= 86_400_000) return `${ms / 86_400_000}d`;if (ms >= 3_600_000) return `${ms / 3_600_000}h`;@@ -313,17 +319,35 @@ export async function createLoop({ prompt, interval, name, cwd, providerId, timereturn loop;}-function startLoopTimer(loop, runImmediately = false) {+// setTimeout/setInterval handles share one id space in Node, so a single+// clear covers whichever of the pending first-fire timeout or the steady+// interval is stored on the active entry.+function clearLoopTimer(active) {+ clearTimeout(active.timer);+ clearInterval(active.timer);+}++// firstDelayMs (restore path) delays the first fire past one full interval's+// worth of waiting: the first tick runs once, then the steady interval takes over.+function startLoopTimer(loop, runImmediately = false, firstDelayMs = null) {const runWithLogging = () => runIteration(loop.id);- const timer = setInterval(runWithLogging, loop.intervalMs);- activeLoops.set(loop.id, {- timer,+ const active = {+ timer: null,runId: null,running: false,iterationCount: loop.iterationCount || 0,lastResult: null,history: []- });+ };+ if (firstDelayMs === null) {+ active.timer = setInterval(runWithLogging, loop.intervalMs);+ } else {+ active.timer = setTimeout(() => {+ active.timer = setInterval(runWithLogging, loop.intervalMs);+ runWithLogging();+ }, Math.min(Math.max(firstDelayMs, 0), MAX_TIMER_DELAY_MS));+ }+ activeLoops.set(loop.id, active);if (runImmediately) {setTimeout(runWithLogging, 100);@@ -334,7 +358,7 @@ export async function stopLoop(id) {const active = activeLoops.get(id);if (!active) throw new ServerError(`Loop ${id} is not running`, { status: 409, code: 'INVALID_STATE' });- clearInterval(active.timer);+ clearLoopTimer(active);activeLoops.delete(id);await withLoopsTail(async () => {@@ -376,7 +400,7 @@ export async function resumeLoop(id) {export async function deleteLoop(id) {if (activeLoops.has(id)) {const active = activeLoops.get(id);- clearInterval(active.timer);+ clearLoopTimer(active);activeLoops.delete(id);}@@ -446,7 +470,7 @@ export async function updateLoop(id, updates) {if (activeLoops.has(id) && updates.interval) {const active = activeLoops.get(id);- clearInterval(active.timer);+ clearLoopTimer(active);active.timer = setInterval(() => runIteration(id), updatedRecord.intervalMs);}@@ -488,11 +512,20 @@ export async function restoreLoops() {await ensureDir(LOOPS_OUTPUT_DIR);const loops = await loadLoops();let restored = 0;+ let overdue = 0;+ const now = Date.now();for (const loop of loops) {- if (loop.status === 'running') {- startLoopTimer(loop, false);- restored++;+ if (loop.status !== 'running') continue;+ // Honor the persisted lastRun so a restart does not push the next+ // iteration out by a full interval. An overdue loop gets one staggered+ // catch-up run, never one per missed interval.+ let firstDelay = null;+ if (Number.isFinite(loop.lastRun)) {+ const remaining = loop.lastRun + loop.intervalMs - now;+ firstDelay = remaining > 0 ? remaining : CATCHUP_BASE_DELAY_MS + overdue++ * CATCHUP_STAGGER_MS;}+ startLoopTimer(loop, false, firstDelay);+ restored++;}if (restored > 0) {console.log(`🔄 Restored ${restored} active loop(s)`);diff --git a/server/services/loops.test.js b/server/services/loops.test.jsindex 9a8cbc5ead..f72b654454 100644--- a/server/services/loops.test.js+++ b/server/services/loops.test.js@@ -46,6 +46,7 @@ import { runPromptThroughProvider, resolveProviderAndModel } from './promptRunneimport { getActiveProvider, listSelectableProviders } from './providers.js';import {createLoop,+ restoreLoops,stopLoop,triggerLoop,updateLoop,@@ -588,6 +589,66 @@ describe('loops.js', () => {});});+ describe('restoreLoops honors the persisted lastRun', () => {+ const HOUR = 3_600_000;+ const rec = (id, lastRun, intervalMs = 24 * HOUR) => ({+ id, name: id, prompt: 'p', intervalMs, status: 'running', lastRun, iterationCount: 1+ });+ const stubDisk = (loops) => {+ tryReadFile.mockImplementation(async (path) =>+ String(path).endsWith('loops.json') ? JSON.stringify(loops) : null+ );+ createdLoopIds.push(...loops.map(l => l.id));+ };++ it('fires at lastRun + interval, then every interval', async () => {+ stubDisk([rec('l1', Date.now() - 23 * HOUR)]);+ await restoreLoops();++ await vi.advanceTimersByTimeAsync(HOUR - 1000);+ expect(mockRunPrompt).not.toHaveBeenCalled();+ await vi.advanceTimersByTimeAsync(1000);+ expect(mockRunPrompt).toHaveBeenCalledTimes(1);+ await vi.advanceTimersByTimeAsync(24 * HOUR);+ expect(mockRunPrompt).toHaveBeenCalledTimes(2);+ });++ it('runs one staggered catch-up per overdue loop, then resumes the interval', async () => {+ stubDisk([rec('o1', Date.now() - 100 * HOUR, HOUR), rec('o2', Date.now() - 100 * HOUR, HOUR)]);+ await restoreLoops();++ await vi.advanceTimersByTimeAsync(1000);+ expect(mockRunPrompt).toHaveBeenCalledTimes(1);+ await vi.advanceTimersByTimeAsync(9_000);+ expect(mockRunPrompt).toHaveBeenCalledTimes(1);+ await vi.advanceTimersByTimeAsync(1000);+ expect(mockRunPrompt).toHaveBeenCalledTimes(2);++ // No further catch-ups for the missed intervals; next runs are an interval later.+ await vi.advanceTimersByTimeAsync(HOUR - 12_000);+ expect(mockRunPrompt).toHaveBeenCalledTimes(2);+ await vi.advanceTimersByTimeAsync(12_000);+ expect(mockRunPrompt).toHaveBeenCalledTimes(4);+ });++ it('stopLoop before the first fire cancels it', async () => {+ stubDisk([rec('s1', Date.now() - 100 * HOUR, HOUR)]);+ await restoreLoops();+ await stopLoop('s1');+ await vi.advanceTimersByTimeAsync(2 * HOUR);+ expect(mockRunPrompt).not.toHaveBeenCalled();+ });++ it('keeps the full interval for a loop that never completed an iteration', async () => {+ stubDisk([rec('n1', null, HOUR)]);+ await restoreLoops();+ await vi.advanceTimersByTimeAsync(HOUR - 1000);+ expect(mockRunPrompt).not.toHaveBeenCalled();+ await vi.advanceTimersByTimeAsync(1000);+ expect(mockRunPrompt).toHaveBeenCalledTimes(1);+ });+ });+// ===========================================================================// Early rejection recovery (#9649) — if loadLoops rejects or setup fails// before the main run, active.running must be reset so subsequent ticks