Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: file provider-failure investigations durably when CoS is stopped (#11758)
[._.]/ Adam Eivy43b344490ff2
diff --git a/server/services/autoFixer.js b/server/services/autoFixer.jsindex 030242d0fa..0cfbc5cc54 100644--- a/server/services/autoFixer.js+++ b/server/services/autoFixer.js@@ -77,9 +77,6 @@ function tripCircuit(errorKey) {return recent.length > CIRCUIT_MAX_FAILURES;}-// Store pending tasks when CoS is not running (for later pickup)-const pendingAutoFixTasks = [];-// Collapse a (possibly multi-line) error string to one capped line for logging// — the single-line logging convention forbids multi-line blobs; the untruncated// text always remains in the run record's `error` field.@@ -424,20 +421,6 @@ export function initAutoFixer() {console.log('🔧 Auto-fixer initialized');}-/**- * Get pending autofix tasks (for CoS to pick up when it starts)- */-export function getPendingAutoFixTasks() {- return [...pendingAutoFixTasks];-}--/**- * Clear pending autofix tasks after they've been processed- */-export function clearPendingAutoFixTasks() {- pendingAutoFixTasks.length = 0;-}-/*** Test-only: drop all deferred timers + dedupe entries so the next call* starts from a clean slate. Production code paths never call this.@@ -449,7 +432,6 @@ export function _resetAutoFixerForTests() {recentErrors.clear();failureTimestamps.clear();escalatedKeys.clear();- pendingAutoFixTasks.length = 0;// This module's task-creation path now goes through the SHARED investigation// circuit (investigationTaskProducer.js), so a reset that stopped at the local// maps would leave earlier cases' creations counted against later ones — every@@ -568,43 +550,25 @@ async function createAIProviderInvestigationTask(error) {// matches the other two task producers in this file.};- // If CoS is running, create the task immediately. The approval verdict is- // resolved INSIDE this branch: on the other path the task is only queued for- // later pickup, so a verdict computed now would be read against a backlog that- // is stale by the time it is filed — and the read (both task files) buys- // nothing.- if (isRunning()) {- // Unattended by default, matching every other investigation producer (#3714):- // a provider failure is exactly the diagnosis CoS exists to do for itself, and- // gating it behind an approval only meant the queue filled with approvals while- // the provider stayed broken. The loop guards are unchanged and still upstream- // of here — `isDuplicateError` collapses a repeat inside the dedupe window and- // `tripCircuit` suppresses the task entirely past CIRCUIT_MAX_FAILURES.- const { task } = await fileInvestigationTask({- ...taskData,- fingerprint: investigationFingerprint({- category: diagnostics.category,- kind: 'provider-failure',- scope: ctx.provider- })- });- console.log(`✅ AI provider investigation task created: ${task.id} [tier ${diagnostics.tier}: ${diagnostics.fixStrategy}]`);- return task;- }-- // Otherwise, store for later pickup- console.log(`📋 CoS not running - queuing AI provider investigation task [tier ${diagnostics.tier}: ${diagnostics.fixStrategy}]`);- pendingAutoFixTasks.push({+ // Filed durably whether or not the CoS daemon is running: `addTask` persists+ // without it and CoS picks the task up when it next runs. (An in-memory queue+ // here once had no consumer and silently dropped every failure while stopped.)+ // Unattended by default, matching every other investigation producer (#3714):+ // a provider failure is exactly the diagnosis CoS exists to do for itself, and+ // gating it behind an approval only meant the queue filled with approvals while+ // the provider stayed broken. The loop guards are unchanged and still upstream+ // of here — `isDuplicateError` collapses a repeat inside the dedupe window and+ // `tripCircuit` suppresses the task entirely past CIRCUIT_MAX_FAILURES.+ const { task } = await fileInvestigationTask({...taskData,- createdAt: Date.now(),- error: {- code: error.code,- message: error.message,- context: ctx- }+ fingerprint: investigationFingerprint({+ category: diagnostics.category,+ kind: 'provider-failure',+ scope: ctx.provider+ })});-- return null;+ console.log(`✅ AI provider investigation task filed: ${task.id} [tier ${diagnostics.tier}: ${diagnostics.fixStrategy}]`);+ return task;}/**diff --git a/server/services/autoFixer.test.js b/server/services/autoFixer.test.jsindex 539cc30c20..522e1bfc2e 100644--- a/server/services/autoFixer.test.js+++ b/server/services/autoFixer.test.js@@ -30,8 +30,6 @@ const cos = await import('./cos.js');const {noteFallbackHandled,initAutoFixer,- getPendingAutoFixTasks,- clearPendingAutoFixTasks,_resetAutoFixerForTests,classifyFixTier,buildFixDiagnostics,@@ -44,6 +42,11 @@ const { errorEvents } = await import('../lib/errorHandler.js');// listener to be attached, regardless of which test file boots first.initAutoFixer();+// Tasks handed to the durable store (addTask's first argument).+function filedTasks() {+ return cos.addTask.mock.calls.map(([task]) => task);+}+function emitProviderFailure({ provider = 'Primary CLI', model = 'm-1', runId = 'run-1' } = {}) {errorEvents.emit('error', {code: 'AI_PROVIDER_EXECUTION_FAILED',@@ -266,18 +269,20 @@ describe('autoFixer — defer + noteFallbackHandled', () => {expect(cos.addTask).toHaveBeenCalledTimes(1);});- it('queues to pendingAutoFixTasks when CoS is not running', async () => {- clearPendingAutoFixTasks();+ it('files a durable investigation task when CoS is not running', async () => {cos.isRunning.mockReturnValue(false);emitProviderFailure({ provider: 'Primary CLI', model: 'm-1' });await vi.advanceTimersByTimeAsync(5500);- expect(cos.addTask).not.toHaveBeenCalled();- const pending = getPendingAutoFixTasks();- expect(pending).toHaveLength(1);- expect(pending[0].description).toBe('Investigate AI provider failure: Primary CLI (m-1)');- clearPendingAutoFixTasks();+ expect(cos.addTask).toHaveBeenCalledTimes(1);+ expect(filedTasks()[0].description).toContain('Investigate AI provider failure: Primary CLI (m-1)');+ expect(filedTasks()[0].isInvestigation).toBe(true);++ // A repeat inside the dedupe window files nothing new.+ emitProviderFailure({ provider: 'Primary CLI', model: 'm-1', runId: 'run-2' });+ await vi.advanceTimersByTimeAsync(5500);+ expect(cos.addTask).toHaveBeenCalledTimes(1);});// The body built by buildAIProviderErrorContext has always fallen back to@@ -285,7 +290,6 @@ describe('autoFixer — defer + noteFallbackHandled', () => {// no attribution filed a task literally called// "Investigate AI provider failure: undefined (undefined)".it('never titles a task with a literal undefined provider or model', async () => {- clearPendingAutoFixTasks();cos.isRunning.mockReturnValue(false);// Emitted raw, not through emitProviderFailure — that helper defaults@@ -300,10 +304,9 @@ describe('autoFixer — defer + noteFallbackHandled', () => {});await vi.advanceTimersByTimeAsync(5500);- const pending = getPendingAutoFixTasks();+ const pending = filedTasks();expect(pending).toHaveLength(1);expect(pending[0].description).not.toMatch(/undefined/);- clearPendingAutoFixTasks();});it('clears the dedupe entry when deferred task creation fails so future failures can still raise tasks', async () => {@@ -702,20 +705,18 @@ describe('autoFixer — diagnostics ride on the created task record (issue #2328}});- it('attaches diagnostics to the pending record when CoS is not running', async () => {- clearPendingAutoFixTasks();+ it('attaches diagnostics to the filed task when CoS is not running', async () => {cos.isRunning.mockReturnValue(false);emitProviderFailure({ provider: 'LM Studio', model: 'qwen', runId: 'p-1' });await vi.advanceTimersByTimeAsync(5500);- const pending = getPendingAutoFixTasks();+ const pending = filedTasks();expect(pending).toHaveLength(1);expect(pending[0].diagnostics).toMatchObject({triggerEvent: 'AI_PROVIDER_EXECUTION_FAILED',target: 'LM Studio (qwen)',tier: FIX_TIERS.ESCALATE, // no errorAnalysis category → unknown → escalate});- clearPendingAutoFixTasks();});it('attaches diagnostics to the generic critical-error fix task', async () => {@@ -843,13 +844,11 @@ describe('autoFixer — stale deployed build in the investigation body', () => {installState.getBootCommit.mockReturnValue(null);installState.getInstallState.mockClear();installState.getInstallState.mockResolvedValue(null);- clearPendingAutoFixTasks();_resetAutoFixerForTests();});afterEach(() => {vi.useRealTimers();- clearPendingAutoFixTasks();_resetAutoFixerForTests();});@@ -864,7 +863,7 @@ describe('autoFixer — stale deployed build in the investigation body', () => {emitProviderFailure({ provider: 'Ollama', model: 'm-1' });await vi.advanceTimersByTimeAsync(5500);- const { context } = getPendingAutoFixTasks()[0];+ const { context } = filedTasks()[0];expect(context).toContain('## Deployed Build: STALE');expect(context).toContain('git log aaaaaaa..bbbbbbb');expect(context).toContain('1. Rule out the stale deployed build');@@ -884,7 +883,7 @@ describe('autoFixer — stale deployed build in the investigation body', () => {emitProviderFailure({ provider: 'Ollama', model: 'm-1' });await vi.advanceTimersByTimeAsync(5500);- const { context } = getPendingAutoFixTasks()[0];+ const { context } = filedTasks()[0];expect(context).not.toContain('Deployed Build');expect(context).toContain('1. Check if the AI provider is configured correctly');});@@ -899,6 +898,6 @@ describe('autoFixer — stale deployed build in the investigation body', () => {await vi.advanceTimersByTimeAsync(5500);expect(installState.getInstallState).not.toHaveBeenCalled();- expect(getPendingAutoFixTasks()[0].context).not.toContain('Deployed Build');+ expect(filedTasks()[0].context).not.toContain('Deployed Build');});});