Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: file provider-failure investigations durably when CoS is stopped (#11758)

[._.]/ Adam Eivy43b344490ff2
diff --git a/server/services/autoFixer.js b/server/services/autoFixer.js
index 030242d0fa..0cfbc5cc54 100644
--- a/server/services/autoFixer.js
+++ b/server/services/autoFixer.js
@@ -77,9 +77,6 @@ function tripCircuit(errorKey) {
return recent.length > CIRCUIT_MAX_FAILURES;
}
-// Store pending tasks when CoS is not running (for later pickup)
-const pendingAutoFixTasks = [];
-
// Collapse a (possibly multi-line) error string to one capped line for logging
// — the single-line logging convention forbids multi-line blobs; the untruncated
// text always remains in the run record's `error` field.
@@ -424,20 +421,6 @@ export function initAutoFixer() {
console.log('🔧 Auto-fixer initialized');
}
-/**
- * Get pending autofix tasks (for CoS to pick up when it starts)
- */
-export function getPendingAutoFixTasks() {
- return [...pendingAutoFixTasks];
-}
-
-/**
- * Clear pending autofix tasks after they've been processed
- */
-export function clearPendingAutoFixTasks() {
- pendingAutoFixTasks.length = 0;
-}
-
/**
* Test-only: drop all deferred timers + dedupe entries so the next call
* starts from a clean slate. Production code paths never call this.
@@ -449,7 +432,6 @@ export function _resetAutoFixerForTests() {
recentErrors.clear();
failureTimestamps.clear();
escalatedKeys.clear();
- pendingAutoFixTasks.length = 0;
// This module's task-creation path now goes through the SHARED investigation
// circuit (investigationTaskProducer.js), so a reset that stopped at the local
// maps would leave earlier cases' creations counted against later ones — every
@@ -568,43 +550,25 @@ async function createAIProviderInvestigationTask(error) {
// matches the other two task producers in this file.
};
- // If CoS is running, create the task immediately. The approval verdict is
- // resolved INSIDE this branch: on the other path the task is only queued for
- // later pickup, so a verdict computed now would be read against a backlog that
- // is stale by the time it is filed — and the read (both task files) buys
- // nothing.
- if (isRunning()) {
- // Unattended by default, matching every other investigation producer (#3714):
- // a provider failure is exactly the diagnosis CoS exists to do for itself, and
- // gating it behind an approval only meant the queue filled with approvals while
- // the provider stayed broken. The loop guards are unchanged and still upstream
- // of here — `isDuplicateError` collapses a repeat inside the dedupe window and
- // `tripCircuit` suppresses the task entirely past CIRCUIT_MAX_FAILURES.
- const { task } = await fileInvestigationTask({
- ...taskData,
- fingerprint: investigationFingerprint({
- category: diagnostics.category,
- kind: 'provider-failure',
- scope: ctx.provider
- })
- });
- console.log(`✅ AI provider investigation task created: ${task.id} [tier ${diagnostics.tier}: ${diagnostics.fixStrategy}]`);
- return task;
- }
-
- // Otherwise, store for later pickup
- console.log(`📋 CoS not running - queuing AI provider investigation task [tier ${diagnostics.tier}: ${diagnostics.fixStrategy}]`);
- pendingAutoFixTasks.push({
+ // Filed durably whether or not the CoS daemon is running: `addTask` persists
+ // without it and CoS picks the task up when it next runs. (An in-memory queue
+ // here once had no consumer and silently dropped every failure while stopped.)
+ // Unattended by default, matching every other investigation producer (#3714):
+ // a provider failure is exactly the diagnosis CoS exists to do for itself, and
+ // gating it behind an approval only meant the queue filled with approvals while
+ // the provider stayed broken. The loop guards are unchanged and still upstream
+ // of here — `isDuplicateError` collapses a repeat inside the dedupe window and
+ // `tripCircuit` suppresses the task entirely past CIRCUIT_MAX_FAILURES.
+ const { task } = await fileInvestigationTask({
...taskData,
- createdAt: Date.now(),
- error: {
- code: error.code,
- message: error.message,
- context: ctx
- }
+ fingerprint: investigationFingerprint({
+ category: diagnostics.category,
+ kind: 'provider-failure',
+ scope: ctx.provider
+ })
});
-
- return null;
+ console.log(`✅ AI provider investigation task filed: ${task.id} [tier ${diagnostics.tier}: ${diagnostics.fixStrategy}]`);
+ return task;
}
/**
diff --git a/server/services/autoFixer.test.js b/server/services/autoFixer.test.js
index 539cc30c20..522e1bfc2e 100644
--- a/server/services/autoFixer.test.js
+++ b/server/services/autoFixer.test.js
@@ -30,8 +30,6 @@ const cos = await import('./cos.js');
const {
noteFallbackHandled,
initAutoFixer,
- getPendingAutoFixTasks,
- clearPendingAutoFixTasks,
_resetAutoFixerForTests,
classifyFixTier,
buildFixDiagnostics,
@@ -44,6 +42,11 @@ const { errorEvents } = await import('../lib/errorHandler.js');
// listener to be attached, regardless of which test file boots first.
initAutoFixer();
+// Tasks handed to the durable store (addTask's first argument).
+function filedTasks() {
+ return cos.addTask.mock.calls.map(([task]) => task);
+}
+
function emitProviderFailure({ provider = 'Primary CLI', model = 'm-1', runId = 'run-1' } = {}) {
errorEvents.emit('error', {
code: 'AI_PROVIDER_EXECUTION_FAILED',
@@ -266,18 +269,20 @@ describe('autoFixer — defer + noteFallbackHandled', () => {
expect(cos.addTask).toHaveBeenCalledTimes(1);
});
- it('queues to pendingAutoFixTasks when CoS is not running', async () => {
- clearPendingAutoFixTasks();
+ it('files a durable investigation task when CoS is not running', async () => {
cos.isRunning.mockReturnValue(false);
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1' });
await vi.advanceTimersByTimeAsync(5500);
- expect(cos.addTask).not.toHaveBeenCalled();
- const pending = getPendingAutoFixTasks();
- expect(pending).toHaveLength(1);
- expect(pending[0].description).toBe('Investigate AI provider failure: Primary CLI (m-1)');
- clearPendingAutoFixTasks();
+ expect(cos.addTask).toHaveBeenCalledTimes(1);
+ expect(filedTasks()[0].description).toContain('Investigate AI provider failure: Primary CLI (m-1)');
+ expect(filedTasks()[0].isInvestigation).toBe(true);
+
+ // A repeat inside the dedupe window files nothing new.
+ emitProviderFailure({ provider: 'Primary CLI', model: 'm-1', runId: 'run-2' });
+ await vi.advanceTimersByTimeAsync(5500);
+ expect(cos.addTask).toHaveBeenCalledTimes(1);
});
// The body built by buildAIProviderErrorContext has always fallen back to
@@ -285,7 +290,6 @@ describe('autoFixer — defer + noteFallbackHandled', () => {
// no attribution filed a task literally called
// "Investigate AI provider failure: undefined (undefined)".
it('never titles a task with a literal undefined provider or model', async () => {
- clearPendingAutoFixTasks();
cos.isRunning.mockReturnValue(false);
// Emitted raw, not through emitProviderFailure — that helper defaults
@@ -300,10 +304,9 @@ describe('autoFixer — defer + noteFallbackHandled', () => {
});
await vi.advanceTimersByTimeAsync(5500);
- const pending = getPendingAutoFixTasks();
+ const pending = filedTasks();
expect(pending).toHaveLength(1);
expect(pending[0].description).not.toMatch(/undefined/);
- clearPendingAutoFixTasks();
});
it('clears the dedupe entry when deferred task creation fails so future failures can still raise tasks', async () => {
@@ -702,20 +705,18 @@ describe('autoFixer — diagnostics ride on the created task record (issue #2328
}
});
- it('attaches diagnostics to the pending record when CoS is not running', async () => {
- clearPendingAutoFixTasks();
+ it('attaches diagnostics to the filed task when CoS is not running', async () => {
cos.isRunning.mockReturnValue(false);
emitProviderFailure({ provider: 'LM Studio', model: 'qwen', runId: 'p-1' });
await vi.advanceTimersByTimeAsync(5500);
- const pending = getPendingAutoFixTasks();
+ const pending = filedTasks();
expect(pending).toHaveLength(1);
expect(pending[0].diagnostics).toMatchObject({
triggerEvent: 'AI_PROVIDER_EXECUTION_FAILED',
target: 'LM Studio (qwen)',
tier: FIX_TIERS.ESCALATE, // no errorAnalysis category → unknown → escalate
});
- clearPendingAutoFixTasks();
});
it('attaches diagnostics to the generic critical-error fix task', async () => {
@@ -843,13 +844,11 @@ describe('autoFixer — stale deployed build in the investigation body', () => {
installState.getBootCommit.mockReturnValue(null);
installState.getInstallState.mockClear();
installState.getInstallState.mockResolvedValue(null);
- clearPendingAutoFixTasks();
_resetAutoFixerForTests();
});
afterEach(() => {
vi.useRealTimers();
- clearPendingAutoFixTasks();
_resetAutoFixerForTests();
});
@@ -864,7 +863,7 @@ describe('autoFixer — stale deployed build in the investigation body', () => {
emitProviderFailure({ provider: 'Ollama', model: 'm-1' });
await vi.advanceTimersByTimeAsync(5500);
- const { context } = getPendingAutoFixTasks()[0];
+ const { context } = filedTasks()[0];
expect(context).toContain('## Deployed Build: STALE');
expect(context).toContain('git log aaaaaaa..bbbbbbb');
expect(context).toContain('1. Rule out the stale deployed build');
@@ -884,7 +883,7 @@ describe('autoFixer — stale deployed build in the investigation body', () => {
emitProviderFailure({ provider: 'Ollama', model: 'm-1' });
await vi.advanceTimersByTimeAsync(5500);
- const { context } = getPendingAutoFixTasks()[0];
+ const { context } = filedTasks()[0];
expect(context).not.toContain('Deployed Build');
expect(context).toContain('1. Check if the AI provider is configured correctly');
});
@@ -899,6 +898,6 @@ describe('autoFixer — stale deployed build in the investigation body', () => {
await vi.advanceTimersByTimeAsync(5500);
expect(installState.getInstallState).not.toHaveBeenCalled();
- expect(getPendingAutoFixTasks()[0].context).not.toContain('Deployed Build');
+ expect(filedTasks()[0].context).not.toContain('Deployed Build');
});
});