Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

refactor: classify a Persistent Mind turn failure in one place (#11113)

Adam Eivy2d1e764fe3d9
diff --git a/server/services/persistentMindSupervisor.js b/server/services/persistentMindSupervisor.js
index 0bbd1e9090..fa691fe065 100644
--- a/server/services/persistentMindSupervisor.js
+++ b/server/services/persistentMindSupervisor.js
@@ -775,6 +775,60 @@ async function runAdmittedPersistentMindTurn() {
}
}
+// The one place a failed turn is classified. `usageLimit`, `contextBudget` and
+// a call denial are mutually exclusive causes, and the park message, status,
+// retire/retry flags, usage-limit probe and log line all follow from which one
+// applies — so they are decided together here and the `catch` only acts on the
+// result.
+//
+// Persistent Mind has no automatic provider fallback pool. A hard usage-limit /
+// quota exhaustion would otherwise climb failureCount and keep burning
+// scheduled wakes; it autopauses and probes for recovery (transient
+// rate-limits / network blips stay on the interrupted + backoff path).
+// Local context-budget / known-window-below-request failures are not
+// transient either: retrying the same wake only climbs failureCount, so they
+// autopause with an actionable reason until the operator raises the window or
+// shrinks mind context. A refusal already named its own cause and the status it
+// belongs under; only a revoked temporary route retires the wake, exactly as
+// the pre-turn resolution does.
+function classifyPersistentMindTurnFailure(error, { signal, runStartedAt }) {
+ const denied = isPersistentMindCallDenial(error);
+ const aborted = signal.aborted;
+ const detail = aborted
+ ? String(signal.reason || 'Persistent mind turn interrupted')
+ : errorMessage(error);
+ const nonDeniedFault = !denied && !aborted;
+ const usageLimit = nonDeniedFault && isHardProviderUsageLimitError(error);
+ const contextBudget = nonDeniedFault && !usageLimit && isContextBudgetFitnessError(error);
+ const held = denied && error.retryDisposition === 'hold' && error.requiresResubmission !== true;
+ const base = {
+ message: detail,
+ status: held ? 'paused' : ((denied && error.deniedStatus) || 'interrupted'),
+ consumedAttempt: runStartedAt != null,
+ retireWake: denied && error.requiresResubmission === true,
+ retryAt: denied && error.retryDisposition === 'reset-wait' ? error.retryAt : null,
+ markUsageLimit: usageLimit,
+ logMessage: `Persistent mind turn ${denied ? 'refused a further provider call' : 'interrupted'}: ${detail}`,
+ };
+ if (usageLimit) {
+ return {
+ ...base,
+ message: PROVIDER_USAGE_LIMIT_PAUSE_REASON,
+ status: 'paused',
+ logMessage: `Persistent mind autopaused on provider usage limit: ${detail}`,
+ };
+ }
+ if (contextBudget) {
+ return {
+ ...base,
+ message: contextBudgetPauseReasonFrom(error),
+ status: 'paused',
+ logMessage: `Persistent mind autopaused on local context budget: ${detail}`,
+ };
+ }
+ return base;
+}
+
// Admission owns the global/action reservations until the claimed turn's
// provider work, error handling, and endpoint cleanup have all settled — with
// one exception: once a LOCAL-model turn has secured its endpoint-scoped slot
@@ -1019,56 +1073,23 @@ async function runClaimedPersistentMindTurn(turn, mind, globalSlot) {
}, generation);
} catch (error) {
if (generation === runtimeGeneration) {
- const denied = isPersistentMindCallDenial(error);
- const message = controller.signal.aborted
- ? String(controller.signal.reason || 'Persistent mind turn interrupted')
- : errorMessage(error);
- // Persistent Mind has no automatic provider fallback pool. A hard
- // usage-limit / quota exhaustion would otherwise climb failureCount and
- // keep burning scheduled wakes; autopause, then probe for recovery.
- // Transient rate-limits / network blips stay on the interrupted +
- // backoff path below.
- const usageLimit = !denied && !controller.signal.aborted && isHardProviderUsageLimitError(error);
- // Local context-budget / known-window-below-request failures are not
- // transient: retrying the same wake only climbs failureCount. Autopause
- // with an actionable reason (token counts + numCtx hint) and wait for
- // the operator to raise the window or shrink mind context.
- const contextBudget = !denied && !controller.signal.aborted && !usageLimit
- && isContextBudgetFitnessError(error);
- // A refusal already named its own cause and the status it belongs
- // under; only a revoked temporary route retires the wake, exactly as
- // the pre-turn resolution does.
- await parkActiveTurn(
- turn.id,
- usageLimit
- ? PROVIDER_USAGE_LIMIT_PAUSE_REASON
- : (contextBudget ? contextBudgetPauseReasonFrom(error) : message),
- (usageLimit || contextBudget || (denied && error.retryDisposition === 'hold' && error.requiresResubmission !== true)) ? 'paused' : ((denied && error.deniedStatus) || 'interrupted'),
- {
- consumedAttempt: runStartedAt != null,
- retireWake: denied && error.requiresResubmission === true,
- retryAt: denied && error.retryDisposition === 'reset-wait' ? error.retryAt : null,
- },
- );
- if (usageLimit) {
+ const failure = classifyPersistentMindTurnFailure(error, { signal: controller.signal, runStartedAt });
+ await parkActiveTurn(turn.id, failure.message, failure.status, {
+ consumedAttempt: failure.consumedAttempt,
+ retireWake: failure.retireWake,
+ retryAt: failure.retryAt,
+ });
+ if (failure.markUsageLimit) {
const providerId = (await loadState()).config?.persistentMindProfile?.providerId
|| null;
if (typeof providerId === 'string' && providerId) {
- await markProviderUsageLimit(providerId, { message }).catch((markError) => {
+ await markProviderUsageLimit(providerId, { message: errorMessage(error) }).catch((markError) => {
console.error(`❌ Failed to mark provider usage limit for mind probe: ${markError.message}`);
});
}
scheduleUsageLimitProbe(0);
}
- emitLog(
- 'warn',
- usageLimit
- ? `Persistent mind autopaused on provider usage limit: ${message}`
- : contextBudget
- ? `Persistent mind autopaused on local context budget: ${message}`
- : `Persistent mind turn ${denied ? 'refused a further provider call' : 'interrupted'}: ${message}`,
- { turnId: turn.id },
- );
+ emitLog('warn', failure.logMessage, { turnId: turn.id });
}
} finally {
release();
diff --git a/server/services/persistentMindSupervisor.test.js b/server/services/persistentMindSupervisor.test.js
index 18937dd99b..288c46e819 100644
--- a/server/services/persistentMindSupervisor.test.js
+++ b/server/services/persistentMindSupervisor.test.js
@@ -1,5 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { createDefaultPersistentMindState, normalizePersistentMindState, PERSISTENT_MIND_LIMITS } from '../lib/persistentMind.js';
+import { buildPersistentMindCallDenial } from '../lib/persistentMindTrajectory.js';
import { PROVIDER_USAGE_LIMIT_PAUSE_REASON } from '../lib/persistentMindUsageLimit.js';
import { contextBudgetPauseReasonFrom } from '../lib/persistentMindContextBudget.js';
import {
@@ -1753,6 +1754,48 @@ it('does not auto-resume pauses that are not usage-limit autopauses', async () =
});
});
+ describe('refused provider call parking', () => {
+ const echoProfileAdapter = () => vi.fn(async ({ profile }) => ({ ok: true, provider: profile.provider }));
+ const drainDeniedTurn = async (denial) => {
+ const run = vi.fn(async () => { throw denial; });
+ await supervisor.registerPersistentMindTurnAdapter({ prepare: echoProfileAdapter(), run });
+ await supervisor.setPersistentMindEnabled(true);
+ await supervisor.startPersistentMind();
+ await supervisor.enqueuePersistentMindMessage({ id: 'message-denied', text: 'Think.' });
+ await supervisor.drainPersistentMind();
+ return run;
+ };
+
+ it('pauses a held refusal until an operator resumes it, with no backoff gate', async () => {
+ const run = await drainDeniedTurn(buildPersistentMindCallDenial({
+ reason: 'Maintenance hold is active', status: 'waiting', disposition: 'hold',
+ }));
+
+ expect(mock.root.persistentMind).toMatchObject({
+ status: 'paused',
+ pauseReason: 'Maintenance hold is active',
+ nextEligibleWakeAt: null,
+ activeTurn: null,
+ });
+ await supervisor.drainPersistentMind();
+ expect(run).toHaveBeenCalledTimes(1);
+ });
+
+ it('parks a reset-wait refusal under its own status and wakes at the reset time', async () => {
+ const retryAt = new Date(Date.now() + 3_600_000).toISOString();
+ await drainDeniedTurn(buildPersistentMindCallDenial({
+ reason: 'Daily allowance exhausted', status: 'waiting', disposition: 'reset-wait', retryAt,
+ }));
+
+ expect(mock.root.persistentMind).toMatchObject({
+ status: 'waiting',
+ pauseReason: 'Daily allowance exhausted',
+ nextEligibleWakeAt: retryAt,
+ activeTurn: null,
+ });
+ });
+ });
+
describe('usage-limit retry time', () => {
const PROBE_ID = 'cos-persistent-mind-usage-limit-probe';
beforeEach(() => mock.events.clear());