Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: retain scheduled cron runs and release owned audit retry holds

[._.]/ Adam Eivy3fcb804b018e
diff --git a/server/services/agentManagement.js b/server/services/agentManagement.js
index 2ff22d7746..27dbb47df6 100644
--- a/server/services/agentManagement.js
+++ b/server/services/agentManagement.js
@@ -1559,13 +1559,20 @@ export async function handleOrphanedTask(taskId, agentId, getTaskByIdFn, { agent
return {};
});
const targetBranch = resolveTaskTargetBranch(resumePatch);
- emitLog('info', `🔓 Completing interrupted retry transition for task ${taskId}${targetBranch ? ` — resuming ${targetBranch}` : ''}`, {
- taskId, agentId, branchName: targetBranch
- });
- await updateTask(taskId, {
+ const result = await updateTask(taskId, {
status: 'pending',
metadata: { ...resumePatch, ...clearedRetryHoldMetadata() }
- }, task.taskType || 'user');
+ }, task.taskType || 'user', {
+ expectedStatus: 'in_progress', expectedMetadata: { [RETRY_HOLD_KEY]: task.metadata[RETRY_HOLD_KEY] },
+ });
+ if (result?.error) {
+ emitLog('error', `Interrupted retry transition for task ${taskId} failed: ${result.error}`, { taskId, agentId });
+ return;
+ }
+ if (result?.statusChanged) return;
+ emitLog('info', `🔓 Completed interrupted retry transition for task ${taskId}${targetBranch ? ` — resuming ${targetBranch}` : ''}`, {
+ taskId, agentId, branchName: targetBranch
+ });
return;
}
diff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.js
index 187a784590..f60ea36eb4 100644
--- a/server/services/agentManagement.test.js
+++ b/server/services/agentManagement.test.js
@@ -1932,7 +1932,21 @@ describe('the orphan sweep finishes an interrupted retry transition (#3373)', ()
retryPendingCleanup: undefined,
retryPendingSince: undefined,
},
- }, 'user');
+ }, 'user', { expectedStatus: 'in_progress', expectedMetadata: { retryPendingCleanup: 'agent-dead' } });
+ });
+
+ it('recovers extended Deep with an owned hold release and reports rejected writes', async () => {
+ const task = heldTask();
+ task.metadata = { ...task.metadata, auditDepth: 'deep', auditWorkflow: 'extended-v1' };
+ getTaskById.mockResolvedValue(task);
+ updateTask.mockResolvedValueOnce({ error: 'Write rejected' });
+ const { emitLog } = await import('./cosEvents.js');
+ await handleOrphanedTask(task.id, 'unknown-reset', getTaskById);
+ expect(updateTask).toHaveBeenCalledWith(task.id, expect.objectContaining({ status: 'pending' }), 'user', {
+ expectedStatus: 'in_progress', expectedMetadata: { retryPendingCleanup: 'agent-dead' },
+ });
+ expect(emitLog).toHaveBeenCalledWith('error', expect.stringContaining('Write rejected'), expect.anything());
+ expect(emitLog).not.toHaveBeenCalledWith('info', expect.stringContaining('Completed interrupted retry'), expect.anything());
});
// The verdict was already reached and already budgeted a retry — this is not a
@@ -1956,7 +1970,7 @@ describe('the orphan sweep finishes an interrupted retry transition (#3373)', ()
await handleOrphanedTask('task-1', 'agent-dead', getTaskById, { agentMetadata: null });
expect(committedDuringRun).not.toHaveBeenCalled();
- expect(updateTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ status: 'pending' }), 'user');
+ expect(updateTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ status: 'pending' }), 'user', { expectedStatus: 'in_progress', expectedMetadata: { retryPendingCleanup: 'agent-dead' } });
});
// The sweep is NOT owner-scoped by design — whoever armed the hold is gone by
@@ -1966,7 +1980,7 @@ describe('the orphan sweep finishes an interrupted retry transition (#3373)', ()
await handleOrphanedTask('task-1', 'unknown-reset', getTaskById, { agentMetadata: null });
- expect(updateTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ status: 'pending' }), 'user');
+ expect(updateTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ status: 'pending' }), 'user', { expectedStatus: 'in_progress', expectedMetadata: { retryPendingCleanup: 'agent-someone-else' } });
});
// A user-terminated (or budget-exhausted) task is blocked and never held, so the
diff --git a/server/services/agentWorktreeCleanup.js b/server/services/agentWorktreeCleanup.js
index e7778caca0..01754b3d6a 100644
--- a/server/services/agentWorktreeCleanup.js
+++ b/server/services/agentWorktreeCleanup.js
@@ -987,10 +987,13 @@ export async function releaseRetryHold({ agentId, task, success, agentMetadata }
const result = await updateTask(task.id, {
status: 'pending',
metadata: { ...patch, ...clearedRetryHoldMetadata() }
- }, taskType).catch(err => {
+ }, taskType, {
+ expectedStatus: 'in_progress', expectedMetadata: { retryPendingCleanup: persisted.metadata.retryPendingCleanup },
+ }).catch(err => {
emitLog('warn', `Failed to release retry hold on task ${task.id}: ${err.message}`, { taskId: task.id, agentId });
return { error: err.message };
});
+ if (result?.statusChanged) return {};
if (result?.error) {
// The hold is still on disk, so the orphan sweep finishes the transition —
// the task is delayed, never stranded.
diff --git a/server/services/cleanupAgentWorktree.test.js b/server/services/cleanupAgentWorktree.test.js
index 280038b954..91dbba6312 100644
--- a/server/services/cleanupAgentWorktree.test.js
+++ b/server/services/cleanupAgentWorktree.test.js
@@ -1757,7 +1757,7 @@ describe('releaseRetryHold', () => {
retryPendingCleanup: undefined,
retryPendingSince: undefined,
}
- }, 'user');
+ }, 'user', { expectedStatus: 'in_progress', expectedMetadata: { retryPendingCleanup: 'agent-x' } });
});
// Nothing survived cleanup — the task still has to become spawnable, it just
@@ -1771,7 +1771,7 @@ describe('releaseRetryHold', () => {
expect(updateTask).toHaveBeenCalledWith('task-1', {
status: 'pending',
metadata: { retryPendingCleanup: undefined, retryPendingSince: undefined }
- }, 'user');
+ }, 'user', { expectedStatus: 'in_progress', expectedMetadata: { retryPendingCleanup: 'agent-x' } });
});
// A hold armed before the marker carried an id (legacy shape) is releasable by
@@ -1781,7 +1781,7 @@ describe('releaseRetryHold', () => {
await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false, agentMetadata });
- expect(updateTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ status: 'pending' }), 'user');
+ expect(updateTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ status: 'pending' }), 'user', { expectedStatus: 'in_progress', expectedMetadata: { retryPendingCleanup: 'true' } });
});
// The whole point of stamping the owner: attempt A's slow cleanup lands after
diff --git a/server/services/cos.test.js b/server/services/cos.test.js
index da122121c4..afea20f36e 100644
--- a/server/services/cos.test.js
+++ b/server/services/cos.test.js
@@ -1703,7 +1703,8 @@ describe('cos.js source — priority + capacity invariants', () => {
expect(
fnBody,
'queue path must call prepareManagedAppImprovementTask so applyPlanIdMetadata runs + the full prompt is used'
- ).toMatch(/prepareManagedAppImprovementTask\s*\(/);
+ ).toMatch(/prepareTask\s*\(/);
+ expect(GEN_SRC).toContain('prepareTask = prepareManagedAppImprovementTask');
// Match the call shape, not the specific variable name — `task` could
// legitimately be renamed (e.g. `queuedTask`) in a behavior-preserving
@@ -1714,7 +1715,8 @@ describe('cos.js source — priority + capacity invariants', () => {
expect(
fnBody,
'queue path must persist via addTask with raw:true so the enriched task object survives serialization'
- ).toMatch(/addTask\s*\(\s*\w+\s*,\s*['"]internal['"]\s*,\s*\{\s*raw:\s*true\b/);
+ ).toMatch(/persistTask\s*\(\s*\w+\s*,\s*['"]internal['"]\s*,\s*\{\s*raw:\s*true\b/);
+ expect(GEN_SRC).toContain('persistTask = addTask');
// The old buggy path called `getTaskDescription` to build a one-line
// description and then passed app/context/approvalRequired fields to
diff --git a/server/services/cosTaskGenerator.cronQueue.test.js b/server/services/cosTaskGenerator.cronQueue.test.js
new file mode 100644
index 0000000000..ebf77001e1
--- /dev/null
+++ b/server/services/cosTaskGenerator.cronQueue.test.js
@@ -0,0 +1,97 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('./cosEvents.js', () => ({ cosEvents: { emit: vi.fn(), on: vi.fn() }, emitLog: vi.fn() }));
+vi.mock('./apps.js', async (original) => ({
+ ...(await original()), getActiveApps: vi.fn(async () => [{ id: 'example-app', name: 'Example App' }])
+}));
+vi.mock('./appActivity.js', async (original) => ({
+ ...(await original()), loadAppActivity: vi.fn(async () => ({ apps: {} }))
+}));
+vi.mock('./taskSchedule.js', async (original) => ({
+ ...(await original()), getDueTasks: vi.fn(), recordExecution: vi.fn(async () => {})
+}));
+
+import { queueEligibleImprovementTasks } from './cosTaskGenerator.js';
+import { getDueTasks, recordExecution } from './taskSchedule.js';
+import { loadAppActivity } from './appActivity.js';
+import { cosEvents } from './cosEvents.js';
+
+const state = { config: { improvementEnabled: true, appReviewCooldownMs: 60_000 } };
+const dueCron = taskType => ({ taskType, reason: 'cron-due', interval: { type: 'cron' } });
+const drain = { taskType: 'security', reason: 'perpetual-drain', interval: { type: 'on-demand', perpetual: true } };
+const queuedTask = (analysisType, status = 'pending') => ({
+ id: `sys-example-${analysisType}`, status, metadata: { app: 'example-app', analysisType }
+});
+let due;
+let prepareTask;
+let persistTask;
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ due = [dueCron('do-release')];
+ getDueTasks.mockImplementation(async appId => appId ? due : []);
+ loadAppActivity.mockResolvedValue({ apps: {} });
+ prepareTask = vi.fn(async taskType => ({
+ task: { description: `Run ${taskType}\nFull instructions`, metadata: { app: 'example-app', analysisType: taskType } },
+ pendingPerpetualDispatch: null
+ }));
+ persistTask = vi.fn(async task => ({ ...task }));
+});
+const run = (tasks = [], options = {}) => queueEligibleImprovementTasks(state, { tasks }, { prepareTask, persistTask, ...options });
+
+describe('scheduled managed-app queue admission', () => {
+ it.each(['pending', 'in_progress', 'blocked'])('captures a clock slot while unrelated work is %s', async status => {
+ await run([queuedTask('observability', status)]);
+ expect(persistTask).toHaveBeenCalledWith(expect.objectContaining({
+ priority: 'LOW', description: 'Run do-release\nFull instructions', metadata: { app: 'example-app', analysisType: 'do-release' }
+ }), 'internal', { raw: true, ignoreTaskId: null, suppressDequeue: true });
+ expect(recordExecution).toHaveBeenCalledExactlyOnceWith('task:do-release', 'example-app');
+ expect(cosEvents.emit).toHaveBeenCalledWith('cos:dequeue-requested');
+ });
+
+ it('captures every simultaneous cron, including an app override, during app cooldown', async () => {
+ due = [dueCron('do-release'), { ...dueCron('pr-watcher'), interval: { type: 'on-demand' } }, drain];
+ loadAppActivity.mockResolvedValue({ apps: { 'example-app': { cooldownUntil: new Date(Date.now() + 60_000).toISOString() } } });
+ await run();
+ expect(persistTask.mock.calls.map(([task]) => task.metadata.analysisType)).toEqual(['do-release', 'pr-watcher']);
+ expect(recordExecution).toHaveBeenCalledTimes(2);
+ // A later tick sees the persisted tasks even after their clock window closes.
+ due = [drain];
+ await run(persistTask.mock.calls.map(([task]) => ({ ...task, status: 'pending' })));
+ expect(persistTask).toHaveBeenCalledTimes(2);
+ });
+
+ it.each(['pending', 'in_progress', 'blocked'])('keeps same-type %s dedup while allowing a sibling clock slot', async status => {
+ due = [dueCron('do-release'), dueCron('pr-watcher')];
+ await run([queuedTask('do-release', status)]);
+ expect(prepareTask.mock.calls.map(([type]) => type)).toEqual(['pr-watcher']);
+ expect(recordExecution).toHaveBeenCalledExactlyOnceWith('task:pr-watcher', 'example-app');
+ });
+
+ it('does not consume cadence for generator declines or disk-level duplicate races', async () => {
+ due = [dueCron('do-release'), dueCron('pr-watcher')];
+ prepareTask.mockResolvedValueOnce({ task: null });
+ persistTask.mockResolvedValueOnce({ duplicate: true });
+ await run();
+ expect(recordExecution).not.toHaveBeenCalled();
+ expect(cosEvents.emit).not.toHaveBeenCalled();
+ });
+
+ it('retains one background drain per app and holds it behind unrelated work', async () => {
+ due = [drain, { ...drain, taskType: 'observability' }];
+ await run([queuedTask('existing-work')]);
+ expect(persistTask).not.toHaveBeenCalled();
+ await run();
+ expect(persistTask).toHaveBeenCalledTimes(1);
+ expect(recordExecution).toHaveBeenCalledExactlyOnceWith('task:security', 'example-app');
+ });
+
+ it('forwards completion exclusions and respects a disabled improvement gate', async () => {
+ await run([queuedTask('do-release', 'in_progress')], { ignoreTaskId: 'sys-example-do-release', wakeAfterRecord: false });
+ expect(prepareTask).toHaveBeenCalledWith('do-release', expect.anything(), state, { ignoreTaskId: 'sys-example-do-release' });
+ expect(cosEvents.emit).not.toHaveBeenCalled();
+ persistTask.mockClear();
+ await queueEligibleImprovementTasks({ config: { improvementEnabled: false } }, { tasks: [] }, { prepareTask, persistTask });
+ expect(persistTask).not.toHaveBeenCalled();
+ });
+});
diff --git a/server/services/cosTaskGenerator.js b/server/services/cosTaskGenerator.js
index 4181cdf773..f6a21e98ed 100644
--- a/server/services/cosTaskGenerator.js
+++ b/server/services/cosTaskGenerator.js
@@ -33,6 +33,7 @@ import { sanitizeTaskMetadata, PIPELINE_STAGE_BEHAVIOR_FLAGS, MAX_TOTAL_SPAWNS,
import { PATHS } from '../lib/fileUtils.js';
import { renderOrPrependSection } from '../lib/promptSectionRenderer.js';
import { currentTaskTypeName } from '../lib/scheduledTaskTypes.js';
+import { TASK_READINESS_REASON } from '../lib/taskReadinessReasons.js';
import { isPlainObject } from '../lib/objects.js';
import { hasQuotaBurnProvenance, isManualOnDemandRequest, onDemandRequestMetadata } from '../lib/quotaBurnOrigin.js';
import { isAutoApprovableInvestigation } from '../lib/investigationTasks.js';
@@ -1231,7 +1232,10 @@ export async function queueDueInstallWideImprovementTasks({
* Called during every evaluation to ensure system tasks are queued even when user tasks exist
* Tasks are queued to COS-TASKS.md and will be picked up in Priority 2
*/
-export async function queueEligibleImprovementTasks(state, cosTaskData, { ignoreTaskId = null, wakeAfterRecord = true, perpetualContinuation = null } = {}) {
+export async function queueEligibleImprovementTasks(state, cosTaskData, {
+ ignoreTaskId = null, wakeAfterRecord = true, perpetualContinuation = null,
+ prepareTask = prepareManagedAppImprovementTask, persistTask = addTask
+} = {}) {
const taskSchedule = await import('./taskSchedule.js');
const { getDueTasks, getNextTaskType, recordExecution } = taskSchedule;
@@ -1274,10 +1278,17 @@ export async function queueEligibleImprovementTasks(state, cosTaskData, { ignore
// Queue eligible improvement tasks for all managed apps (including PortOS)
const apps = await getActiveApps().catch(() => []);
for (const app of apps) {
- // One pending improvement per app at a time — sibling types must wait
- // until the current task drains, otherwise they queue faster than they
- // can run (per-project concurrency limit + cooldown after each completion).
- if (appsWithPendingImprovement.has(app.id)) {
+ // Capture every due clock slot before applying the background-drain cap.
+ // Cron eligibility lasts one minute: a busy app or a sibling cron must not
+ // discard that occurrence. Persist it now; spawn admission still enforces
+ // concurrency, budgets and cooldowns when the task actually executes.
+ // Use the readiness reason rather than the global interval: an app may
+ // supply its own cron while the global task remains on-demand.
+ const continuingTaskType = perpetualContinuation?.appId === app.id ? perpetualContinuation.taskType : null;
+ const dueTasks = await getDueTasks(app.id, { continuingTaskType });
+ const cronTasks = dueTasks.filter(task => task.reason === TASK_READINESS_REASON.CRON_DUE);
+ const appBusy = appsWithPendingImprovement.has(app.id);
+ if (appBusy && cronTasks.length === 0) {
if (appsWithBlockedImprovement.has(app.id)) {
emitLog('info', `⛔ Skipping improvement queue for ${app.name}: failure-blocked improvement task ${appsWithBlockedImprovement.get(app.id)} occupies the app slot — resolve or delete it to resume`, { appId: app.id });
} else {
@@ -1305,70 +1316,67 @@ export async function queueEligibleImprovementTasks(state, cosTaskData, { ignore
// bypass the back-to-back refill fired right after a perpetual run reads its
// OWN app as "on cooldown" and skips the re-queue, and the drain stalls.
//
- // When the app IS on cooldown, constrain the pick to a due perpetual task
- // (`perpetualOnly`). This is what makes a MIXED schedule converge: if the app
- // also has a due cron/custom type (e.g. pr-watcher), the unconstrained
- // `getNextTaskType` would return that higher-priority NON-exempt type first,
- // we'd see a non-perpetual pick, and we'd skip the whole app for the cooldown
- // window — stranding the perpetual drain behind it. Asking perpetual-only
- // returns the due perpetual drain (or null → leave the cooled-down app
- // alone). When NOT on cooldown, the normal full-priority pick runs.
+ // Once clock slots have been captured, cooldown only narrows the
+ // background pick to a perpetual drain, whose park is its own throttle.
const onCooldown = isAppActivityOnCooldown(appActivity, state.config.appReviewCooldownMs);
- const nextTypeResult = await getNextTaskType(app.id, {
- perpetualOnly: onCooldown,
- continuingTaskType: perpetualContinuation?.appId === app.id ? perpetualContinuation.taskType : null
- }).catch(() => null);
- if (!nextTypeResult) continue;
- const nextType = nextTypeResult.taskType;
-
- const taskKey = `app:${app.id}:${nextType}`;
- if (existingTaskTypes.has(taskKey)) {
- if (blockedTaskTypes.has(taskKey)) {
- emitLog('info', `⛔ Skipping ${nextType} for ${app.name}: blocked task ${blockedTaskTypes.get(taskKey)} of this type already exists — resolve or delete it to resume`, { appId: app.id });
- } else {
- emitLog('debug', `Improvement task ${nextType} for ${app.name} already queued`);
+ const nextTypeResult = cronTasks.length === 0
+ ? await getNextTaskType(app.id, {
+ perpetualOnly: onCooldown, continuingTaskType, dueTasks
+ }).catch(() => null)
+ : null;
+ const nextTypes = cronTasks.length > 0
+ ? cronTasks.map(task => task.taskType)
+ : nextTypeResult ? [nextTypeResult.taskType] : [];
+
+ for (const nextType of nextTypes) {
+ const taskKey = `app:${app.id}:${nextType}`;
+ if (existingTaskTypes.has(taskKey)) {
+ if (blockedTaskTypes.has(taskKey)) {
+ emitLog('info', `⛔ Skipping ${nextType} for ${app.name}: blocked task ${blockedTaskTypes.get(taskKey)} of this type already exists — resolve or delete it to resume`, { appId: app.id });
+ } else {
+ emitLog('debug', `Improvement task ${nextType} for ${app.name} already queued`);
+ }
+ continue;
}
- continue;
- }
-
- // Route through the rich generator so `applyPlanIdMetadata` runs — it
- // scans open `claim/<slug>` branches + PRs and excludes in-flight slugs
- // from the pick. The old stub path skipped this and let two plan-task
- // agents claim the same slug (2026-05-21 incident). The generator
- // returns null on plan-gate / precondition skip; we silently continue.
- // Regression-pinned in cos.test.js.
- const prepared = await prepareManagedAppImprovementTask(nextType, app, state, { ignoreTaskId });
- if (!prepared.task) continue;
- const { task, pendingPerpetualDispatch } = prepared;
-
- // Queue-path invariants override the generator's direct-spawn defaults
- // (which use MEDIUM priority + `app-improve-*` id).
- task.priority = 'LOW';
- task.priorityValue = PRIORITY_VALUES.LOW;
- task.id = `sys-${app.id.slice(0, 8)}-${nextType}-${Date.now().toString(36)}`;
-
- // Queue path: addTask's raw branch persists the task through COS-TASKS.md,
- // which folds multi-line descriptions to first line + full body in
- // metadata.prompt (#4153). On-demand path: emits unpersisted task on
- // task:ready, renders full description via getTaskPrompt.
- const newTask = await addTask(task, 'internal', { raw: true, ignoreTaskId, suppressDequeue: true });
- if (newTask?.duplicate) continue;
- // Recorded from `pendingPerpetualDispatch` — the RECORD prepare returned —
- // never from `task` or `newTask`: addTask hands back a different object
- // whenever the description is multi-line, so recording must not depend on
- // which one the caller is holding (#6871).
- await recordDeferredPerpetualDispatch(pendingPerpetualDispatch, taskSchedule);
- if (wakeAfterRecord) cosEvents.emit('cos:dequeue-requested');
- await recordExecution(`task:${nextType}`, app.id);
-
- emitLog('info', `Queued improvement task: ${nextType} for ${app.name}`, { taskId: newTask.id, appId: app.id });
- existingTaskTypes.add(taskKey);
- appsWithPendingImprovement.add(app.id);
- queued++;
-
- // Only queue one task per app per evaluation to avoid flooding
+ // Route through the rich generator so `applyPlanIdMetadata` runs — it
+ // scans open `claim/<slug>` branches + PRs and excludes in-flight slugs
+ // from the pick. The old stub path skipped this and let two plan-task
+ // agents claim the same slug (2026-05-21 incident). The generator
+ // returns null on plan-gate / precondition skip; we silently continue.
+ // Regression-pinned in cos.test.js.
+ const prepared = await prepareTask(nextType, app, state, { ignoreTaskId });
+ if (!prepared.task) continue;
+ const { task, pendingPerpetualDispatch } = prepared;
+
+ // Queue-path invariants override the generator's direct-spawn defaults
+ // (which use MEDIUM priority + `app-improve-*` id).
+ task.priority = 'LOW';
+ task.priorityValue = PRIORITY_VALUES.LOW;
+ task.id = `sys-${app.id.slice(0, 8)}-${nextType}-${Date.now().toString(36)}`;
+
+ // Queue path: addTask's raw branch persists the task through COS-TASKS.md,
+ // which folds multi-line descriptions to first line + full body in
+ // metadata.prompt (#4153). On-demand path: emits unpersisted task on
+ // task:ready, renders full description via getTaskPrompt.
+ const newTask = await persistTask(task, 'internal', { raw: true, ignoreTaskId, suppressDequeue: true });
+ if (newTask?.duplicate) continue;
+ // Recorded from `pendingPerpetualDispatch` — the RECORD prepare returned —
+ // never from `task` or `newTask`: addTask hands back a different object
+ // whenever the description is multi-line, so recording must not depend on
+ // which one the caller is holding (#6871).
+ await recordDeferredPerpetualDispatch(pendingPerpetualDispatch, taskSchedule);
+ if (wakeAfterRecord) cosEvents.emit('cos:dequeue-requested');
+
+ await recordExecution(`task:${nextType}`, app.id);
+
+ emitLog('info', `Queued improvement task: ${nextType} for ${app.name}`, { taskId: newTask.id, appId: app.id });
+ existingTaskTypes.add(taskKey);
+ appsWithPendingImprovement.add(app.id);
+ queued++;
+ }
+ // Background drains still enqueue at most one task per app per evaluation.
}
if (queued > 0) {
diff --git a/server/services/cosTaskGenerator.test.js b/server/services/cosTaskGenerator.test.js
index 12ff08e5eb..6ba349b018 100644
--- a/server/services/cosTaskGenerator.test.js
+++ b/server/services/cosTaskGenerator.test.js
@@ -1623,7 +1623,7 @@ describe('ignoreTaskId reaches the in-flight-counting gates (#3179)', () => {
it('queueEligibleImprovementTasks passes its ignoreTaskId down to the generator', () => {
// It already forwards the same id to addTask and buildImprovementDedupSets;
// the generator was the one path that dropped it.
- expect(GEN_SRC).toMatch(/prepareManagedAppImprovementTask\(nextType, app, state, \{[\s\S]*ignoreTaskId[\s\S]*\}\)/);
+ expect(GEN_SRC).toMatch(/prepareTask\(nextType, app, state, \{[\s\S]*ignoreTaskId[\s\S]*\}\)/);
});
it('the perpetual gate hands ignoreTaskId to the work detector', () => {
diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.js
index b54829af28..fef41285cb 100644
--- a/server/services/cosTaskStore.js
+++ b/server/services/cosTaskStore.js
@@ -782,7 +782,14 @@ async function writeTaskUpdateLocked(taskId, updates, taskType, { now, suppressD
if (updates.status === 'pending' && isLegacyDeepAudit(current.metadata)) return { error: 'Historical Deep certification tasks are read-only; start a new Deep audit' };
if (updates.status === 'pending' && current.metadata?.auditDepth === 'deep' && isRetryHeld(current.metadata)) {
- return { error: 'Deep checkpoint cleanup is still pending; resume after cleanup finishes' };
+ // Extended audits retry normally after cleanup. Only the guarded release of
+ // this exact persisted hold may make one spawnable; a bare Resume/Run must
+ // still wait, and legacy certification records remain read-only above.
+ const releasesCurrentHold = expectedStatus === 'in_progress'
+ && expectedMetadata?.[RETRY_HOLD_KEY] === current.metadata[RETRY_HOLD_KEY]
+ && Object.hasOwn(updates.metadata || {}, RETRY_HOLD_KEY)
+ && updates.metadata[RETRY_HOLD_KEY] === undefined;
+ if (!releasesCurrentHold) return { error: 'Deep checkpoint cleanup is still pending; resume after cleanup finishes' };
}
const updatedMetadata = mergeUpdateMetadata(tasks[taskIndex].metadata, updates);
diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.js
index 06884ce0e4..09f0102267 100644
--- a/server/services/cosTaskStore.test.js
+++ b/server/services/cosTaskStore.test.js
@@ -93,9 +93,15 @@ vi.mock('./cosState.js', () => ({
ROOT_DIR: '/root'
}));
-vi.mock('./cos.js', () => ({ isRunning: vi.fn(() => false) }));
+vi.mock('./cos.js', () => ({
+ isRunning: vi.fn(() => false),
+ getAgentRecord: vi.fn(),
+ getTaskById: (...args) => getTaskById(...args),
+ updateTask: (...args) => updateTask(...args),
+}));
vi.mock('./cosEvents.js', () => ({
+ emitLog: vi.fn(),
cosEvents: { emit: (name, payload) => mock.events.push({ name, payload }) }
}));
@@ -246,6 +252,27 @@ describe('cosTaskStore.updateTask expectedStatus', () => {
} });
});
+ it('releases an extended Deep retry through the real persisted adapter, preserving the branch', async () => {
+ const { releaseRetryHold } = await import('./agentWorktreeCleanup.js');
+ await addTask({ id: 'sys-extended-retry', description: 'Example audit' }, 'internal');
+ const task = await updateTask('sys-extended-retry', { status: 'in_progress', metadata: {
+ auditDepth: 'deep', auditWorkflow: 'extended-v1', retryPendingCleanup: 'agent-owner',
+ retryPendingSince: '2026-01-01T00:00:00.000Z', existingBranch: 'cos/retained',
+ } }, 'internal');
+ expect(await updateTask(task.id, { status: 'pending', metadata: { retryPendingCleanup: undefined } }, 'internal'))
+ .toMatchObject({ error: expect.stringContaining('cleanup') });
+ const patch = { status: 'pending', metadata: { retryPendingCleanup: undefined } };
+ expect(await updateTask(task.id, patch, 'internal', {
+ expectedStatus: 'in_progress', expectedMetadata: { retryPendingCleanup: 'agent-stale' },
+ })).toMatchObject({ statusChanged: true });
+ await releaseRetryHold({ agentId: 'agent-owner', task: { ...task, taskType: 'internal' }, success: false, agentMetadata: {} });
+ const persisted = await getTaskById(task.id);
+ expect(persisted).toMatchObject({ status: 'pending', metadata: { existingBranch: 'cos/retained', auditWorkflow: 'extended-v1' } });
+ expect(persisted.metadata).not.toHaveProperty('retryPendingCleanup');
+ expect(persisted.metadata).not.toHaveProperty('retryPendingSince');
+ expect(mock.events).toContainEqual(expect.objectContaining({ name: 'tasks:changed', payload: expect.objectContaining({ action: 'requeued' }) }));
+ });
+
it('does not apply a stale terminal update after a task has been claimed', async () => {
const task = await addTask({ id: 'sys-rl-cas', description: 'Resolve a PR' }, 'internal');
await updateTask(task.id, { status: 'in_progress' }, 'internal');
diff --git a/server/services/taskSchedule.js b/server/services/taskSchedule.js
index 0aff94c61a..09d229066a 100644
--- a/server/services/taskSchedule.js
+++ b/server/services/taskSchedule.js
@@ -1104,8 +1104,8 @@ export async function getDueTasks(appId = null, { continuingTaskType = null } =
* When `perpetualOnly` is true, return a due perpetual drain or null; never a
* cron type that would mask it.
*/
-export async function getNextTaskType(appId = null, { perpetualOnly = false, continuingTaskType = null } = {}) {
- const dueTasks = await getDueTasks(appId, { continuingTaskType });
+export async function getNextTaskType(appId = null, { perpetualOnly = false, continuingTaskType = null, dueTasks = null } = {}) {
+ dueTasks ??= await getDueTasks(appId, { continuingTaskType });
const perpetualDue = dueTasks.filter(t => t.interval.perpetual === true);
if (perpetualOnly) {