Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix(cos): stop failure-artifact reaper from overwriting revived tasks or expiring running investigations (#12059)

[._.]/ Adam Eivy608dc751e297
diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.js
index 8632cf38c9..346de67bb7 100644
--- a/server/services/cosTaskStore.js
+++ b/server/services/cosTaskStore.js
@@ -1181,6 +1181,9 @@ export function isReapableBlockedFailure(task, { now = Date.now(), maxAgeMs = DE
export function isReapableInvestigation(task, tasksById) {
if (!isInvestigationTask(task)) return false;
if (task.status === 'completed') return false; // already terminal
+ // An in-progress investigation has a live agent diagnosing it; reaping it
+ // would let a later finalize regress the status or spawn a second run (#12059).
+ if (task.status === 'in_progress') return false;
const affected = Array.isArray(task.metadata?.affectedTasks) ? task.metadata.affectedTasks : [];
if (affected.length === 0) return false;
return affected.every((id) => {
@@ -1230,11 +1233,13 @@ export async function sweepResolvedFailureTasks({
let staleBlocks = 0;
let investigations = 0;
for (const { task, type, reason } of targets) {
+ // Guard the flip under the write lock: a Resume/Run or dequeue that landed
+ // after the snapshot must not be overwritten by a stale auto-expire (#12059).
const updated = await updateTask(task.id, {
status: 'completed',
metadata: { resolution: 'auto-expired', autoExpiredReason: reason, autoExpiredAt: new Date(now).toISOString() }
- }, type, { now });
- if (updated?.error) continue;
+ }, type, { now, expectedStatus: task.status });
+ if (updated?.error || updated?.statusChanged) continue;
if (reason === 'stale-failure-block') staleBlocks++; else investigations++;
}
diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.js
index 2e73e064e3..29078ac208 100644
--- a/server/services/cosTaskStore.test.js
+++ b/server/services/cosTaskStore.test.js
@@ -30,6 +30,7 @@ const mock = vi.hoisted(() => ({
parseCalls: 0,
// When set, atomicWrite rejects with it (a write that dies before the rename).
atomicFailure: null,
+ afterWrite: null,
state: null,
events: [],
// Controls the mocked codeReview.js for resolveTaskChallengeWithRecheck (#2471).
@@ -61,6 +62,8 @@ vi.mock('../lib/fileUtils.js', async (importOriginal) => {
if (mock.atomicFailure) throw mock.atomicFailure;
mock.files.set(p, content);
mock.mtimes.set(p, (mock.mtimes.get(p) || 0) + 1000);
+ // Lets a test mutate the file between two reaper flips (snapshot race).
+ if (mock.afterWrite && mock.afterWrite(p)) mock.afterWrite = null;
})
};
});
@@ -162,6 +165,7 @@ const baseState = () => ({
beforeEach(() => {
mock.atomicFailure = null;
+ mock.afterWrite = null;
mock.files = new Map();
mock.mtimes = new Map();
mock.parseCalls = 0;
@@ -2323,6 +2327,11 @@ describe('cosTaskStore — stale failure-artifact reaper (#2619)', () => {
expect(isReapableInvestigation(investigation(), new Map())).toBe(true);
});
+ it('never reaps an in-progress investigation, even when its origins are all completed', () => {
+ const byId = new Map([['task-a', { id: 'task-a', status: 'completed' }]]);
+ expect(isReapableInvestigation(investigation({ status: 'in_progress' }), byId)).toBe(false);
+ });
+
it('does not reap while any originating task is still live', () => {
const byId = new Map([['task-a', { id: 'task-a', status: 'blocked' }]]);
expect(isReapableInvestigation(investigation(), byId)).toBe(false);
@@ -2421,6 +2430,44 @@ describe('cosTaskStore — stale failure-artifact reaper (#2619)', () => {
expect(after.tasks.find(t => t.id === 'sys-inv2').status).toBe('pending');
});
+ it('leaves an in-progress investigation running even after its origin completes', async () => {
+ await addTask({ description: 'origin fixed by hand', id: 'task-origin3', priority: 'HIGH' }, 'user', { now: NOW });
+ await addTask({
+ description: '[Auto] Investigate agent failure [fp]: running',
+ id: 'sys-running',
+ isInvestigation: true,
+ affectedTasks: ['task-origin3']
+ }, 'internal', { now: NOW });
+ await updateTask('sys-running', { status: 'in_progress' }, 'internal', { now: NOW });
+ await updateTask('task-origin3', { status: 'completed' }, 'user', { now: NOW });
+ const res = await sweepResolvedFailureTasks({ now: NOW });
+ expect(res.reaped).toBe(0);
+ const after = await getCosTasks();
+ expect(after.tasks.find(t => t.id === 'sys-running').status).toBe('in_progress');
+ });
+
+ it('does not auto-expire a target that was revived between the snapshot and its flip', async () => {
+ await addTask({ description: 'stale one', id: 'task-race1', priority: 'HIGH' }, 'user', { now: NOW });
+ await addTask({ description: 'stale two', id: 'task-race2', priority: 'HIGH' }, 'user', { now: NOW });
+ for (const id of ['task-race1', 'task-race2']) {
+ await updateTask(id, { status: 'blocked', metadata: { blockedCategory: 'max-retries', blockedAt: daysAgo(20) } }, 'user', { now: NOW });
+ }
+ // After the first flip lands, the user revives the second target (Resume
+ // → in_progress). Its flip was planned from the pre-revive snapshot.
+ mock.afterWrite = (p) => {
+ if (p !== USER_FILE) return false;
+ // Revive whichever target the sweep has not flipped yet.
+ mock.files.set(p, mock.files.get(p).replace(/^(- )\[!\](.*#task-race)/m, '$1[~]$2'));
+ mock.mtimes.set(p, (mock.mtimes.get(p) || 0) + 1000);
+ return true;
+ };
+ const res = await sweepResolvedFailureTasks({ now: NOW });
+ expect(res).toMatchObject({ reaped: 1, staleBlocks: 1 });
+ const after = await getUserTasks();
+ const statuses = after.tasks.filter(t => t.id.startsWith('task-race')).map(t => t.status).sort();
+ expect(statuses).toEqual(['completed', 'in_progress']);
+ });
+
it('bounds the number of flips per sweep to the limit', async () => {
for (let i = 0; i < 4; i++) {
await addTask({ description: `old ${i}`, id: `task-b${i}`, priority: 'LOW' }, 'user', { now: NOW });