Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: CoS dequeue no longer re-admits tasks that are already spawning (#11847)

Adam Eivy4cfca74ce044
diff --git a/server/services/cos.js b/server/services/cos.js
index 28a24af94a..a023cfc22e 100644
--- a/server/services/cos.js
+++ b/server/services/cos.js
@@ -48,7 +48,7 @@ import { mergePersistentMindPlaybook } from '../lib/persistentMindPlaybook.js';
import { mergePersistentMindThinkingPresets } from '../lib/persistentMindThinkingPresets.js';
// Dependency-free leaf holding the shared agent maps + the runner-mode flag,
// read by `isRunnerHolding` below.
-import { useRunner } from './agentState.js';
+import { useRunner, spawningTasks } from './agentState.js';
// Shared state management (extracted to avoid circular deps)
import { loadState, saveState, withStateLock, loadConfig, saveConfig, withConfigLock, ensureDirectories, isImprovementEnabled, canQueueImprovementTasks, SCRIPTS_DIR, isDaemonRunning, setDaemonRunning } from './cosState.js';
@@ -1088,6 +1088,7 @@ async function spawnDequeuePriority1UserTasks(ctx) {
await admitPendingUserTasks({ pendingUserTasks, instanceId }, {
capacityExhausted: () => capacity.spawned >= capacity.availableSlots,
+ isOwned: (task) => capacity.isTaskOwned(task),
canSpawn: (task) => capacity.canSpawn(task),
emitSpawn: (task) => cosEvents.emit('task:ready', task),
trackSpawn: (task) => capacity.trackSpawn(task),
@@ -1121,6 +1122,7 @@ async function spawnDequeuePriority2AutoApproved(ctx, { taskSchedule }) {
spawnProjectCounts: capacity.spawnProjectCounts,
instanceId,
}, {
+ isOwned: (task) => capacity.isTaskOwned(task),
canSpawn: (task, ceiling) => capacity.canSpawn(task, ceiling),
trackSpawn: (task) => capacity.trackSpawn(task),
emitSpawn: (task) => {
@@ -1249,6 +1251,7 @@ export async function dequeueNextTask({ ignoreTaskId = null } = {}) {
// to every CoS-subscribed socket client.
const loggedFullEndpoints = new Set();
const capacity = createDequeueCapacity(state, {
+ inFlightTaskIds: spawningTasks,
agentsByProject: countRunningAgentsByProject(state.agents),
localEndpointCounts: countRunningAgentsByLocalEndpoint(state.agents, localSlots.endpointForAgent),
localEndpointLimit: localSlots.limit,
diff --git a/server/services/cos.test.js b/server/services/cos.test.js
index afea20f36e..22570328dc 100644
--- a/server/services/cos.test.js
+++ b/server/services/cos.test.js
@@ -499,6 +499,54 @@ function makeLocalSlotCapacity(state, { activeProvider = CLOUD_PROVIDER, limit =
const userBuckets = (tasks) => ({ onDemand: [], user: tasks, autoSystem: [], idle: [] });
+describe('dequeueNextTask — task ownership beyond registered agents (#11847)', () => {
+ it('never admits an in-flight spawn, and still counts it against the global slots', () => {
+ // 4-slot cap: 1 running agent + 2 spawns in flight (no agent record yet)
+ // leaves one real slot, which the queued task — not the in-flight ones — gets.
+ const state = makeState({ maxConcurrentAgents: 4, runningAgents: [makeRunningAgent()] });
+ const capacity = createDequeueCapacity(state, { inFlightTaskIds: new Set(['spawning-a', 'spawning-b']) });
+ expect(capacity.availableSlots).toBe(1);
+
+ const buckets = {
+ onDemand: [],
+ user: [task('spawning-a'), task('spawning-b'), task('queued-c'), task('queued-d')],
+ autoSystem: [],
+ idle: [],
+ };
+ expect(priorityDequeue(buckets, capacity).map(t => t.id)).toEqual(['queued-c']);
+ });
+
+ it('admits nothing when running agents plus in-flight spawns fill maxConcurrentAgents', () => {
+ const state = makeState({ maxConcurrentAgents: 2, runningAgents: [makeRunningAgent()] });
+ const capacity = createDequeueCapacity(state, { inFlightTaskIds: ['spawning-a'] });
+ expect(capacity.availableSlots).toBe(0);
+ expect(priorityDequeue({ onDemand: [], user: [task('queued-b')], autoSystem: [], idle: [] }, capacity)).toEqual([]);
+ });
+
+ it('does not double-count an in-flight spawn whose agent is already running', () => {
+ const state = makeState({
+ maxConcurrentAgents: 3,
+ runningAgents: [{ ...makeRunningAgent(), taskId: 'spawning-a' }],
+ });
+ const capacity = createDequeueCapacity(state, { inFlightTaskIds: ['spawning-a'] });
+ expect(capacity.availableSlots).toBe(2);
+ });
+
+ it('admits a task id at most once per cycle across tiers', () => {
+ const state = makeState({ maxConcurrentAgents: 5 });
+ const capacity = createDequeueCapacity(state, {});
+ const buckets = {
+ onDemand: [task('dup-1')],
+ user: [task('dup-1'), task('user-2')],
+ autoSystem: [task('dup-1'), task('user-2'), task('sys-3')],
+ idle: [],
+ };
+ expect(priorityDequeue(buckets, capacity).map(t => t.id)).toEqual(['dup-1', 'user-2', 'sys-3']);
+ expect(capacity.spawnedTaskIds).toEqual(['dup-1', 'user-2', 'sys-3']);
+ expect(capacity.spawned).toBe(3);
+ });
+});
+
describe('dequeueNextTask — per-local-endpoint agent cap (#4834)', () => {
it('serializes two queued tasks that resolve to the SAME local endpoint', () => {
// Global cap 5, nothing running: both tasks clear the global and
diff --git a/server/services/cosDequeue.js b/server/services/cosDequeue.js
index f183602315..6276626e14 100644
--- a/server/services/cosDequeue.js
+++ b/server/services/cosDequeue.js
@@ -29,7 +29,8 @@ import { emitLog } from './cosEvents.js';
* per-project tally, and exposes the exact `canSpawn` / `trackSpawn` closure the
* scheduler uses to enforce the global slot cap AND the per-project cap.
*
- * `availableSlots` = global cap minus currently-running agents (may be 0 or
+ * `availableSlots` = global cap minus currently-running agents minus in-flight
+ * spawns that have no running agent record yet (may be 0 or
* negative if a config change shrank the cap below live load — callers still
* guard with `availableSlots <= 0`). `perProjectLimit` falls back to the global
* cap when `maxConcurrentAgentsPerProject` is unset/0, matching the scheduler's
@@ -41,6 +42,16 @@ import { emitLog } from './cosEvents.js';
* buckets into the `_self` project key (PortOS-on-itself work) so app-less tasks
* can't bypass the per-project cap.
*
+ * Task ownership is tracked here too, so an id is admitted at most once. Two
+ * kinds of owner are invisible to `hasActiveTaskOwner` (it reads registered
+ * agents only): a task admitted earlier in THIS cycle (Priority 0 persists its
+ * task as pending, so Priority 2 reads it back with no agent yet), and a task
+ * whose spawn is still in flight from an earlier cycle (`inFlightTaskIds`, the
+ * `spawningTasks` set — passed in so this stays pure). Re-admitting either one
+ * spent a global/project/autonomous slot on a duplicate `task:ready` that the
+ * spawn guard then discarded. `isTaskOwned` exposes the same predicate so the
+ * admission passes can skip an owned task before their gates record deferrals.
+ *
* The THIRD cap is per local inference endpoint (issue #4834): a single GPU
* can't hold N model contexts at once, so agents whose provider resolves to the
* same local endpoint dispatch `localEndpointLimit` at a time and the rest stay
@@ -68,6 +79,7 @@ import { emitLog } from './cosEvents.js';
* Idle review prechecks global/project capacity before preparation below.
*/
export function createDequeueCapacity(state, {
+ inFlightTaskIds = [],
agentsByProject = {},
localEndpointCounts = {},
localEndpointLimit = Infinity,
@@ -75,7 +87,8 @@ export function createDequeueCapacity(state, {
onLocalEndpointHold = null,
} = {}) {
const runningAgents = Object.values(state.agents).filter(a => a.status === 'running').length;
- const availableSlots = state.config.maxConcurrentAgents - runningAgents;
+ const inFlight = new Set(inFlightTaskIds);
+ const availableSlots = state.config.maxConcurrentAgents - runningAgents - countUnregisteredInFlight(inFlight, state.agents);
const perProjectLimit = state.config.maxConcurrentAgentsPerProject || state.config.maxConcurrentAgents;
// A caller passing 0/NaN would wedge every local-endpoint task forever; floor
// at 1 so the cap degrades to "serialize", never to "never dispatch".
@@ -87,10 +100,13 @@ export function createDequeueCapacity(state, {
const spawnedTaskIds = [];
let spawned = 0;
+ // App eligibility probes have no task id yet; only real tasks can have an
+ // owner (a running legacy agent may also have no taskId).
+ const isTaskOwned = (task) => Boolean(task?.id)
+ && (hasActiveTaskOwner(task.id, state.agents) || inFlight.has(task.id) || spawnedTaskIds.includes(task.id));
+
const admit = (task, ceiling, gateLocalEndpoint) => {
- // App eligibility probes have no task id yet; only real tasks can have
- // an owner (a running legacy agent may also have no taskId).
- if (task.id && hasActiveTaskOwner(task.id, state.agents)) return false;
+ if (isTaskOwned(task)) return false;
if (spawned >= ceiling) return false;
const project = task.metadata?.app || '_self';
if ((spawnProjectCounts[project] || 0) >= perProjectLimit) return false;
@@ -129,6 +145,7 @@ export function createDequeueCapacity(state, {
canSpawn,
canSpawnCommitted,
trackSpawn,
+ isTaskOwned,
// Live read of the running spawn count — a getter so callers always see the
// current total after trackSpawn mutations rather than a stale snapshot.
get spawned() { return spawned; },
@@ -136,6 +153,20 @@ export function createDequeueCapacity(state, {
};
}
+/**
+ * How many in-flight spawns hold a slot that `runningAgents` does not count yet:
+ * a task sits in `spawningTasks` for the whole spawn, but its agent only shows
+ * as `running` in state once registered. Both engines subtract this from their
+ * free-slot count so skipping the in-flight task cannot over-spawn past
+ * `maxConcurrentAgents`.
+ */
+export function countUnregisteredInFlight(inFlightTaskIds, agents = {}) {
+ const running = new Set(Object.values(agents).filter(a => a.status === 'running').map(a => a.taskId));
+ let count = 0;
+ for (const id of inFlightTaskIds) if (!running.has(id)) count++;
+ return count;
+}
+
/**
* Count running agents grouped by the local inference endpoint they occupy
* (issue #4834). `endpointForAgent` maps a running agent to its local endpoint
diff --git a/server/services/cosTaskGenerator.js b/server/services/cosTaskGenerator.js
index 87f204f0be..083e2e362d 100644
--- a/server/services/cosTaskGenerator.js
+++ b/server/services/cosTaskGenerator.js
@@ -53,7 +53,8 @@ import { getActiveApps, getAppTaskTypeOverrides } from './apps.js';
// The single Priority-0 on-demand loop body, shared with the dequeueNextTask
// engine in cos.js so the two can no longer drift (#6618).
import { drainOnDemandRequests } from './onDemandDrain.js';
-import { admitFeatureAgentTasks, admitIdleReviewTask, isUserTaskRunnableUnattended } from './cosDequeue.js';
+import { admitFeatureAgentTasks, admitIdleReviewTask, countUnregisteredInFlight, isUserTaskRunnableUnattended } from './cosDequeue.js';
+import { spawningTasks } from './agentState.js';
import { resolveAgentProviderPin } from './appTaskProviderPin.js';
import { getTaskTypeConfidence } from './taskLearning.js';
import { classifySafetyKind, requiresSafetyApproval } from './taskLearning/safetyKind.js';
@@ -575,16 +576,19 @@ export async function admitAutoApprovedSystemTasks(ctx, adapter) {
spawnProjectCounts,
instanceId,
} = ctx;
- const { canSpawn, trackSpawn, emitSpawn, onDefer = async () => {} } = adapter;
+ const { canSpawn, trackSpawn, emitSpawn, isOwned = () => false, onDefer = async () => {} } = adapter;
if (!cosTaskData.exists) return [];
+ // A task already spawning or admitted this cycle is not a candidate: letting
+ // it reach the gates would record a capacity/cooldown deferral for work that
+ // is already running.
const autoApproved = [
...(cosTaskData.autoApproved || []),
...(state.config.autoApproveInvestigations
? (cosTaskData.grouped?.pending || []).filter((task) => isAutoApprovableInvestigation(task, state.config))
: [])
- ];
+ ].filter((task) => !isOwned(task));
// One hook set feeds both walks, so the dry-run plan and the execute pass
// cannot disagree about what each gate means.
const gateHooks = {
@@ -660,11 +664,13 @@ async function recordAutoApprovedDeferral({ type, task, appId, project }, state,
* capacity denies a candidate.
*/
export async function admitPendingUserTasks({ pendingUserTasks, instanceId }, adapter) {
- const { capacityExhausted, canSpawn, emitSpawn, trackSpawn, onDefer = async () => {} } = adapter;
+ const { capacityExhausted, canSpawn, emitSpawn, trackSpawn, isOwned = () => false, onDefer = async () => {} } = adapter;
const admitted = [];
for (const task of pendingUserTasks) {
if (capacityExhausted()) break;
+ // Already spawning or admitted this cycle — not deferred, just not a candidate.
+ if (isOwned(task)) continue;
// Not runnable here: the task is pinned to another instance (#4520), or a
// federated peer holds a live lease on it (#1650) and is working it on the
// other machine. Skip it during candidate selection so it doesn't consume
@@ -905,7 +911,8 @@ export async function evaluateTasks(options) {
// Count running agents and available slots (global + per-project)
const runningAgentEntries = Object.values(state.agents).filter(a => a.status === 'running');
const runningAgents = runningAgentEntries.length;
- const availableSlots = state.config.maxConcurrentAgents - runningAgents;
+ // In-flight spawns hold a slot before their agent registers as running.
+ const availableSlots = state.config.maxConcurrentAgents - runningAgents - countUnregisteredInFlight(spawningTasks, state.agents);
const perProjectLimit = state.config.maxConcurrentAgentsPerProject || state.config.maxConcurrentAgents;
const agentsByProject = countRunningAgentsByProject(state.agents);
@@ -931,7 +938,10 @@ export async function evaluateTasks(options) {
// Helper: check if a task can spawn (within both global and per-project limits).
// `ceiling` defaults to the global slot count; autonomous sections pass the
// lower `autonomousSlotCeiling` so the CoS action budget (#711) caps them.
+ const admittedTaskIds = new Set();
+ const isTaskOwned = (task) => Boolean(task?.id) && (admittedTaskIds.has(task.id) || spawningTasks.has(task.id));
const canSpawnTask = (task, ceiling = availableSlots) => {
+ if (isTaskOwned(task)) return false;
if (tasksToSpawn.length >= ceiling) return false;
const project = task.metadata?.app || '_self';
return (spawnProjectCounts[project] || 0) < perProjectLimit;
@@ -941,6 +951,7 @@ export async function evaluateTasks(options) {
// one without the other (#9609).
const admitSpawn = (task) => {
tasksToSpawn.push(task);
+ if (task.id) admittedTaskIds.add(task.id);
const project = task.metadata?.app || '_self';
spawnProjectCounts[project] = (spawnProjectCounts[project] || 0) + 1;
};
@@ -948,7 +959,7 @@ export async function evaluateTasks(options) {
// passes also serve dequeueNextTask (cos.js), which dispatches and counts
// through different objects, so they take emit and track as separate hooks;
// here `admitSpawn` already does both, so the track hook is inert.
- const spawnAdapter = Object.freeze({ canSpawn: canSpawnTask, emitSpawn: admitSpawn, trackSpawn: () => {} });
+ const spawnAdapter = Object.freeze({ canSpawn: canSpawnTask, emitSpawn: admitSpawn, trackSpawn: () => {}, isOwned: isTaskOwned });
// Check if there are pending user tasks (even if on cooldown). If user tasks
// exist, don't run self-improvement — wait for user tasks to be ready.
diff --git a/server/services/cosTaskGenerator.test.js b/server/services/cosTaskGenerator.test.js
index acbc485e6d..55f5f24db1 100644
--- a/server/services/cosTaskGenerator.test.js
+++ b/server/services/cosTaskGenerator.test.js
@@ -100,6 +100,7 @@ import * as prReviewerPipeline from './prReviewerPipeline.js';
import { cosEvents } from './cosEvents.js';
import { DEFAULT_TASK_INTERVALS, getTaskInterval } from './taskSchedule.js';
import { MAX_TOTAL_SPAWNS } from '../lib/validation.js';
+import { createDequeueCapacity } from './cosDequeue.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
const GEN_SRC = readFileSync(join(__dirname, 'cosTaskGenerator.js'), 'utf-8');
@@ -2167,3 +2168,71 @@ describe('admitAutoApprovedSystemTasks dry-run/execute parity', () => {
vi.restoreAllMocks();
});
});
+
+// A Priority 0 on-demand task is persisted as pending + auto-approved, so
+// Priority 2 reads the same id back before any agent is registered (#11847).
+describe('auto-approved admission skips tasks already owned this cycle (#11847)', () => {
+ const ctx = (tasks, capacity) => ({
+ state: { config: { appReviewCooldownMs: 60000 } },
+ cosTaskData: { exists: true, autoApproved: tasks },
+ taskSchedule: { tasks: {} },
+ cosAutonomyMode: 'execute',
+ autonomousSlotCeiling: capacity.availableSlots,
+ alreadySpawned: capacity.spawned,
+ perProjectLimit: capacity.perProjectLimit,
+ spawnProjectCounts: capacity.spawnProjectCounts,
+ instanceId: 'local',
+ });
+ const adapterFor = (capacity, emitted, deferred = []) => ({
+ isOwned: (t) => capacity.isTaskOwned(t),
+ canSpawn: (t, ceiling) => capacity.canSpawn(t, ceiling),
+ trackSpawn: (t) => capacity.trackSpawn(t),
+ emitSpawn: (t) => emitted.push(t.id),
+ onDefer: async (d) => { deferred.push(d.task.id); },
+ });
+ const makeCapacity = (inFlightTaskIds = []) => createDequeueCapacity(
+ { config: { maxConcurrentAgents: 4, maxConcurrentAgentsPerProject: null }, agents: {} },
+ { inFlightTaskIds }
+ );
+
+ beforeEach(() => {
+ vi.spyOn(appActivity, 'isAppOnCooldown').mockResolvedValue(false);
+ });
+
+ it('does not emit task:ready a second time for the id Priority 0 just admitted', async () => {
+ const capacity = makeCapacity();
+ const onDemand = { id: 'on-demand-1', metadata: { app: 'appA' } };
+ expect(capacity.canSpawnCommitted(onDemand)).toBe(true);
+ capacity.trackSpawn(onDemand);
+
+ const emitted = [];
+ const deferred = [];
+ await admitAutoApprovedSystemTasks(
+ ctx([onDemand, { id: 'other-2', metadata: { app: 'appB' } }], capacity),
+ adapterFor(capacity, emitted, deferred)
+ );
+
+ expect(emitted).toEqual(['other-2']);
+ expect(deferred).toEqual([]);
+ expect(capacity.spawnedTaskIds).toEqual(['on-demand-1', 'other-2']);
+ });
+
+ it('skips a task whose spawn is in flight and spends no slot on it', async () => {
+ const capacity = makeCapacity(['spawning-1']);
+ expect(capacity.availableSlots).toBe(3);
+
+ const emitted = [];
+ await admitAutoApprovedSystemTasks(
+ ctx([
+ { id: 'spawning-1', metadata: { app: 'appA' } },
+ { id: 'a', metadata: { app: 'appB' } },
+ { id: 'b', metadata: { app: 'appC' } },
+ { id: 'c', metadata: { app: 'appD' } },
+ ], capacity),
+ adapterFor(capacity, emitted)
+ );
+
+ expect(emitted).toEqual(['a', 'b', 'c']);
+ expect(capacity.spawned).toBe(3);
+ });
+});