Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: CoS dequeue no longer re-admits tasks that are already spawning (#11847)
Adam Eivy4cfca74ce044
diff --git a/server/services/cos.js b/server/services/cos.jsindex 28a24af94a..a023cfc22e 100644--- a/server/services/cos.js+++ b/server/services/cos.js@@ -48,7 +48,7 @@ import { mergePersistentMindPlaybook } from '../lib/persistentMindPlaybook.js';import { mergePersistentMindThinkingPresets } from '../lib/persistentMindThinkingPresets.js';// Dependency-free leaf holding the shared agent maps + the runner-mode flag,// read by `isRunnerHolding` below.-import { useRunner } from './agentState.js';+import { useRunner, spawningTasks } from './agentState.js';// Shared state management (extracted to avoid circular deps)import { loadState, saveState, withStateLock, loadConfig, saveConfig, withConfigLock, ensureDirectories, isImprovementEnabled, canQueueImprovementTasks, SCRIPTS_DIR, isDaemonRunning, setDaemonRunning } from './cosState.js';@@ -1088,6 +1088,7 @@ async function spawnDequeuePriority1UserTasks(ctx) {await admitPendingUserTasks({ pendingUserTasks, instanceId }, {capacityExhausted: () => capacity.spawned >= capacity.availableSlots,+ isOwned: (task) => capacity.isTaskOwned(task),canSpawn: (task) => capacity.canSpawn(task),emitSpawn: (task) => cosEvents.emit('task:ready', task),trackSpawn: (task) => capacity.trackSpawn(task),@@ -1121,6 +1122,7 @@ async function spawnDequeuePriority2AutoApproved(ctx, { taskSchedule }) {spawnProjectCounts: capacity.spawnProjectCounts,instanceId,}, {+ isOwned: (task) => capacity.isTaskOwned(task),canSpawn: (task, ceiling) => capacity.canSpawn(task, ceiling),trackSpawn: (task) => capacity.trackSpawn(task),emitSpawn: (task) => {@@ -1249,6 +1251,7 @@ export async function dequeueNextTask({ ignoreTaskId = null } = {}) {// to every CoS-subscribed socket client.const loggedFullEndpoints = new Set();const capacity = createDequeueCapacity(state, {+ inFlightTaskIds: spawningTasks,agentsByProject: countRunningAgentsByProject(state.agents),localEndpointCounts: countRunningAgentsByLocalEndpoint(state.agents, localSlots.endpointForAgent),localEndpointLimit: localSlots.limit,diff --git a/server/services/cos.test.js b/server/services/cos.test.jsindex afea20f36e..22570328dc 100644--- a/server/services/cos.test.js+++ b/server/services/cos.test.js@@ -499,6 +499,54 @@ function makeLocalSlotCapacity(state, { activeProvider = CLOUD_PROVIDER, limit =const userBuckets = (tasks) => ({ onDemand: [], user: tasks, autoSystem: [], idle: [] });+describe('dequeueNextTask — task ownership beyond registered agents (#11847)', () => {+ it('never admits an in-flight spawn, and still counts it against the global slots', () => {+ // 4-slot cap: 1 running agent + 2 spawns in flight (no agent record yet)+ // leaves one real slot, which the queued task — not the in-flight ones — gets.+ const state = makeState({ maxConcurrentAgents: 4, runningAgents: [makeRunningAgent()] });+ const capacity = createDequeueCapacity(state, { inFlightTaskIds: new Set(['spawning-a', 'spawning-b']) });+ expect(capacity.availableSlots).toBe(1);++ const buckets = {+ onDemand: [],+ user: [task('spawning-a'), task('spawning-b'), task('queued-c'), task('queued-d')],+ autoSystem: [],+ idle: [],+ };+ expect(priorityDequeue(buckets, capacity).map(t => t.id)).toEqual(['queued-c']);+ });++ it('admits nothing when running agents plus in-flight spawns fill maxConcurrentAgents', () => {+ const state = makeState({ maxConcurrentAgents: 2, runningAgents: [makeRunningAgent()] });+ const capacity = createDequeueCapacity(state, { inFlightTaskIds: ['spawning-a'] });+ expect(capacity.availableSlots).toBe(0);+ expect(priorityDequeue({ onDemand: [], user: [task('queued-b')], autoSystem: [], idle: [] }, capacity)).toEqual([]);+ });++ it('does not double-count an in-flight spawn whose agent is already running', () => {+ const state = makeState({+ maxConcurrentAgents: 3,+ runningAgents: [{ ...makeRunningAgent(), taskId: 'spawning-a' }],+ });+ const capacity = createDequeueCapacity(state, { inFlightTaskIds: ['spawning-a'] });+ expect(capacity.availableSlots).toBe(2);+ });++ it('admits a task id at most once per cycle across tiers', () => {+ const state = makeState({ maxConcurrentAgents: 5 });+ const capacity = createDequeueCapacity(state, {});+ const buckets = {+ onDemand: [task('dup-1')],+ user: [task('dup-1'), task('user-2')],+ autoSystem: [task('dup-1'), task('user-2'), task('sys-3')],+ idle: [],+ };+ expect(priorityDequeue(buckets, capacity).map(t => t.id)).toEqual(['dup-1', 'user-2', 'sys-3']);+ expect(capacity.spawnedTaskIds).toEqual(['dup-1', 'user-2', 'sys-3']);+ expect(capacity.spawned).toBe(3);+ });+});+describe('dequeueNextTask — per-local-endpoint agent cap (#4834)', () => {it('serializes two queued tasks that resolve to the SAME local endpoint', () => {// Global cap 5, nothing running: both tasks clear the global anddiff --git a/server/services/cosDequeue.js b/server/services/cosDequeue.jsindex f183602315..6276626e14 100644--- a/server/services/cosDequeue.js+++ b/server/services/cosDequeue.js@@ -29,7 +29,8 @@ import { emitLog } from './cosEvents.js';* per-project tally, and exposes the exact `canSpawn` / `trackSpawn` closure the* scheduler uses to enforce the global slot cap AND the per-project cap.*- * `availableSlots` = global cap minus currently-running agents (may be 0 or+ * `availableSlots` = global cap minus currently-running agents minus in-flight+ * spawns that have no running agent record yet (may be 0 or* negative if a config change shrank the cap below live load — callers still* guard with `availableSlots <= 0`). `perProjectLimit` falls back to the global* cap when `maxConcurrentAgentsPerProject` is unset/0, matching the scheduler's@@ -41,6 +42,16 @@ import { emitLog } from './cosEvents.js';* buckets into the `_self` project key (PortOS-on-itself work) so app-less tasks* can't bypass the per-project cap.*+ * Task ownership is tracked here too, so an id is admitted at most once. Two+ * kinds of owner are invisible to `hasActiveTaskOwner` (it reads registered+ * agents only): a task admitted earlier in THIS cycle (Priority 0 persists its+ * task as pending, so Priority 2 reads it back with no agent yet), and a task+ * whose spawn is still in flight from an earlier cycle (`inFlightTaskIds`, the+ * `spawningTasks` set — passed in so this stays pure). Re-admitting either one+ * spent a global/project/autonomous slot on a duplicate `task:ready` that the+ * spawn guard then discarded. `isTaskOwned` exposes the same predicate so the+ * admission passes can skip an owned task before their gates record deferrals.+ ** The THIRD cap is per local inference endpoint (issue #4834): a single GPU* can't hold N model contexts at once, so agents whose provider resolves to the* same local endpoint dispatch `localEndpointLimit` at a time and the rest stay@@ -68,6 +79,7 @@ import { emitLog } from './cosEvents.js';* Idle review prechecks global/project capacity before preparation below.*/export function createDequeueCapacity(state, {+ inFlightTaskIds = [],agentsByProject = {},localEndpointCounts = {},localEndpointLimit = Infinity,@@ -75,7 +87,8 @@ export function createDequeueCapacity(state, {onLocalEndpointHold = null,} = {}) {const runningAgents = Object.values(state.agents).filter(a => a.status === 'running').length;- const availableSlots = state.config.maxConcurrentAgents - runningAgents;+ const inFlight = new Set(inFlightTaskIds);+ const availableSlots = state.config.maxConcurrentAgents - runningAgents - countUnregisteredInFlight(inFlight, state.agents);const perProjectLimit = state.config.maxConcurrentAgentsPerProject || state.config.maxConcurrentAgents;// A caller passing 0/NaN would wedge every local-endpoint task forever; floor// at 1 so the cap degrades to "serialize", never to "never dispatch".@@ -87,10 +100,13 @@ export function createDequeueCapacity(state, {const spawnedTaskIds = [];let spawned = 0;+ // App eligibility probes have no task id yet; only real tasks can have an+ // owner (a running legacy agent may also have no taskId).+ const isTaskOwned = (task) => Boolean(task?.id)+ && (hasActiveTaskOwner(task.id, state.agents) || inFlight.has(task.id) || spawnedTaskIds.includes(task.id));+const admit = (task, ceiling, gateLocalEndpoint) => {- // App eligibility probes have no task id yet; only real tasks can have- // an owner (a running legacy agent may also have no taskId).- if (task.id && hasActiveTaskOwner(task.id, state.agents)) return false;+ if (isTaskOwned(task)) return false;if (spawned >= ceiling) return false;const project = task.metadata?.app || '_self';if ((spawnProjectCounts[project] || 0) >= perProjectLimit) return false;@@ -129,6 +145,7 @@ export function createDequeueCapacity(state, {canSpawn,canSpawnCommitted,trackSpawn,+ isTaskOwned,// Live read of the running spawn count — a getter so callers always see the// current total after trackSpawn mutations rather than a stale snapshot.get spawned() { return spawned; },@@ -136,6 +153,20 @@ export function createDequeueCapacity(state, {};}+/**+ * How many in-flight spawns hold a slot that `runningAgents` does not count yet:+ * a task sits in `spawningTasks` for the whole spawn, but its agent only shows+ * as `running` in state once registered. Both engines subtract this from their+ * free-slot count so skipping the in-flight task cannot over-spawn past+ * `maxConcurrentAgents`.+ */+export function countUnregisteredInFlight(inFlightTaskIds, agents = {}) {+ const running = new Set(Object.values(agents).filter(a => a.status === 'running').map(a => a.taskId));+ let count = 0;+ for (const id of inFlightTaskIds) if (!running.has(id)) count++;+ return count;+}+/*** Count running agents grouped by the local inference endpoint they occupy* (issue #4834). `endpointForAgent` maps a running agent to its local endpointdiff --git a/server/services/cosTaskGenerator.js b/server/services/cosTaskGenerator.jsindex 87f204f0be..083e2e362d 100644--- a/server/services/cosTaskGenerator.js+++ b/server/services/cosTaskGenerator.js@@ -53,7 +53,8 @@ import { getActiveApps, getAppTaskTypeOverrides } from './apps.js';// The single Priority-0 on-demand loop body, shared with the dequeueNextTask// engine in cos.js so the two can no longer drift (#6618).import { drainOnDemandRequests } from './onDemandDrain.js';-import { admitFeatureAgentTasks, admitIdleReviewTask, isUserTaskRunnableUnattended } from './cosDequeue.js';+import { admitFeatureAgentTasks, admitIdleReviewTask, countUnregisteredInFlight, isUserTaskRunnableUnattended } from './cosDequeue.js';+import { spawningTasks } from './agentState.js';import { resolveAgentProviderPin } from './appTaskProviderPin.js';import { getTaskTypeConfidence } from './taskLearning.js';import { classifySafetyKind, requiresSafetyApproval } from './taskLearning/safetyKind.js';@@ -575,16 +576,19 @@ export async function admitAutoApprovedSystemTasks(ctx, adapter) {spawnProjectCounts,instanceId,} = ctx;- const { canSpawn, trackSpawn, emitSpawn, onDefer = async () => {} } = adapter;+ const { canSpawn, trackSpawn, emitSpawn, isOwned = () => false, onDefer = async () => {} } = adapter;if (!cosTaskData.exists) return [];+ // A task already spawning or admitted this cycle is not a candidate: letting+ // it reach the gates would record a capacity/cooldown deferral for work that+ // is already running.const autoApproved = [...(cosTaskData.autoApproved || []),...(state.config.autoApproveInvestigations? (cosTaskData.grouped?.pending || []).filter((task) => isAutoApprovableInvestigation(task, state.config)): [])- ];+ ].filter((task) => !isOwned(task));// One hook set feeds both walks, so the dry-run plan and the execute pass// cannot disagree about what each gate means.const gateHooks = {@@ -660,11 +664,13 @@ async function recordAutoApprovedDeferral({ type, task, appId, project }, state,* capacity denies a candidate.*/export async function admitPendingUserTasks({ pendingUserTasks, instanceId }, adapter) {- const { capacityExhausted, canSpawn, emitSpawn, trackSpawn, onDefer = async () => {} } = adapter;+ const { capacityExhausted, canSpawn, emitSpawn, trackSpawn, isOwned = () => false, onDefer = async () => {} } = adapter;const admitted = [];for (const task of pendingUserTasks) {if (capacityExhausted()) break;+ // Already spawning or admitted this cycle — not deferred, just not a candidate.+ if (isOwned(task)) continue;// Not runnable here: the task is pinned to another instance (#4520), or a// federated peer holds a live lease on it (#1650) and is working it on the// other machine. Skip it during candidate selection so it doesn't consume@@ -905,7 +911,8 @@ export async function evaluateTasks(options) {// Count running agents and available slots (global + per-project)const runningAgentEntries = Object.values(state.agents).filter(a => a.status === 'running');const runningAgents = runningAgentEntries.length;- const availableSlots = state.config.maxConcurrentAgents - runningAgents;+ // In-flight spawns hold a slot before their agent registers as running.+ const availableSlots = state.config.maxConcurrentAgents - runningAgents - countUnregisteredInFlight(spawningTasks, state.agents);const perProjectLimit = state.config.maxConcurrentAgentsPerProject || state.config.maxConcurrentAgents;const agentsByProject = countRunningAgentsByProject(state.agents);@@ -931,7 +938,10 @@ export async function evaluateTasks(options) {// Helper: check if a task can spawn (within both global and per-project limits).// `ceiling` defaults to the global slot count; autonomous sections pass the// lower `autonomousSlotCeiling` so the CoS action budget (#711) caps them.+ const admittedTaskIds = new Set();+ const isTaskOwned = (task) => Boolean(task?.id) && (admittedTaskIds.has(task.id) || spawningTasks.has(task.id));const canSpawnTask = (task, ceiling = availableSlots) => {+ if (isTaskOwned(task)) return false;if (tasksToSpawn.length >= ceiling) return false;const project = task.metadata?.app || '_self';return (spawnProjectCounts[project] || 0) < perProjectLimit;@@ -941,6 +951,7 @@ export async function evaluateTasks(options) {// one without the other (#9609).const admitSpawn = (task) => {tasksToSpawn.push(task);+ if (task.id) admittedTaskIds.add(task.id);const project = task.metadata?.app || '_self';spawnProjectCounts[project] = (spawnProjectCounts[project] || 0) + 1;};@@ -948,7 +959,7 @@ export async function evaluateTasks(options) {// passes also serve dequeueNextTask (cos.js), which dispatches and counts// through different objects, so they take emit and track as separate hooks;// here `admitSpawn` already does both, so the track hook is inert.- const spawnAdapter = Object.freeze({ canSpawn: canSpawnTask, emitSpawn: admitSpawn, trackSpawn: () => {} });+ const spawnAdapter = Object.freeze({ canSpawn: canSpawnTask, emitSpawn: admitSpawn, trackSpawn: () => {}, isOwned: isTaskOwned });// Check if there are pending user tasks (even if on cooldown). If user tasks// exist, don't run self-improvement — wait for user tasks to be ready.diff --git a/server/services/cosTaskGenerator.test.js b/server/services/cosTaskGenerator.test.jsindex acbc485e6d..55f5f24db1 100644--- a/server/services/cosTaskGenerator.test.js+++ b/server/services/cosTaskGenerator.test.js@@ -100,6 +100,7 @@ import * as prReviewerPipeline from './prReviewerPipeline.js';import { cosEvents } from './cosEvents.js';import { DEFAULT_TASK_INTERVALS, getTaskInterval } from './taskSchedule.js';import { MAX_TOTAL_SPAWNS } from '../lib/validation.js';+import { createDequeueCapacity } from './cosDequeue.js';const __dirname = dirname(fileURLToPath(import.meta.url));const GEN_SRC = readFileSync(join(__dirname, 'cosTaskGenerator.js'), 'utf-8');@@ -2167,3 +2168,71 @@ describe('admitAutoApprovedSystemTasks dry-run/execute parity', () => {vi.restoreAllMocks();});});++// A Priority 0 on-demand task is persisted as pending + auto-approved, so+// Priority 2 reads the same id back before any agent is registered (#11847).+describe('auto-approved admission skips tasks already owned this cycle (#11847)', () => {+ const ctx = (tasks, capacity) => ({+ state: { config: { appReviewCooldownMs: 60000 } },+ cosTaskData: { exists: true, autoApproved: tasks },+ taskSchedule: { tasks: {} },+ cosAutonomyMode: 'execute',+ autonomousSlotCeiling: capacity.availableSlots,+ alreadySpawned: capacity.spawned,+ perProjectLimit: capacity.perProjectLimit,+ spawnProjectCounts: capacity.spawnProjectCounts,+ instanceId: 'local',+ });+ const adapterFor = (capacity, emitted, deferred = []) => ({+ isOwned: (t) => capacity.isTaskOwned(t),+ canSpawn: (t, ceiling) => capacity.canSpawn(t, ceiling),+ trackSpawn: (t) => capacity.trackSpawn(t),+ emitSpawn: (t) => emitted.push(t.id),+ onDefer: async (d) => { deferred.push(d.task.id); },+ });+ const makeCapacity = (inFlightTaskIds = []) => createDequeueCapacity(+ { config: { maxConcurrentAgents: 4, maxConcurrentAgentsPerProject: null }, agents: {} },+ { inFlightTaskIds }+ );++ beforeEach(() => {+ vi.spyOn(appActivity, 'isAppOnCooldown').mockResolvedValue(false);+ });++ it('does not emit task:ready a second time for the id Priority 0 just admitted', async () => {+ const capacity = makeCapacity();+ const onDemand = { id: 'on-demand-1', metadata: { app: 'appA' } };+ expect(capacity.canSpawnCommitted(onDemand)).toBe(true);+ capacity.trackSpawn(onDemand);++ const emitted = [];+ const deferred = [];+ await admitAutoApprovedSystemTasks(+ ctx([onDemand, { id: 'other-2', metadata: { app: 'appB' } }], capacity),+ adapterFor(capacity, emitted, deferred)+ );++ expect(emitted).toEqual(['other-2']);+ expect(deferred).toEqual([]);+ expect(capacity.spawnedTaskIds).toEqual(['on-demand-1', 'other-2']);+ });++ it('skips a task whose spawn is in flight and spends no slot on it', async () => {+ const capacity = makeCapacity(['spawning-1']);+ expect(capacity.availableSlots).toBe(3);++ const emitted = [];+ await admitAutoApprovedSystemTasks(+ ctx([+ { id: 'spawning-1', metadata: { app: 'appA' } },+ { id: 'a', metadata: { app: 'appB' } },+ { id: 'b', metadata: { app: 'appC' } },+ { id: 'c', metadata: { app: 'appD' } },+ ], capacity),+ adapterFor(capacity, emitted)+ );++ expect(emitted).toEqual(['a', 'b', 'c']);+ expect(capacity.spawned).toBe(3);+ });+});