⭐️⭐️⭐️ Open-source, self-hosted alternative to Linear and Plane. Project management and issue tracking where teams and AI agents work side by side to plan and ship products.
TypeScript agenticagileai-agentsbug-reportingbug-trackerganttissue-trackerjirajira-alternativekanbanlinearlinear-alternativeopen-sourceproduct-managementproject-managementroadmapself-hostedsprint-planningteam-collaborationtrello-alternative
fix(api): enforce workspace run admission atomically (#747)
I'm Groot 🌳0b0558b2b854
diff --git a/apps/api/src/modules/agents/core/__tests__/integration/older-delayed-limit.test.ts b/apps/api/src/modules/agents/core/__tests__/integration/older-delayed-limit.test.tsnew file mode 100644index 00000000..11563ab5--- /dev/null+++ b/apps/api/src/modules/agents/core/__tests__/integration/older-delayed-limit.test.ts@@ -0,0 +1,83 @@+import { beforeEach, describe, expect, it } from 'bun:test';+import { db, agentRun } from '@repo/db';+import { eq, sql } from 'drizzle-orm';+import { authedApi, createProject } from '#tests/helpers/app';+import { signUpTestUser } from '#tests/helpers/auth';+import { resetDb } from '#tests/helpers/db';+import { createAgent } from '#tests/helpers/agents';+import { claimDueRuns, enqueueAgentRun, markRunExecuting, scheduleRunRetry } from '../../run-queue';+import { processAgentRuns } from '../../run-poller';++describe('older delayed workspace limit regression', () => {+ beforeEach(resetDb);++ for (const retry of [false, true])+ it(`keeps an older delayed run queued during newer ${retry ? 'retry backoff' : 'execution'}`, async () => {+ const owner = await signUpTestUser({ name: 'Review fixture' });+ const api = authedApi(owner.cookie);+ expect((await createProject(api, { key: 'MKT', name: 'Review fixture' })).status).toBe(201);+ expect(+ (await api.workspaces({ workspaceId: owner.workspaceId! }).patch({ maxConcurrentRuns: 1 }))+ .status,+ ).toBe(200);+ const project = await api.projects({ projectKey: 'MKT' }).get();+ expect(project.status).toBe(200);+ const createdAgent = await createAgent(api, 'MKT', {+ name: 'Bot',+ username: 'bot',+ kind: 'internal',+ });+ expect(createdAgent.status).toBe(201);+ const agent = createdAgent.data!.agent;+ const makeIssue = async (title: string) => {+ const response = await api+ .projects({ projectKey: 'MKT' })+ .issues.post({ columnId: project.data!.columns[0].id, title });+ expect(response.status).toBe(201);+ return response.data!;+ };+ const olderIssue = await makeIssue('Older delayed run');+ await enqueueAgentRun({+ agentId: agent.id,+ projectId: agent.projects[0].id,+ issueId: olderIssue.id,+ sourceActivityId: null,+ prompt: 'Older delayed run',+ delaySeconds: 3600,+ });+ const newerIssue = await makeIssue('Newer immediate run');+ await enqueueAgentRun({+ agentId: agent.id,+ projectId: agent.projects[0].id,+ issueId: newerIssue.id,+ sourceActivityId: null,+ prompt: 'Newer immediate run',+ });+ const queue = () => api['agent-runs'].queue.get({ query: {} });+ const initial = await queue();+ expect(initial.status).toBe(200);+ const older = initial.data!.items.find((run) => run.issueId === olderIssue.id)!;+ const newer = initial.data!.items.find((run) => run.issueId === newerIssue.id)!;+ expect(older.id).toBeLessThan(newer.id);+ const claims = await claimDueRuns();+ expect(claims.map((run) => run.id)).toEqual([newer.id]);+ expect(await markRunExecuting(newer.id, claims[0].attempts)).toBe(true);+ if (retry) await scheduleRunRetry(newer.id, 60_000, 'model unavailable');++ // Advance only the delayed fixture's queue clock, as existing run-limit tests do.+ await db+ .update(agentRun)+ .set({ nextAttemptAt: sql`now()` })+ .where(eq(agentRun.id, older.id));+ await processAgentRuns();+ const after = await queue();+ expect(after.status).toBe(200);+ const observedOlder = after.data!.items.find((run) => run.id === older.id)!;+ const observedNewer = after.data!.items.find((run) => run.id === newer.id)!;+ const flight = await api.projects({ projectKey: 'MKT' })['agent-runs']['in-flight'].get();+ expect(flight.status).toBe(200);+ expect(flight.data).toMatchObject({ running: 1, limit: 1 });+ expect(observedNewer.executionState).toBe(retry ? 'queued' : 'running');+ expect(observedOlder).toMatchObject({ status: 'pending', attempts: 0, lastError: null });+ });+});diff --git a/apps/api/src/modules/agents/core/__tests__/integration/run-limits.test.ts b/apps/api/src/modules/agents/core/__tests__/integration/run-limits.test.tsindex fa29751c..2168aff3 100644--- a/apps/api/src/modules/agents/core/__tests__/integration/run-limits.test.ts+++ b/apps/api/src/modules/agents/core/__tests__/integration/run-limits.test.ts@@ -6,7 +6,14 @@ import { signUpTestUser } from '#tests/helpers/auth';import { resetDb } from '#tests/helpers/db';import { clearLimits, setLimits } from '#tests/helpers/limits';import { createAgent } from '#tests/helpers/agents';-import { claimDueRuns, deferRun, enqueueAgentRun, scheduleRunRetry } from '../../run-queue';+import {+ admitWorkspaceRun,+ claimDueRuns,+ deferRun,+ enqueueAgentRun,+ scheduleRunRetry,+ type ClaimedRun,+} from '../../run-queue';import { processAgentRuns } from '../../run-poller';// The ceilings on the runs in flight at once: the instance's, counted where the poller@@ -43,6 +50,7 @@ async function setup({ external = false } = {}) {return issue.id;};+ let firstClaim: ClaimedRun | undefined;if (external) {const runner = (await createAgent(asOwner, 'MKT', { name: 'Runner', username: 'runner', kind: 'external' })@@ -54,11 +62,11 @@ async function setup({ external = false } = {}) {.where(eq(agentRun.issueId, issueId));} else {await enqueue('first');- await claimDueRuns();+ [firstClaim] = await claimDueRuns();}const issueId = await enqueue('second');const [second] = await db.select().from(agentRun).where(eq(agentRun.issueId, issueId));- return { second, asOwner, workspaceId: owner.workspaceId!, enqueue };+ return { second, asOwner, workspaceId: owner.workspaceId!, enqueue, firstClaim };}// Reads the second run back after one poll. A run turned away for a full queue keeps@@ -150,6 +158,23 @@ describe('agent run limits', () => {expect((await claimDueRuns()).map((run) => run.id)).toEqual([first.id]);});+ it('admits only one run when replicas race for a workspace slot', async () => {+ const { second, workspaceId, firstClaim, asOwner } = await setup();+ await asOwner.workspaces({ workspaceId }).patch({ maxConcurrentRuns: 1 });+ const [later] = await claimDueRuns();+ // Both runs are claimed before either replica tries to start execution.+ const claims = [later, firstClaim!];+ const admitted = await Promise.all(claims.map((run) => admitWorkspaceRun(run, workspaceId, 1)));+ expect(admitted).toEqual([false, true]);+ const queue = await asOwner['agent-runs'].queue.get({ query: {} });+ expect(queue.status).toBe(200);+ expect(queue.data!.items.filter((run) => run.executionState === 'running')).toHaveLength(1);+ expect(queue.data!.items.find((run) => run.id === second.id)).toMatchObject({+ attempts: 0,+ lastError: null,+ });+ });+it('leaves the runs of external agents out of both limits', async () => {const { second, asOwner, workspaceId } = await setup({ external: true });await asOwner.god['agent-settings'].put({ maxConcurrentRuns: 1 });diff --git a/apps/api/src/modules/agents/core/run-poller.ts b/apps/api/src/modules/agents/core/run-poller.tsindex 8111c902..1a11c29e 100644--- a/apps/api/src/modules/agents/core/run-poller.ts+++ b/apps/api/src/modules/agents/core/run-poller.ts@@ -9,10 +9,10 @@ import { runRefusal } from './run-refusal';import {agentRunConfig,claimDueRuns,+ admitWorkspaceRun,countRunsAhead,deferRun,markRunFailed,- markRunExecuting,markRunSuccess,runRetryDelayMs,scheduleRunRetry,@@ -26,9 +26,6 @@ import { runThreadId } from './runtime/thread-ids';// the only thing a run is built from, so nothing about which project or which bot user// it acts as can be handed in from outside.-// How long a run waits when its workspace has no free slot.-const DEFERRED_RETRY_SECONDS = 30;-export async function processAgentRuns(): Promise<void> {const runs = await claimDueRuns();await Promise.all(runs.map(processRun));@@ -40,7 +37,7 @@ async function processRun(run: ClaimedRun): Promise<void> {const limits = await getLimits(workspaceId);const runLimit = await getRunLimit(workspaceId, limits);if (runLimit > 0 && (await countRunsAhead(workspaceId, run.id)) >= runLimit) {- await deferRun(run.id, DEFERRED_RETRY_SECONDS);+ await deferRun(run.id, 30);return;}const wait = await runAdmissionDelay(run);@@ -70,8 +67,8 @@ async function processRun(run: ClaimedRun): Promise<void> {);if (run.executionDeadline <= performance.now()) return;try {+ if (!(await admitWorkspaceRun(run, workspaceId, runLimit))) return;await agentRunStarted(run);- if (!(await markRunExecuting(run.id, run.attempts))) return;const timeoutMs = Math.floor(Math.min(configuredTimeoutMs, run.executionDeadline - performance.now()),);diff --git a/apps/api/src/modules/agents/core/run-queue.ts b/apps/api/src/modules/agents/core/run-queue.tsindex c795f068..9c855bc2 100644--- a/apps/api/src/modules/agents/core/run-queue.ts+++ b/apps/api/src/modules/agents/core/run-queue.ts@@ -98,18 +98,31 @@ export const holdsSlot = and(eq(aiAgent.kind, 'internal'),);-// Runs of this workspace's agents that hold a slot and were queued before this one.-// Counting only the older runs is what keeps a ceiling from turning away every run of a-// burst at once: each yields to the ones ahead of it and the rest come back on their next-// attempt.-export async function countRunsAhead(workspaceId: number, runId: number): Promise<number> {- const [row] = await db+// Active runs and retry backoff keep their slots regardless of queue order. Among+// claims awaiting admission, lower ids take precedence so a burst can make progress.+export async function countRunsAhead(+ workspaceId: number,+ runId: number,+ executor: DbExecutor = db,+): Promise<number> {+ const [row] = await executor.select({ count: sql<number>`count(*)::int` }).from(agentRun).innerJoin(aiAgent, eq(aiAgent.id, agentRun.agentId)).innerJoin(project, eq(project.id, agentRun.projectId)).innerJoin(team, eq(team.id, project.teamId))- .where(and(eq(team.workspaceId, workspaceId), holdsSlot, lt(agentRun.id, runId)));+ .where(+ and(+ eq(team.workspaceId, workspaceId),+ holdsSlot,+ sql`${agentRun.id} <> ${runId}`,+ or(+ lt(agentRun.id, runId),+ isNotNull(agentRun.executionStartedAt),+ isNull(agentRun.claimedAt),+ ),+ ),+ );return row?.count ?? 0;}@@ -468,8 +481,13 @@ export async function markRunFailed(id: number, error: string, output?: string):// attempt, and nothing has been recorded on the issue yet, so the run leaves no trace of// having been picked up at all: a first claim's started_at goes too, so the waiting run// holds no slot in countRunsAhead and can be canceled as not started.-export async function deferRun(id: number, delaySeconds: number): Promise<void> {- await db+export async function deferRun(+ id: number,+ delaySeconds: number,+ executor: DbExecutor = db,+ attempts?: number,+): Promise<void> {+ await executor.update(agentRun).set({executionStartedAt: null,@@ -478,7 +496,19 @@ export async function deferRun(id: number, delaySeconds: number): Promise<void>startedAt: sql`CASE WHEN ${agentRun.attempts} = 1 THEN NULL ELSE ${agentRun.startedAt} END`,nextAttemptAt: sql`now() + make_interval(secs => ${delaySeconds})`,})- .where(and(eq(agentRun.id, id), eq(agentRun.status, 'pending')));+ .where(+ and(+ eq(agentRun.id, id),+ eq(agentRun.status, 'pending'),+ attempts === undefined+ ? undefined+ : and(+ eq(agentRun.attempts, attempts),+ isNotNull(agentRun.claimedAt),+ gt(agentRun.nextAttemptAt, sql`now()`),+ ),+ ),+ );}export const executionState = sql<'queued' | 'running' | 'success' | 'failed' | 'canceled'>`@@ -487,8 +517,12 @@ export const executionState = sql<'queued' | 'running' | 'success' | 'failed' |THEN 'running'ELSE 'queued' END`;-export async function markRunExecuting(id: number, attempts: number): Promise<boolean> {- const rows = await db+export async function markRunExecuting(+ id: number,+ attempts: number,+ executor: DbExecutor = db,+): Promise<boolean> {+ const rows = await executor.update(agentRun).set({ executionStartedAt: new Date() }).where(@@ -503,6 +537,23 @@ export async function markRunExecuting(id: number, attempts: number): Promise<boreturn rows.length > 0;}+// Share the claim lock: another replica cannot admit a later claim between the+// workspace count and this run reserving its execution slot.+export async function admitWorkspaceRun(+ run: ClaimedRun,+ workspaceId: number,+ limit: number,+): Promise<boolean> {+ return db.transaction(async (tx) => {+ await tx.execute(sql`select pg_advisory_xact_lock(${RUN_CLAIM_LOCK_NS}, 0)`);+ if (limit > 0 && (await countRunsAhead(workspaceId, run.id, tx)) >= limit) {+ await deferRun(run.id, 30, tx, run.attempts);+ return false;+ }+ return markRunExecuting(run.id, run.attempts, tx);+ });+}+export interface AgentRunRow {id: number;status: string;