Skip to content

⭐️⭐️⭐️ Open-source, self-hosted alternative to Linear and Plane. Project management and issue tracking where teams and AI agents work side by side to plan and ship products.

fix(api): enforce workspace run admission atomically (#747)

I'm Groot 🌳0b0558b2b854
diff --git a/apps/api/src/modules/agents/core/__tests__/integration/older-delayed-limit.test.ts b/apps/api/src/modules/agents/core/__tests__/integration/older-delayed-limit.test.ts
new file mode 100644
index 00000000..11563ab5
--- /dev/null
+++ b/apps/api/src/modules/agents/core/__tests__/integration/older-delayed-limit.test.ts
@@ -0,0 +1,83 @@
+import { beforeEach, describe, expect, it } from 'bun:test';
+import { db, agentRun } from '@repo/db';
+import { eq, sql } from 'drizzle-orm';
+import { authedApi, createProject } from '#tests/helpers/app';
+import { signUpTestUser } from '#tests/helpers/auth';
+import { resetDb } from '#tests/helpers/db';
+import { createAgent } from '#tests/helpers/agents';
+import { claimDueRuns, enqueueAgentRun, markRunExecuting, scheduleRunRetry } from '../../run-queue';
+import { processAgentRuns } from '../../run-poller';
+
+describe('older delayed workspace limit regression', () => {
+ beforeEach(resetDb);
+
+ for (const retry of [false, true])
+ it(`keeps an older delayed run queued during newer ${retry ? 'retry backoff' : 'execution'}`, async () => {
+ const owner = await signUpTestUser({ name: 'Review fixture' });
+ const api = authedApi(owner.cookie);
+ expect((await createProject(api, { key: 'MKT', name: 'Review fixture' })).status).toBe(201);
+ expect(
+ (await api.workspaces({ workspaceId: owner.workspaceId! }).patch({ maxConcurrentRuns: 1 }))
+ .status,
+ ).toBe(200);
+ const project = await api.projects({ projectKey: 'MKT' }).get();
+ expect(project.status).toBe(200);
+ const createdAgent = await createAgent(api, 'MKT', {
+ name: 'Bot',
+ username: 'bot',
+ kind: 'internal',
+ });
+ expect(createdAgent.status).toBe(201);
+ const agent = createdAgent.data!.agent;
+ const makeIssue = async (title: string) => {
+ const response = await api
+ .projects({ projectKey: 'MKT' })
+ .issues.post({ columnId: project.data!.columns[0].id, title });
+ expect(response.status).toBe(201);
+ return response.data!;
+ };
+ const olderIssue = await makeIssue('Older delayed run');
+ await enqueueAgentRun({
+ agentId: agent.id,
+ projectId: agent.projects[0].id,
+ issueId: olderIssue.id,
+ sourceActivityId: null,
+ prompt: 'Older delayed run',
+ delaySeconds: 3600,
+ });
+ const newerIssue = await makeIssue('Newer immediate run');
+ await enqueueAgentRun({
+ agentId: agent.id,
+ projectId: agent.projects[0].id,
+ issueId: newerIssue.id,
+ sourceActivityId: null,
+ prompt: 'Newer immediate run',
+ });
+ const queue = () => api['agent-runs'].queue.get({ query: {} });
+ const initial = await queue();
+ expect(initial.status).toBe(200);
+ const older = initial.data!.items.find((run) => run.issueId === olderIssue.id)!;
+ const newer = initial.data!.items.find((run) => run.issueId === newerIssue.id)!;
+ expect(older.id).toBeLessThan(newer.id);
+ const claims = await claimDueRuns();
+ expect(claims.map((run) => run.id)).toEqual([newer.id]);
+ expect(await markRunExecuting(newer.id, claims[0].attempts)).toBe(true);
+ if (retry) await scheduleRunRetry(newer.id, 60_000, 'model unavailable');
+
+ // Advance only the delayed fixture's queue clock, as existing run-limit tests do.
+ await db
+ .update(agentRun)
+ .set({ nextAttemptAt: sql`now()` })
+ .where(eq(agentRun.id, older.id));
+ await processAgentRuns();
+ const after = await queue();
+ expect(after.status).toBe(200);
+ const observedOlder = after.data!.items.find((run) => run.id === older.id)!;
+ const observedNewer = after.data!.items.find((run) => run.id === newer.id)!;
+ const flight = await api.projects({ projectKey: 'MKT' })['agent-runs']['in-flight'].get();
+ expect(flight.status).toBe(200);
+ expect(flight.data).toMatchObject({ running: 1, limit: 1 });
+ expect(observedNewer.executionState).toBe(retry ? 'queued' : 'running');
+ expect(observedOlder).toMatchObject({ status: 'pending', attempts: 0, lastError: null });
+ });
+});
diff --git a/apps/api/src/modules/agents/core/__tests__/integration/run-limits.test.ts b/apps/api/src/modules/agents/core/__tests__/integration/run-limits.test.ts
index fa29751c..2168aff3 100644
--- a/apps/api/src/modules/agents/core/__tests__/integration/run-limits.test.ts
+++ b/apps/api/src/modules/agents/core/__tests__/integration/run-limits.test.ts
@@ -6,7 +6,14 @@ import { signUpTestUser } from '#tests/helpers/auth';
import { resetDb } from '#tests/helpers/db';
import { clearLimits, setLimits } from '#tests/helpers/limits';
import { createAgent } from '#tests/helpers/agents';
-import { claimDueRuns, deferRun, enqueueAgentRun, scheduleRunRetry } from '../../run-queue';
+import {
+ admitWorkspaceRun,
+ claimDueRuns,
+ deferRun,
+ enqueueAgentRun,
+ scheduleRunRetry,
+ type ClaimedRun,
+} from '../../run-queue';
import { processAgentRuns } from '../../run-poller';
// The ceilings on the runs in flight at once: the instance's, counted where the poller
@@ -43,6 +50,7 @@ async function setup({ external = false } = {}) {
return issue.id;
};
+ let firstClaim: ClaimedRun | undefined;
if (external) {
const runner = (
await createAgent(asOwner, 'MKT', { name: 'Runner', username: 'runner', kind: 'external' })
@@ -54,11 +62,11 @@ async function setup({ external = false } = {}) {
.where(eq(agentRun.issueId, issueId));
} else {
await enqueue('first');
- await claimDueRuns();
+ [firstClaim] = await claimDueRuns();
}
const issueId = await enqueue('second');
const [second] = await db.select().from(agentRun).where(eq(agentRun.issueId, issueId));
- return { second, asOwner, workspaceId: owner.workspaceId!, enqueue };
+ return { second, asOwner, workspaceId: owner.workspaceId!, enqueue, firstClaim };
}
// Reads the second run back after one poll. A run turned away for a full queue keeps
@@ -150,6 +158,23 @@ describe('agent run limits', () => {
expect((await claimDueRuns()).map((run) => run.id)).toEqual([first.id]);
});
+ it('admits only one run when replicas race for a workspace slot', async () => {
+ const { second, workspaceId, firstClaim, asOwner } = await setup();
+ await asOwner.workspaces({ workspaceId }).patch({ maxConcurrentRuns: 1 });
+ const [later] = await claimDueRuns();
+ // Both runs are claimed before either replica tries to start execution.
+ const claims = [later, firstClaim!];
+ const admitted = await Promise.all(claims.map((run) => admitWorkspaceRun(run, workspaceId, 1)));
+ expect(admitted).toEqual([false, true]);
+ const queue = await asOwner['agent-runs'].queue.get({ query: {} });
+ expect(queue.status).toBe(200);
+ expect(queue.data!.items.filter((run) => run.executionState === 'running')).toHaveLength(1);
+ expect(queue.data!.items.find((run) => run.id === second.id)).toMatchObject({
+ attempts: 0,
+ lastError: null,
+ });
+ });
+
it('leaves the runs of external agents out of both limits', async () => {
const { second, asOwner, workspaceId } = await setup({ external: true });
await asOwner.god['agent-settings'].put({ maxConcurrentRuns: 1 });
diff --git a/apps/api/src/modules/agents/core/run-poller.ts b/apps/api/src/modules/agents/core/run-poller.ts
index 8111c902..1a11c29e 100644
--- a/apps/api/src/modules/agents/core/run-poller.ts
+++ b/apps/api/src/modules/agents/core/run-poller.ts
@@ -9,10 +9,10 @@ import { runRefusal } from './run-refusal';
import {
agentRunConfig,
claimDueRuns,
+ admitWorkspaceRun,
countRunsAhead,
deferRun,
markRunFailed,
- markRunExecuting,
markRunSuccess,
runRetryDelayMs,
scheduleRunRetry,
@@ -26,9 +26,6 @@ import { runThreadId } from './runtime/thread-ids';
// the only thing a run is built from, so nothing about which project or which bot user
// it acts as can be handed in from outside.
-// How long a run waits when its workspace has no free slot.
-const DEFERRED_RETRY_SECONDS = 30;
-
export async function processAgentRuns(): Promise<void> {
const runs = await claimDueRuns();
await Promise.all(runs.map(processRun));
@@ -40,7 +37,7 @@ async function processRun(run: ClaimedRun): Promise<void> {
const limits = await getLimits(workspaceId);
const runLimit = await getRunLimit(workspaceId, limits);
if (runLimit > 0 && (await countRunsAhead(workspaceId, run.id)) >= runLimit) {
- await deferRun(run.id, DEFERRED_RETRY_SECONDS);
+ await deferRun(run.id, 30);
return;
}
const wait = await runAdmissionDelay(run);
@@ -70,8 +67,8 @@ async function processRun(run: ClaimedRun): Promise<void> {
);
if (run.executionDeadline <= performance.now()) return;
try {
+ if (!(await admitWorkspaceRun(run, workspaceId, runLimit))) return;
await agentRunStarted(run);
- if (!(await markRunExecuting(run.id, run.attempts))) return;
const timeoutMs = Math.floor(
Math.min(configuredTimeoutMs, run.executionDeadline - performance.now()),
);
diff --git a/apps/api/src/modules/agents/core/run-queue.ts b/apps/api/src/modules/agents/core/run-queue.ts
index c795f068..9c855bc2 100644
--- a/apps/api/src/modules/agents/core/run-queue.ts
+++ b/apps/api/src/modules/agents/core/run-queue.ts
@@ -98,18 +98,31 @@ export const holdsSlot = and(
eq(aiAgent.kind, 'internal'),
);
-// Runs of this workspace's agents that hold a slot and were queued before this one.
-// Counting only the older runs is what keeps a ceiling from turning away every run of a
-// burst at once: each yields to the ones ahead of it and the rest come back on their next
-// attempt.
-export async function countRunsAhead(workspaceId: number, runId: number): Promise<number> {
- const [row] = await db
+// Active runs and retry backoff keep their slots regardless of queue order. Among
+// claims awaiting admission, lower ids take precedence so a burst can make progress.
+export async function countRunsAhead(
+ workspaceId: number,
+ runId: number,
+ executor: DbExecutor = db,
+): Promise<number> {
+ const [row] = await executor
.select({ count: sql<number>`count(*)::int` })
.from(agentRun)
.innerJoin(aiAgent, eq(aiAgent.id, agentRun.agentId))
.innerJoin(project, eq(project.id, agentRun.projectId))
.innerJoin(team, eq(team.id, project.teamId))
- .where(and(eq(team.workspaceId, workspaceId), holdsSlot, lt(agentRun.id, runId)));
+ .where(
+ and(
+ eq(team.workspaceId, workspaceId),
+ holdsSlot,
+ sql`${agentRun.id} <> ${runId}`,
+ or(
+ lt(agentRun.id, runId),
+ isNotNull(agentRun.executionStartedAt),
+ isNull(agentRun.claimedAt),
+ ),
+ ),
+ );
return row?.count ?? 0;
}
@@ -468,8 +481,13 @@ export async function markRunFailed(id: number, error: string, output?: string):
// attempt, and nothing has been recorded on the issue yet, so the run leaves no trace of
// having been picked up at all: a first claim's started_at goes too, so the waiting run
// holds no slot in countRunsAhead and can be canceled as not started.
-export async function deferRun(id: number, delaySeconds: number): Promise<void> {
- await db
+export async function deferRun(
+ id: number,
+ delaySeconds: number,
+ executor: DbExecutor = db,
+ attempts?: number,
+): Promise<void> {
+ await executor
.update(agentRun)
.set({
executionStartedAt: null,
@@ -478,7 +496,19 @@ export async function deferRun(id: number, delaySeconds: number): Promise<void>
startedAt: sql`CASE WHEN ${agentRun.attempts} = 1 THEN NULL ELSE ${agentRun.startedAt} END`,
nextAttemptAt: sql`now() + make_interval(secs => ${delaySeconds})`,
})
- .where(and(eq(agentRun.id, id), eq(agentRun.status, 'pending')));
+ .where(
+ and(
+ eq(agentRun.id, id),
+ eq(agentRun.status, 'pending'),
+ attempts === undefined
+ ? undefined
+ : and(
+ eq(agentRun.attempts, attempts),
+ isNotNull(agentRun.claimedAt),
+ gt(agentRun.nextAttemptAt, sql`now()`),
+ ),
+ ),
+ );
}
export const executionState = sql<'queued' | 'running' | 'success' | 'failed' | 'canceled'>`
@@ -487,8 +517,12 @@ export const executionState = sql<'queued' | 'running' | 'success' | 'failed' |
THEN 'running'
ELSE 'queued' END`;
-export async function markRunExecuting(id: number, attempts: number): Promise<boolean> {
- const rows = await db
+export async function markRunExecuting(
+ id: number,
+ attempts: number,
+ executor: DbExecutor = db,
+): Promise<boolean> {
+ const rows = await executor
.update(agentRun)
.set({ executionStartedAt: new Date() })
.where(
@@ -503,6 +537,23 @@ export async function markRunExecuting(id: number, attempts: number): Promise<bo
return rows.length > 0;
}
+// Share the claim lock: another replica cannot admit a later claim between the
+// workspace count and this run reserving its execution slot.
+export async function admitWorkspaceRun(
+ run: ClaimedRun,
+ workspaceId: number,
+ limit: number,
+): Promise<boolean> {
+ return db.transaction(async (tx) => {
+ await tx.execute(sql`select pg_advisory_xact_lock(${RUN_CLAIM_LOCK_NS}, 0)`);
+ if (limit > 0 && (await countRunsAhead(workspaceId, run.id, tx)) >= limit) {
+ await deferRun(run.id, 30, tx, run.attempts);
+ return false;
+ }
+ return markRunExecuting(run.id, run.attempts, tx);
+ });
+}
+
export interface AgentRunRow {
id: number;
status: string;