Skip to content

⭐️⭐️⭐️ Open-source, self-hosted alternative to Linear and Plane. Project management and issue tracking where teams and AI agents work side by side to plan and ship products.

fix(api): keep agent execution within its claimed lease (#646)

I'm Groot 🌳b6610ae958d0
diff --git a/apps/api/src/modules/agents/core/__tests__/integration/run-timeout.test.ts b/apps/api/src/modules/agents/core/__tests__/integration/run-timeout.test.ts
index ddfe49df..e8b66271 100644
--- a/apps/api/src/modules/agents/core/__tests__/integration/run-timeout.test.ts
+++ b/apps/api/src/modules/agents/core/__tests__/integration/run-timeout.test.ts
@@ -1,7 +1,7 @@
import { describe, it, expect, beforeEach, afterEach } from 'bun:test';
import { db, agentRun } from '@repo/db';
import { eq, sql } from 'drizzle-orm';
-import { setAgentSettings } from '#modules/settings/service';
+import { getAgentSettings, setAgentSettings } from '#modules/settings/service';
import { authedApi, createProject } from '#tests/helpers/app';
import { signUpTestUser } from '#tests/helpers/auth';
import { resetDb } from '#tests/helpers/db';
@@ -11,23 +11,39 @@ import { createCredential } from '#tests/helpers/integrations';
import { claimDueRuns, enqueueAgentRun } from '../../run-queue';
import { processAgentRuns } from '../../run-poller';
import { getMastra } from '../../runtime/observability';
+import { setRunAdmission } from '../../run-admission';
-// A run that outlasts its time limit. The model endpoint accepts the request and never
-// answers, so the run ends only because its abort signal fires. The instance maximum is
-// set through the service, which skips the route's 10-second floor.
+// The service sets limits below the route's 10-second floor to keep the tests short.
describe('agent run timeout', () => {
let endpoint: ReturnType<typeof Bun.serve>;
+ let previousLease: string | undefined;
+ let previousSettings: Awaited<ReturnType<typeof getAgentSettings>>;
+ let releaseModel: (() => void) | undefined;
+ let processing: Promise<void> | undefined;
beforeEach(async () => {
await resetDb();
+ previousLease = process.env.AGENT_RUN_LEASE_SECONDS;
+ previousSettings = await getAgentSettings();
endpoint = Bun.serve({ port: 0, fetch: () => new Promise<Response>(() => {}) });
});
afterEach(async () => {
clearLimits();
- await endpoint.stop(true);
- await getMastra().observability.flush();
+ setRunAdmission();
+ if (previousLease === undefined) delete process.env.AGENT_RUN_LEASE_SECONDS;
+ else process.env.AGENT_RUN_LEASE_SECONDS = previousLease;
+ releaseModel?.();
+ try {
+ if (processing) await processing;
+ } finally {
+ await endpoint.stop(true);
+ await setAgentSettings(previousSettings);
+ await getMastra().observability.flush();
+ releaseModel = undefined;
+ processing = undefined;
+ }
});
async function queueRun(runTimeoutSec: number | null = null) {
@@ -115,4 +131,80 @@ describe('agent run timeout', () => {
`)) as unknown as { leaseSeconds: number }[];
expect(leaseSeconds).toBeGreaterThan(1800);
});
+
+ it('keeps a raised instance maximum within the lease already claimed', async () => {
+ await endpoint.stop(true);
+ let modelTimer: ReturnType<typeof setTimeout> | undefined;
+ let finishResponse!: () => void;
+ const responseReady = new Promise<void>((resolve) => {
+ finishResponse = resolve;
+ });
+ releaseModel = () => {
+ clearTimeout(modelTimer);
+ finishResponse();
+ };
+ endpoint = Bun.serve({
+ port: 0,
+ async fetch() {
+ modelTimer = setTimeout(finishResponse, 3000);
+ await responseReady;
+ return Response.json({
+ id: 'timeout-test',
+ object: 'chat.completion',
+ created: 0,
+ model: 'any-model',
+ choices: [
+ {
+ index: 0,
+ message: { role: 'assistant', content: 'Done.' },
+ finish_reason: 'stop',
+ },
+ ],
+ });
+ },
+ });
+ const issueId = await queueRun();
+ process.env.AGENT_RUN_LEASE_SECONDS = '1';
+ await setAgentSettings({ maxRunSeconds: 1 });
+ setRunAdmission(async () => {
+ await setAgentSettings({ maxRunSeconds: 1800 });
+ return 0;
+ });
+
+ processing = processAgentRuns();
+ await processing;
+
+ expect(await runOf(issueId)).toMatchObject({
+ status: 'failed',
+ attempts: 1,
+ lastError: 'Timed out after 1s',
+ });
+ }, 10_000);
+
+ it('does not start a run after admission exhausts the claim execution budget', async () => {
+ let modelCalls = 0;
+ await endpoint.stop(true);
+ endpoint = Bun.serve({
+ port: 0,
+ fetch() {
+ modelCalls++;
+ return new Response(null, { status: 500 });
+ },
+ });
+ const issueId = await queueRun();
+ process.env.AGENT_RUN_LEASE_SECONDS = '1';
+ await setAgentSettings({ maxRunSeconds: 1 });
+ setRunAdmission(async () => {
+ await new Promise((resolve) => setTimeout(resolve, 1200));
+ return 0;
+ });
+
+ await processAgentRuns();
+
+ expect(modelCalls).toBe(0);
+ expect(await runOf(issueId)).toMatchObject({
+ status: 'pending',
+ executionStartedAt: null,
+ });
+ });
});
diff --git a/apps/api/src/modules/agents/core/run-poller.ts b/apps/api/src/modules/agents/core/run-poller.ts
index 1d75ab36..8d5657dc 100644
--- a/apps/api/src/modules/agents/core/run-poller.ts
+++ b/apps/api/src/modules/agents/core/run-poller.ts
@@ -62,14 +62,23 @@ async function processRun(run: ClaimedRun): Promise<void> {
// The issue's timeline entries are written here, where the agent's work actually
// starts and ends. A failure that will be retried is not the end of the run, so only
// the last attempt logs one.
- const timeoutMs = runTimeoutMs(
+ const configuredTimeoutMs = runTimeoutMs(
run.agentRunTimeoutSec,
(await getAgentSettings()).maxRunSeconds,
maxRunSeconds,
);
+ if (run.executionDeadline <= performance.now()) return;
try {
await agentRunStarted(run);
if (!(await markRunExecuting(run.id, run.attempts))) return;
+ const timeoutMs = Math.floor(
+ Math.min(configuredTimeoutMs, run.executionDeadline - performance.now()),
+ );
+ if (timeoutMs <= 0) {
+ await recordAgentRunFinished(run, 'failed');
+ await markRunFailed(run.id, 'Claim execution budget exhausted');
+ return;
+ }
const result = await runAgent(run.agentId, run.projectId, framePrompt(run), {
callerUserId: run.agentUserId,
threadId: runThreadId(run),
diff --git a/apps/api/src/modules/agents/core/run-queue.ts b/apps/api/src/modules/agents/core/run-queue.ts
index 112624d2..3fc69b52 100644
--- a/apps/api/src/modules/agents/core/run-queue.ts
+++ b/apps/api/src/modules/agents/core/run-queue.ts
@@ -109,6 +109,8 @@ export interface ClaimedRun {
agentUserId: string;
// The agent's own run time limit, null when it uses the instance's maximum.
agentRunTimeoutSec: number | null;
+ // Monotonic deadline that leaves the claim's margin for recording the outcome.
+ executionDeadline: number;
// The agent's own handle, so the prompt can tell whether the text addressed it.
agentUsername: string;
// The issue's human-readable key ("MKT-42") and title, read inline so the poller can
@@ -142,6 +144,7 @@ export async function claimDueRuns(): Promise<ClaimedRun[]> {
agentRunConfig.leaseSeconds(),
(await getAgentSettings()).maxRunSeconds + LEASE_MARGIN_SECONDS,
);
+ const executionDeadline = performance.now() + (leaseSeconds - LEASE_MARGIN_SECONDS) * 1000;
const rows = await db.execute(sql`
UPDATE agent_run r
SET attempts = r.attempts + 1,
@@ -191,6 +194,7 @@ export async function claimDueRuns(): Promise<ClaimedRun[]> {
return Promise.all(
runs.map(async (run) => ({
...run,
+ executionDeadline,
threadContext: await loadThreadContext(run.sourceActivityId),
})),
);