⭐️⭐️⭐️ Open-source, self-hosted alternative to Linear and Plane. Project management and issue tracking where teams and AI agents work side by side to plan and ship products.
TypeScript agenticagileai-agentsbug-reportingbug-trackerganttissue-trackerjirajira-alternativekanbanlinearlinear-alternativeopen-sourceproduct-managementproject-managementroadmapself-hostedsprint-planningteam-collaborationtrello-alternative
fix(api): keep agent execution within its claimed lease (#646)
I'm Groot 🌳b6610ae958d0
diff --git a/apps/api/src/modules/agents/core/__tests__/integration/run-timeout.test.ts b/apps/api/src/modules/agents/core/__tests__/integration/run-timeout.test.tsindex ddfe49df..e8b66271 100644--- a/apps/api/src/modules/agents/core/__tests__/integration/run-timeout.test.ts+++ b/apps/api/src/modules/agents/core/__tests__/integration/run-timeout.test.ts@@ -1,7 +1,7 @@import { describe, it, expect, beforeEach, afterEach } from 'bun:test';import { db, agentRun } from '@repo/db';import { eq, sql } from 'drizzle-orm';-import { setAgentSettings } from '#modules/settings/service';+import { getAgentSettings, setAgentSettings } from '#modules/settings/service';import { authedApi, createProject } from '#tests/helpers/app';import { signUpTestUser } from '#tests/helpers/auth';import { resetDb } from '#tests/helpers/db';@@ -11,23 +11,39 @@ import { createCredential } from '#tests/helpers/integrations';import { claimDueRuns, enqueueAgentRun } from '../../run-queue';import { processAgentRuns } from '../../run-poller';import { getMastra } from '../../runtime/observability';+import { setRunAdmission } from '../../run-admission';-// A run that outlasts its time limit. The model endpoint accepts the request and never-// answers, so the run ends only because its abort signal fires. The instance maximum is-// set through the service, which skips the route's 10-second floor.+// The service sets limits below the route's 10-second floor to keep the tests short.describe('agent run timeout', () => {let endpoint: ReturnType<typeof Bun.serve>;+ let previousLease: string | undefined;+ let previousSettings: Awaited<ReturnType<typeof getAgentSettings>>;+ let releaseModel: (() => void) | undefined;+ let processing: Promise<void> | undefined;beforeEach(async () => {await resetDb();+ previousLease = process.env.AGENT_RUN_LEASE_SECONDS;+ previousSettings = await getAgentSettings();endpoint = Bun.serve({ port: 0, fetch: () => new Promise<Response>(() => {}) });});afterEach(async () => {clearLimits();- await endpoint.stop(true);- await getMastra().observability.flush();+ setRunAdmission();+ if (previousLease === undefined) delete process.env.AGENT_RUN_LEASE_SECONDS;+ else process.env.AGENT_RUN_LEASE_SECONDS = previousLease;+ releaseModel?.();+ try {+ if (processing) await processing;+ } finally {+ await endpoint.stop(true);+ await setAgentSettings(previousSettings);+ await getMastra().observability.flush();+ releaseModel = undefined;+ processing = undefined;+ }});async function queueRun(runTimeoutSec: number | null = null) {@@ -115,4 +131,80 @@ describe('agent run timeout', () => {`)) as unknown as { leaseSeconds: number }[];expect(leaseSeconds).toBeGreaterThan(1800);});++ it('keeps a raised instance maximum within the lease already claimed', async () => {+ await endpoint.stop(true);+ let modelTimer: ReturnType<typeof setTimeout> | undefined;+ let finishResponse!: () => void;+ const responseReady = new Promise<void>((resolve) => {+ finishResponse = resolve;+ });+ releaseModel = () => {+ clearTimeout(modelTimer);+ finishResponse();+ };+ endpoint = Bun.serve({+ port: 0,+ async fetch() {+ modelTimer = setTimeout(finishResponse, 3000);+ await responseReady;+ return Response.json({+ id: 'timeout-test',+ object: 'chat.completion',+ created: 0,+ model: 'any-model',+ choices: [+ {+ index: 0,+ message: { role: 'assistant', content: 'Done.' },+ finish_reason: 'stop',+ },+ ],+ });+ },+ });+ const issueId = await queueRun();+ process.env.AGENT_RUN_LEASE_SECONDS = '1';+ await setAgentSettings({ maxRunSeconds: 1 });+ setRunAdmission(async () => {+ await setAgentSettings({ maxRunSeconds: 1800 });+ return 0;+ });++ processing = processAgentRuns();+ await processing;++ expect(await runOf(issueId)).toMatchObject({+ status: 'failed',+ attempts: 1,+ lastError: 'Timed out after 1s',+ });+ }, 10_000);++ it('does not start a run after admission exhausts the claim execution budget', async () => {+ let modelCalls = 0;+ await endpoint.stop(true);+ endpoint = Bun.serve({+ port: 0,+ fetch() {+ modelCalls++;+ return new Response(null, { status: 500 });+ },+ });+ const issueId = await queueRun();+ process.env.AGENT_RUN_LEASE_SECONDS = '1';+ await setAgentSettings({ maxRunSeconds: 1 });+ setRunAdmission(async () => {+ await new Promise((resolve) => setTimeout(resolve, 1200));+ return 0;+ });++ await processAgentRuns();++ expect(modelCalls).toBe(0);+ expect(await runOf(issueId)).toMatchObject({+ status: 'pending',+ executionStartedAt: null,+ });+ });});diff --git a/apps/api/src/modules/agents/core/run-poller.ts b/apps/api/src/modules/agents/core/run-poller.tsindex 1d75ab36..8d5657dc 100644--- a/apps/api/src/modules/agents/core/run-poller.ts+++ b/apps/api/src/modules/agents/core/run-poller.ts@@ -62,14 +62,23 @@ async function processRun(run: ClaimedRun): Promise<void> {// The issue's timeline entries are written here, where the agent's work actually// starts and ends. A failure that will be retried is not the end of the run, so only// the last attempt logs one.- const timeoutMs = runTimeoutMs(+ const configuredTimeoutMs = runTimeoutMs(run.agentRunTimeoutSec,(await getAgentSettings()).maxRunSeconds,maxRunSeconds,);+ if (run.executionDeadline <= performance.now()) return;try {await agentRunStarted(run);if (!(await markRunExecuting(run.id, run.attempts))) return;+ const timeoutMs = Math.floor(+ Math.min(configuredTimeoutMs, run.executionDeadline - performance.now()),+ );+ if (timeoutMs <= 0) {+ await recordAgentRunFinished(run, 'failed');+ await markRunFailed(run.id, 'Claim execution budget exhausted');+ return;+ }const result = await runAgent(run.agentId, run.projectId, framePrompt(run), {callerUserId: run.agentUserId,threadId: runThreadId(run),diff --git a/apps/api/src/modules/agents/core/run-queue.ts b/apps/api/src/modules/agents/core/run-queue.tsindex 112624d2..3fc69b52 100644--- a/apps/api/src/modules/agents/core/run-queue.ts+++ b/apps/api/src/modules/agents/core/run-queue.ts@@ -109,6 +109,8 @@ export interface ClaimedRun {agentUserId: string;// The agent's own run time limit, null when it uses the instance's maximum.agentRunTimeoutSec: number | null;+ // Monotonic deadline that leaves the claim's margin for recording the outcome.+ executionDeadline: number;// The agent's own handle, so the prompt can tell whether the text addressed it.agentUsername: string;// The issue's human-readable key ("MKT-42") and title, read inline so the poller can@@ -142,6 +144,7 @@ export async function claimDueRuns(): Promise<ClaimedRun[]> {agentRunConfig.leaseSeconds(),(await getAgentSettings()).maxRunSeconds + LEASE_MARGIN_SECONDS,);+ const executionDeadline = performance.now() + (leaseSeconds - LEASE_MARGIN_SECONDS) * 1000;const rows = await db.execute(sql`UPDATE agent_run rSET attempts = r.attempts + 1,@@ -191,6 +194,7 @@ export async function claimDueRuns(): Promise<ClaimedRun[]> {return Promise.all(runs.map(async (run) => ({...run,+ executionDeadline,threadContext: await loadThreadContext(run.sourceActivityId),})),);