⭐️⭐️⭐️ Open-source, self-hosted alternative to Linear and Plane. Project management and issue tracking where teams and AI agents work side by side to plan and ship products.
TypeScript agenticagileai-agentsbug-reportingbug-trackerganttissue-trackerjirajira-alternativekanbanlinearlinear-alternativeopen-sourceproduct-managementproject-managementroadmapself-hostedsprint-planningteam-collaborationtrello-alternative
fix(api): an agent run that reaches maxSteps ends as failed (#525)
Pasqual Tronconedc5a94f10ae4
diff --git a/apps/api/src/modules/agents/core/__tests__/integration/run-step-limit.test.ts b/apps/api/src/modules/agents/core/__tests__/integration/run-step-limit.test.tsnew file mode 100644index 00000000..9ea951e5--- /dev/null+++ b/apps/api/src/modules/agents/core/__tests__/integration/run-step-limit.test.ts@@ -0,0 +1,138 @@+import { describe, it, expect, beforeEach, afterEach } from 'bun:test';+import { authedApi } from '#tests/helpers/app';+import { signUpTestUser } from '#tests/helpers/auth';+import { resetDb } from '#tests/helpers/db';+import { createAgent } from '#tests/helpers/agents';+import { createCredential } from '#tests/helpers/integrations';+import { claimDueRuns, enqueueAgentRun } from '../../run-queue';+import { processAgentRuns } from '../../run-poller';+import { STEP_LIMIT_MESSAGE, streamAgent, type AgentRunEvent } from '../../runtime';++// An OpenAI-compatible model that asks for a tool on every call, so the run ends only+// because it reaches maxSteps; with `answers` set it replies once and stops instead.+let answers = false;+let calls = 0;++function reply(stream: boolean): Response {+ calls++;+ const toolCall = {+ id: `call_${calls}`,+ type: 'function',+ function: { name: 'get_project', arguments: '{}' },+ };+ const text = answers ? 'Done.' : `step ${calls}`;+ const finish = answers ? 'stop' : 'tool_calls';+ if (!stream) {+ const message = {+ role: 'assistant',+ content: text,+ ...(answers ? {} : { tool_calls: [toolCall] }),+ };+ return Response.json({+ id: 'x',+ object: 'chat.completion',+ created: 0,+ model: 'm',+ choices: [{ index: 0, finish_reason: finish, message }],+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },+ });+ }+ const chunk = (choice: object, extra: object = {}) =>+ `data: ${JSON.stringify({ id: 'x', object: 'chat.completion.chunk', created: 0, model: 'm', choices: [{ index: 0, ...choice }], ...extra })}\n\n`;+ let body = chunk({ delta: { role: 'assistant', content: text } });+ if (!answers) body += chunk({ delta: { tool_calls: [{ index: 0, ...toolCall }] } });+ body += chunk(+ { delta: {}, finish_reason: finish },+ { usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } },+ );+ return new Response(`${body}data: [DONE]\n\n`, {+ headers: { 'content-type': 'text/event-stream' },+ });+}++describe('agent run step limit', () => {+ let endpoint: ReturnType<typeof Bun.serve>;++ beforeEach(async () => {+ await resetDb();+ answers = false;+ calls = 0;+ endpoint = Bun.serve({+ port: 0,+ fetch: async (req) => reply(Boolean(((await req.json()) as { stream?: boolean }).stream)),+ });+ });++ afterEach(async () => {+ await endpoint.stop(true);+ });++ async function setup() {+ const owner = await signUpTestUser({ name: 'Owner' });+ const asOwner = authedApi(owner.cookie);+ await asOwner.projects.post({ key: 'MKT', name: 'Marketing' });+ const view = await asOwner.projects({ projectKey: 'MKT' }).get();+ const credentialId = await createCredential(asOwner, 'MKT', {+ integrationKey: 'openai-compatible',+ credential: { apiKey: 'sk-test', baseUrl: `http://localhost:${endpoint.port}/v1` },+ });+ const agent = (+ await createAgent(asOwner, 'MKT', {+ name: 'Bot',+ username: 'bot',+ kind: 'internal',+ modelCredentialId: credentialId,+ model: 'any-model',+ maxSteps: 2,+ })+ ).data!.agent;+ return { owner, asOwner, view, agent };+ }++ async function runOnIssue() {+ const { asOwner, view, agent } = await setup();+ const issue = (+ await asOwner+ .projects({ projectKey: 'MKT' })+ .issues.post({ columnId: view.data!.columns[0].id, title: 'Task' })+ ).data!;+ await enqueueAgentRun({+ agentId: agent.id,+ projectId: agent.projects[0].id,+ issueId: issue.id,+ sourceActivityId: null,+ prompt: 'do it',+ });+ await processAgentRuns();+ const runs = await asOwner+ .teams({ teamId: agent.teamId })+ ['ai-agents']({ agentId: agent.id })+ .runs.get({ query: {} });+ return runs.data!.items.find((run) => run.issueId === issue.id)!;+ }++ it('ends a run that reaches maxSteps as failed and does not retry it', async () => {+ const run = await runOnIssue();+ expect(calls).toBe(2);+ expect(run).toMatchObject({ status: 'failed', attempts: 1, lastError: STEP_LIMIT_MESSAGE });+ expect((await claimDueRuns()).find((r) => r.id === run.id)).toBeUndefined();+ });++ it('keeps a run that answers within maxSteps a success', async () => {+ answers = true;+ const run = await runOnIssue();+ expect(run).toMatchObject({ status: 'success', lastError: null, output: 'Done.' });+ });++ it('ends a chat answer that reaches maxSteps with an error', async () => {+ const { owner, agent } = await setup();+ const events: AgentRunEvent[] = [];+ for await (const event of streamAgent(agent.id, agent.projects[0].id, 'hello', {+ callerUserId: owner.userId,+ })) {+ events.push(event);+ }+ expect(events.at(-2)).toEqual({ type: 'error', message: STEP_LIMIT_MESSAGE });+ expect(events.at(-1)?.type).toBe('done');+ });+});diff --git a/apps/api/src/modules/agents/core/run-poller.ts b/apps/api/src/modules/agents/core/run-poller.tsindex efc114f9..cb584401 100644--- a/apps/api/src/modules/agents/core/run-poller.ts+++ b/apps/api/src/modules/agents/core/run-poller.ts@@ -16,7 +16,7 @@ import {scheduleRunRetry,type ClaimedRun,} from './run-queue';-import { runAgent } from './runtime';+import { STEP_LIMIT_MESSAGE, runAgent } from './runtime';import { runThreadId } from './runtime/thread-ids';// Drains the agent_run queue. The runtime, the model credentials and the limits are@@ -65,6 +65,11 @@ async function processRun(run: ClaimedRun): Promise<void> {await markRunFailed(run.id, `Timed out after ${Math.round(timeoutMs / 1000)}s`);return;}+ if (result.stepLimit) {+ await recordAgentRunFinished(run, 'failed');+ await markRunFailed(run.id, STEP_LIMIT_MESSAGE);+ return;+ }await recordAgentRunFinished(run, 'success');await markRunSuccess(run.id, result.text, result.usage);} catch (error) {diff --git a/apps/api/src/modules/agents/core/runtime/index.ts b/apps/api/src/modules/agents/core/runtime/index.tsindex 115048e9..99a1cc00 100644--- a/apps/api/src/modules/agents/core/runtime/index.ts+++ b/apps/api/src/modules/agents/core/runtime/index.ts@@ -154,7 +154,9 @@ async function buildAgent(// caller can continue the conversation; it is null when memory is off.//// `aborted` is true when opts.abortSignal fired: Mastra then resolves with the partial-// text instead of rejecting, so only this tells it from a normal finish.+// text instead of rejecting, so only this tells it from a normal finish. `stepLimit` is+// true when the run used its last step still calling tools: Mastra resolves that as a+// finish too, with the text of the steps and no answer.export async function runAgent(agentId: number,projectId: number,@@ -165,6 +167,7 @@ export async function runAgent(threadId: string | null;usage: ContextUsage | null;aborted: boolean;+ stepLimit: boolean;}> {const { agent, row, options, threadId } = await prepareRun(agentId, projectId, prompt, opts);const result = await agent.generate(prompt, { ...options, abortSignal: opts.abortSignal });@@ -177,9 +180,20 @@ export async function runAgent(threadId,usage,aborted: opts.abortSignal?.aborted ?? false,+ stepLimit: reachedStepLimit(result.finishReason, result.steps.length, options.maxSteps),};}+export const STEP_LIMIT_MESSAGE = 'Reached the step limit';++function reachedStepLimit(+ finishReason: string | undefined,+ steps: number,+ maxSteps: number,+): boolean {+ return finishReason === 'tool-calls' && steps >= maxSteps;+}+// What the model reported about the last call of an answer, as the pair the chat keeps.// Mastra's counts already hold the tokens read from cache inside `inputTokens`, so they// are taken as they are. Null for a model that reports none, which the chat shows as a@@ -242,6 +256,11 @@ export async function* streamAgent(break;}}+ if (+ reachedStepLimit(await result.finishReason, (await result.steps).length, options.maxSteps)+ ) {+ yield { type: 'error', message: STEP_LIMIT_MESSAGE };+ }// An answer the member stopped never reaches here: the stream is aborted and the// thread keeps the number of the answer before it.if (threadId) await recordContextUsage(threadId, row.id, contextOf(usage));