Your always-on AI coworkers that move between text, calls, and Slack.
TypeScript
fix: make agent run timeout configurable (#101)
Twil3akine79b4db192f60
diff --git a/.env.example b/.env.exampleindex 24e72d6..c5051ce 100644--- a/.env.example+++ b/.env.example@@ -21,6 +21,8 @@ INTELLIGENCE_API_KEY=OPENAI_API_KEY=OPENAI_BASE_URL=https://api.openai.com/v1OPENAI_MODEL=+# Maximum duration for Dot turns and scheduled/background agent runs (milliseconds).+# AGENT_RUN_TIMEOUT_MS=90000# Keep the CLI-generated CPK_TELEMETRY_ID with its matching project key.# The SDK uses it to connect OpenDots usage to the Intelligence account.# CPK_TELEMETRY_ID=diff --git a/compose.yml b/compose.ymlindex be199b6..ecac471 100644--- a/compose.yml+++ b/compose.yml@@ -25,6 +25,7 @@ services:SLACK_TEAM_ID: ${SLACK_TEAM_ID:-}SLACK_USER_IDS: ${SLACK_USER_IDS:-}SLACK_DOT_ID: ${SLACK_DOT_ID:-}+ AGENT_RUN_TIMEOUT_MS: ${AGENT_RUN_TIMEOUT_MS:-90000}OPENAI_API_KEY: ${OPENAI_API_KEY:-}OPENAI_BASE_URL: ${OPENAI_BASE_URL:-https://api.openai.com/v1}OPENAI_MODEL: ${OPENAI_MODEL:-}diff --git a/src/server/dot-agent.ts b/src/server/dot-agent.tsindex 9d7f9ad..f5e34ab 100644--- a/src/server/dot-agent.ts+++ b/src/server/dot-agent.ts@@ -23,7 +23,10 @@ import { Observable } from 'rxjs';import { z } from 'zod';import { Store } from './store.js';import { WorkspaceStore } from './workspace.js';-import type { PlatformConfig } from './platform-config.js';+import {+ DEFAULT_AGENT_RUN_TIMEOUT_MS,+ type PlatformConfig,+} from './platform-config.js';import { browserResponse } from './research.js';import { answerObserver, type SetupTelemetry } from './setup-telemetry.js';const channelError = () => ({@@ -31,7 +34,6 @@ const channelError = () => ({message:'OpenDots could not complete this request. Please check the app and try again.',});-const TURN_TIME_LIMIT_MS = 90_000;export class DotAgent extends AbstractAgent {private inner?: BuiltInAgent;private controller?: AbortController;@@ -68,6 +70,8 @@ export class DotAgent extends AbstractAgent {let timedOut = false;let finished = false;let configurationFailure = false;+ const turnTimeLimitMs =+ this.config.agentRunTimeoutMs ?? DEFAULT_AGENT_RUN_TIMEOUT_MS;const observe = answerObserver((event) =>this.setupTelemetry?.capture(event),);@@ -75,10 +79,10 @@ export class DotAgent extends AbstractAgent {timedOut = true;observe({ type: EventType.RUN_ERROR });this.abortRun();- }, TURN_TIME_LIMIT_MS);+ }, turnTimeLimitMs);const timeLimitError = () => ({type: EventType.RUN_ERROR,- message: `This turn reached the ${TURN_TIME_LIMIT_MS / 1000} second time limit and was stopped. Try a smaller request.`,+ message: `This turn reached the ${turnTimeLimitMs / 1000} second time limit and was stopped. Try a smaller request.`,});const fail = (error: unknown) => {if (!configurationFailure) observe({ type: EventType.RUN_ERROR });diff --git a/src/server/index.ts b/src/server/index.tsindex 5fc19c8..f9b4f23 100644--- a/src/server/index.ts+++ b/src/server/index.ts@@ -10,6 +10,7 @@ import { resolveAppOrigins } from './app-origin.js';import { WorkspaceStore } from './workspace.js';import { Platform } from './platform.js';import {+ agentRunTimeoutMsFromEnv,intelligenceApiKeyFromEnv,intelligenceWsUrlFromEnv,type PlatformConfig,@@ -37,6 +38,7 @@ const config: PlatformConfig = {apiKey: process.env.OPENAI_API_KEY,model: process.env.OPENAI_MODEL,baseUrl: process.env.OPENAI_BASE_URL ?? 'https://api.openai.com/v1',+ agentRunTimeoutMs: agentRunTimeoutMsFromEnv(process.env),webSearchProvider: webSearchProvider(process.env.WEB_SEARCH_PROVIDER),parallelApiKey: process.env.PARALLEL_API_KEY,browserUrl: process.env.BROWSER_URL,@@ -85,6 +87,7 @@ const runner = new Runner(});return { text, sources: [], sample: false };},+ config.agentRunTimeoutMs,);const wsOrigin = new URL(config.intelligenceWsUrl ?? 'wss://realtime.intelligence.copilotkit.ai',diff --git a/src/server/platform-config.ts b/src/server/platform-config.tsindex 22ea448..f065582 100644--- a/src/server/platform-config.ts+++ b/src/server/platform-config.ts@@ -7,6 +7,8 @@ const INTELLIGENCE_API_KEY_ENV_NAMES = ['INTELLIGENCE_API_KEY',] as const;export const INTELLIGENCE_KEY_MISSING_LABEL = `${INTELLIGENCE_API_KEY_ENV_NAMES[1]} (or ${INTELLIGENCE_API_KEY_ENV_NAMES[0]})`;+export const DEFAULT_AGENT_RUN_TIMEOUT_MS = 90_000;+const MAX_TIMER_DELAY_MS = 2_147_483_647;export function intelligenceApiKeyFromEnv(env: Record<string, string | undefined>,@@ -23,6 +25,19 @@ export function intelligenceWsUrlFromEnv(]);}+export function agentRunTimeoutMsFromEnv(+ env: Record<string, string | undefined>,+): number {+ const raw = env.AGENT_RUN_TIMEOUT_MS?.trim();+ if (!raw) return DEFAULT_AGENT_RUN_TIMEOUT_MS;+ const value = Number(raw);+ if (!Number.isInteger(value) || value < 1 || value > MAX_TIMER_DELAY_MS)+ throw new Error(+ `AGENT_RUN_TIMEOUT_MS must be an integer between 1 and ${MAX_TIMER_DELAY_MS}.`,+ );+ return value;+}+function firstNonEmptyEnvValue(env: Record<string, string | undefined>,names: readonly string[],@@ -41,6 +56,7 @@ export interface PlatformConfig extends WebConfig {model?: string;apiKey?: string;baseUrl: string;+ agentRunTimeoutMs?: number;computerSupervisorUrl?: string;computerSupervisorToken?: string;computerToken?: string;diff --git a/src/server/runner.ts b/src/server/runner.tsindex e5a047e..c9de2d6 100644--- a/src/server/runner.ts+++ b/src/server/runner.ts@@ -1,7 +1,8 @@-import { Store } from './store.js';+import { DEFAULT_WORKER_LEASE_MS, Store } from './store.js';import type { Result, Memory } from '../shared/types.js';import type { Claim } from './store.js';import { research, type Config } from './research.js';+import { DEFAULT_AGENT_RUN_TIMEOUT_MS } from './platform-config.js';export class Runner {private timer?: ReturnType<typeof setInterval>;private active = new Map<string, AbortController>();@@ -14,6 +15,7 @@ export class Runner {signal: AbortSignal,progress: (text: string) => void,) => Promise<Result>,+ private runTimeoutMs = DEFAULT_AGENT_RUN_TIMEOUT_MS,) {}start() {if (!this.timer) {@@ -53,7 +55,11 @@ export class Runner {}private async runTick() {if (this.active.size) return;- const claim = this.store.claim();+ // Leave time to persist the outcome after the configured run timeout.+ const claim = this.store.claim(+ Date.now(),+ Math.max(DEFAULT_WORKER_LEASE_MS, this.runTimeoutMs * 2),+ );if (!claim) return;const controller = new AbortController();this.active.set(claim.id, controller);@@ -68,9 +74,11 @@ export class Runner {const timeout = setTimeout(() =>controller.abort(- new Error('Research exceeded the 90 second time limit.'),+ new Error(+ `Research exceeded the ${this.runTimeoutMs / 1000} second time limit.`,+ ),),- 90_000,+ this.runTimeoutMs,);try {const settings = this.store.settings();diff --git a/src/server/store.ts b/src/server/store.tsindex 36af42e..05d7d86 100644--- a/src/server/store.ts+++ b/src/server/store.ts@@ -14,6 +14,7 @@ import type {} from '../shared/types.js';export type Claim = Task & { lease: string };+export const DEFAULT_WORKER_LEASE_MS = 180_000;const defaults: Settings = {name: 'Dot',paused: false,@@ -206,7 +207,10 @@ export class Store {);return this.task(id);}- claim(now = Date.now()): Claim | null {+ claim(+ now = Date.now(),+ leaseDurationMs = DEFAULT_WORKER_LEASE_MS,+ ): Claim | null {return this.transaction(() => {const settings = this.settings();if (settings.paused || !settings.researchAllowed) return null;@@ -230,7 +234,7 @@ export class Store {.prepare("UPDATE tasks SET status='running', lease=?, leaseUntil=?, nextRunAt=NULL, error=NULL, updatedAt=? WHERE id=?",)- .run(lease, now + 180_000, now, task.id);+ .run(lease, now + leaseDurationMs, now, task.id);this.db.prepare("INSERT INTO runs VALUES (?, ?, 'running', ?, NULL, NULL, NULL)",diff --git a/tests/agent-run-timeout.test.ts b/tests/agent-run-timeout.test.tsnew file mode 100644index 0000000..6074393--- /dev/null+++ b/tests/agent-run-timeout.test.ts@@ -0,0 +1,223 @@+import { afterEach, expect, it, vi } from 'vitest';+import { mkdtempSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { EventType, type RunAgentInput } from '@ag-ui/core';+import { lastValueFrom, toArray } from 'rxjs';+import { DotAgent } from '../src/server/dot-agent.js';+import {+ DEFAULT_AGENT_RUN_TIMEOUT_MS,+ agentRunTimeoutMsFromEnv,+} from '../src/server/platform-config.js';+import { Runner } from '../src/server/runner.js';+import { Store } from '../src/server/store.js';+import { WorkspaceStore } from '../src/server/workspace.js';++const resources: Array<{ close(): void }> = [];+function sharedStores() {+ const dir = mkdtempSync(join(tmpdir(), 'opendots-timeout-'));+ const path = join(dir, 'test.sqlite');+ const first = new Store(path);+ const second = new Store(path);+ resources.push(first, second, {+ close: () => rmSync(dir, { recursive: true, force: true }),+ });+ return { first, second };+}+afterEach(() => {+ vi.restoreAllMocks();+ vi.useRealTimers();+ resources.splice(0).forEach((resource) => resource.close());+});++it('uses 90 seconds by default and accepts a positive integer override', () => {+ expect(agentRunTimeoutMsFromEnv({})).toBe(DEFAULT_AGENT_RUN_TIMEOUT_MS);+ expect(agentRunTimeoutMsFromEnv({ AGENT_RUN_TIMEOUT_MS: ' 240000 ' })).toBe(+ 240_000,+ );+});++it('rejects invalid timeout overrides at startup', () => {+ for (const value of ['0', '-1', '1.5', 'not-a-number', '2147483648'])+ expect(() =>+ agentRunTimeoutMsFromEnv({ AGENT_RUN_TIMEOUT_MS: value }),+ ).toThrow(/AGENT_RUN_TIMEOUT_MS/);+});++it('uses the configured timeout for interactive Dot turns', async () => {+ vi.useFakeTimers();+ const store = new Store(':memory:');+ const workspace = new WorkspaceStore(':memory:', 'owner');+ resources.push(store, workspace);+ const dot = workspace.dots()[0];+ workspace.bindThread('thread', dot.id, 'Timeout test');+ const agent = new DotAgent(+ store,+ workspace,+ {+ intelligenceKey: 'fixture',+ apiKey: 'fixture',+ model: 'custom-model',+ baseUrl: 'https://unused.invalid/v1',+ runtimeUrl: '',+ voiceName: 'marin',+ slackUsers: [],+ agentRunTimeoutMs: 1_250,+ },+ dot.id,+ );+ const input: RunAgentInput = {+ threadId: 'thread',+ runId: 'run',+ state: {},+ context: [],+ messages: [{ id: 'user', role: 'user', content: 'Keep working.' }],+ tools: [],+ forwardedProps: {},+ };+ vi.spyOn(globalThis, 'fetch').mockImplementation(+ (_url, init) =>+ new Promise((_resolve, reject) => {+ init?.signal?.addEventListener(+ 'abort',+ () => reject(init.signal?.reason),+ { once: true },+ );+ }),+ );+ const finished = lastValueFrom(agent.run(input).pipe(toArray()));+ await vi.advanceTimersByTimeAsync(1_251);+ const events = await finished;+ expect(events).toEqual(+ expect.arrayContaining([+ expect.objectContaining({+ type: EventType.RUN_ERROR,+ message: expect.stringContaining('1.25 second time limit'),+ }),+ ]),+ );+});++it('uses the configured timeout for scheduled/background runs', async () => {+ vi.useFakeTimers();+ const store = new Store(':memory:');+ resources.push(store);+ let signal: AbortSignal | undefined;+ const runner = new Runner(+ store,+ { mode: 'sample', baseUrl: '' },+ (_claim, _memories, runSignal) =>+ new Promise((_resolve, reject) => {+ signal = runSignal;+ runSignal.addEventListener('abort', () => reject(runSignal.reason), {+ once: true,+ });+ }),+ 1_250,+ );+ store.createTask('Long-running task');+ const pending = runner.tick();+ await vi.advanceTimersByTimeAsync(1_251);+ await pending;+ expect(signal?.aborted).toBe(true);+ expect(store.tasks()[0].status).toBe('failed');+});++it('completes a longer background run after another worker checks the old lease deadline', async () => {+ vi.useFakeTimers();+ const { first, second } = sharedStores();+ const result = {+ text: 'Finished after 200 seconds',+ sources: [],+ sample: true,+ };+ let signal: AbortSignal | undefined;+ const runner = new Runner(+ first,+ { mode: 'sample', baseUrl: '' },+ (_claim, _memories, runSignal) =>+ new Promise((resolve, reject) => {+ signal = runSignal;+ const timer = setTimeout(() => resolve(result), 200_000);+ runSignal.addEventListener(+ 'abort',+ () => {+ clearTimeout(timer);+ reject(runSignal.reason);+ },+ { once: true },+ );+ }),+ 240_000,+ );+ const competingExecute = vi.fn().mockResolvedValue(result);+ const competing = new Runner(+ second,+ { mode: 'sample', baseUrl: '' },+ competingExecute,+ );+ const task = first.createTask('Long-running task');+ const pending = runner.tick();+ const lease = first.task(task.id)?.lease;+ try {+ await vi.advanceTimersByTimeAsync(180_001);+ await competing.tick();+ expect(second.task(task.id)?.status).toBe('running');+ expect(second.task(task.id)?.lease).toBe(lease);+ expect(signal?.aborted).toBe(false);+ expect(competingExecute).not.toHaveBeenCalled();++ await vi.advanceTimersByTimeAsync(20_000);+ await pending;+ expect(second.task(task.id)?.status).toBe('completed');+ expect(second.detail(task.id)?.runs).toEqual([+ expect.objectContaining({ status: 'completed', result }),+ ]);+ } finally {+ runner.stop();+ competing.stop();+ await pending;+ }+});++it('still times out a longer background run after another worker checks its lease', async () => {+ vi.useFakeTimers();+ const { first, second } = sharedStores();+ let signal: AbortSignal | undefined;+ const runner = new Runner(+ first,+ { mode: 'sample', baseUrl: '' },+ (_claim, _memories, runSignal) =>+ new Promise((_resolve, reject) => {+ signal = runSignal;+ runSignal.addEventListener('abort', () => reject(runSignal.reason), {+ once: true,+ });+ }),+ 240_000,+ );+ const competing = new Runner(second, { mode: 'sample', baseUrl: '' });+ const task = first.createTask('Long-running task');+ const pending = runner.tick();+ try {+ await vi.advanceTimersByTimeAsync(180_001);+ await competing.tick();+ expect(second.task(task.id)?.status).toBe('running');+ expect(signal?.aborted).toBe(false);++ await vi.advanceTimersByTimeAsync(60_000);+ await pending;+ expect(signal?.aborted).toBe(true);+ expect(second.task(task.id)?.status).toBe('failed');+ expect(second.detail(task.id)?.runs).toEqual([+ expect.objectContaining({+ status: 'failed',+ error: 'Research exceeded the 240 second time limit.',+ }),+ ]);+ } finally {+ runner.stop();+ competing.stop();+ await pending;+ }+});diff --git a/tests/store.test.ts b/tests/store.test.tsindex 54e4335..edb4fa0 100644--- a/tests/store.test.ts+++ b/tests/store.test.ts@@ -78,34 +78,39 @@ describe('durable task lifecycle', () => {store.action(task.id, 'run');expect(store.claim(Date.now())?.id).toBe(task.id);});- it('holds expired work until the owner retries it', () => {- const { store, path } = fixture();- store.createTask('Recover me');- const now = Date.now();- const old = store.claim(now)!;- const restarted = new Store(path);- try {- expect(restarted.claim(now + 180_001)).toBeNull();- expect(restarted.task(old.id)?.status).toBe('interrupted');- expect(restarted.detail(old.id)?.runs[0].status).toBe('interrupted');- restarted.action(old.id, 'run');- const recovered = restarted.claim(now + 180_001)!;- expect(recovered.id).toBe(old.id);- expect(recovered.lease).not.toBe(old.lease);- expect(- store.finish(old, { text: 'Old', sources: [], sample: true }),- ).toBe(false);- expect(- restarted.finish(recovered, {- text: 'New',- sources: [],- sample: true,- }),- ).toBe(true);- } finally {- restarted.close();- }- });+ it.each([180_000, 480_000])(+ 'holds expired work until the owner retries it with a %i ms lease',+ (leaseDurationMs) => {+ const { store, path } = fixture();+ store.createTask('Recover me');+ const now = Date.now();+ const old = store.claim(now, leaseDurationMs)!;+ const restarted = new Store(path);+ try {+ expect(restarted.claim(now + leaseDurationMs - 1)).toBeNull();+ expect(restarted.task(old.id)?.status).toBe('running');+ expect(restarted.claim(now + leaseDurationMs + 1)).toBeNull();+ expect(restarted.task(old.id)?.status).toBe('interrupted');+ expect(restarted.detail(old.id)?.runs[0].status).toBe('interrupted');+ restarted.action(old.id, 'run');+ const recovered = restarted.claim(now + leaseDurationMs + 1)!;+ expect(recovered.id).toBe(old.id);+ expect(recovered.lease).not.toBe(old.lease);+ expect(+ store.finish(old, { text: 'Old', sources: [], sample: true }),+ ).toBe(false);+ expect(+ restarted.finish(recovered, {+ text: 'New',+ sources: [],+ sample: true,+ }),+ ).toBe(true);+ } finally {+ restarted.close();+ }+ },+ );it('claims other queued work while an expired run waits for review', () => {const { store } = fixture();const interrupted = store.createTask('Create the first page');