Skip to content

Your always-on AI coworkers that move between text, calls, and Slack.

TypeScript

fix: make agent run timeout configurable (#101)

Twil3akine79b4db192f60
diff --git a/.env.example b/.env.example
index 24e72d6..c5051ce 100644
--- a/.env.example
+++ b/.env.example
@@ -21,6 +21,8 @@ INTELLIGENCE_API_KEY=
OPENAI_API_KEY=
OPENAI_BASE_URL=https://api.openai.com/v1
OPENAI_MODEL=
+# Maximum duration for Dot turns and scheduled/background agent runs (milliseconds).
+# AGENT_RUN_TIMEOUT_MS=90000
# Keep the CLI-generated CPK_TELEMETRY_ID with its matching project key.
# The SDK uses it to connect OpenDots usage to the Intelligence account.
# CPK_TELEMETRY_ID=
diff --git a/compose.yml b/compose.yml
index be199b6..ecac471 100644
--- a/compose.yml
+++ b/compose.yml
@@ -25,6 +25,7 @@ services:
SLACK_TEAM_ID: ${SLACK_TEAM_ID:-}
SLACK_USER_IDS: ${SLACK_USER_IDS:-}
SLACK_DOT_ID: ${SLACK_DOT_ID:-}
+ AGENT_RUN_TIMEOUT_MS: ${AGENT_RUN_TIMEOUT_MS:-90000}
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
OPENAI_BASE_URL: ${OPENAI_BASE_URL:-https://api.openai.com/v1}
OPENAI_MODEL: ${OPENAI_MODEL:-}
diff --git a/src/server/dot-agent.ts b/src/server/dot-agent.ts
index 9d7f9ad..f5e34ab 100644
--- a/src/server/dot-agent.ts
+++ b/src/server/dot-agent.ts
@@ -23,7 +23,10 @@ import { Observable } from 'rxjs';
import { z } from 'zod';
import { Store } from './store.js';
import { WorkspaceStore } from './workspace.js';
-import type { PlatformConfig } from './platform-config.js';
+import {
+ DEFAULT_AGENT_RUN_TIMEOUT_MS,
+ type PlatformConfig,
+} from './platform-config.js';
import { browserResponse } from './research.js';
import { answerObserver, type SetupTelemetry } from './setup-telemetry.js';
const channelError = () => ({
@@ -31,7 +34,6 @@ const channelError = () => ({
message:
'OpenDots could not complete this request. Please check the app and try again.',
});
-const TURN_TIME_LIMIT_MS = 90_000;
export class DotAgent extends AbstractAgent {
private inner?: BuiltInAgent;
private controller?: AbortController;
@@ -68,6 +70,8 @@ export class DotAgent extends AbstractAgent {
let timedOut = false;
let finished = false;
let configurationFailure = false;
+ const turnTimeLimitMs =
+ this.config.agentRunTimeoutMs ?? DEFAULT_AGENT_RUN_TIMEOUT_MS;
const observe = answerObserver((event) =>
this.setupTelemetry?.capture(event),
);
@@ -75,10 +79,10 @@ export class DotAgent extends AbstractAgent {
timedOut = true;
observe({ type: EventType.RUN_ERROR });
this.abortRun();
- }, TURN_TIME_LIMIT_MS);
+ }, turnTimeLimitMs);
const timeLimitError = () => ({
type: EventType.RUN_ERROR,
- message: `This turn reached the ${TURN_TIME_LIMIT_MS / 1000} second time limit and was stopped. Try a smaller request.`,
+ message: `This turn reached the ${turnTimeLimitMs / 1000} second time limit and was stopped. Try a smaller request.`,
});
const fail = (error: unknown) => {
if (!configurationFailure) observe({ type: EventType.RUN_ERROR });
diff --git a/src/server/index.ts b/src/server/index.ts
index 5fc19c8..f9b4f23 100644
--- a/src/server/index.ts
+++ b/src/server/index.ts
@@ -10,6 +10,7 @@ import { resolveAppOrigins } from './app-origin.js';
import { WorkspaceStore } from './workspace.js';
import { Platform } from './platform.js';
import {
+ agentRunTimeoutMsFromEnv,
intelligenceApiKeyFromEnv,
intelligenceWsUrlFromEnv,
type PlatformConfig,
@@ -37,6 +38,7 @@ const config: PlatformConfig = {
apiKey: process.env.OPENAI_API_KEY,
model: process.env.OPENAI_MODEL,
baseUrl: process.env.OPENAI_BASE_URL ?? 'https://api.openai.com/v1',
+ agentRunTimeoutMs: agentRunTimeoutMsFromEnv(process.env),
webSearchProvider: webSearchProvider(process.env.WEB_SEARCH_PROVIDER),
parallelApiKey: process.env.PARALLEL_API_KEY,
browserUrl: process.env.BROWSER_URL,
@@ -85,6 +87,7 @@ const runner = new Runner(
});
return { text, sources: [], sample: false };
},
+ config.agentRunTimeoutMs,
);
const wsOrigin = new URL(
config.intelligenceWsUrl ?? 'wss://realtime.intelligence.copilotkit.ai',
diff --git a/src/server/platform-config.ts b/src/server/platform-config.ts
index 22ea448..f065582 100644
--- a/src/server/platform-config.ts
+++ b/src/server/platform-config.ts
@@ -7,6 +7,8 @@ const INTELLIGENCE_API_KEY_ENV_NAMES = [
'INTELLIGENCE_API_KEY',
] as const;
export const INTELLIGENCE_KEY_MISSING_LABEL = `${INTELLIGENCE_API_KEY_ENV_NAMES[1]} (or ${INTELLIGENCE_API_KEY_ENV_NAMES[0]})`;
+export const DEFAULT_AGENT_RUN_TIMEOUT_MS = 90_000;
+const MAX_TIMER_DELAY_MS = 2_147_483_647;
export function intelligenceApiKeyFromEnv(
env: Record<string, string | undefined>,
@@ -23,6 +25,19 @@ export function intelligenceWsUrlFromEnv(
]);
}
+export function agentRunTimeoutMsFromEnv(
+ env: Record<string, string | undefined>,
+): number {
+ const raw = env.AGENT_RUN_TIMEOUT_MS?.trim();
+ if (!raw) return DEFAULT_AGENT_RUN_TIMEOUT_MS;
+ const value = Number(raw);
+ if (!Number.isInteger(value) || value < 1 || value > MAX_TIMER_DELAY_MS)
+ throw new Error(
+ `AGENT_RUN_TIMEOUT_MS must be an integer between 1 and ${MAX_TIMER_DELAY_MS}.`,
+ );
+ return value;
+}
+
function firstNonEmptyEnvValue(
env: Record<string, string | undefined>,
names: readonly string[],
@@ -41,6 +56,7 @@ export interface PlatformConfig extends WebConfig {
model?: string;
apiKey?: string;
baseUrl: string;
+ agentRunTimeoutMs?: number;
computerSupervisorUrl?: string;
computerSupervisorToken?: string;
computerToken?: string;
diff --git a/src/server/runner.ts b/src/server/runner.ts
index e5a047e..c9de2d6 100644
--- a/src/server/runner.ts
+++ b/src/server/runner.ts
@@ -1,7 +1,8 @@
-import { Store } from './store.js';
+import { DEFAULT_WORKER_LEASE_MS, Store } from './store.js';
import type { Result, Memory } from '../shared/types.js';
import type { Claim } from './store.js';
import { research, type Config } from './research.js';
+import { DEFAULT_AGENT_RUN_TIMEOUT_MS } from './platform-config.js';
export class Runner {
private timer?: ReturnType<typeof setInterval>;
private active = new Map<string, AbortController>();
@@ -14,6 +15,7 @@ export class Runner {
signal: AbortSignal,
progress: (text: string) => void,
) => Promise<Result>,
+ private runTimeoutMs = DEFAULT_AGENT_RUN_TIMEOUT_MS,
) {}
start() {
if (!this.timer) {
@@ -53,7 +55,11 @@ export class Runner {
}
private async runTick() {
if (this.active.size) return;
- const claim = this.store.claim();
+ // Leave time to persist the outcome after the configured run timeout.
+ const claim = this.store.claim(
+ Date.now(),
+ Math.max(DEFAULT_WORKER_LEASE_MS, this.runTimeoutMs * 2),
+ );
if (!claim) return;
const controller = new AbortController();
this.active.set(claim.id, controller);
@@ -68,9 +74,11 @@ export class Runner {
const timeout = setTimeout(
() =>
controller.abort(
- new Error('Research exceeded the 90 second time limit.'),
+ new Error(
+ `Research exceeded the ${this.runTimeoutMs / 1000} second time limit.`,
+ ),
),
- 90_000,
+ this.runTimeoutMs,
);
try {
const settings = this.store.settings();
diff --git a/src/server/store.ts b/src/server/store.ts
index 36af42e..05d7d86 100644
--- a/src/server/store.ts
+++ b/src/server/store.ts
@@ -14,6 +14,7 @@ import type {
} from '../shared/types.js';
export type Claim = Task & { lease: string };
+export const DEFAULT_WORKER_LEASE_MS = 180_000;
const defaults: Settings = {
name: 'Dot',
paused: false,
@@ -206,7 +207,10 @@ export class Store {
);
return this.task(id);
}
- claim(now = Date.now()): Claim | null {
+ claim(
+ now = Date.now(),
+ leaseDurationMs = DEFAULT_WORKER_LEASE_MS,
+ ): Claim | null {
return this.transaction(() => {
const settings = this.settings();
if (settings.paused || !settings.researchAllowed) return null;
@@ -230,7 +234,7 @@ export class Store {
.prepare(
"UPDATE tasks SET status='running', lease=?, leaseUntil=?, nextRunAt=NULL, error=NULL, updatedAt=? WHERE id=?",
)
- .run(lease, now + 180_000, now, task.id);
+ .run(lease, now + leaseDurationMs, now, task.id);
this.db
.prepare(
"INSERT INTO runs VALUES (?, ?, 'running', ?, NULL, NULL, NULL)",
diff --git a/tests/agent-run-timeout.test.ts b/tests/agent-run-timeout.test.ts
new file mode 100644
index 0000000..6074393
--- /dev/null
+++ b/tests/agent-run-timeout.test.ts
@@ -0,0 +1,223 @@
+import { afterEach, expect, it, vi } from 'vitest';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { EventType, type RunAgentInput } from '@ag-ui/core';
+import { lastValueFrom, toArray } from 'rxjs';
+import { DotAgent } from '../src/server/dot-agent.js';
+import {
+ DEFAULT_AGENT_RUN_TIMEOUT_MS,
+ agentRunTimeoutMsFromEnv,
+} from '../src/server/platform-config.js';
+import { Runner } from '../src/server/runner.js';
+import { Store } from '../src/server/store.js';
+import { WorkspaceStore } from '../src/server/workspace.js';
+
+const resources: Array<{ close(): void }> = [];
+function sharedStores() {
+ const dir = mkdtempSync(join(tmpdir(), 'opendots-timeout-'));
+ const path = join(dir, 'test.sqlite');
+ const first = new Store(path);
+ const second = new Store(path);
+ resources.push(first, second, {
+ close: () => rmSync(dir, { recursive: true, force: true }),
+ });
+ return { first, second };
+}
+afterEach(() => {
+ vi.restoreAllMocks();
+ vi.useRealTimers();
+ resources.splice(0).forEach((resource) => resource.close());
+});
+
+it('uses 90 seconds by default and accepts a positive integer override', () => {
+ expect(agentRunTimeoutMsFromEnv({})).toBe(DEFAULT_AGENT_RUN_TIMEOUT_MS);
+ expect(agentRunTimeoutMsFromEnv({ AGENT_RUN_TIMEOUT_MS: ' 240000 ' })).toBe(
+ 240_000,
+ );
+});
+
+it('rejects invalid timeout overrides at startup', () => {
+ for (const value of ['0', '-1', '1.5', 'not-a-number', '2147483648'])
+ expect(() =>
+ agentRunTimeoutMsFromEnv({ AGENT_RUN_TIMEOUT_MS: value }),
+ ).toThrow(/AGENT_RUN_TIMEOUT_MS/);
+});
+
+it('uses the configured timeout for interactive Dot turns', async () => {
+ vi.useFakeTimers();
+ const store = new Store(':memory:');
+ const workspace = new WorkspaceStore(':memory:', 'owner');
+ resources.push(store, workspace);
+ const dot = workspace.dots()[0];
+ workspace.bindThread('thread', dot.id, 'Timeout test');
+ const agent = new DotAgent(
+ store,
+ workspace,
+ {
+ intelligenceKey: 'fixture',
+ apiKey: 'fixture',
+ model: 'custom-model',
+ baseUrl: 'https://unused.invalid/v1',
+ runtimeUrl: '',
+ voiceName: 'marin',
+ slackUsers: [],
+ agentRunTimeoutMs: 1_250,
+ },
+ dot.id,
+ );
+ const input: RunAgentInput = {
+ threadId: 'thread',
+ runId: 'run',
+ state: {},
+ context: [],
+ messages: [{ id: 'user', role: 'user', content: 'Keep working.' }],
+ tools: [],
+ forwardedProps: {},
+ };
+ vi.spyOn(globalThis, 'fetch').mockImplementation(
+ (_url, init) =>
+ new Promise((_resolve, reject) => {
+ init?.signal?.addEventListener(
+ 'abort',
+ () => reject(init.signal?.reason),
+ { once: true },
+ );
+ }),
+ );
+ const finished = lastValueFrom(agent.run(input).pipe(toArray()));
+ await vi.advanceTimersByTimeAsync(1_251);
+ const events = await finished;
+ expect(events).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ type: EventType.RUN_ERROR,
+ message: expect.stringContaining('1.25 second time limit'),
+ }),
+ ]),
+ );
+});
+
+it('uses the configured timeout for scheduled/background runs', async () => {
+ vi.useFakeTimers();
+ const store = new Store(':memory:');
+ resources.push(store);
+ let signal: AbortSignal | undefined;
+ const runner = new Runner(
+ store,
+ { mode: 'sample', baseUrl: '' },
+ (_claim, _memories, runSignal) =>
+ new Promise((_resolve, reject) => {
+ signal = runSignal;
+ runSignal.addEventListener('abort', () => reject(runSignal.reason), {
+ once: true,
+ });
+ }),
+ 1_250,
+ );
+ store.createTask('Long-running task');
+ const pending = runner.tick();
+ await vi.advanceTimersByTimeAsync(1_251);
+ await pending;
+ expect(signal?.aborted).toBe(true);
+ expect(store.tasks()[0].status).toBe('failed');
+});
+
+it('completes a longer background run after another worker checks the old lease deadline', async () => {
+ vi.useFakeTimers();
+ const { first, second } = sharedStores();
+ const result = {
+ text: 'Finished after 200 seconds',
+ sources: [],
+ sample: true,
+ };
+ let signal: AbortSignal | undefined;
+ const runner = new Runner(
+ first,
+ { mode: 'sample', baseUrl: '' },
+ (_claim, _memories, runSignal) =>
+ new Promise((resolve, reject) => {
+ signal = runSignal;
+ const timer = setTimeout(() => resolve(result), 200_000);
+ runSignal.addEventListener(
+ 'abort',
+ () => {
+ clearTimeout(timer);
+ reject(runSignal.reason);
+ },
+ { once: true },
+ );
+ }),
+ 240_000,
+ );
+ const competingExecute = vi.fn().mockResolvedValue(result);
+ const competing = new Runner(
+ second,
+ { mode: 'sample', baseUrl: '' },
+ competingExecute,
+ );
+ const task = first.createTask('Long-running task');
+ const pending = runner.tick();
+ const lease = first.task(task.id)?.lease;
+ try {
+ await vi.advanceTimersByTimeAsync(180_001);
+ await competing.tick();
+ expect(second.task(task.id)?.status).toBe('running');
+ expect(second.task(task.id)?.lease).toBe(lease);
+ expect(signal?.aborted).toBe(false);
+ expect(competingExecute).not.toHaveBeenCalled();
+
+ await vi.advanceTimersByTimeAsync(20_000);
+ await pending;
+ expect(second.task(task.id)?.status).toBe('completed');
+ expect(second.detail(task.id)?.runs).toEqual([
+ expect.objectContaining({ status: 'completed', result }),
+ ]);
+ } finally {
+ runner.stop();
+ competing.stop();
+ await pending;
+ }
+});
+
+it('still times out a longer background run after another worker checks its lease', async () => {
+ vi.useFakeTimers();
+ const { first, second } = sharedStores();
+ let signal: AbortSignal | undefined;
+ const runner = new Runner(
+ first,
+ { mode: 'sample', baseUrl: '' },
+ (_claim, _memories, runSignal) =>
+ new Promise((_resolve, reject) => {
+ signal = runSignal;
+ runSignal.addEventListener('abort', () => reject(runSignal.reason), {
+ once: true,
+ });
+ }),
+ 240_000,
+ );
+ const competing = new Runner(second, { mode: 'sample', baseUrl: '' });
+ const task = first.createTask('Long-running task');
+ const pending = runner.tick();
+ try {
+ await vi.advanceTimersByTimeAsync(180_001);
+ await competing.tick();
+ expect(second.task(task.id)?.status).toBe('running');
+ expect(signal?.aborted).toBe(false);
+
+ await vi.advanceTimersByTimeAsync(60_000);
+ await pending;
+ expect(signal?.aborted).toBe(true);
+ expect(second.task(task.id)?.status).toBe('failed');
+ expect(second.detail(task.id)?.runs).toEqual([
+ expect.objectContaining({
+ status: 'failed',
+ error: 'Research exceeded the 240 second time limit.',
+ }),
+ ]);
+ } finally {
+ runner.stop();
+ competing.stop();
+ await pending;
+ }
+});
diff --git a/tests/store.test.ts b/tests/store.test.ts
index 54e4335..edb4fa0 100644
--- a/tests/store.test.ts
+++ b/tests/store.test.ts
@@ -78,34 +78,39 @@ describe('durable task lifecycle', () => {
store.action(task.id, 'run');
expect(store.claim(Date.now())?.id).toBe(task.id);
});
- it('holds expired work until the owner retries it', () => {
- const { store, path } = fixture();
- store.createTask('Recover me');
- const now = Date.now();
- const old = store.claim(now)!;
- const restarted = new Store(path);
- try {
- expect(restarted.claim(now + 180_001)).toBeNull();
- expect(restarted.task(old.id)?.status).toBe('interrupted');
- expect(restarted.detail(old.id)?.runs[0].status).toBe('interrupted');
- restarted.action(old.id, 'run');
- const recovered = restarted.claim(now + 180_001)!;
- expect(recovered.id).toBe(old.id);
- expect(recovered.lease).not.toBe(old.lease);
- expect(
- store.finish(old, { text: 'Old', sources: [], sample: true }),
- ).toBe(false);
- expect(
- restarted.finish(recovered, {
- text: 'New',
- sources: [],
- sample: true,
- }),
- ).toBe(true);
- } finally {
- restarted.close();
- }
- });
+ it.each([180_000, 480_000])(
+ 'holds expired work until the owner retries it with a %i ms lease',
+ (leaseDurationMs) => {
+ const { store, path } = fixture();
+ store.createTask('Recover me');
+ const now = Date.now();
+ const old = store.claim(now, leaseDurationMs)!;
+ const restarted = new Store(path);
+ try {
+ expect(restarted.claim(now + leaseDurationMs - 1)).toBeNull();
+ expect(restarted.task(old.id)?.status).toBe('running');
+ expect(restarted.claim(now + leaseDurationMs + 1)).toBeNull();
+ expect(restarted.task(old.id)?.status).toBe('interrupted');
+ expect(restarted.detail(old.id)?.runs[0].status).toBe('interrupted');
+ restarted.action(old.id, 'run');
+ const recovered = restarted.claim(now + leaseDurationMs + 1)!;
+ expect(recovered.id).toBe(old.id);
+ expect(recovered.lease).not.toBe(old.lease);
+ expect(
+ store.finish(old, { text: 'Old', sources: [], sample: true }),
+ ).toBe(false);
+ expect(
+ restarted.finish(recovered, {
+ text: 'New',
+ sources: [],
+ sample: true,
+ }),
+ ).toBe(true);
+ } finally {
+ restarted.close();
+ }
+ },
+ );
it('claims other queued work while an expired run waits for review', () => {
const { store } = fixture();
const interrupted = store.createTask('Create the first page');