Skip to content

armgit / Orkas

Public Imported from GitHub
0

Orkas is an open-source (MIT), local-first AI team for your desktop: a Commander turns your goal into to-dos and coordinates 16 built-in specialist agents plus Claude Code, Codex, OpenCode, OpenClaw and Hermes, with projects, scheduled automations and connectors. Free app for macOS & Windows; built-in models or your own keys.

Avoid wedged tool calls

wytb205766220d7
diff --git a/src/core-agent/src/agent/runner.ts b/src/core-agent/src/agent/runner.ts
index 8e5f4829..125976b5 100644
--- a/src/core-agent/src/agent/runner.ts
+++ b/src/core-agent/src/agent/runner.ts
@@ -13,7 +13,7 @@ import type { CoreAgentConfig } from "../config/schema.js";
import type { EvolutionConfig } from "../evolution/types.js";
import type { LLMProvider, CompletionResult } from "../providers/base.js";
import { ProviderRegistry } from "../providers/registry.js";
-import type { AgentTool, ToolContext, ToolResult } from "../tools/base.js";
+import type { AgentTool, ToolContext, ToolProgress, ToolResult } from "../tools/base.js";
import { toToolDefinition } from "../tools/base.js";
import { getBuiltinTools } from "../tools/builtin.js";
import { SkillStore } from "../evolution/skill-store.js";
@@ -26,6 +26,7 @@ const RETRY_BASE_DELAY_MS = 1_000;
const RETRY_MAX_DELAY_MS = 30_000;
const RETRY_AFTER_MAX_DELAY_MS = 120_000;
const RETRY_JITTER_RATIO = 0.2;
+const TOOL_HEARTBEAT_TIMEOUT_GRACE_MS = 30_000;
function retryDelayMs(err: unknown, attempt: number): number {
if (err instanceof RateLimitError && err.retryAfterMs != null) {
@@ -399,6 +400,9 @@ export class AgentRunner {
return true;
};
const abortedToolMessage = "Tool execution aborted: Run aborted";
+ const toolIdleTimeoutMs = this.config.agent.toolIdleTimeoutMs;
+ const stalledToolMessage =
+ `Tool execution stalled after ${toolIdleTimeoutMs}ms without substantive progress`;
try {
if (params.signal?.aborted) {
addToolResultOnce(abortedToolMessage, undefined, true);
@@ -415,14 +419,19 @@ export class AgentRunner {
const progressQueue: Array<Extract<AgentRunEvent, { type: "tool_progress" }>> = [];
let notifyProgress: (() => void) | null = null;
let acceptingProgress = true;
+ let toolStalled = false;
+ const toolAbort = createChildAbortController(params.signal);
+ const toolIdle = createToolIdleWatchdog(toolIdleTimeoutMs);
const toolCtx: ToolContext = {
workingDir: params.workingDir,
- signal: params.signal,
+ signal: toolAbort.signal,
state: toolState,
emitProgress: (progress) => {
if (!acceptingProgress) return;
const message = String(progress?.message || "").trim();
if (!message) return;
+ const idleDelayMs = toolIdleDelayForProgress(progress, toolIdleTimeoutMs);
+ if (idleDelayMs != null) toolIdle.reset(idleDelayMs);
progressQueue.push({
type: "tool_progress",
id: call.id,
@@ -457,11 +466,22 @@ export class AgentRunner {
const progressWait = new Promise<"progress">((resolve) => {
notifyProgress = () => resolve("progress");
});
- const waits: Array<Promise<ToolCompletion | "progress" | "abort">> = [toolPromise, progressWait];
+ const waits: Array<Promise<ToolCompletion | "progress" | "abort" | "tool_idle">> = [
+ toolPromise,
+ progressWait,
+ toolIdle.promise,
+ ];
if (abortWait.promise) waits.push(abortWait.promise);
const raced = await Promise.race(waits);
if (raced === "progress") continue;
+ if (raced === "tool_idle") {
+ toolStalled = true;
+ toolAbort.abort();
+ break;
+ }
if (raced === "abort") {
+ acceptingProgress = false;
+ progressQueue.length = 0;
addToolResultOnce(abortedToolMessage, undefined, true);
yield {
type: "tool_end",
@@ -477,12 +497,27 @@ export class AgentRunner {
}
} finally {
abortWait.cleanup();
+ toolIdle.cancel();
+ toolAbort.cleanup();
notifyProgress = null;
acceptingProgress = false;
}
while (progressQueue.length) {
yield progressQueue.shift()!;
}
+ if (toolStalled) {
+ addToolResultOnce(stalledToolMessage, undefined, true);
+ yield {
+ type: "tool_end",
+ id: call.id,
+ name: call.name,
+ result: stalledToolMessage,
+ isError: true,
+ };
+ permanentToolErrors++;
+ log.warn(`Tool ${call.name} stalled: ${stalledToolMessage}`);
+ continue;
+ }
if (completed.ok !== true) throw completed.err;
const toolResult = completed.result;
if (addToolResultOnce(toolResult.content, toolResult.images, toolResult.isError)) {
@@ -847,6 +882,76 @@ function waitForAbort(signal: AbortSignal | undefined): {
return { promise, cleanup };
}
+function createChildAbortController(parent: AbortSignal | undefined): {
+ signal: AbortSignal;
+ abort: () => void;
+ cleanup: () => void;
+} {
+ const controller = new AbortController();
+ let cleanup = () => undefined;
+ if (parent) {
+ const onAbort = () => controller.abort();
+ if (parent.aborted) {
+ onAbort();
+ } else {
+ parent.addEventListener("abort", onAbort, { once: true });
+ cleanup = () => parent.removeEventListener("abort", onAbort);
+ }
+ }
+ return {
+ signal: controller.signal,
+ abort: () => controller.abort(),
+ cleanup,
+ };
+}
+
+function createToolIdleWatchdog(timeoutMs: number): {
+ promise: Promise<"tool_idle">;
+ reset: (nextTimeoutMs?: number) => void;
+ cancel: () => void;
+} {
+ let timer: NodeJS.Timeout | null = null;
+ let settled = false;
+ let resolveIdle!: (value: "tool_idle") => void;
+ const promise = new Promise<"tool_idle">((resolve) => {
+ resolveIdle = resolve;
+ });
+ const reset = (nextTimeoutMs = timeoutMs) => {
+ if (settled) return;
+ if (timer) clearTimeout(timer);
+ timer = setTimeout(() => {
+ settled = true;
+ resolveIdle("tool_idle");
+ }, nextTimeoutMs);
+ if (typeof timer.unref === "function") timer.unref();
+ };
+ const cancel = () => {
+ settled = true;
+ if (timer) clearTimeout(timer);
+ timer = null;
+ };
+ reset();
+ return { promise, reset, cancel };
+}
+
+function isHeartbeatProgress(progress: ToolProgress): boolean {
+ return progress.data?.heartbeat === true;
+}
+
+function toolIdleDelayForProgress(progress: ToolProgress, defaultTimeoutMs: number): number | null {
+ if (!isHeartbeatProgress(progress)) return defaultTimeoutMs;
+ const declaredTimeoutMs = finiteProgressNumber(progress.data?.timeoutMs);
+ const elapsedMs = finiteProgressNumber(progress.data?.elapsedMs);
+ if (declaredTimeoutMs == null || elapsedMs == null) return null;
+ const remainingMs = declaredTimeoutMs - elapsedMs;
+ if (remainingMs <= 0) return null;
+ return Math.max(defaultTimeoutMs, remainingMs + TOOL_HEARTBEAT_TIMEOUT_GRACE_MS);
+}
+
+function finiteProgressNumber(value: unknown): number | null {
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
+}
+
/**
* Build the skills guidance block injected into the system prompt.
* Tells the agent about skill_manage and metacognition tools.
diff --git a/src/core-agent/src/config/schema.ts b/src/core-agent/src/config/schema.ts
index acdae905..d49d265d 100644
--- a/src/core-agent/src/config/schema.ts
+++ b/src/core-agent/src/config/schema.ts
@@ -65,6 +65,8 @@ export const AgentConfigSchema = z.object({
maxRetries: z.number().int().min(0).default(3),
/** Maximum number of tool-use loop iterations per run. */
maxToolLoops: z.number().int().positive().default(50),
+ /** Max time a tool may run without completing or reporting substantive progress. */
+ toolIdleTimeoutMs: z.number().int().positive().default(600_000),
/** System prompt override or additions. */
systemPrompt: z.string().optional(),
/** Thinking/reasoning level: off, low, high. */
diff --git a/src/core-agent/src/sandbox/executor.ts b/src/core-agent/src/sandbox/executor.ts
index 8ed11915..1f56920f 100644
--- a/src/core-agent/src/sandbox/executor.ts
+++ b/src/core-agent/src/sandbox/executor.ts
@@ -28,6 +28,8 @@ export interface SandboxConfig {
allowNetwork?: boolean;
/** Environment variables to pass through. */
env?: Record<string, string>;
+ /** Abort signal used by the runner to stop a wedged tool without aborting the whole turn. */
+ signal?: AbortSignal;
/** Shell to use (default: /bin/sh on POSIX, PowerShell on Windows). */
shell?: string;
}
@@ -444,6 +446,7 @@ export class SandboxExecutor {
let stderrTruncated = false;
let settled = false;
let forceSettleTimer: NodeJS.Timeout | null = null;
+ let abortListener: (() => void) | null = null;
const env = buildSandboxEnv(this.config.env);
@@ -463,6 +466,13 @@ export class SandboxExecutor {
windowsHide: true,
});
+ const abortSignal = this.config.signal;
+ if (abortSignal) {
+ abortListener = () => killChild();
+ if (abortSignal.aborted) abortListener();
+ else abortSignal.addEventListener("abort", abortListener, { once: true });
+ }
+
// Close stdin immediately — no interactive input
child.stdin.end();
@@ -507,6 +517,7 @@ export class SandboxExecutor {
settled = true;
clearTimeout(timeoutId);
if (forceSettleTimer) clearTimeout(forceSettleTimer);
+ if (abortSignal && abortListener) abortSignal.removeEventListener("abort", abortListener);
let stdout = decodeProcessOutput(Buffer.concat(stdoutChunks), process.platform, env);
let stderr = decodeProcessOutput(Buffer.concat(stderrChunks), process.platform, env);
if (stdoutTruncated) stdout += "\n... [output truncated by sandbox]";
@@ -540,6 +551,7 @@ export class SandboxExecutor {
if (settled) return;
clearTimeout(timeoutId);
if (forceSettleTimer) clearTimeout(forceSettleTimer);
+ if (abortSignal && abortListener) abortSignal.removeEventListener("abort", abortListener);
settled = true;
const stdout = decodeProcessOutput(Buffer.concat(stdoutChunks), process.platform, env);
resolve({
diff --git a/src/core-agent/src/tools/base.ts b/src/core-agent/src/tools/base.ts
index e459d47e..6c295d62 100644
--- a/src/core-agent/src/tools/base.ts
+++ b/src/core-agent/src/tools/base.ts
@@ -15,7 +15,9 @@ export type ToolProgress = {
phase?: string;
/** Human-readable status text for the process rail. */
message: string;
- /** Optional structured metadata for renderers / logs. */
+ /** Optional structured metadata for renderers / logs.
+ * Set `heartbeat: true` for "still running" keepalive updates that should
+ * not reset the runner's tool-idle watchdog. */
data?: Record<string, unknown>;
};
diff --git a/src/core-agent/src/tools/builtin.ts b/src/core-agent/src/tools/builtin.ts
index 2f86e163..f6d1f4b0 100644
--- a/src/core-agent/src/tools/builtin.ts
+++ b/src/core-agent/src/tools/builtin.ts
@@ -103,6 +103,7 @@ export const bashTool: AgentTool = defineTool({
workingDir: ctx.workingDir ?? ".",
timeoutMs,
env: ctx.state.sandboxEnv as Record<string, string> | undefined,
+ signal: ctx.signal,
});
if (input.run_in_background === true) {
@@ -151,7 +152,7 @@ function startBashHeartbeat(ctx: ToolContext, timeoutMs: number): () => void {
ctx.emitProgress?.({
phase: "running",
message: `Command still running (${formatDuration(elapsedMs)} elapsed; timeout ${formatDuration(timeoutMs)})`,
- data: { elapsedMs, timeoutMs },
+ data: { elapsedMs, timeoutMs, heartbeat: true },
});
}, BASH_PROGRESS_INTERVAL_MS);
if (typeof timer.unref === "function") timer.unref();
diff --git a/src/core-agent/test/agent-runner.test.ts b/src/core-agent/test/agent-runner.test.ts
index 8938d83f..542fec61 100644
--- a/src/core-agent/test/agent-runner.test.ts
+++ b/src/core-agent/test/agent-runner.test.ts
@@ -501,6 +501,147 @@ describe("AgentRunner", () => {
});
});
+ it("runStream converts a heartbeat-only wedged tool into an error result and continues", async () => {
+ const mockProvider = createMockProvider([
+ {
+ content: [
+ { type: "tool_use", id: "call_1", name: "wedged_tool", input: {} },
+ ],
+ stopReason: "tool_use",
+ usage: { inputTokens: 5, outputTokens: 3, totalTokens: 8 },
+ model: "mock-model",
+ },
+ {
+ content: [{ type: "text", text: "continued after tool stall" }],
+ stopReason: "end_turn",
+ usage: { inputTokens: 7, outputTokens: 5, totalTokens: 12 },
+ model: "mock-model",
+ },
+ ]);
+
+ const registry = new ProviderRegistry();
+ registry.registerFactory("mock", () => mockProvider);
+ const wedgedTool = defineTool({
+ name: "wedged_tool",
+ description: "Only emits keepalive progress and never resolves",
+ inputSchema: { type: "object", properties: {} },
+ async execute(_input, ctx) {
+ const timer = setInterval(() => {
+ ctx.emitProgress?.({
+ phase: "running",
+ message: "still running",
+ data: { heartbeat: true },
+ });
+ }, 5);
+ ctx.signal?.addEventListener("abort", () => clearInterval(timer), { once: true });
+ return new Promise(() => undefined);
+ },
+ });
+ const config = createConfig({
+ agent: {
+ defaultProvider: "mock",
+ defaultModel: "mock-model",
+ toolIdleTimeoutMs: 30,
+ },
+ });
+
+ const runner = new AgentRunner({ config, providers: registry, tools: [wedgedTool] });
+ const collected: Array<{ type: string; [k: string]: unknown }> = [];
+ for await (const ev of runner.runStream({ message: "go" })) {
+ collected.push(ev as { type: string; [k: string]: unknown });
+ }
+
+ const toolEnd = collected.find((e) => e.type === "tool_end");
+ expect(toolEnd).toMatchObject({
+ type: "tool_end",
+ id: "call_1",
+ name: "wedged_tool",
+ isError: true,
+ result: "Tool execution stalled after 30ms without substantive progress",
+ });
+ const done = collected[collected.length - 1] as { type: string; result?: { text?: string; meta?: { aborted?: boolean; permanentToolErrors?: number } } };
+ expect(done.type).toBe("done");
+ expect(done.result?.text).toBe("continued after tool stall");
+ expect(done.result?.meta?.aborted).toBeUndefined();
+ expect(done.result?.meta?.permanentToolErrors).toBe(1);
+ const toolResults = runner.getSession().getMessages().flatMap((msg) =>
+ msg.content.filter((content) => content.type === "tool_result"),
+ );
+ expect(toolResults).toHaveLength(1);
+ expect(toolResults[0]).toMatchObject({
+ type: "tool_result",
+ toolUseId: "call_1",
+ content: "Tool execution stalled after 30ms without substantive progress",
+ isError: true,
+ });
+ });
+
+ it("runStream lets heartbeat-only tools finish when they advertise their own timeout", async () => {
+ const mockProvider = createMockProvider([
+ {
+ content: [
+ { type: "tool_use", id: "call_1", name: "bounded_tool", input: {} },
+ ],
+ stopReason: "tool_use",
+ usage: { inputTokens: 5, outputTokens: 3, totalTokens: 8 },
+ model: "mock-model",
+ },
+ {
+ content: [{ type: "text", text: "bounded tool completed" }],
+ stopReason: "end_turn",
+ usage: { inputTokens: 7, outputTokens: 5, totalTokens: 12 },
+ model: "mock-model",
+ },
+ ]);
+
+ const registry = new ProviderRegistry();
+ registry.registerFactory("mock", () => mockProvider);
+ const boundedTool = defineTool({
+ name: "bounded_tool",
+ description: "Emits keepalive progress with a declared timeout before resolving",
+ inputSchema: { type: "object", properties: {} },
+ async execute(_input, ctx) {
+ const startedAt = Date.now();
+ const timer = setInterval(() => {
+ ctx.emitProgress?.({
+ phase: "running",
+ message: "still running",
+ data: { heartbeat: true, elapsedMs: Date.now() - startedAt, timeoutMs: 500 },
+ });
+ }, 5);
+ await new Promise((resolve) => setTimeout(resolve, 80));
+ clearInterval(timer);
+ return { content: "ok" };
+ },
+ });
+ const config = createConfig({
+ agent: {
+ defaultProvider: "mock",
+ defaultModel: "mock-model",
+ toolIdleTimeoutMs: 30,
+ },
+ });
+
+ const runner = new AgentRunner({ config, providers: registry, tools: [boundedTool] });
+ const collected: Array<{ type: string; [k: string]: unknown }> = [];
+ for await (const ev of runner.runStream({ message: "go" })) {
+ collected.push(ev as { type: string; [k: string]: unknown });
+ }
+
+ const toolEnd = collected.find((e) => e.type === "tool_end");
+ expect(toolEnd).toMatchObject({
+ type: "tool_end",
+ id: "call_1",
+ name: "bounded_tool",
+ isError: undefined,
+ result: "ok",
+ });
+ const done = collected[collected.length - 1] as { type: string; result?: { text?: string; meta?: { permanentToolErrors?: number } } };
+ expect(done.type).toBe("done");
+ expect(done.result?.text).toBe("bounded tool completed");
+ expect(done.result?.meta?.permanentToolErrors).toBeUndefined();
+ });
+
it("runStream forwards tool input deltas before tool execution", async () => {
const mockProvider = createMockProvider([
{
diff --git a/src/main/model/core-agent/local-tools.ts b/src/main/model/core-agent/local-tools.ts
index 774fe323..1efcfc05 100644
--- a/src/main/model/core-agent/local-tools.ts
+++ b/src/main/model/core-agent/local-tools.ts
@@ -455,6 +455,7 @@ async function executeDirectOrkasCli(
const startedAt = Date.now();
let heartbeat: NodeJS.Timeout | null = null;
let settleTimer: NodeJS.Timeout | null = null;
+ let abortListener: (() => void) | null = null;
const child = spawn(invocation.nodePath, [invocation.scriptPath, ...invocation.args], {
cwd: workingDir,
@@ -470,6 +471,7 @@ async function executeDirectOrkasCli(
clearTimeout(timeout);
if (heartbeat) clearInterval(heartbeat);
if (settleTimer) clearTimeout(settleTimer);
+ if (ctx.signal && abortListener) ctx.signal.removeEventListener('abort', abortListener);
resolve(result);
};
@@ -530,13 +532,19 @@ async function executeDirectOrkasCli(
}, timeoutMs);
if (typeof timeout.unref === 'function') timeout.unref();
+ if (ctx.signal) {
+ abortListener = () => killChild();
+ if (ctx.signal.aborted) abortListener();
+ else ctx.signal.addEventListener('abort', abortListener, { once: true });
+ }
+
if (ctx.emitProgress) {
heartbeat = setInterval(() => {
const elapsedMs = Date.now() - startedAt;
ctx.emitProgress?.({
phase: 'running',
message: `Command still running (${formatBashDuration(elapsedMs)} elapsed; timeout ${formatBashDuration(timeoutMs)})`,
- data: { elapsedMs, timeoutMs },
+ data: { elapsedMs, timeoutMs, heartbeat: true },
});
}, BASH_PROGRESS_INTERVAL_MS);
if (typeof heartbeat.unref === 'function') heartbeat.unref();