Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: skip refused tool calls in quality checklist

ogsamrate025f46e27cd
diff --git a/apps/app/src/app/lib/good-session.ts b/apps/app/src/app/lib/good-session.ts
index 8359c555..100c8916 100644
--- a/apps/app/src/app/lib/good-session.ts
+++ b/apps/app/src/app/lib/good-session.ts
@@ -292,7 +292,10 @@ const EDIT_TOOLS = new Set([
const SHELL_TOOLS = new Set(["bash", "shell", "run", "exec", "terminal", "execute", "bash_output"]);
const PATCH_FILE = /^\*\*\* (?:Add|Update|Delete) File: (.+)$|^\*\*\* Move to: (.+)$/gm;
-type ToolCall = { tool: string; input: Record<string, unknown>; state: string; failed: boolean };
+type ToolCall = { tool: string; input: Record<string, unknown>; state: string; failed: boolean; notRun: boolean };
+
+/** A call the engine or the user refused, so nothing ran (bad arguments, an unknown tool, a denied permission). */
+const NOT_RUN = /invalid arguments|no tool named|unknown tool|tool .{0,40}not (found|available)|permission .{0,20}(denied|rejected)|user (rejected|denied|dismissed)/i;
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -310,11 +313,13 @@ function toolCalls(message: UIMessage): ToolCall[] {
if (part.type === "dynamic-tool") tool = str(record.toolName);
else if (part.type.startsWith("tool-")) tool = part.type.slice(5);
else continue;
+ const failed = record.state === "output-error";
calls.push({
tool: tool.toLowerCase().replace(/^.*[.:]/, ""),
input: isRecord(record.input) ? record.input : {},
state: str(record.state),
- failed: record.state === "output-error",
+ failed,
+ notRun: failed && NOT_RUN.test(str(record.errorText)),
});
}
return calls;
@@ -374,6 +379,7 @@ export function goodSessionChecklist(input: GoodSessionInput): GoodSessionCheckl
const dbBuilt = calls.some((call) => SHELL_TOOLS.has(call.tool) && DB_SETUP.test(shellCommand(call.input)));
for (const call of calls) {
+ if (call.notRun) continue;
if (EDIT_TOOLS.has(call.tool)) {
for (const path of editedPaths(call.input)) {
if (outsideProject(path, input.workspaceRoot)) {
diff --git a/apps/app/src/react-app/domains/quality/good-session.tsx b/apps/app/src/react-app/domains/quality/good-session.tsx
index f7a5e997..8132596f 100644
--- a/apps/app/src/react-app/domains/quality/good-session.tsx
+++ b/apps/app/src/react-app/domains/quality/good-session.tsx
@@ -125,14 +125,14 @@ export function GoodSessionChecklistBar(props: GoodSessionChecklistBarProps) {
useEffect(() => {
const previous = wasRunning.current;
wasRunning.current = { sessionId: props.sessionId, running: props.turnRunning };
- if (previous.sessionId !== props.sessionId) return;
+ if (previous.sessionId !== props.sessionId || !hasPrompt) return;
const alreadyNudged = nudgedSessions().includes(props.sessionId);
if (!shouldNudge({ wasRunning: previous.running, running: props.turnRunning, checklist, alreadyNudged })) return;
const nudge = goodSessionNudge(checklist);
if (!nudge) return;
rememberNudged(props.sessionId);
toast(nudge.title, { id: `good-session-nudge:${props.sessionId}`, description: nudge.body, duration: 12_000 });
- }, [checklist, props.sessionId, props.turnRunning]);
+ }, [checklist, hasPrompt, props.sessionId, props.turnRunning]);
if (!hasPrompt) return null;
return (
diff --git a/apps/app/tests/good-session.test.tsx b/apps/app/tests/good-session.test.tsx
index 8f37f914..1c91d06c 100644
--- a/apps/app/tests/good-session.test.tsx
+++ b/apps/app/tests/good-session.test.tsx
@@ -144,6 +144,25 @@ describe("the live checklist", () => {
expect(states(transcript([tool("write", { filePath: "README.md" })])).by.code).toBe("fail");
});
+ test("calls the engine refused never ran: bad arguments, unknown tool, denied permission", () => {
+ const refused = (toolName: string, input: Record<string, unknown>, errorText: string) =>
+ ({ type: "dynamic-tool", toolName, toolCallId: `r${toolName}`, state: "output-error", input, errorText });
+ const messages = transcript([
+ refused("write", { content: "x" }, 'Invalid arguments for tool "write": - path: Missing key'),
+ refused("bash", { command: "npm test" }, 'No tool named "bash" is currently available.'),
+ refused("bash", { command: "cd /etc && make" }, "The user rejected permission to use this specific tool call."),
+ ]);
+ const { by } = states(messages);
+ expect(by.code).toBe("fail");
+ expect(by.ran).toBe("fail");
+ expect(by.project).toBe("pass");
+ });
+
+ test("the v2 engine's shell tool and path key", () => {
+ const { result } = states(transcript([tool("write", { path: "src/a.ts", content: "x" }), tool("shell", { command: "npm test" })]));
+ expect(result.good).toBe(true);
+ });
+
test("a failing test run still counts as running something", () => {
expect(states(transcript([tool("bash", { command: "pytest" }, "output-error")])).by.ran).toBe("pass");
});