Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
// @vitest-environment node
import { beforeEach, describe, expect, test, vi } from "vitest";
import { aiServiceClientConnect } from "@/api";
import { AIChatMessageRole } from "@/types/proto-es/v1/ai_service_pb";
import { runAgentLoop } from "./agentLoop";
import type { ToolExecutionResult, ToolExecutor } from "./types";
vi.mock("@/api", () => ({
aiServiceClientConnect: {
chat: vi.fn(),
},
}));
describe("runAgentLoop", () => {
beforeEach(() => {
vi.mocked(aiServiceClientConnect.chat).mockReset();
});
test("treats a non-AbortError response after user cancellation as aborted", async () => {
let rejectChat!: (error: Error) => void;
vi.mocked(aiServiceClientConnect.chat).mockImplementation(
() =>
new Promise((_, reject) => {
rejectChat = reject;
})
);
const controller = new AbortController();
const onUsage = vi.fn();
const run = runAgentLoop(
[{ role: "user", content: "Check the project" }],
[],
vi.fn(),
{ onUsage },
controller.signal
);
await vi.waitFor(() =>
expect(aiServiceClientConnect.chat).toHaveBeenCalled()
);
controller.abort();
rejectChat(new Error("Transport closed"));
expect(await run).toEqual({ kind: "aborted", totalTokensUsed: 0 });
expect(onUsage).not.toHaveBeenCalled();
});
test("does not execute tool calls returned after cancellation", async () => {
let resolveChat!: (
response: Awaited<ReturnType<typeof aiServiceClientConnect.chat>>
) => void;
vi.mocked(aiServiceClientConnect.chat).mockImplementation(
() =>
new Promise((resolve) => {
resolveChat = resolve;
})
);
const controller = new AbortController();
const executeTool = vi.fn();
const onAssistantMessage = vi.fn();
const onUsage = vi.fn();
const run = runAgentLoop(
[{ role: "user", content: "Check the project" }],
[{ name: "test_tool", description: "test", parametersSchema: {} }],
executeTool,
{ onAssistantMessage, onUsage },
controller.signal
);
await vi.waitFor(() =>
expect(aiServiceClientConnect.chat).toHaveBeenCalled()
);
controller.abort();
resolveChat({
content: "",
toolCalls: [
{ id: "tool-1", name: "test_tool", arguments: "{}", metadata: "" },
],
usage: { totalTokens: 7, inputTokens: 5, outputTokens: 2 },
} as never);
expect(await run).toEqual({ kind: "aborted", totalTokensUsed: 7 });
expect(onUsage).toHaveBeenCalledExactlyOnceWith({
purpose: "agent",
toolNames: ["test_tool"],
totalTokens: 7,
inputTokens: 5,
outputTokens: 2,
});
expect(executeTool).not.toHaveBeenCalled();
expect(onAssistantMessage).not.toHaveBeenCalled();
});
test.each([
{
name: "tool result",
result: { kind: "tool_result", result: "late result" },
},
{
name: "completion",
result: { kind: "done", text: "Finished", success: true },
},
{
name: "request for input",
result: {
kind: "ask_user",
ask: { toolCallId: "tool-1", prompt: "Which database?", kind: "input" },
},
},
] satisfies { name: string; result: ToolExecutionResult }[])(
"does not publish a late $name after aborting a tool",
async ({ result }) => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "",
toolCalls: [
{
id: "tool-1",
name: "test_tool",
arguments: "{}",
metadata: "",
},
],
} as never);
let resolveTool!: (result: ToolExecutionResult) => void;
const executeTool: ToolExecutor = vi.fn(
() =>
new Promise<ToolExecutionResult>((resolve) => {
resolveTool = resolve;
})
);
const controller = new AbortController();
const onToolResult = vi.fn();
const onText = vi.fn();
const onError = vi.fn();
const run = runAgentLoop(
[{ role: "user", content: "Do the task" }],
[{ name: "test_tool", description: "test", parametersSchema: {} }],
executeTool,
{ onToolResult, onText, onError },
controller.signal
);
await vi.waitFor(() => expect(executeTool).toHaveBeenCalledTimes(1));
controller.abort();
resolveTool(result);
expect(await run).toEqual({ kind: "aborted", totalTokensUsed: 0 });
expect(onToolResult).not.toHaveBeenCalled();
expect(onText).not.toHaveBeenCalled();
expect(onError).not.toHaveBeenCalled();
expect(aiServiceClientConnect.chat).toHaveBeenCalledTimes(1);
}
);
test("returns a completed outcome for explicit done", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "",
toolCalls: [
{
id: "tool-1",
name: "done",
arguments: JSON.stringify({
text: "Finished successfully",
success: true,
}),
metadata: "",
},
],
} as never);
const executeTool: ToolExecutor = vi.fn(
async (_name, args: Record<string, unknown>) => ({
kind: "done" as const,
text: String(args.text),
success: args.success !== false,
})
);
const onText = vi.fn();
const onAssistantMessage = vi.fn();
const onToolResult = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
executeTool,
{ onText, onAssistantMessage, onToolResult }
);
expect(outcome).toEqual({
kind: "completed",
text: "Finished successfully",
success: true,
totalTokensUsed: 0,
});
expect(onToolResult).toHaveBeenCalledWith(
"tool-1",
JSON.stringify({
text: "Finished successfully",
success: true,
})
);
expect(onText).toHaveBeenCalledWith("Finished successfully");
expect(onAssistantMessage).toHaveBeenCalledWith({
role: "assistant",
content: "",
toolCalls: [
{
id: "tool-1",
name: "done",
arguments: JSON.stringify({
text: "Finished successfully",
success: true,
}),
metadata: "",
},
],
});
});
test("forwards a successful handoff with the final answer", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "",
toolCalls: [
{ id: "done-1", name: "done", arguments: "{}", metadata: "" },
],
} as never);
const handoff = {
kind: "review-project-issues" as const,
project: "projects/sample",
};
const onText = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
vi.fn(async () => ({
kind: "done" as const,
text: "Review the queue",
success: true,
handoff,
})),
{ onText }
);
expect(outcome).toEqual({
kind: "completed",
text: "Review the queue",
success: true,
handoff,
totalTokensUsed: 0,
});
expect(onText).toHaveBeenCalledWith("Review the queue", handoff);
});
test("forwards response actions only for a successful final answer", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "",
toolCalls: [
{ id: "done-1", name: "done", arguments: "{}", metadata: "" },
],
} as never);
const actions = [
{
kind: "open" as const,
target: {
kind: "review-project-issues" as const,
project: "projects/sample",
},
},
{ kind: "continue" as const, prompt: "What should I check next?" },
];
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
vi.fn(async () => ({
kind: "done" as const,
text: "Review the queue",
success: true,
actions,
}))
);
expect(outcome).toMatchObject({ kind: "completed", actions });
});
test("drops a proposed handoff when the final answer fails", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "",
toolCalls: [
{ id: "done-1", name: "done", arguments: "{}", metadata: "" },
],
} as never);
const onText = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
vi.fn(async () => ({
kind: "done" as const,
text: "The request failed",
success: false,
handoff: {
kind: "review-project-issues" as const,
project: "projects/sample",
},
})),
{ onText }
);
expect(outcome).toEqual({
kind: "completed",
text: "The request failed",
success: false,
totalTokensUsed: 0,
});
expect(onText).toHaveBeenCalledExactlyOnceWith("The request failed");
});
test("accumulates token usage across repeated provider calls", async () => {
vi.mocked(aiServiceClientConnect.chat)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "tool-1",
name: "search_api",
arguments: JSON.stringify({ service: "SQLService" }),
metadata: "",
},
],
usage: {
totalTokens: 11,
inputTokens: 9,
outputTokens: 2,
cachedInputTokens: 3,
},
} as never)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "tool-2",
name: "done",
arguments: JSON.stringify({
text: "Final answer",
success: true,
}),
metadata: "",
},
],
usage: {
totalTokens: 7,
},
} as never);
const executeTool: ToolExecutor = vi.fn(async (name, args) => {
if (name === "done") {
return {
kind: "done" as const,
text: String(args.text),
success: args.success !== false,
};
}
return {
kind: "tool_result" as const,
result: JSON.stringify({ ok: true }),
};
});
const onUsage = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
executeTool,
{ onUsage }
);
expect(outcome).toEqual({
kind: "completed",
text: "Final answer",
success: true,
totalTokensUsed: 18,
});
expect(onUsage.mock.calls).toEqual([
[
{
purpose: "agent",
toolNames: ["search_api"],
totalTokens: 11,
inputTokens: 9,
outputTokens: 2,
cachedInputTokens: 3,
},
],
[{ purpose: "agent", toolNames: ["done"], totalTokens: 7 }],
]);
});
test.each([
{ usage: undefined, expected: {} },
{
usage: {
totalTokens: 0,
inputTokens: 0,
outputTokens: 0,
cachedInputTokens: 0,
},
expected: {
totalTokens: 0,
inputTokens: 0,
outputTokens: 0,
cachedInputTokens: 0,
},
},
])(
"reports unknown usage separately from explicit zero: $usage",
async ({ usage, expected }) => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "Done",
toolCalls: [],
usage,
} as never);
const onUsage = vi.fn();
await runAgentLoop(
[{ role: "user", content: "ping" }],
[],
vi.fn(),
{ onUsage },
undefined,
{ allowPlainTextCompletion: true }
);
expect(onUsage).toHaveBeenCalledExactlyOnceWith({
purpose: "agent",
toolNames: [],
...expected,
});
}
);
test("stops after cumulative provider usage reaches the run budget", async () => {
vi.mocked(aiServiceClientConnect.chat)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "tool-1",
name: "lookup",
arguments: "{}",
metadata: "",
},
],
usage: { totalTokens: 60_000 },
} as never)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "done-1",
name: "done",
arguments: JSON.stringify({ text: "Done", success: true }),
metadata: "",
},
],
} as never);
const outcome = await runAgentLoop(
[
{ role: "system", content: "system" },
{ role: "user", content: "Inspect" },
],
[{ name: "lookup", description: "Lookup", parametersSchema: {} }],
vi.fn(async (name, args) =>
name === "done"
? { kind: "done" as const, text: String(args.text), success: true }
: { kind: "tool_result" as const, result: "ok" }
)
);
expect(outcome.kind).toBe("completed");
expect(outcome).toMatchObject({ success: false, totalTokensUsed: 60_000 });
expect(aiServiceClientConnect.chat).toHaveBeenCalledOnce();
});
test("refuses a request that exceeds the serialized payload budget", async () => {
const outcome = await runAgentLoop(
[
{ role: "system", content: "s".repeat(300_000) },
{ role: "user", content: "Inspect" },
],
[],
vi.fn()
);
expect(outcome).toMatchObject({ kind: "completed", success: false });
expect(aiServiceClientConnect.chat).not.toHaveBeenCalled();
});
test("prunes completed tool output while retaining the current question", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "done-1",
name: "done",
arguments: JSON.stringify({ text: "Finished", success: true }),
metadata: "",
},
],
} as never);
const largeResult = "schema".repeat(50_000);
const outcome = await runAgentLoop(
[
{ role: "system", content: "system" },
{ role: "user", content: "Inspect the schema" },
{
role: "assistant",
content: "Reading schema",
toolCalls: [{ id: "old-tool", name: "get_schema", arguments: "{}" }],
},
{ role: "tool", toolCallId: "old-tool", content: largeResult },
{ role: "user", content: "Now summarize it" },
],
[],
vi.fn(async () => ({
kind: "done" as const,
text: "Finished",
success: true,
}))
);
expect(outcome).toMatchObject({ kind: "completed", success: true });
const request = vi.mocked(aiServiceClientConnect.chat).mock.calls[0]?.[0];
const sentMessages = request?.messages?.map((message) => message.content);
expect(sentMessages).toContain("Now summarize it");
expect(sentMessages?.join(" ")).toContain(
"omitted to fit the context limit"
);
expect(sentMessages?.join(" ")).not.toContain(largeResult);
});
test("limits provider attempts within one agent run", async () => {
vi.mocked(aiServiceClientConnect.chat).mockImplementation(async () => {
const callNumber = vi.mocked(aiServiceClientConnect.chat).mock.calls
.length;
return {
content: "",
toolCalls: [
{
id: `tool-${callNumber}`,
name: "lookup",
arguments: "{}",
metadata: "",
},
],
} as never;
});
const outcome = await runAgentLoop(
[{ role: "user", content: "Inspect" }],
[{ name: "lookup", description: "Lookup", parametersSchema: {} }],
vi.fn(async () => ({ kind: "tool_result" as const, result: "ok" }))
);
expect(outcome).toMatchObject({ kind: "completed", success: false });
expect(aiServiceClientConnect.chat).toHaveBeenCalledTimes(12);
});
test("replays completed tool flows with matched tool results", async () => {
vi.mocked(aiServiceClientConnect.chat)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "tool-1",
name: "done",
arguments: JSON.stringify({
text: "Finished successfully",
success: true,
}),
metadata: "",
},
],
} as never)
.mockResolvedValueOnce({
content: "Follow-up answer",
toolCalls: [],
} as never);
const executeTool: ToolExecutor = vi.fn(
async (_name, args: Record<string, unknown>) => ({
kind: "done" as const,
text: String(args.text),
success: args.success !== false,
})
);
const completed = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
executeTool
);
expect(completed.kind).toBe("completed");
await runAgentLoop(
[
{ role: "system", content: "system" },
{
role: "assistant",
content: "",
toolCalls: [
{
id: "tool-1",
name: "done",
arguments: JSON.stringify({
text: "Finished successfully",
success: true,
}),
metadata: "",
},
],
},
{
role: "tool",
toolCallId: "tool-1",
content: JSON.stringify({
text: "Finished successfully",
success: true,
}),
},
{ role: "assistant", content: "Finished successfully" },
{ role: "user", content: "What next?" },
],
[],
vi.fn()
);
expect(aiServiceClientConnect.chat).toHaveBeenCalledTimes(2);
const secondCallMessages = vi.mocked(aiServiceClientConnect.chat).mock
.calls[1]?.[0].messages;
expect(secondCallMessages).toEqual(
expect.arrayContaining([
expect.objectContaining({
role: AIChatMessageRole.AI_CHAT_MESSAGE_ROLE_ASSISTANT,
toolCalls: [
expect.objectContaining({
id: "tool-1",
name: "done",
}),
],
}),
expect.objectContaining({
role: AIChatMessageRole.AI_CHAT_MESSAGE_ROLE_TOOL,
toolCallId: "tool-1",
content: JSON.stringify({
text: "Finished successfully",
success: true,
}),
}),
])
);
});
test("returns an error when the assistant replies with plain text instead of done", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "Here is the answer",
toolCalls: [],
} as never);
const executeTool: ToolExecutor = vi.fn();
const onText = vi.fn();
const onError = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
executeTool,
{ onText, onError }
);
expect(outcome.kind).toBe("error");
if (outcome.kind !== "error") {
throw new Error(`Expected error outcome, got ${outcome.kind}`);
}
expect(outcome.error.message).toBe(
"Agent must finish with done({ text, success, actions }) or ask_user(...), not plain assistant text."
);
expect(outcome.totalTokensUsed).toBe(0);
expect(onError).toHaveBeenCalledWith(outcome.error);
expect(onText).not.toHaveBeenCalled();
expect(executeTool).not.toHaveBeenCalled();
});
test("asks for a terminal tool call after a plain-text reply to a database count", async () => {
vi.mocked(aiServiceClientConnect.chat)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "count-1",
name: "get_database_count",
arguments: "{}",
metadata: "",
},
],
} as never)
.mockResolvedValueOnce({
content: "The project has 26 active databases.",
toolCalls: [],
} as never)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "done-1",
name: "done",
arguments: JSON.stringify({
text: "The project has 26 active databases.",
success: true,
}),
metadata: "",
},
],
} as never);
const executeTool: ToolExecutor = vi.fn(async (name, args) =>
name === "done"
? {
kind: "done" as const,
text: String(args.text),
success: true,
}
: {
kind: "tool_result" as const,
result: JSON.stringify({ count: 26 }),
}
);
const onText = vi.fn();
const onError = vi.fn();
const tools = ["get_database_count", "call_api", "ask_user", "done"].map(
(name) => ({ name, description: name, parametersSchema: {} })
);
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
tools,
executeTool,
{ onText, onError }
);
expect(outcome).toEqual({
kind: "completed",
text: "The project has 26 active databases.",
success: true,
totalTokensUsed: 0,
});
expect(onText).toHaveBeenCalledOnce();
expect(onText).toHaveBeenCalledWith("The project has 26 active databases.");
expect(onError).not.toHaveBeenCalled();
expect(executeTool).toHaveBeenCalledTimes(2);
expect(
vi
.mocked(aiServiceClientConnect.chat)
.mock.calls[2]?.[0]?.toolDefinitions?.map((tool) => tool.name)
).toEqual(["ask_user", "done"]);
});
test("recovers an empty model response by requesting a terminal answer", async () => {
vi.mocked(aiServiceClientConnect.chat)
.mockResolvedValueOnce({ content: "", toolCalls: [] } as never)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "done-1",
name: "done",
arguments: JSON.stringify({
text: "Issue is canceled.",
success: true,
}),
metadata: "",
},
],
} as never);
const executeTool: ToolExecutor = vi.fn(async (_name, args) => ({
kind: "done" as const,
text: String(args.text),
success: true,
}));
const outcome = await runAgentLoop(
[{ role: "user", content: "Summarize the selected issue" }],
["ask_user", "done"].map((name) => ({
name,
description: name,
parametersSchema: {},
})),
executeTool,
undefined,
undefined,
{ allowPlainTextCompletion: true }
);
expect(outcome).toEqual({
kind: "completed",
text: "Issue is canceled.",
success: true,
totalTokensUsed: 0,
});
expect(aiServiceClientConnect.chat).toHaveBeenCalledTimes(2);
expect(
vi
.mocked(aiServiceClientConnect.chat)
.mock.calls[1]?.[0]?.messages?.at(-1)?.content
).toMatch(/previous response was empty/i);
});
test("still rejects plain text after one terminal-tool retry", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "Answer without done",
toolCalls: [],
} as never);
const executeTool: ToolExecutor = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[{ name: "done", description: "done", parametersSchema: {} }],
executeTool
);
expect(outcome.kind).toBe("error");
expect(aiServiceClientConnect.chat).toHaveBeenCalledTimes(2);
expect(executeTool).not.toHaveBeenCalled();
});
test("reports a rejected API request instead of a terminal-tool error", async () => {
const apiError = "cannot change status from DONE to OPEN for issue 101";
vi.mocked(aiServiceClientConnect.chat)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({
operationId: "IssueService/BatchUpdateIssuesStatus",
}),
metadata: "",
},
],
} as never)
.mockResolvedValue({
content: "The issue was reopened.",
toolCalls: [],
} as never);
const executeTool: ToolExecutor = vi.fn(async () => ({
kind: "tool_result" as const,
result: JSON.stringify({ status: 400, error: apiError }),
}));
const onText = vi.fn();
const onError = vi.fn();
const outcome = await runAgentLoop(
[{ role: "user", content: "Reopen the issue" }],
["call_api", "done", "ask_user"].map((name) => ({
name,
description: name,
parametersSchema: {},
})),
executeTool,
{ onText, onError }
);
expect(outcome).toEqual({
kind: "completed",
text: apiError,
success: false,
totalTokensUsed: 0,
});
expect(onText).toHaveBeenCalledExactlyOnceWith(apiError);
expect(onError).not.toHaveBeenCalled();
expect(executeTool).toHaveBeenCalledTimes(1);
expect(aiServiceClientConnect.chat).toHaveBeenCalledTimes(3);
});
test("reports an API rejection after permission approval from replayed history", async () => {
const apiError = "cannot change status from DONE to OPEN for issue 101";
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "This issue cannot be reopened.",
toolCalls: [],
} as never);
const executeTool: ToolExecutor = vi.fn();
const onText = vi.fn();
const outcome = await runAgentLoop(
[
{ role: "user", content: "Reopen the issue" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({
operationId: "IssueService/BatchUpdateIssuesStatus",
}),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 400, error: apiError }),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
executeTool,
{ onText }
);
expect(outcome).toEqual({
kind: "completed",
text: apiError,
success: false,
totalTokensUsed: 0,
});
expect(onText).toHaveBeenCalledExactlyOnceWith(apiError);
expect(executeTool).not.toHaveBeenCalled();
});
test("does not treat a rejected API call as a successful plain-text completion", async () => {
const apiError = "cannot change status from DONE to OPEN for issue 101";
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "The issue was reopened.",
toolCalls: [],
} as never);
const outcome = await runAgentLoop(
[
{ role: "user", content: "Reopen the issue" },
{
role: "assistant",
toolCalls: [{ id: "api-1", name: "call_api", arguments: "{}" }],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 400, error: apiError }),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn(),
undefined,
undefined,
{ allowPlainTextCompletion: true }
);
expect(outcome).toEqual({
kind: "completed",
text: apiError,
success: false,
totalTokensUsed: 0,
});
expect(aiServiceClientConnect.chat).toHaveBeenCalledTimes(1);
});
test("requires done after a failed call followed by a successful call", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "Recovered answer without done",
toolCalls: [],
} as never);
const outcome = await runAgentLoop(
[
{ role: "user", content: "Update the issue" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({
operationId: "IssueService/BatchUpdateIssuesStatus",
body: { issues: ["projects/a/issues/101"], status: "OPEN" },
}),
},
{
id: "api-2",
name: "call_api",
arguments: JSON.stringify({
body: {
parent: "projects/a",
status: "OPEN",
issues: ["projects/a/issues/101"],
},
operationId: "IssueService/BatchUpdateIssuesStatus",
}),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 400, error: "Initial failure" }),
},
{
role: "tool",
toolCallId: "api-2",
content: JSON.stringify({ status: 200, response: {} }),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn()
);
expect(outcome.kind).toBe("error");
expect(aiServiceClientConnect.chat).toHaveBeenCalledTimes(2);
});
test("does not substitute an earlier API error for a missing done", async () => {
const apiError = "cannot change status from DONE to OPEN for issue 101";
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "Both issues were reopened.",
toolCalls: [],
} as never);
const outcome = await runAgentLoop(
[
{ role: "user", content: "Reopen two issues" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({
operationId: "IssueService/BatchUpdateIssuesStatus",
body: { issues: ["projects/a/issues/101"], status: "OPEN" },
}),
},
{
id: "api-2",
name: "call_api",
arguments: JSON.stringify({
operationId: "IssueService/BatchUpdateIssuesStatus",
body: { issues: ["projects/a/issues/102"], status: "OPEN" },
}),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 400, error: apiError }),
},
{
role: "tool",
toolCallId: "api-2",
content: JSON.stringify({ status: 200, response: {} }),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn()
);
expect(outcome.kind).toBe("error");
});
test("requires done after a failed write followed by another successful write", async () => {
const first = {
operationId: "IssueService/BatchUpdateIssuesStatus",
body: { issues: ["projects/a/issues/101"], status: "OPEN" },
};
const second = {
operationId: "IssueService/BatchUpdateIssuesStatus",
body: { issues: ["projects/a/issues/101"], status: "DONE" },
};
const apiError = "First request failed";
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "Both requests succeeded.",
toolCalls: [],
} as never);
const outcome = await runAgentLoop(
[
{ role: "user", content: "Run both requests" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify(first),
},
{
id: "api-2",
name: "call_api",
arguments: JSON.stringify(second),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 400, error: apiError }),
},
{
role: "tool",
toolCallId: "api-2",
content: JSON.stringify({ status: 200, response: {} }),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn()
);
expect(outcome.kind).toBe("error");
});
test.each([
{
operationId: "SQLService/Query",
failure: { status: 400, error: "Unknown column missing" },
correctedStatement: "SELECT id FROM t WHERE id = 1",
},
{
operationId: "bytebase.v1.SQLService.Query",
failure: {
status: 200,
response: { results: [{ error: "Unknown column missing" }] },
},
correctedStatement: "SELECT t.id FROM t JOIN u ON t.id = u.t_id",
},
])(
"preserves a failed $operationId attempt after a successful retry",
async ({ operationId, failure, correctedStatement }) => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "",
toolCalls: [
{
id: "done-1",
name: "done",
arguments: JSON.stringify({
text: "The corrected query returned one row.",
success: true,
}),
metadata: "",
},
],
} as never);
const onToolResult = vi.fn();
const outcome = await runAgentLoop(
[
{
role: "user",
content: "Query the database and correct any SQL error",
},
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({
operationId,
body: {
name: "instances/i/databases/db",
statement: "SELECT missing FROM t",
},
}),
},
{
id: "api-2",
name: "call_api",
arguments: JSON.stringify({
operationId,
body: {
name: "instances/i/databases/db",
statement: correctedStatement,
},
}),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify(failure),
},
{
role: "tool",
toolCallId: "api-2",
content: JSON.stringify({
status: 200,
response: { results: [{}] },
}),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn(async () => ({
kind: "done" as const,
text: "The corrected query returned one row.",
success: true,
})),
{ onToolResult }
);
expect(outcome).toEqual({
kind: "completed",
text: "The corrected query returned one row.",
success: true,
totalTokensUsed: 0,
});
expect(onToolResult).toHaveBeenCalledWith(
"done-1",
JSON.stringify({
text: "The corrected query returned one row.",
success: true,
failedApiCallCount: 1,
})
);
}
);
test("requires done after a failed query followed by a successful query on another database", async () => {
const apiError = "Unknown column missing";
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "The query returned rows.",
toolCalls: [],
} as never);
const outcome = await runAgentLoop(
[
{ role: "user", content: "Query both databases" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({
operationId: "SQLService/Query",
body: {
name: "instances/i/databases/db-one",
statement: "SELECT missing FROM t",
},
}),
},
{
id: "api-2",
name: "call_api",
arguments: JSON.stringify({
operationId: "SQLService/Query",
body: {
name: "instances/i/databases/db-two",
statement: "SELECT id FROM t",
},
}),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 400, error: apiError }),
},
{
role: "tool",
toolCallId: "api-2",
content: JSON.stringify({ status: 200, response: { results: [{}] } }),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn()
);
expect(outcome.kind).toBe("error");
});
test("requires done after a failed write followed by a successful query", async () => {
const apiError = "Update denied";
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "The database was updated.",
toolCalls: [],
} as never);
const outcome = await runAgentLoop(
[
{ role: "user", content: "Update and verify the database" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({
operationId: "DatabaseService/UpdateDatabase",
body: {
database: { name: "instances/i/databases/db" },
},
}),
},
{
id: "api-2",
name: "call_api",
arguments: JSON.stringify({
operationId: "SQLService/Query",
body: {
name: "instances/i/databases/db",
statement: "SELECT id FROM t",
},
}),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 403, error: apiError }),
},
{
role: "tool",
toolCallId: "api-2",
content: JSON.stringify({ status: 200, response: { results: [{}] } }),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn()
);
expect(outcome.kind).toBe("error");
});
test.each([
"UPDATE t SET id = 2",
"SELECT id INTO backup FROM t",
"SELECT id FROM t FOR UPDATE",
"SELECT mutate_data()",
"SELECT id FROM t; DELETE FROM t",
"EXPLAIN ANALYZE UPDATE t SET id = 2",
])(
"does not infer task completion from SQL syntax: %s",
async (statement) => {
const apiError = "Statement failed";
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "The statements succeeded.",
toolCalls: [],
} as never);
const outcome = await runAgentLoop(
[
{ role: "user", content: "Run the statements" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({
operationId: "SQLService/Query",
body: { name: "instances/i/databases/db", statement },
}),
},
{
id: "api-2",
name: "call_api",
arguments: JSON.stringify({
operationId: "SQLService/Query",
body: {
name: "instances/i/databases/db",
statement: "SELECT id FROM t",
},
}),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 400, error: apiError }),
},
{
role: "tool",
toolCallId: "api-2",
content: JSON.stringify({
status: 200,
response: { results: [{}] },
}),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn()
);
expect(outcome.kind).toBe("error");
}
);
test("requires done after a rejected write followed by a successful read", async () => {
const apiError = "cannot change status from DONE to OPEN for issue 101";
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "The issue was reopened.",
toolCalls: [],
} as never);
const outcome = await runAgentLoop(
[
{ role: "user", content: "Reopen the issue" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({
operationId: "IssueService/BatchUpdateIssuesStatus",
body: { issues: ["projects/a/issues/101"], status: "OPEN" },
}),
},
{
id: "api-2",
name: "call_api",
arguments: JSON.stringify({
operationId: "IssueService/GetIssue",
body: { name: "projects/a/issues/101" },
}),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 400, error: apiError }),
},
{
role: "tool",
toolCallId: "api-2",
content: JSON.stringify({
status: 200,
response: { status: "DONE" },
}),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn()
);
expect(outcome.kind).toBe("error");
});
test("counts failed API attempts when the agent completes successfully", async () => {
const apiError = "cannot change status from DONE to OPEN for issue 101";
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "",
toolCalls: [
{
id: "done-1",
name: "done",
arguments: JSON.stringify({
text: "The issue was reopened.",
success: true,
}),
metadata: "",
},
],
} as never);
const onToolResult = vi.fn();
const outcome = await runAgentLoop(
[
{ role: "user", content: "Reopen the issue" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({
operationId: "IssueService/BatchUpdateIssuesStatus",
body: { issues: ["projects/a/issues/101"], status: "OPEN" },
}),
},
{
id: "api-2",
name: "call_api",
arguments: JSON.stringify({
operationId: "IssueService/GetIssue",
body: { name: "projects/a/issues/101" },
}),
},
{
id: "api-3",
name: "call_api",
arguments: JSON.stringify({
operationId: "IssueService/BatchUpdateIssuesStatus",
body: { issues: ["projects/a/issues/101"], status: "OPEN" },
}),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 400, error: apiError }),
},
{
role: "tool",
toolCallId: "api-2",
content: JSON.stringify({ status: 403, error: "Read denied" }),
},
{
role: "tool",
toolCallId: "api-3",
content: JSON.stringify({ status: 200, response: {} }),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn(async () => ({
kind: "done" as const,
text: "The issue was reopened.",
success: true,
})),
{ onToolResult }
);
expect(outcome).toEqual({
kind: "completed",
text: "The issue was reopened.",
success: true,
totalTokensUsed: 0,
});
expect(onToolResult).toHaveBeenCalledWith(
"done-1",
JSON.stringify({
text: "The issue was reopened.",
success: true,
failedApiCallCount: 2,
})
);
});
test.each(["SQLService/Query", "bytebase.v1.SQLService.Query"])(
"reports a query result error for %s inside a successful API response",
async (operationId) => {
const queryError = "permission denied for relation records";
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "The query returned rows.",
toolCalls: [],
} as never);
const outcome = await runAgentLoop(
[
{ role: "user", content: "Query the database" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({ operationId }),
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({
status: 200,
response: { results: [{ error: queryError }] },
}),
},
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn()
);
expect(outcome).toEqual({
kind: "completed",
text: queryError,
success: false,
totalTokensUsed: 0,
});
}
);
test("does not reuse an API rejection from an earlier user turn", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "Answer without done",
toolCalls: [],
} as never);
const outcome = await runAgentLoop(
[
{ role: "user", content: "Reopen the issue" },
{
role: "assistant",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: "{}",
},
],
},
{
role: "tool",
toolCallId: "api-1",
content: JSON.stringify({ status: 400, error: "Not allowed" }),
},
{ role: "user", content: "What is next?" },
],
[{ name: "done", description: "done", parametersSchema: {} }],
vi.fn()
);
expect(outcome.kind).toBe("error");
expect(aiServiceClientConnect.chat).toHaveBeenCalledTimes(2);
});
test("does not execute a non-terminal tool requested during finalization", async () => {
vi.mocked(aiServiceClientConnect.chat)
.mockResolvedValueOnce({
content: "The project has 26 active databases.",
toolCalls: [],
} as never)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "api-1",
name: "call_api",
arguments: JSON.stringify({ operationId: "DeleteDatabase" }),
metadata: "",
},
],
} as never);
const executeTool: ToolExecutor = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
["call_api", "done"].map((name) => ({
name,
description: name,
parametersSchema: {},
})),
executeTool
);
expect(outcome.kind).toBe("error");
expect(executeTool).not.toHaveBeenCalled();
});
test("completes a read-only starter when the model answers in plain text after a tool result", async () => {
vi.mocked(aiServiceClientConnect.chat)
.mockResolvedValueOnce({
content: "",
toolCalls: [
{
id: "tool-open-issues",
name: "get_project_open_issues",
arguments: "{}",
metadata: "",
},
],
} as never)
.mockResolvedValueOnce({
content: "You have two open Issues in this Project.",
toolCalls: [],
} as never);
const executeTool: ToolExecutor = vi.fn(async () => ({
kind: "tool_result" as const,
result: JSON.stringify({ issues: [] }),
}));
const onText = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
executeTool,
{ onText },
undefined,
{ allowPlainTextCompletion: true }
);
expect(outcome).toEqual({
kind: "completed",
text: "You have two open Issues in this Project.",
success: true,
totalTokensUsed: 0,
});
expect(onText).toHaveBeenCalledWith(
"You have two open Issues in this Project."
);
});
test("returns awaiting_user without synthesizing an immediate tool result", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "",
toolCalls: [
{
id: "tool-ask",
name: "ask_user",
arguments: JSON.stringify({
prompt: "Which environment should I use?",
kind: "choose",
options: [
{ label: "Production", value: "prod" },
{ label: "Staging", value: "staging" },
],
}),
metadata: "",
},
],
} as never);
const executeTool: ToolExecutor = vi.fn(
async (_name, _args, toolCallId) => ({
kind: "ask_user" as const,
ask: {
toolCallId,
prompt: "Which environment should I use?",
kind: "choose" as const,
options: [
{ label: "Production", value: "prod" },
{ label: "Staging", value: "staging" },
],
},
})
);
const onToolResult = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
executeTool,
{ onToolResult }
);
expect(outcome).toEqual({
kind: "awaiting_user",
ask: {
toolCallId: "tool-ask",
prompt: "Which environment should I use?",
kind: "choose",
options: [
{ label: "Production", value: "prod" },
{ label: "Staging", value: "staging" },
],
},
totalTokensUsed: 0,
});
expect(onToolResult).not.toHaveBeenCalled();
});
test("records skipped tool results after ask_user and does not execute later tool calls", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "",
toolCalls: [
{
id: "tool-ask",
name: "ask_user",
arguments: JSON.stringify({
prompt: "Which environment should I use?",
kind: "input",
}),
metadata: "",
},
{
id: "tool-late",
name: "search_api",
arguments: JSON.stringify({ service: "SQLService" }),
metadata: "",
},
],
} as never);
const executeTool: ToolExecutor = vi.fn(async (name, _args, toolCallId) => {
if (name === "ask_user") {
return {
kind: "ask_user" as const,
ask: {
toolCallId,
prompt: "Which environment should I use?",
kind: "input" as const,
},
};
}
return {
kind: "tool_result" as const,
result: "should not run",
};
});
const onToolResult = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
executeTool,
{ onToolResult }
);
expect(outcome).toEqual({
kind: "awaiting_user",
ask: {
toolCallId: "tool-ask",
prompt: "Which environment should I use?",
kind: "input",
},
totalTokensUsed: 0,
});
expect(executeTool).toHaveBeenCalledTimes(1);
expect(executeTool).toHaveBeenCalledWith(
"ask_user",
{
prompt: "Which environment should I use?",
kind: "input",
},
"tool-ask"
);
expect(onToolResult).toHaveBeenCalledTimes(1);
expect(onToolResult).toHaveBeenCalledWith(
"tool-late",
JSON.stringify({
skipped: true,
blockedByToolCallId: "tool-ask",
blockedByKind: "ask_user",
reason: "Skipped because this assistant turn already emitted ask_user",
})
);
});
test("records skipped tool results after done and does not execute later tool calls", async () => {
vi.mocked(aiServiceClientConnect.chat).mockResolvedValue({
content: "",
toolCalls: [
{
id: "tool-done",
name: "done",
arguments: JSON.stringify({
text: "Finished successfully",
success: true,
}),
metadata: "",
},
{
id: "tool-late",
name: "navigate",
arguments: JSON.stringify({ path: "/projects/demo" }),
metadata: "",
},
],
} as never);
const executeTool: ToolExecutor = vi.fn(async (name, args) => {
if (name === "done") {
return {
kind: "done" as const,
text: String(args.text),
success: args.success !== false,
};
}
return {
kind: "tool_result" as const,
result: "should not run",
};
});
const onToolResult = vi.fn();
const onText = vi.fn();
const outcome = await runAgentLoop(
[{ role: "system", content: "system" }],
[],
executeTool,
{ onToolResult, onText }
);
expect(outcome).toEqual({
kind: "completed",
text: "Finished successfully",
success: true,
totalTokensUsed: 0,
});
expect(executeTool).toHaveBeenCalledTimes(1);
expect(executeTool).toHaveBeenCalledWith(
"done",
{
text: "Finished successfully",
success: true,
},
"tool-done"
);
expect(onToolResult).toHaveBeenNthCalledWith(
1,
"tool-done",
JSON.stringify({
text: "Finished successfully",
success: true,
})
);
expect(onToolResult).toHaveBeenNthCalledWith(
2,
"tool-late",
JSON.stringify({
skipped: true,
blockedByToolCallId: "tool-done",
blockedByKind: "done",
reason: "Skipped because this assistant turn already emitted done",
})
);
expect(onText).toHaveBeenCalledWith("Finished successfully");
});
});