Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
import { formatAgentMessages, Providers } from '@librechat/agents';
import type { ToolExecuteBatchRequest } from '@librechat/agents';
import type { Response as ServerResponse } from 'express';
import type { InputItem, ResponseContext } from '../types';
import {
buildAggregatedResponse,
convertInputToMessages,
createAggregatorEventHandlers,
createResponseAggregator,
createResponsesEventHandlers,
validateResponseRequest,
createResponseContext,
buildResponsesUsage,
} from '../service';
import { clientToolDeferralContent, createClientToolExecuteHandler } from '../clientTools';
import { extractMessageContent } from '~/protection/adapters/messages';
import { buildResponse, createResponseTracker } from '../handlers';
import { buildRunToolSet } from '~/agents/tools';
describe('response usage aggregation', () => {
const context: ResponseContext = {
responseId: 'resp_test',
model: 'agent_test',
createdAt: 1778317637,
};
it('accumulates usage across parent and subagent model calls', () => {
const aggregator = createResponseAggregator();
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_chat_model_end.handle('on_chat_model_end', {
output: {
usage_metadata: {
input_tokens: 100,
output_tokens: 40,
input_token_details: { cache_read: 10 },
},
},
});
handlers.on_chat_model_end.handle('on_chat_model_end', {
output: {
usage_metadata: {
input_tokens: 25,
output_tokens: 15,
cache_read_input_tokens: 5,
},
},
});
expect(aggregator.usage).toEqual({
inputTokens: 125,
outputTokens: 55,
reasoningTokens: 0,
cachedTokens: 15,
});
});
it('builds one normalized wire total with an identity-free child breakdown', () => {
const usage = buildResponsesUsage([
{ input_tokens: 100, output_tokens: 40, provider: 'openAI' },
{
input_tokens: 25,
output_tokens: 10,
provider: 'openAI',
usage_type: 'subagent',
input_token_details: { cache_read: 5 },
},
]);
expect(usage).toEqual({
input_tokens: 125,
output_tokens: 50,
total_tokens: 175,
input_tokens_details: { cached_tokens: 5 },
output_tokens_details: { reasoning_tokens: 0 },
primary: { input_tokens: 100, output_tokens: 40, total_tokens: 140 },
subagent: { input_tokens: 25, output_tokens: 10, total_tokens: 35 },
});
const response = buildAggregatedResponse(context, createResponseAggregator(), usage);
expect(response.usage).toEqual(usage);
});
it('uses the normalized override in the completed streaming event', () => {
const writes: string[] = [];
const res = {
write: (chunk: string) => {
writes.push(chunk);
},
} as unknown as ServerResponse;
const tracker = createResponseTracker();
const usage = buildResponsesUsage([
{ input_tokens: 100, output_tokens: 40, provider: 'openAI' },
{
input_tokens: 25,
output_tokens: 10,
provider: 'openAI',
usage_type: 'subagent',
},
]);
createResponsesEventHandlers({ res, context, tracker }).finalizeStream(usage);
const completed = writes.find((chunk) => chunk.startsWith('data: {'));
expect(JSON.parse(completed?.slice(6) ?? '{}').response.usage).toEqual(usage);
});
});
/**
* A caller-executed tool is never run by the server, so `on_tool_end` — where a
* server-run call gets its terminating events — cannot fire for it. The
* streaming lifecycle still has to end the item, otherwise the caller that must
* run the tool is handed an item that never closes.
*/
describe('streaming lifecycle of a caller-executed tool call', () => {
const context: ResponseContext = {
responseId: 'resp_test',
model: 'agent_test',
createdAt: 1778317637,
};
const CALL_ID = 'toolu_stream_1';
const CLIENT_TOOL = 'submit_sql';
/** Collects the events one streamed run writes, as parsed payloads. */
function recorder(): { res: ServerResponse; events: Array<Record<string, unknown>> } {
const events: Array<Record<string, unknown>> = [];
const res = {
write: (chunk: string) => {
if (chunk.startsWith('data: ') && !chunk.includes('[DONE]')) {
events.push(JSON.parse(chunk.slice(6)));
}
},
} as unknown as ServerResponse;
return { res, events };
}
/** The terminating events for the call itself; the tool result is its own
* `function_call_output` item and closes separately. */
function terminatingEvents(events: Array<Record<string, unknown>>): string[] {
return events
.filter(
(event) =>
event.type === 'response.function_call_arguments.done' ||
(event.type === 'response.output_item.done' &&
(event.item as { type?: string } | undefined)?.type === 'function_call'),
)
.map((event) => event.type as string);
}
/** Streams one call to `toolName`, then ends the run as the graph would. */
function streamToolCall(toolName: string): Array<Record<string, unknown>> {
const { res, events } = recorder();
const { handlers, finalizeStream } = createResponsesEventHandlers({
res,
context,
tracker: createResponseTracker(),
clientToolNames: new Set([CLIENT_TOOL]),
});
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: CALL_ID, name: toolName }] },
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ id: CALL_ID, index: 0, args: '{"sql":' }] },
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: '"SELECT 1"}' }] },
});
finalizeStream();
return events;
}
const streamClientToolCall = (): Array<Record<string, unknown>> => streamToolCall(CLIENT_TOOL);
it('terminates the arguments and the item before the response completes', () => {
const types = streamClientToolCall().map((event) => event.type);
expect(types).toEqual([
'response.output_item.added',
'response.function_call_arguments.delta',
'response.function_call_arguments.delta',
'response.function_call_arguments.done',
'response.output_item.done',
'response.completed',
]);
});
it('reports the assembled arguments on the terminating events', () => {
const events = streamClientToolCall();
const argumentsDone = events.find(
(event) => event.type === 'response.function_call_arguments.done',
);
expect(argumentsDone).toMatchObject({
call_id: CALL_ID,
arguments: '{"sql":"SELECT 1"}',
});
});
it('leaves no item in progress inside a completed response', () => {
const events = streamClientToolCall();
const completed = events.find((event) => event.type === 'response.completed') as {
response: { status: string; output: Array<{ type: string; status?: string }> };
};
expect(completed.response.status).toBe('completed');
expect(completed.response.output).toEqual([
expect.objectContaining({ type: 'function_call', status: 'completed' }),
]);
});
/**
* The first tool call of a message is announced before its arguments arrive,
* but a second one is announced after them: the SDK opens a single
* `tool_calls` run step per message, so any later call gets its own step only
* at model end.
*
* An argument fragment that arrives before its call is announced has no item
* to write to, so it is dropped. `on_chat_model_end` is what puts those
* arguments back, and the only way it can tell they were dropped is by
* checking what the tracker actually holds.
*/
it('recovers the arguments of a call announced after its own fragments', () => {
const { res, events } = recorder();
const { handlers, finalizeStream } = createResponsesEventHandlers({
res,
context,
tracker: createResponseTracker(),
clientToolNames: new Set([CLIENT_TOOL]),
});
const FIRST = 'toolu_first';
const SECOND = 'toolu_second';
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: FIRST, name: CLIENT_TOOL }] },
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ id: FIRST, index: 1, args: '{"sql":"A"}' }] },
});
/* The second block opens with the id, and its fragments follow, all before
the run step that announces it. */
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: {
type: 'tool_calls',
tool_calls: [{ id: SECOND, name: CLIENT_TOOL, index: 2, args: '' }],
},
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ index: 2, args: '{"sql":"B"}' }] },
});
handlers.on_run_step.handle('on_run_step', {
id: 'step_2',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: SECOND, name: CLIENT_TOOL }] },
});
handlers.on_chat_model_end.handle('on_chat_model_end', {
output: {
tool_calls: [
{ id: FIRST, args: { sql: 'A' } },
{ id: SECOND, args: { sql: 'B' } },
],
},
});
finalizeStream();
const completed = events.find((event) => event.type === 'response.completed') as {
response: { output: Array<{ call_id?: string; arguments?: string }> };
};
expect(completed.response.output).toEqual([
expect.objectContaining({ call_id: FIRST, arguments: '{"sql":"A"}' }),
expect.objectContaining({ call_id: SECOND, arguments: '{"sql":"B"}' }),
]);
});
/** The first call's fragments are already streamed, so the backfill must not
* append the model-end copy on top of them. */
it('does not duplicate arguments a call already streamed', () => {
const { res, events } = recorder();
const { handlers, finalizeStream } = createResponsesEventHandlers({
res,
context,
tracker: createResponseTracker(),
clientToolNames: new Set([CLIENT_TOOL]),
});
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: CALL_ID, name: CLIENT_TOOL }] },
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ id: CALL_ID, index: 0, args: '{"sql":' }] },
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ index: 0, args: '"SELECT 1"}' }] },
});
handlers.on_chat_model_end.handle('on_chat_model_end', {
output: { tool_calls: [{ id: CALL_ID, args: { sql: 'SELECT 1' } }] },
});
finalizeStream();
const argumentsDone = events.find(
(event) => event.type === 'response.function_call_arguments.done',
);
expect(argumentsDone).toMatchObject({
call_id: CALL_ID,
arguments: '{"sql":"SELECT 1"}',
});
});
/**
* A call the server answered itself must not look like one handed back for
* the caller to run: the model was told to re-issue it, so a caller that
* executed it too would run a side-effecting tool twice.
*/
it('reports a deferred call as answered, not as handed back', () => {
const { res, events } = recorder();
const { handlers, finalizeStream, emitClientToolDeferral } = createResponsesEventHandlers({
res,
context,
tracker: createResponseTracker(),
clientToolNames: new Set([CLIENT_TOOL]),
});
const execute = createClientToolExecuteHandler({
delegate: { handle: () => {} },
clientToolNames: new Set([CLIENT_TOOL]),
responseId: 'resp_test',
onDeferred: emitClientToolDeferral,
});
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: CALL_ID, name: CLIENT_TOOL }] },
});
execute.handle('on_tool_execute', {
toolCalls: [{ id: CALL_ID, name: CLIENT_TOOL, args: {} }],
resolve: () => {},
} as unknown as ToolExecuteBatchRequest);
finalizeStream();
const completed = events.find((event) => event.type === 'response.completed') as {
response: { output: Array<{ type: string; call_id?: string; output?: string }> };
};
const answer = completed.response.output.find((item) => item.type === 'function_call_output');
expect(answer).toMatchObject({
call_id: CALL_ID,
output: clientToolDeferralContent(CLIENT_TOOL),
});
});
/**
* A mixed batch answers the client call server-side, which can terminate it
* through `on_tool_end` after all. Finalization must not emit a second pair
* for a call that already closed.
*/
it('does not re-terminate a call on_tool_end already closed', () => {
const { res, events } = recorder();
const { handlers, finalizeStream } = createResponsesEventHandlers({
res,
context,
tracker: createResponseTracker(),
clientToolNames: new Set([CLIENT_TOOL]),
});
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: CALL_ID, name: CLIENT_TOOL }] },
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ id: CALL_ID, index: 0, args: '{}' }] },
});
handlers.on_tool_end.handle('on_tool_end', { tool_call_id: CALL_ID, output: 'done' });
finalizeStream();
expect(terminatingEvents(events)).toEqual([
'response.function_call_arguments.done',
'response.output_item.done',
]);
});
/**
* Scoping, pinned: a server tool's call is `on_tool_end`'s to terminate, and
* this module never sees that event because the controller replaces the
* handler instead of composing with it. Terminating such a call from
* finalization would paper over that separate bug AND change the event stream
* of every request that declares no client tool, so it deliberately does not.
*/
it('leaves a server tool call to on_tool_end', () => {
expect(terminatingEvents(streamToolCall('bash_tool'))).toEqual([]);
});
});
describe('convertInputToMessages', () => {
// ── String input shorthand ─────────────────────────────────────────
it('converts a string input to a single user message', () => {
const result = convertInputToMessages('Hello');
expect(result).toEqual([{ role: 'user', content: 'Hello' }]);
});
// ── Empty input array ──────────────────────────────────────────────
it('returns an empty array for empty input', () => {
const result = convertInputToMessages([]);
expect(result).toEqual([]);
});
// ── Role mapping ───────────────────────────────────────────────────
it('maps developer role to system', () => {
const input: InputItem[] = [
{ type: 'message', role: 'developer', content: 'You are helpful.' },
];
expect(convertInputToMessages(input)).toEqual([
{ role: 'system', content: 'You are helpful.' },
]);
});
it('maps system role to system', () => {
const input: InputItem[] = [{ type: 'message', role: 'system', content: 'System prompt.' }];
expect(convertInputToMessages(input)).toEqual([{ role: 'system', content: 'System prompt.' }]);
});
it('maps user role to user', () => {
const input: InputItem[] = [{ type: 'message', role: 'user', content: 'Hi' }];
expect(convertInputToMessages(input)).toEqual([{ role: 'user', content: 'Hi' }]);
});
it('maps assistant role to assistant', () => {
const input: InputItem[] = [{ type: 'message', role: 'assistant', content: 'Hello!' }];
expect(convertInputToMessages(input)).toEqual([{ role: 'assistant', content: 'Hello!' }]);
});
it('defaults unknown roles to user', () => {
const input = [
{ type: 'message', role: 'unknown_role', content: 'test' },
] as unknown as InputItem[];
expect(convertInputToMessages(input)[0].role).toBe('user');
});
// ── input_text content blocks ──────────────────────────────────────
it('converts input_text blocks to text blocks', () => {
const input: InputItem[] = [
{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'Hello world' }],
},
];
const result = convertInputToMessages(input);
expect(result).toEqual([{ role: 'user', content: [{ type: 'text', text: 'Hello world' }] }]);
});
// ── output_text content blocks (the original bug) ──────────────────
it('converts output_text blocks to text blocks', () => {
const input: InputItem[] = [
{
type: 'message',
role: 'assistant',
content: [{ type: 'output_text', text: 'I can help!', annotations: [], logprobs: [] }],
},
];
const result = convertInputToMessages(input);
expect(result).toEqual([
{ role: 'assistant', content: [{ type: 'text', text: 'I can help!' }] },
]);
});
// ── refusal content blocks ─────────────────────────────────────────
it('converts refusal blocks to text blocks', () => {
const input: InputItem[] = [
{
type: 'message',
role: 'assistant',
content: [{ type: 'refusal', refusal: 'I cannot do that.' }],
},
];
const result = convertInputToMessages(input);
expect(result).toEqual([
{ role: 'assistant', content: [{ type: 'text', text: 'I cannot do that.' }] },
]);
});
// ── input_image content blocks ─────────────────────────────────────
it('converts input_image blocks to image_url blocks', () => {
const input: InputItem[] = [
{
type: 'message',
role: 'user',
content: [
{ type: 'input_image', image_url: 'https://example.com/img.png', detail: 'high' },
],
},
];
const result = convertInputToMessages(input);
expect(result).toEqual([
{
role: 'user',
content: [
{
type: 'image_url',
image_url: { url: 'https://example.com/img.png', detail: 'high' },
},
],
},
]);
});
// ── input_file content blocks ──────────────────────────────────────
it('converts input_file blocks to text placeholders', () => {
const input: InputItem[] = [
{
type: 'message',
role: 'user',
content: [{ type: 'input_file', filename: 'report.pdf', file_id: 'f_123' }],
},
];
const result = convertInputToMessages(input);
expect(result).toEqual([
{ role: 'user', content: [{ type: 'text', text: '[File: report.pdf]' }] },
]);
});
it('uses "unknown" for input_file without filename', () => {
const input: InputItem[] = [
{
type: 'message',
role: 'user',
content: [{ type: 'input_file', file_id: 'f_123' }],
},
];
const result = convertInputToMessages(input);
expect(result).toEqual([
{ role: 'user', content: [{ type: 'text', text: '[File: unknown]' }] },
]);
});
// ── Null / undefined filtering ─────────────────────────────────────
it('filters out null elements in content arrays', () => {
const input = [
{
type: 'message',
role: 'user',
content: [null, { type: 'input_text', text: 'valid' }, undefined],
},
] as unknown as InputItem[];
const result = convertInputToMessages(input);
expect(result).toEqual([{ role: 'user', content: [{ type: 'text', text: 'valid' }] }]);
});
// ── Missing text field defaults to empty string ────────────────────
it('defaults to empty string when text field is missing on input_text', () => {
const input = [
{
type: 'message',
role: 'user',
content: [{ type: 'input_text' }],
},
] as unknown as InputItem[];
const result = convertInputToMessages(input);
expect(result).toEqual([{ role: 'user', content: [{ type: 'text', text: '' }] }]);
});
it('defaults to empty string when text field is missing on output_text', () => {
const input = [
{
type: 'message',
role: 'assistant',
content: [{ type: 'output_text' }],
},
] as unknown as InputItem[];
const result = convertInputToMessages(input);
expect(result).toEqual([{ role: 'assistant', content: [{ type: 'text', text: '' }] }]);
});
it('defaults to empty string when refusal field is missing on refusal block', () => {
const input = [
{
type: 'message',
role: 'assistant',
content: [{ type: 'refusal' }],
},
] as unknown as InputItem[];
const result = convertInputToMessages(input);
expect(result).toEqual([{ role: 'assistant', content: [{ type: 'text', text: '' }] }]);
});
// ── Unknown block types are filtered out ───────────────────────────
it('filters out unknown content block types', () => {
const input = [
{
type: 'message',
role: 'user',
content: [
{ type: 'input_text', text: 'keep me' },
{ type: 'some_future_type', data: 'ignore' },
],
},
] as unknown as InputItem[];
const result = convertInputToMessages(input);
expect(result).toEqual([{ role: 'user', content: [{ type: 'text', text: 'keep me' }] }]);
});
// ── Mixed valid/invalid content in same array ──────────────────────
it('handles mixed valid and invalid content blocks', () => {
const input = [
{
type: 'message',
role: 'assistant',
content: [
{ type: 'output_text', text: 'Hello', annotations: [], logprobs: [] },
null,
{ type: 'unknown_type' },
{ type: 'refusal', refusal: 'No can do' },
],
},
] as unknown as InputItem[];
const result = convertInputToMessages(input);
expect(result).toEqual([
{
role: 'assistant',
content: [
{ type: 'text', text: 'Hello' },
{ type: 'text', text: 'No can do' },
],
},
]);
});
// ── Non-array, non-string content defaults to empty string ─────────
it('defaults to empty string for non-array non-string content', () => {
const input = [{ type: 'message', role: 'user', content: 42 }] as unknown as InputItem[];
const result = convertInputToMessages(input);
expect(result).toEqual([{ role: 'user', content: '' }]);
});
// ── Function call items ────────────────────────────────────────────
it('pairs a function_call with its output into one assistant tool_call part', () => {
const input: InputItem[] = [
{
type: 'function_call',
id: 'fc_1',
call_id: 'call_abc',
name: 'get_weather',
arguments: '{"city":"NYC"}',
},
{ type: 'function_call_output', call_id: 'call_abc', output: '{"temp":72}' },
];
const result = convertInputToMessages(input);
expect(result).toEqual([
{
role: 'assistant',
content: [
{
type: 'tool_call',
tool_call: {
id: 'call_abc',
name: 'get_weather',
args: '{"city":"NYC"}',
output: '{"temp":72}',
},
},
],
},
]);
});
it('never emits a tool-role message, which formats as a stray SystemMessage', () => {
const input: InputItem[] = [
{ type: 'message', role: 'user', content: 'Run it' },
{
type: 'function_call',
id: 'fc_1',
call_id: 'call_abc',
name: 'get_weather',
arguments: '{}',
},
{ type: 'function_call_output', call_id: 'call_abc', output: 'ok' },
];
const result = convertInputToMessages(input);
expect(result.map((message) => message.role)).toEqual(['user', 'assistant']);
});
it('keeps a parallel batch on a single assistant turn', () => {
const input: InputItem[] = [
{ type: 'function_call', id: 'fc_1', call_id: 'call_a', name: 'first', arguments: '{}' },
{ type: 'function_call', id: 'fc_2', call_id: 'call_b', name: 'second', arguments: '{}' },
{ type: 'function_call_output', call_id: 'call_a', output: 'a' },
{ type: 'function_call_output', call_id: 'call_b', output: 'b' },
];
const result = convertInputToMessages(input);
expect(result).toHaveLength(1);
expect(result[0].content).toEqual([
{ type: 'tool_call', tool_call: { id: 'call_a', name: 'first', args: '{}', output: 'a' } },
{ type: 'tool_call', tool_call: { id: 'call_b', name: 'second', args: '{}', output: 'b' } },
]);
});
it('starts a new assistant turn when a message separates two calls', () => {
const input: InputItem[] = [
{ type: 'function_call', id: 'fc_1', call_id: 'call_a', name: 'first', arguments: '{}' },
{ type: 'function_call_output', call_id: 'call_a', output: 'a' },
{ type: 'message', role: 'user', content: 'And again' },
{ type: 'function_call', id: 'fc_2', call_id: 'call_b', name: 'second', arguments: '{}' },
{ type: 'function_call_output', call_id: 'call_b', output: 'b' },
];
const result = convertInputToMessages(input);
expect(result.map((message) => message.role)).toEqual(['assistant', 'user', 'assistant']);
});
// ── Item references are skipped ────────────────────────────────────
it('skips item_reference items', () => {
const input: InputItem[] = [
{ type: 'item_reference', id: 'ref_123' },
{ type: 'message', role: 'user', content: 'Hello' },
];
const result = convertInputToMessages(input);
expect(result).toEqual([{ role: 'user', content: 'Hello' }]);
});
// ── Multi-turn conversation (the real-world scenario) ──────────────
it('handles a full multi-turn conversation with output_text blocks', () => {
const input: InputItem[] = [
{
type: 'message',
role: 'developer',
content: [{ type: 'input_text', text: 'You are a helpful assistant.' }],
},
{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'What is 2+2?' }],
},
{
type: 'message',
role: 'assistant',
content: [{ type: 'output_text', text: '2+2 is 4.', annotations: [], logprobs: [] }],
},
{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'And 3+3?' }],
},
];
const result = convertInputToMessages(input);
expect(result).toEqual([
{ role: 'system', content: [{ type: 'text', text: 'You are a helpful assistant.' }] },
{ role: 'user', content: [{ type: 'text', text: 'What is 2+2?' }] },
{ role: 'assistant', content: [{ type: 'text', text: '2+2 is 4.' }] },
{ role: 'user', content: [{ type: 'text', text: 'And 3+3?' }] },
]);
});
});
describe('tool call argument attribution', () => {
/** Reproduces the SQL Console construct run: an MCP lookup, then a caller-declared submit_sql. */
it('keeps arguments with their own call when a run makes two calls in separate steps', () => {
const aggregator = createResponseAggregator();
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'toolu_lookup', name: 'list_tables' }],
},
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: {
type: 'tool_calls',
tool_calls: [{ id: 'toolu_lookup', index: 1, args: '{"database":' }],
},
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ index: 1, args: '"default"}' }] },
});
handlers.on_run_step.handle('on_run_step', {
id: 'step_2',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'toolu_submit', name: 'submit_sql' }] },
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_2',
delta: {
type: 'tool_calls',
tool_calls: [{ id: 'toolu_submit', index: 1, args: '{"sql":' }],
},
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_2',
delta: { type: 'tool_calls', tool_calls: [{ index: 1, args: '"SELECT 1"}' }] },
});
expect(aggregator.toolCalls.get('toolu_lookup')?.arguments).toBe('{"database":"default"}');
expect(aggregator.toolCalls.get('toolu_submit')?.arguments).toBe('{"sql":"SELECT 1"}');
});
it('separates two calls made in one step under provider content-block indexes', () => {
const aggregator = createResponseAggregator();
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: {
type: 'tool_calls',
tool_calls: [
{ id: 'toolu_a', name: 'list_tables' },
{ id: 'toolu_b', name: 'submit_sql' },
],
},
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: {
type: 'tool_calls',
tool_calls: [
{ id: 'toolu_a', index: 1, args: '{"a":1}' },
{ id: 'toolu_b', index: 2, args: '{"sql":' },
],
},
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ index: 2, args: '"SELECT 2"}' }] },
});
expect(aggregator.toolCalls.get('toolu_a')?.arguments).toBe('{"a":1}');
expect(aggregator.toolCalls.get('toolu_b')?.arguments).toBe('{"sql":"SELECT 2"}');
});
it('attributes an index-less chunk to the only call in the step', () => {
const aggregator = createResponseAggregator();
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'toolu_only', name: 'submit_sql' }] },
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ args: '{"sql":"SELECT 3"}' }] },
});
expect(aggregator.toolCalls.get('toolu_only')?.arguments).toBe('{"sql":"SELECT 3"}');
});
it('drops an unattributable chunk rather than guessing between two calls', () => {
const aggregator = createResponseAggregator();
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: {
type: 'tool_calls',
tool_calls: [
{ id: 'toolu_a', name: 'list_tables' },
{ id: 'toolu_b', name: 'submit_sql' },
],
},
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ index: 7, args: '{"sql":"nope"}' }] },
});
expect(aggregator.toolCalls.get('toolu_a')?.arguments).toBe('');
expect(aggregator.toolCalls.get('toolu_b')?.arguments).toBe('');
});
it('seeds arguments a provider delivers on the step instead of as deltas', () => {
const aggregator = createResponseAggregator();
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'toolu_whole', name: 'submit_sql', args: '{"sql":"SELECT 4"}' }],
},
});
expect(aggregator.toolCalls.get('toolu_whole')?.arguments).toBe('{"sql":"SELECT 4"}');
});
it('ignores a non-string args object rather than stringifying it', () => {
const aggregator = createResponseAggregator();
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'toolu_obj', name: 'submit_sql', args: {} }],
},
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ id: 'toolu_obj', args: { sql: 'SELECT 5' } }] },
});
expect(aggregator.toolCalls.get('toolu_obj')?.arguments).toBe('');
});
});
describe('tool call arguments from the completed model message', () => {
const SQL = 'SELECT\n created_at\nFROM default.analytics_test_v2\nLIMIT 100';
/**
* The SQL Console construct run: the provider sent list_tables as fragments and submit_sql
* whole, so only the completed message carries the query.
*/
it('fills arguments for a call the provider delivered without fragments', () => {
const aggregator = createResponseAggregator();
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: {
type: 'tool_calls',
tool_calls: [{ id: 'toolu_lookup', name: 'list_tables' }],
},
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: {
type: 'tool_calls',
tool_calls: [{ id: 'toolu_lookup', index: 1, args: '{"database":"default"}' }],
},
});
handlers.on_chat_model_end.handle('on_chat_model_end', {
output: {
tool_calls: [{ id: 'toolu_lookup', args: { database: 'default' } }],
},
});
handlers.on_run_step.handle('on_run_step', {
id: 'step_2',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'toolu_submit', name: 'submit_sql' }] },
});
handlers.on_chat_model_end.handle('on_chat_model_end', {
output: { tool_calls: [{ id: 'toolu_submit', args: { sql: SQL } }] },
});
expect(aggregator.toolCalls.get('toolu_lookup')?.arguments).toBe('{"database":"default"}');
expect(JSON.parse(aggregator.toolCalls.get('toolu_submit')?.arguments ?? '{}')).toEqual({
sql: SQL,
});
});
it('leaves fragment-accumulated arguments untouched', () => {
const aggregator = createResponseAggregator();
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_run_step.handle('on_run_step', {
id: 'step_1',
stepDetails: { type: 'tool_calls', tool_calls: [{ id: 'toolu_a', name: 'submit_sql' }] },
});
handlers.on_run_step_delta.handle('on_run_step_delta', {
id: 'step_1',
delta: { type: 'tool_calls', tool_calls: [{ id: 'toolu_a', args: '{"sql":"SELECT 1"}' }] },
});
handlers.on_chat_model_end.handle('on_chat_model_end', {
output: { tool_calls: [{ id: 'toolu_a', args: { sql: 'SELECT 999' } }] },
});
expect(aggregator.toolCalls.get('toolu_a')?.arguments).toBe('{"sql":"SELECT 1"}');
});
it('ignores completed calls the run never announced, and usage still accumulates', () => {
const aggregator = createResponseAggregator();
const handlers = createAggregatorEventHandlers(aggregator);
handlers.on_chat_model_end.handle('on_chat_model_end', {
output: {
usage_metadata: { input_tokens: 10, output_tokens: 2 },
tool_calls: [{ id: 'toolu_unknown', args: { sql: 'SELECT 1' } }],
},
});
expect(aggregator.toolCalls.size).toBe(0);
expect(aggregator.usage.inputTokens).toBe(10);
});
});
/**
* The continuation a caller-executed tool needs. `previous_response_id` cannot
* carry a tool exchange — a turn is persisted as text, so neither item of the
* pair survives it — which leaves replaying both items in `input` as the only
* supported shape. See the `clientTools` module docstring.
*/
describe('client tool continuation replay', () => {
const CALL_ID = 'call_submit_1';
const TOOL_NAME = 'submit_sql';
const replayInput: InputItem[] = [
{ type: 'message', role: 'user', content: 'Run the query' },
{
type: 'function_call',
id: 'fc_1',
call_id: CALL_ID,
name: TOOL_NAME,
arguments: '{"sql":"SELECT 1"}',
},
{ type: 'function_call_output', call_id: CALL_ID, output: 'submitted' },
];
it('pairs a replayed function_call with its output on one assistant turn', () => {
expect(convertInputToMessages(replayInput)).toEqual([
{ role: 'user', content: 'Run the query' },
{
role: 'assistant',
content: [
{
type: 'tool_call',
tool_call: {
id: CALL_ID,
name: TOOL_NAME,
args: '{"sql":"SELECT 1"}',
output: 'submitted',
},
},
],
},
]);
});
/**
* The conversion is only half the contract: the controller hands these
* messages to `formatAgentMessages`, which reads a tool exchange ONLY from a
* `tool_call` content part. Asserting the converted shape in isolation is
* what let the earlier OpenAI-style `{ role: 'tool' }` message pass its unit
* test and still fail every live replay — `formatMessage` has no branch for
* that role, so it became a SystemMessage mid-conversation and Anthropic
* rejected the payload ("System messages are only permitted as the first
* passed message"). This test formats what the controller formats.
*/
it('formats the replay into a paired tool_use and tool result', () => {
const toolSet = buildRunToolSet({
toolDefinitions: [{ name: TOOL_NAME, description: 'caller-executed', schema: {} }],
} as never);
expect(toolSet.has(TOOL_NAME)).toBe(true);
const { messages } = formatAgentMessages(
convertInputToMessages(replayInput) as never,
{},
toolSet,
undefined,
{ provider: Providers.ANTHROPIC },
);
/** The invariant `_convertMessagesToAnthropicPayload` enforces before it
* throws: nothing past the first message may be a system message. */
expect(messages.slice(1).map((message) => message._getType())).not.toContain('system');
expect(messages.map((message) => message._getType())).toEqual(['human', 'ai', 'tool']);
/** The call rides on the AIMessage as a parsed `tool_calls` entry — the
* provider integration renders it as the tool-use block. */
const toolCalls = (messages[1] as unknown as { tool_calls?: Array<Record<string, unknown>> })
.tool_calls;
expect(toolCalls).toEqual([{ id: CALL_ID, name: TOOL_NAME, args: { sql: 'SELECT 1' } }]);
const toolResult = messages[2] as unknown as { tool_call_id: string; content: unknown };
expect(toolResult.tool_call_id).toBe(CALL_ID);
expect(toolResult.content).toBe('submitted');
});
/**
* The replayed arguments and result moved out of a message's own `content`
* and into a `tool_call` part, and the controller runs the content filter
* over these same messages (`extractMessageContent`) before the run starts.
* Caller-supplied text must stay reachable by that traversal: a shape the
* filter cannot see is a filter that silently stops covering the replay.
*/
it('keeps replayed tool text visible to the content-filter traversal', () => {
const fragments = [...extractMessageContent(convertInputToMessages(replayInput) as never)].map(
(fragment) => fragment.text,
);
expect(fragments).toContain('{"sql":"SELECT 1"}');
expect(fragments).toContain('submitted');
});
/**
* What a caller gets wrong by following the OpenAI habit of sending only the
* output: the tool result arrives with no call to answer. Rejected at ingress
* with a message that names the item, rather than reaching the provider and
* coming back as an opaque upstream failure.
*/
it('rejects a bare function_call_output', () => {
const result = validateResponseRequest({
model: 'agent_test',
input: [{ type: 'function_call_output', call_id: CALL_ID, output: 'submitted' }],
});
expect(result.valid).toBe(false);
expect((result as { error: string }).error).toContain(
`function_call_output ${CALL_ID} has no matching function_call`,
);
});
/** Only the caller can answer a call to its own tool, so only there is the
* missing half the caller's mistake. */
it('rejects a function_call to a declared tool whose output was not replayed', () => {
const result = validateResponseRequest({
model: 'agent_test',
input: replayInput.slice(0, 2),
tools: [{ type: 'function', name: TOOL_NAME, parameters: { type: 'object' } }],
});
expect(result.valid).toBe(false);
expect((result as { error: string }).error).toContain(
`function_call ${CALL_ID} has no function_call_output`,
);
});
/**
* The server emits a `function_call` for its own tools but never a
* `function_call_output`, so appending a previous response's `output` to the
* next request's `input` — the usual continuation — carries calls the caller
* cannot answer. Refusing those would reject a transcript the server itself
* produced.
*/
it('accepts an unanswered call to a tool the caller did not declare', () => {
const result = validateResponseRequest({
model: 'agent_test',
input: replayInput.slice(0, 2),
});
expect(result.valid).toBe(true);
});
it('drops an unanswered server call rather than replaying it with no result', () => {
const messages = convertInputToMessages(replayInput.slice(0, 2));
expect(messages).toEqual([{ role: 'user', content: 'Run the query' }]);
});
it.each([
['a duplicate call_id', [replayInput[1], replayInput[1]], 'duplicate function_call call_id'],
[
'non-string arguments',
[{ type: 'function_call', id: 'fc_1', call_id: CALL_ID, name: TOOL_NAME, arguments: {} }],
'requires arguments as a JSON string',
],
[
'a missing call_id',
[{ type: 'function_call', id: 'fc_1', name: TOOL_NAME, arguments: '{}' }],
'requires a non-empty string call_id',
],
])('rejects %s', (_label, input, expected) => {
const result = validateResponseRequest({ model: 'agent_test', input });
expect(result.valid).toBe(false);
expect((result as { error: string }).error).toContain(expected as string);
});
});
describe('reported tools and tool_choice', () => {
const context: ResponseContext = {
responseId: 'resp_test',
model: 'agent_test',
createdAt: 1778317637,
};
/**
* `tool_choice` is never forwarded to the model, so echoing the request's ask
* would tell the caller a directive was applied when the run ignored it.
*/
it.each([
['required', 'required'],
['none', 'none'],
['a specific function', { type: 'function', name: 'submit_sql' }],
])('reports tool_choice as auto when the request asked for %s', (_label, toolChoice) => {
const { request } = validateResponseRequest({
model: 'agent_test',
input: 'hi',
tool_choice: toolChoice,
});
const built = createResponseContext(request!, 'resp_test');
expect(built).not.toHaveProperty('toolChoice');
expect(buildAggregatedResponse(built, createResponseAggregator()).tool_choice).toBe('auto');
expect(buildResponse(built, createResponseTracker(), 'completed').tool_choice).toBe('auto');
});
it('does not report a request tool until the run resolves which ones applied', () => {
const { request } = validateResponseRequest({
model: 'agent_test',
input: 'hi',
tools: [{ type: 'function', name: 'submit_sql' }],
});
const built = createResponseContext(request!, 'resp_test');
expect(built.tools).toBeUndefined();
expect(buildAggregatedResponse(built, createResponseAggregator()).tools).toEqual([]);
});
it('reports the applied tools the run recorded on the context', () => {
const applied: ResponseContext = {
...context,
tools: [{ type: 'function', name: 'submit_sql' }],
};
expect(buildAggregatedResponse(applied, createResponseAggregator()).tools).toEqual([
{ type: 'function', name: 'submit_sql' },
]);
expect(buildResponse(applied, createResponseTracker(), 'completed').tools).toEqual([
{ type: 'function', name: 'submit_sql' },
]);
});
it('rejects a malformed tools entry at ingress', () => {
expect(
validateResponseRequest({ model: 'agent_test', input: 'hi', tools: ['submit_sql'] }),
).toEqual({ valid: false, error: expect.stringContaining('tools[0] must be an object') });
});
});