Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

add orchestration tests for trueforge-core (#431)

Raman Tehland119b58d9de3
diff --git a/.gitignore b/.gitignore
index a90e1c82d..930b3e1e6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,3 +38,5 @@ service-account*.json
# Local Cursor scratch (committed rules under .cursor/rules/ stay tracked)
.cursor/notes/
.cursor/plans/
+
+.vscode
diff --git a/packages/trueforge-core/tests/orchestration/helpers/helpers.ts b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts
new file mode 100644
index 000000000..e4d40d1af
--- /dev/null
+++ b/packages/trueforge-core/tests/orchestration/helpers/helpers.ts
@@ -0,0 +1,212 @@
+import type { ILLM } from '../../../src/core/llm/ILLM';
+import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../../src/core/llm/LLMTypes';
+import { getEmptyUsage } from '../../../src/core/llm/LLMTypes';
+import type { IToolSet, ToolSource } from '../../../src/core/mcp/IMCPServer';
+import { toolResultResponse } from '../../../src/core/mcp/IMCPServer';
+import { ToolSet } from '../../../src/core/mcp/ToolSet';
+import type {
+ AgentThreadExecutionEvent,
+ AgentThreadExecutionResult,
+ AgentThreadSendBatch,
+} from '../../../src/core/runtime/AgentThread.types';
+import type { AgentThreadOrchestrator } from '../../../src/core/runtime/AgentThreadOrchestrator';
+
+export const WRITE_NOTE_TOOL_NAME = 'write_note';
+export const WRITE_NOTE_CALL_ID = 'call-write';
+export const WRITE_NOTE_ARGUMENTS = JSON.stringify({ text: 'hello' });
+export const WRITE_NOTE_RESULT = 'note written';
+
+/** One streamed chunk plus a stop completion. Used when the test needs a text reply and no tool calls. */
+// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O
+export async function* textReplyStream(
+ text: string,
+): AsyncGenerator<ExtendedChatCompletionChunk, RawAssistantMessageWithUsage, unknown> {
+ yield {
+ id: 'chunk-text',
+ object: 'chat.completion.chunk',
+ created: 0,
+ model: 'test-model',
+ choices: [{ index: 0, delta: { role: 'assistant', content: text }, finish_reason: 'stop' }],
+ };
+ return {
+ output: { role: 'assistant', content: text },
+ usage: getEmptyUsage(),
+ finish_reason: 'stop',
+ };
+}
+
+// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O
+export async function* createSubAgentStream() {
+ yield {
+ id: 'chunk-tool',
+ object: 'chat.completion.chunk',
+ created: 0,
+ model: 'test-model',
+ choices: [
+ {
+ index: 0,
+ delta: {
+ role: 'assistant',
+ tool_calls: [
+ {
+ index: 0,
+ id: 'call-sub',
+ type: 'function',
+ function: {
+ name: 'create_sub_agent',
+ arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task' }),
+ },
+ },
+ ],
+ },
+ finish_reason: 'tool_calls',
+ },
+ ],
+ };
+
+ return {
+ output: {
+ role: 'assistant',
+ content: null,
+ tool_calls: [
+ {
+ id: 'call-sub',
+ type: 'function',
+ function: {
+ name: 'create_sub_agent',
+ arguments: JSON.stringify({ name: 'worker', input: 'do the delegated task' }),
+ },
+ },
+ ],
+ },
+ usage: getEmptyUsage(),
+ finish_reason: 'tool_calls',
+ };
+}
+
+// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O
+export async function* writeNoteToolCallStream() {
+ yield {
+ id: 'chunk-write-note',
+ object: 'chat.completion.chunk',
+ created: 0,
+ model: 'test-model',
+ choices: [
+ {
+ index: 0,
+ delta: {
+ role: 'assistant',
+ tool_calls: [
+ {
+ index: 0,
+ id: WRITE_NOTE_CALL_ID,
+ type: 'function',
+ function: {
+ name: WRITE_NOTE_TOOL_NAME,
+ arguments: WRITE_NOTE_ARGUMENTS,
+ },
+ },
+ ],
+ },
+ finish_reason: 'tool_calls',
+ },
+ ],
+ };
+
+ return {
+ output: {
+ role: 'assistant',
+ content: null,
+ tool_calls: [
+ {
+ id: WRITE_NOTE_CALL_ID,
+ type: 'function',
+ function: {
+ name: WRITE_NOTE_TOOL_NAME,
+ arguments: WRITE_NOTE_ARGUMENTS,
+ },
+ },
+ ],
+ },
+ usage: getEmptyUsage(),
+ finish_reason: 'tool_calls',
+ };
+}
+
+function makeWriteNoteSource(callTool: ToolSource['callTool']): ToolSource {
+ return {
+ name: 'notes',
+ id: 'notes',
+ listTools: () =>
+ Promise.resolve({
+ result: {
+ tools: [
+ {
+ name: WRITE_NOTE_TOOL_NAME,
+ description: 'Write a note',
+ inputSchema: {
+ type: 'object',
+ properties: { text: { type: 'string' } },
+ },
+ preload: true,
+ },
+ ],
+ },
+ wasInitialized: undefined,
+ }),
+ callTool,
+ toolCallInfo: () =>
+ Promise.resolve({
+ type: 'mcp',
+ mcp_server_id: 'notes',
+ mcp_server_name: 'notes',
+ original_tool_name: WRITE_NOTE_TOOL_NAME,
+ }),
+ };
+}
+
+/** Approval-gated write_note tool set; `callTool` spy proves allow runs the source and deny does not. */
+export function makeApprovalGatedWriteNoteToolSet(): {
+ toolSet: IToolSet;
+ callTool: jest.Mock;
+} {
+ const callTool = jest.fn(() => Promise.resolve(toolResultResponse({ text: WRITE_NOTE_RESULT })));
+ return {
+ toolSet: new ToolSet({
+ source: makeWriteNoteSource(callTool),
+ selectors: {
+ enableTools: ['@all'],
+ disableTools: [],
+ preloadTools: [],
+ requireApprovalForTools: [WRITE_NOTE_TOOL_NAME],
+ },
+ preload: true,
+ }),
+ callTool,
+ };
+}
+
+/** Consume send() then execute(); return raw events and the generator result. */
+export async function runTurn(input: {
+ orchestrator: AgentThreadOrchestrator;
+ sendBatch: AgentThreadSendBatch;
+ signal?: AbortSignal | undefined;
+}): Promise<{ events: AgentThreadExecutionEvent[]; result: AgentThreadExecutionResult }> {
+ for await (const _event of input.orchestrator.send(input.sendBatch)) {
+ void _event;
+ }
+ const events: AgentThreadExecutionEvent[] = [];
+ const iterator = input.orchestrator.execute({
+ signal: input.signal ?? new AbortController().signal,
+ });
+ let step = await iterator.next();
+ while (!step.done) {
+ events.push(step.value);
+ step = await iterator.next();
+ }
+ return { events, result: step.value };
+}
+
+export function llmCreateInputs(llm: ILLM): unknown[] {
+ return jest.mocked(llm).create.mock.calls.map(call => call[0]);
+}
diff --git a/packages/trueforge-core/tests/orchestration/orchestration.test.ts b/packages/trueforge-core/tests/orchestration/orchestration.test.ts
new file mode 100644
index 000000000..3dceffddb
--- /dev/null
+++ b/packages/trueforge-core/tests/orchestration/orchestration.test.ts
@@ -0,0 +1,88 @@
+/** One root thread, no tools: user message in, text reply out. */
+import { EventType } from '../../src/core/events/schema';
+import { AgentThread } from '../../src/core/runtime/AgentThread';
+import { InternalEventType } from '../../src/core/runtime/AgentThread.types';
+import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator';
+import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing';
+import { makeSilentLogger } from '../core/harnessMocks';
+import { llmCreateInputs, runTurn, textReplyStream } from './helpers/helpers';
+
+const THREAD_ID = 'main';
+const REPLY = 'hello from the mocked model';
+const INSTRUCTION = 'You are running in a test setup.';
+
+const EXPECTED_EVENTS = [
+ { type: EventType.MODEL_MESSAGE, thread_id: THREAD_ID },
+ { type: EventType.MODEL_MESSAGE_DELTA, thread_id: THREAD_ID, content: REPLY },
+ {
+ type: InternalEventType.AGENT_CONTEXT_APPEND,
+ thread_id: THREAD_ID,
+ context: [{ role: 'assistant', content: REPLY }],
+ },
+ { type: InternalEventType.AGENT_DONE, thread_id: THREAD_ID, status: 'done' },
+];
+
+const OUTPUT = {
+ output: { thread_id: THREAD_ID, content: REPLY },
+ required_actions: [],
+};
+
+const EXPECTED_LLM_INPUT = [
+ {
+ stream: true,
+ messages: [
+ { role: 'system', content: expect.stringContaining(INSTRUCTION) },
+ { role: 'user', content: 'hello' },
+ ],
+ },
+];
+
+describe('orchestration: mocked LLM and no tools', () => {
+ it('sends a user message and finishes the thread with a text reply', async () => {
+ const thread = new AgentThread({
+ definition: {
+ modelClient: {
+ create: jest.fn().mockImplementation(() => textReplyStream(REPLY)),
+ createNonStream: jest.fn().mockImplementation(() => textReplyStream(REPLY)),
+ },
+ instruction: INSTRUCTION,
+ messages: undefined,
+ modelParams: undefined,
+ responseFormat: undefined,
+ iterationLimit: undefined,
+ toolSets: undefined,
+ },
+ threadId: THREAD_ID,
+ title: 'orchestration',
+ parent: undefined,
+ agentInfo: undefined,
+ context: undefined,
+ currentContextUsage: undefined,
+ preComputedCompletion: undefined,
+ sandbox: undefined,
+ capabilities: undefined,
+ capabilityState: undefined,
+ tracing: NOOP_AGENT_TRACING,
+ logger: makeSilentLogger(),
+ });
+
+ // Orchestrator owns the thread map and fans send/execute across live threads.
+ // This case has only the root thread, so sub-agent creation must never run.
+ const orchestrator = new AgentThreadOrchestrator({
+ agentThreads: new Map([[thread.threadId, thread]]),
+ createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in no-tool test')),
+ tracing: NOOP_AGENT_TRACING,
+ logger: makeSilentLogger(),
+ });
+
+ const { events, result } = await runTurn({
+ orchestrator,
+ sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
+ });
+
+ expect(events).toMatchObject(EXPECTED_EVENTS);
+ expect(result).toMatchObject(OUTPUT);
+ expect(result.root_agent_error).toBeUndefined();
+ expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_LLM_INPUT);
+ });
+});
diff --git a/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts
new file mode 100644
index 000000000..eff7f395a
--- /dev/null
+++ b/packages/trueforge-core/tests/orchestration/orchestrationApproval.test.ts
@@ -0,0 +1,293 @@
+/** Pause on write_note approval, then resume after allow or deny. */
+import { EventType } from '../../src/core/events/schema';
+import { AgentThread } from '../../src/core/runtime/AgentThread';
+import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types';
+import { AgentThreadOrchestrator } from '../../src/core/runtime/AgentThreadOrchestrator';
+import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing';
+import { makeSilentLogger } from '../core/harnessMocks';
+import {
+ llmCreateInputs,
+ makeApprovalGatedWriteNoteToolSet,
+ runTurn,
+ textReplyStream,
+ WRITE_NOTE_ARGUMENTS,
+ WRITE_NOTE_CALL_ID,
+ WRITE_NOTE_RESULT,
+ WRITE_NOTE_TOOL_NAME,
+ writeNoteToolCallStream,
+} from './helpers/helpers';
+
+const ROOT_ID = 'thread_root';
+const DENY_REASON = 'not allowed in this test';
+/** ToolSet deny → isError path wraps the text payload again for context. */
+const DENY_TOOL_CONTENT = JSON.stringify({
+ error: [{ type: 'text', text: JSON.stringify({ error: `User denied tool call: ${DENY_REASON}` }) }],
+});
+const INSTRUCTION = 'You are running in a test setup.';
+
+const WRITE_NOTE_TOOLS = [{ function: { name: WRITE_NOTE_TOOL_NAME } }];
+
+const EXPECTED_TURN_1_EVENTS = [
+ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID },
+ { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID },
+ {
+ type: InternalEventType.AGENT_CONTEXT_APPEND,
+ thread_id: ROOT_ID,
+ context: [
+ {
+ role: 'assistant',
+ content: null,
+ tool_calls: [
+ {
+ id: WRITE_NOTE_CALL_ID,
+ type: 'function',
+ function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS },
+ },
+ ],
+ },
+ ],
+ },
+ {
+ type: EventType.TOOL_APPROVAL_REQUIRED,
+ thread_id: ROOT_ID,
+ tool_calls: [{ id: WRITE_NOTE_CALL_ID }],
+ },
+];
+
+const TURN_1_OUTPUT = {
+ output: null,
+ required_actions: [
+ {
+ type: EventType.TOOL_APPROVAL_REQUIRED,
+ thread_id: ROOT_ID,
+ tool_calls: [{ id: WRITE_NOTE_CALL_ID }],
+ },
+ ],
+};
+
+const EXPECTED_TURN_1_LLM_INPUT = [
+ {
+ tools: WRITE_NOTE_TOOLS,
+ messages: [
+ { role: 'system', content: expect.stringContaining(INSTRUCTION) },
+ { role: 'user', content: 'hello' },
+ ],
+ },
+];
+
+describe('orchestration: pause then resume on tool approval', () => {
+ describe('allow', () => {
+ const ROOT_FINAL = 'note saved';
+
+ const EXPECTED_TURN_2_EVENTS = [
+ { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID },
+ {
+ type: InternalEventType.AGENT_CONTEXT_APPEND,
+ thread_id: ROOT_ID,
+ context: [{ role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT }],
+ },
+ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID },
+ { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL },
+ {
+ type: InternalEventType.AGENT_CONTEXT_APPEND,
+ thread_id: ROOT_ID,
+ context: [{ role: 'assistant', content: ROOT_FINAL }],
+ },
+ { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' },
+ ];
+
+ const TURN_2_OUTPUT = {
+ output: { thread_id: ROOT_ID, content: ROOT_FINAL },
+ required_actions: [],
+ };
+
+ const EXPECTED_TURN_2_INPUT = {
+ tools: WRITE_NOTE_TOOLS,
+ messages: [
+ { role: 'system', content: expect.stringContaining(INSTRUCTION) },
+ { role: 'user', content: 'hello' },
+ {
+ role: 'assistant',
+ content: null,
+ tool_calls: [
+ {
+ id: WRITE_NOTE_CALL_ID,
+ type: 'function',
+ function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS },
+ },
+ ],
+ },
+ { role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: WRITE_NOTE_RESULT },
+ ],
+ };
+
+ it('pauses for write_note approval, then finishes after allow', async () => {
+ const { orchestrator, thread, callTool } = makeApprovalHarness(ROOT_FINAL);
+
+ const paused = await runTurn({
+ orchestrator,
+ sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
+ });
+ expect(paused.events).toMatchObject(EXPECTED_TURN_1_EVENTS);
+ expect(paused.result).toMatchObject(TURN_1_OUTPUT);
+ expect(paused.result.root_agent_error).toBeUndefined();
+ expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject(EXPECTED_TURN_1_LLM_INPUT);
+ expect(callTool).not.toHaveBeenCalled();
+
+ const resumed = await runTurn({
+ orchestrator,
+ sendBatch: [
+ {
+ type: EventType.USER_TOOL_APPROVAL,
+ thread_id: ROOT_ID,
+ tool_call_id: WRITE_NOTE_CALL_ID,
+ approval: { status: 'allow' },
+ },
+ ],
+ });
+ expect(resumed.events).toMatchObject(EXPECTED_TURN_2_EVENTS);
+ expect(resumed.result).toMatchObject(TURN_2_OUTPUT);
+ expect(resumed.result.root_agent_error).toBeUndefined();
+ expect(callTool).toHaveBeenCalledTimes(1);
+ expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject([
+ ...EXPECTED_TURN_1_LLM_INPUT,
+ EXPECTED_TURN_2_INPUT,
+ ]);
+ });
+ });
+
+ describe('deny', () => {
+ const ROOT_FINAL = 'ok, I will not write the note';
+
+ const EXPECTED_TURN_2_EVENTS = [
+ { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: WRITE_NOTE_CALL_ID },
+ {
+ type: InternalEventType.AGENT_CONTEXT_APPEND,
+ thread_id: ROOT_ID,
+ context: [{ role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: DENY_TOOL_CONTENT }],
+ },
+ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID },
+ { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL },
+ {
+ type: InternalEventType.AGENT_CONTEXT_APPEND,
+ thread_id: ROOT_ID,
+ context: [{ role: 'assistant', content: ROOT_FINAL }],
+ },
+ { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' },
+ ];
+
+ const TURN_2_OUTPUT = {
+ output: { thread_id: ROOT_ID, content: ROOT_FINAL },
+ required_actions: [],
+ };
+
+ const EXPECTED_TURN_2_INPUT = {
+ tools: WRITE_NOTE_TOOLS,
+ messages: [
+ { role: 'system', content: expect.stringContaining(INSTRUCTION) },
+ { role: 'user', content: 'hello' },
+ {
+ role: 'assistant',
+ content: null,
+ tool_calls: [
+ {
+ id: WRITE_NOTE_CALL_ID,
+ type: 'function',
+ function: { name: WRITE_NOTE_TOOL_NAME, arguments: WRITE_NOTE_ARGUMENTS },
+ },
+ ],
+ },
+ { role: 'tool', tool_call_id: WRITE_NOTE_CALL_ID, content: DENY_TOOL_CONTENT },
+ ],
+ };
+
+ it('pauses for write_note approval, then finishes after deny without running the tool', async () => {
+ const { orchestrator, thread, callTool } = makeApprovalHarness(ROOT_FINAL);
+
+ const paused = await runTurn({
+ orchestrator,
+ sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
+ });
+ expect(paused.events).toMatchObject(EXPECTED_TURN_1_EVENTS);
+ expect(paused.result).toMatchObject(TURN_1_OUTPUT);
+ expect(callTool).not.toHaveBeenCalled();
+
+ // Deny is a new turn (send + execute), same as allow — turn 1 already stopped at approval.
+ const resumed = await runTurn({
+ orchestrator,
+ sendBatch: [
+ {
+ type: EventType.USER_TOOL_APPROVAL,
+ thread_id: ROOT_ID,
+ tool_call_id: WRITE_NOTE_CALL_ID,
+ approval: { status: 'deny', reason: DENY_REASON },
+ },
+ ],
+ });
+ expect(resumed.events).toMatchObject(EXPECTED_TURN_2_EVENTS);
+ expect(resumed.result).toMatchObject(TURN_2_OUTPUT);
+ expect(resumed.result.root_agent_error).toBeUndefined();
+ expect(callTool).not.toHaveBeenCalled();
+ expect(llmCreateInputs(thread.definition.modelClient)).toMatchObject([
+ ...EXPECTED_TURN_1_LLM_INPUT,
+ EXPECTED_TURN_2_INPUT,
+ ]);
+ });
+ });
+});
+
+function makeApprovalHarness(finalReply: string): {
+ orchestrator: AgentThreadOrchestrator;
+ thread: AgentThread;
+ callTool: jest.Mock;
+} {
+ const { toolSet, callTool } = makeApprovalGatedWriteNoteToolSet();
+ const agentThreadInput: AgentThreadConstructorInput = {
+ definition: {
+ modelClient: {
+ create: jest
+ .fn()
+ .mockImplementationOnce(() => writeNoteToolCallStream())
+ .mockImplementation(() => textReplyStream(finalReply)),
+ createNonStream: jest.fn(),
+ },
+ instruction: INSTRUCTION,
+ messages: undefined,
+ modelParams: undefined,
+ responseFormat: undefined,
+ iterationLimit: undefined,
+ toolSets: undefined,
+ },
+ threadId: ROOT_ID,
+ title: 'orchestration-approval',
+ parent: undefined,
+ agentInfo: undefined,
+ context: undefined,
+ currentContextUsage: undefined,
+ preComputedCompletion: undefined,
+ sandbox: undefined,
+ capabilities: [
+ {
+ systemToolSets: [toolSet],
+ preSendProcessors: undefined,
+ preLLMProcessors: undefined,
+ preLLMEphemeralProcessors: undefined,
+ postToolCallProcessors: undefined,
+ toolResponseProcessors: undefined,
+ instructionBuilders: undefined,
+ },
+ ],
+ capabilityState: undefined,
+ tracing: NOOP_AGENT_TRACING,
+ logger: makeSilentLogger(),
+ };
+
+ const thread = new AgentThread(agentThreadInput);
+ const orchestrator = new AgentThreadOrchestrator({
+ agentThreads: new Map([[thread.threadId, thread]]),
+ createDynamicSubAgentThread: () => Promise.reject(new Error('unexpected sub-agent in approval test')),
+ tracing: NOOP_AGENT_TRACING,
+ logger: makeSilentLogger(),
+ });
+ return { orchestrator, thread, callTool };
+}
diff --git a/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts
new file mode 100644
index 000000000..6db326c38
--- /dev/null
+++ b/packages/trueforge-core/tests/orchestration/orchestrationWithTools.test.ts
@@ -0,0 +1,210 @@
+import type { AgentDefinition, CreateDynamicSubAgentThread } from '../../src/core';
+import { dynamicSubAgents } from '../../src/core/capabilities/builtins/DynamicSubAgents';
+import { EventType } from '../../src/core/events/schema';
+import type { ILLM } from '../../src/core/llm/ILLM';
+import { AgentThread } from '../../src/core/runtime/AgentThread';
+import { InternalEventType, type AgentThreadConstructorInput } from '../../src/core/runtime/AgentThread.types';
+import {
+ AgentThreadOrchestrator,
+ type AgentThreadOrchestratorInput,
+} from '../../src/core/runtime/AgentThreadOrchestrator';
+import { NOOP_AGENT_TRACING } from '../../src/core/tracing/NoopAgentTracing';
+import { makeSilentLogger } from '../core/harnessMocks';
+import { createSubAgentStream, llmCreateInputs, runTurn, textReplyStream } from './helpers/helpers';
+
+const ROOT_ID = 'thread_root';
+const TOOL_CALL_ID = 'call-sub';
+const CHILD_REPLY = 'hello from the child';
+const ROOT_FINAL = 'How are you?';
+const INSTRUCTION = 'You are running in a test setup.';
+const CHILD_TASK = 'do the delegated task';
+
+const CREATE_SUB_AGENT_ARGS = JSON.stringify({ name: 'worker', input: CHILD_TASK });
+
+/** Root delegates via create_sub_agent; child result returns to parent; root finishes. */
+const EXPECTED_EVENTS = [
+ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID },
+ { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID },
+ {
+ type: InternalEventType.AGENT_CONTEXT_APPEND,
+ thread_id: ROOT_ID,
+ context: [
+ {
+ role: 'assistant',
+ content: null,
+ tool_calls: [
+ {
+ id: TOOL_CALL_ID,
+ type: 'function',
+ function: { name: 'create_sub_agent', arguments: CREATE_SUB_AGENT_ARGS },
+ },
+ ],
+ },
+ ],
+ },
+ // create_sub_agent tool path yields an empty append before THREAD_CREATED.
+ { type: InternalEventType.AGENT_CONTEXT_APPEND, thread_id: ROOT_ID, context: [] },
+ {
+ type: EventType.THREAD_CREATED,
+ title: 'worker',
+ parent: { thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID },
+ },
+ { type: EventType.MODEL_MESSAGE, thread_id: expect.any(String) },
+ { type: EventType.MODEL_MESSAGE_DELTA, thread_id: expect.any(String), content: CHILD_REPLY },
+ {
+ type: InternalEventType.AGENT_CONTEXT_APPEND,
+ thread_id: expect.any(String),
+ context: [{ role: 'assistant', content: CHILD_REPLY }],
+ },
+ { type: EventType.TOOL_RESPONSE, thread_id: ROOT_ID, tool_call_id: TOOL_CALL_ID },
+ {
+ type: InternalEventType.AGENT_CONTEXT_APPEND,
+ thread_id: ROOT_ID,
+ context: [{ role: 'tool', tool_call_id: TOOL_CALL_ID, content: CHILD_REPLY }],
+ },
+ { type: InternalEventType.AGENT_DONE, thread_id: expect.any(String), status: 'done' },
+ { type: EventType.MODEL_MESSAGE, thread_id: ROOT_ID },
+ { type: EventType.MODEL_MESSAGE_DELTA, thread_id: ROOT_ID, content: ROOT_FINAL },
+ {
+ type: InternalEventType.AGENT_CONTEXT_APPEND,
+ thread_id: ROOT_ID,
+ context: [{ role: 'assistant', content: ROOT_FINAL }],
+ },
+ { type: InternalEventType.AGENT_DONE, thread_id: ROOT_ID, status: 'done' },
+];
+
+const OUTPUT = {
+ output: { thread_id: ROOT_ID, content: ROOT_FINAL },
+ required_actions: [],
+};
+
+const EXPECTED_ROOT_LLM_INPUT = [
+ {
+ messages: [
+ { role: 'system', content: expect.stringContaining(INSTRUCTION) },
+ { role: 'user', content: 'hello' },
+ ],
+ },
+ {
+ messages: [
+ { role: 'system', content: expect.stringContaining(INSTRUCTION) },
+ { role: 'user', content: 'hello' },
+ {
+ role: 'assistant',
+ content: null,
+ tool_calls: [
+ {
+ id: TOOL_CALL_ID,
+ type: 'function',
+ function: {
+ name: 'create_sub_agent',
+ arguments: CREATE_SUB_AGENT_ARGS,
+ },
+ },
+ ],
+ },
+ { role: 'tool', tool_call_id: TOOL_CALL_ID, content: CHILD_REPLY },
+ ],
+ },
+];
+
+describe('orchestration: dynamic sub-agent', () => {
+ it('delegates via create_sub_agent, routes child result to parent, then finishes', async () => {
+ let agentThreadInput: AgentThreadConstructorInput = {
+ // AgentDefinition
+ definition: {
+ // This is an instance if ILLM
+ modelClient: {
+ create: jest
+ .fn()
+ .mockImplementationOnce(() => createSubAgentStream())
+ .mockImplementation(() => textReplyStream(ROOT_FINAL)),
+ createNonStream: jest.fn(),
+ },
+ instruction: INSTRUCTION,
+ // Undefined
+ messages: undefined,
+ modelParams: undefined,
+ responseFormat: undefined,
+ iterationLimit: undefined,
+ toolSets: undefined,
+ },
+ threadId: ROOT_ID,
+ title: 'orchestration-with-tools',
+ // Undefined
+ parent: undefined,
+ agentInfo: undefined,
+ context: undefined,
+ currentContextUsage: undefined,
+ preComputedCompletion: undefined,
+ sandbox: undefined,
+ capabilities: [dynamicSubAgents({ sandboxAvailable: false, tracing: NOOP_AGENT_TRACING })],
+ capabilityState: undefined,
+ // Default
+ tracing: NOOP_AGENT_TRACING,
+ logger: makeSilentLogger(),
+ };
+
+ let thread_1 = new AgentThread(agentThreadInput);
+ let childLLM: ILLM | undefined;
+
+ const createSubAgentThread: CreateDynamicSubAgentThread = async ({
+ parentDefinition,
+ request,
+ threadId,
+ parent,
+ }) => {
+ childLLM = {
+ create: jest.fn().mockImplementation(() => textReplyStream(CHILD_REPLY)),
+ createNonStream: jest.fn().mockImplementation(() => textReplyStream(CHILD_REPLY)),
+ };
+
+ const agentDefinition: AgentDefinition = {
+ modelClient: childLLM,
+ instruction: undefined,
+ messages: [{ role: 'user', content: request.input }],
+ modelParams: parentDefinition.modelParams,
+ responseFormat: undefined,
+ iterationLimit: parentDefinition.iterationLimit,
+ toolSets: undefined,
+ };
+ return new AgentThread({
+ definition: agentDefinition,
+ threadId,
+ title: request.name,
+ parent,
+ agentInfo: request,
+ context: undefined,
+ currentContextUsage: undefined,
+ preComputedCompletion: undefined,
+ sandbox: undefined,
+ capabilities: undefined,
+ capabilityState: undefined,
+ tracing: NOOP_AGENT_TRACING,
+ logger: makeSilentLogger(),
+ });
+ };
+
+ let orchestratorInput: AgentThreadOrchestratorInput = {
+ agentThreads: new Map([[thread_1.threadId, thread_1]]),
+ createDynamicSubAgentThread: createSubAgentThread,
+ tracing: NOOP_AGENT_TRACING,
+ logger: makeSilentLogger(),
+ };
+
+ const orchestrator = new AgentThreadOrchestrator(orchestratorInput);
+
+ const { events, result } = await runTurn({
+ orchestrator,
+ sendBatch: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
+ });
+
+ expect(events).toMatchObject(EXPECTED_EVENTS);
+ expect(result).toMatchObject(OUTPUT);
+ expect(result.root_agent_error).toBeUndefined();
+ expect(llmCreateInputs(thread_1.definition.modelClient)).toMatchObject(EXPECTED_ROOT_LLM_INPUT);
+ if (childLLM === undefined) {
+ throw new Error('expected child LLM to be created');
+ }
+ });
+});