Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

19.8 KB
Raw Blame
import { describe, expect, it } from 'vitest';

import type { ToolTimelineEntry } from '../../store/chatRuntimeSlice';
import type { PersistedTranscriptItem } from '../../types/turnState';
import {
  buildProcessingBlocks,
  categorizeTool,
  extractAgentSources,
  extractSearchProvider,
  formatTimelineEntry,
  formatToolName,
  isKnownClientTool,
  stripToolCallEnvelopes,
  summarizeToolGroup,
} from '../toolTimelineFormatting';

function entry(overrides: Partial<ToolTimelineEntry>): ToolTimelineEntry {
  return { id: 'x', name: 'delegate_notion', round: 1, seq: 0, status: 'running', ...overrides };
}

describe('formatTimelineEntry', () => {
  it('falls back to the exact tool_call spec for a malformed bridge name', () => {
    // The bridge unwrap (rule 1) only fires for a string `args.name`; a
    // malformed bridge call (name not a string) falls through to the exact
    // `tool_call` spec (rule 4) instead of the bare "used a tool" fallback,
    // and shows the malformed value verbatim rather than hiding it.
    expect(
      formatTimelineEntry(entry({ name: 'tool_call', argsBuffer: JSON.stringify({ name: 42 }) }))
    ).toEqual({ title: 'Using tools', detail: '42' });
  });
  it('formats integration delegation tools with a user-facing provider label', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'delegate_notion',
          argsBuffer: JSON.stringify({ prompt: 'Find the project brief in Notion.' }),
        })
      )
    ).toEqual({ title: 'Using Notion', detail: 'Find the project brief in Notion.' });
  });

  it('formats spawn_subagent for integrations_agent from toolkit args', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'spawn_subagent',
          argsBuffer: JSON.stringify({
            agent_id: 'integrations_agent',
            prompt:
              'Get my 5 most recent emails. Show subject, sender, date, and a short preview for each.',
            toolkit: 'gmail',
          }),
        })
      )
    ).toEqual({
      title: 'Using Gmail',
      detail:
        'Get my 5 most recent emails. Show subject, sender, date, and a short preview for each.',
    });
  });

  it('formats spawned integration agents with the inherited prompt', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'subagent:integrations_agent',
          sourceToolName: 'delegate_notion',
          detail: 'Search Notion for the latest roadmap.',
        })
      )
    ).toEqual({ title: 'Using Notion', detail: 'Search Notion for the latest roadmap.' });
  });

  it('labels a direct connected-service action by its provider', () => {
    expect(
      formatTimelineEntry(
        entry({ name: 'GMAIL_SEND_EMAIL', argsBuffer: JSON.stringify({ to: '[email protected]' }) })
      )
    ).toEqual({ title: 'Using Gmail', detail: 'Send email · [email protected]' });
    expect(formatTimelineEntry(entry({ name: 'GOOGLE_CALENDAR_CREATE_EVENT' }))).toEqual({
      title: 'Using Google Calendar',
      detail: 'Create event',
    });
  });

  it('never shouts an upper-case action slug, known toolkit or not', () => {
    expect(formatTimelineEntry(entry({ name: 'STRIPE_LIST_CHARGES' }))).toEqual({
      title: 'Using Stripe',
      detail: 'List charges',
    });
    expect(formatTimelineEntry(entry({ name: 'ACME_DO_THING' }))).toEqual({
      title: 'Using Acme',
      detail: 'Do thing',
    });
  });

  it('formats delegate_tools_agent with toolkit context from args', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'delegate_tools_agent',
          argsBuffer: JSON.stringify({
            toolkit: 'github',
            prompt: 'List my open pull requests in GitHub.',
          }),
        })
      )
    ).toEqual({ title: 'Using GitHub', detail: 'List my open pull requests in GitHub.' });
  });

  it('falls back to humanized generic labels for non-integration subagents', () => {
    expect(formatTimelineEntry(entry({ name: 'subagent:orchestrator' }))).toEqual({
      title: 'Planning next steps',
      detail: undefined,
    });
  });

  it('formats composio_list_connections with user-facing copy', () => {
    expect(formatTimelineEntry(entry({ name: 'composio_list_connections' }))).toEqual({
      title: 'Checking your connections',
      detail: undefined,
    });
  });

  it('formats shell tool with truncated command detail', () => {
    expect(
      formatTimelineEntry(
        entry({ name: 'shell', argsBuffer: JSON.stringify({ command: 'cargo test --lib' }) })
      )
    ).toEqual({ title: 'Running command', detail: 'cargo test --lib' });
  });

  it('formats web_fetch with the page as detail', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'web_fetch',
          argsBuffer: JSON.stringify({ url: 'https://docs.example.com/api/v2/users' }),
        })
      )
    ).toEqual({ title: 'Reading webpage', detail: 'docs.example.com/api/v2/users' });
  });

  it('formats web_search with the query as detail', () => {
    expect(
      formatTimelineEntry(
        entry({ name: 'web_search', argsBuffer: JSON.stringify({ query: 'rust async trait' }) })
      )
    ).toEqual({ title: 'Searching the web', detail: 'rust async trait' });
  });

  it('settles a completed web_search into the past tense (provider shows in the search element)', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'web_search',
          status: 'success',
          argsBuffer: JSON.stringify({ query: 'rust async trait' }),
          result: 'Search results for: rust async trait (via Exa)\n1. Some title\n   https://x.dev',
        })
      )
    ).toEqual({ title: 'Searched the web', detail: 'rust async trait' });
  });

  it('settles whichever provider served the search', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'web_search',
          status: 'success',
          argsBuffer: JSON.stringify({ query: 'weather' }),
          result: 'Search results for: weather (via Brave)\n1. Forecast',
        })
      )
    ).toEqual({ title: 'Searched the web', detail: 'weather' });
  });

  it('keeps the running label when no result is present yet', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'web_search',
          status: 'running',
          argsBuffer: JSON.stringify({ query: 'rust async trait' }),
        })
      )
    ).toEqual({ title: 'Searching the web', detail: 'rust async trait' });
  });

  // `web_search_tool` is the name the core actually registers and streams for
  // the canonical search slot; `web_search` is only the settings-family id.
  // Without this the real production row fell through to "Web Search Tool".
  it('formats the streamed web_search_tool name while running', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'web_search_tool',
          status: 'running',
          argsBuffer: JSON.stringify({ query: 'rust async trait' }),
        })
      )
    ).toEqual({ title: 'Searching the web', detail: 'rust async trait' });
  });

  it('attributes a completed web_search_tool from the markdown result', () => {
    // Production renders tool results as markdown (`output_for_llm(true)`),
    // so the marker arrives on the markdown heading line.
    expect(
      formatTimelineEntry(
        entry({
          name: 'web_search_tool',
          status: 'success',
          argsBuffer: JSON.stringify({ query: 'rust async trait' }),
          result: '# Search results — `rust async trait` (via Exa)\n\n## [T](https://x.dev)',
        })
      )
    ).toEqual({ title: 'Searched the web', detail: 'rust async trait' });
  });

  it('attributes a completed web_search_tool that returned no results', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'web_search_tool',
          status: 'success',
          argsBuffer: JSON.stringify({ query: 'zzzz' }),
          result: '_No results for `zzzz`_ (via Exa)',
        })
      )
    ).toEqual({ title: 'Searched the web', detail: 'zzzz' });
  });

  it('formats file_read with shortened path', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'file_read',
          argsBuffer: JSON.stringify({ path: 'crates/openhuman-core/src/agent/progress.rs' }),
        })
      )
    ).toEqual({ title: 'Reading file', detail: '…/agent/progress.rs' });
  });

  it('formats edit tool with file path', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'edit',
          argsBuffer: JSON.stringify({ file_path: 'app/src/components/App.tsx' }),
        })
      )
    ).toEqual({ title: 'Editing file', detail: '…/components/App.tsx' });
  });

  it('formats grep with pattern in title', () => {
    expect(
      formatTimelineEntry(
        entry({ name: 'grep', argsBuffer: JSON.stringify({ pattern: 'SubagentSpawned' }) })
      )
    ).toEqual({ title: 'Searching code', detail: 'SubagentSpawned' });
  });

  it('formats git_operations with subcommand', () => {
    expect(
      formatTimelineEntry(
        entry({ name: 'git_operations', argsBuffer: JSON.stringify({ command: 'diff --stat' }) })
      )
    ).toEqual({ title: 'Running git', detail: 'diff --stat' });
  });

  it('formats glob with pattern detail', () => {
    expect(
      formatTimelineEntry(
        entry({ name: 'glob', argsBuffer: JSON.stringify({ pattern: '**/*.test.ts' }) })
      )
    ).toEqual({ title: 'Finding files', detail: '**/*.test.ts' });
  });

  it('formats list with directory path', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'list',
          argsBuffer: JSON.stringify({ path: 'crates/openhuman-core/src/tools' }),
        })
      )
    ).toEqual({ title: 'Listing folder', detail: '…/src/tools' });
  });

  it('formats browser_open with the page as detail', () => {
    expect(
      formatTimelineEntry(
        entry({
          name: 'browser_open',
          argsBuffer: JSON.stringify({ url: 'https://github.com/tinyhumansai/openhuman' }),
        })
      )
    ).toEqual({ title: 'Opening page', detail: 'github.com/tinyhumansai/openhuman' });
  });
});

describe('extractSearchProvider', () => {
  it('reads the provider from a `(via …)` marker', () => {
    expect(extractSearchProvider('Search results for: q (via Exa)\n1. foo')).toBe('Exa');
    expect(extractSearchProvider('Search results for: q (via Brave)')).toBe('Brave');
    expect(extractSearchProvider('# Search results — `q` (via Querit)')).toBe('Querit');
    expect(extractSearchProvider('# Search results -- `q` (via Tavily)')).toBe('Tavily');
  });

  it('returns undefined when there is no marker or no result', () => {
    expect(extractSearchProvider(undefined)).toBeUndefined();
    expect(extractSearchProvider('')).toBeUndefined();
    expect(extractSearchProvider('No results found for: q')).toBeUndefined();
  });

  it('trims surrounding whitespace in the provider name', () => {
    expect(extractSearchProvider('foo (via  Exa )')).toBe('Exa');
  });

  it('ignores a `(via …)` string that appears only in a result excerpt', () => {
    expect(
      extractSearchProvider('Search results for: q\n1. Title\n   Booked (via SomeAirline) today.')
    ).toBeUndefined();
  });

  it('ignores an implausibly long marker', () => {
    expect(extractSearchProvider(`Search results for: q (via ${'x'.repeat(64)})`)).toBeUndefined();
  });

  it('reads the trailing marker when the echoed query also contains one', () => {
    // The heading echoes the user's query, so a query like `login (via OAuth)`
    // puts a decoy marker ahead of the real one. Only the trailing marker counts.
    expect(extractSearchProvider('Search results for: login (via OAuth) (via Exa)')).toBe('Exa');
  });

  it('reads the marker from an empty-result heading', () => {
    expect(extractSearchProvider('No results found for: q (via Exa)')).toBe('Exa');
    expect(extractSearchProvider('_No results for `q`_ (via Brave)')).toBe('Brave');
    expect(extractSearchProvider('_No results for `q`_ (via Tavily)')).toBe('Tavily');
  });
});

describe('formatToolName', () => {
  it('returns human-readable names for known tools', () => {
    expect(formatToolName('shell')).toBe('Running command');
    expect(formatToolName('web_fetch')).toBe('Reading webpage');
    expect(formatToolName('file_read')).toBe('Reading file');
    expect(formatToolName('edit')).toBe('Editing file');
    expect(formatToolName('grep')).toBe('Searching code');
    expect(formatToolName('git_operations')).toBe('Running git');
    expect(formatToolName('lsp')).toBe('Analyzing code');
  });

  it('falls back to a sentence-cased activity for unknown tools', () => {
    expect(formatToolName('custom_fancy_tool')).toBe('Using custom fancy tool');
  });
});

describe('stripToolCallEnvelopes', () => {
  it('removes a complete <tool_call>…</tool_call> envelope, keeping the prose', () => {
    const out = stripToolCallEnvelopes(
      'Searching now. <tool_call> {"name": "NOTION_SEARCH", "arguments": {"q": "x"}} </tool_call> done.'
    );
    expect(out).toContain('Searching now.');
    expect(out).toContain('done.');
    expect(out).not.toContain('tool_call');
    expect(out).not.toContain('NOTION_SEARCH');
  });

  it('removes a trailing, still-streaming unclosed <tool_call>…', () => {
    const out = stripToolCallEnvelopes('Let me check. <tool_call> {"name": "X", "argum');
    expect(out.trim()).toBe('Let me check.');
    expect(out).not.toContain('tool_call');
  });

  it('leaves text without an envelope untouched', () => {
    expect(stripToolCallEnvelopes('just a normal sentence')).toBe('just a normal sentence');
  });
});

describe('isKnownClientTool', () => {
  it('recognizes built-ins and special agent rows', () => {
    expect(isKnownClientTool('file_read')).toBe(true);
    expect(isKnownClientTool('shell')).toBe(true);
    expect(isKnownClientTool('subagent:orchestrator')).toBe(true);
    // The streamed search-slot name, so the client label wins over the
    // server's humanized "Web Search Tool".
    expect(isKnownClientTool('web_search_tool')).toBe(true);
  });

  it('recognizes Composio actions by their toolkit, and nothing it cannot describe', () => {
    // The registry names a Composio action by its app ("Used Gmail"), so the
    // core's sentence-cased slug does not override it.
    expect(isKnownClientTool('GMAIL_SEND_EMAIL')).toBe(true);
    expect(isKnownClientTool('some_random_mcp_tool')).toBe(false);
  });
});

describe('summarizeToolGroup', () => {
  it('uses the specific label for a single row', () => {
    expect(summarizeToolGroup([entry({ name: 'file_read', argsBuffer: '{"path":"a.ts"}' })])).toBe(
      'Reading file'
    );
  });

  it('counts a homogeneous group', () => {
    expect(
      summarizeToolGroup([
        entry({ id: 'a', name: 'file_read' }),
        entry({ id: 'b', name: 'file_read' }),
      ])
    ).toBe('2 steps · Read file ×2');
  });

  it('lists the distinct steps of a mixed group, most frequent first', () => {
    expect(
      summarizeToolGroup([
        entry({ id: 'a', name: 'file_write' }),
        entry({ id: 'b', name: 'shell' }),
        entry({ id: 'c', name: 'shell' }),
      ])
    ).toBe('3 steps · Ran command ×2, Wrote file');
  });
});

describe('categorizeTool', () => {
  it('maps tools (incl. subagent-prefixed) to a category', () => {
    expect(categorizeTool('grep')).toBe('code');
    expect(categorizeTool('subagent:web_fetch')).toBe('web');
    expect(categorizeTool('GMAIL_SEND_EMAIL')).toBe('app');
  });

  it('categorizes the canonical web-search name, not only its settings id', () => {
    // `web_search` is the UI toggle id; the core expands it to `web_search_tool`
    // (`crates/openhuman-core/src/tools/user_filter.rs:79-80`), and that is the name a
    // timeline row actually carries. The rest of this file already special-cased
    // the canonical name for labels and provider attribution; the category map
    // was the one place that had not, so a real search row categorized as
    // `other` — wrong icon and wrong group summary in the rail, and (since
    // #6169) a row kept on the main transcript that belongs in the rail.
    expect(categorizeTool('web_search_tool')).toBe('web');
    expect(categorizeTool('web_search')).toBe('web');
    expect(categorizeTool('subagent:web_search_tool')).toBe('web');
  });
});

describe('buildProcessingBlocks', () => {
  const tx = (over: Partial<PersistedTranscriptItem> & { kind: PersistedTranscriptItem['kind'] }) =>
    ({ round: 1, seq: 0, ...over }) as PersistedTranscriptItem;

  it('interleaves prose with grouped consecutive tool rows in seq order', () => {
    const entries = [
      entry({ id: 'c1', name: 'file_read' }),
      entry({ id: 'c2', name: 'file_read' }),
    ];
    const transcript: PersistedTranscriptItem[] = [
      tx({ kind: 'thinking', seq: 0, text: 'Let me look' }),
      tx({ kind: 'narration', seq: 1, text: 'Reading the files' }),
      tx({ kind: 'toolCall', seq: 2, callId: 'c1' }),
      tx({ kind: 'toolCall', seq: 3, callId: 'c2' }),
      tx({ kind: 'narration', seq: 4, text: 'Done' }),
    ];
    const blocks = buildProcessingBlocks(transcript, entries);
    expect(blocks.map(b => b.kind)).toEqual(['thinking', 'narration', 'toolGroup', 'narration']);
    const group = blocks[2];
    if (group.kind !== 'toolGroup') throw new Error('expected toolGroup');
    expect(group.summary).toBe('2 steps · Read file ×2');
    expect(group.entries).toHaveLength(2);
  });

  it('skips tool pointers with no matching entry', () => {
    const blocks = buildProcessingBlocks([tx({ kind: 'toolCall', seq: 0, callId: 'missing' })], []);
    expect(blocks).toHaveLength(0);
  });

  it('falls back to a single group when there is no transcript', () => {
    const entries = [entry({ id: 'c1', name: 'shell' })];
    const blocks = buildProcessingBlocks([], entries);
    expect(blocks).toHaveLength(1);
    expect(blocks[0].kind).toBe('toolGroup');
  });
});

describe('formatter null-safety (malformed / legacy snapshot guard)', () => {
  it('does not throw on undefined input', () => {
    // A snake_case (legacy) persisted transcript item yields undefined
    // camelCase fields; the formatter must degrade, not crash the app.
    expect(formatToolName(undefined)).toBe('');
    expect(stripToolCallEnvelopes(undefined)).toBe('');
    expect(stripToolCallEnvelopes(null)).toBe('');
  });
});

describe('extractAgentSources', () => {
  const source = (id: string, url: string): ToolTimelineEntry =>
    entry({ id, name: 'web_fetch', status: 'success', argsBuffer: JSON.stringify({ url }) });

  it('surfaces http(s) sources with hostname titles, deduped in first-seen order', () => {
    const out = extractAgentSources([
      source('a', 'https://example.com/docs'),
      source('b', 'http://blog.example.org/post'),
      source('c', 'https://example.com/docs'), // dup URL — dropped
    ]);
    expect(out).toEqual([
      { id: 'a', title: 'example.com', url: 'https://example.com/docs' },
      { id: 'b', title: 'blog.example.org', url: 'http://blog.example.org/post' },
    ]);
  });

  it('drops non-http(s) URLs so a hostile scheme never reaches an anchor href', () => {
    const out = extractAgentSources([
      source('js', 'javascript:alert(1)'),
      source('data', 'data:text/html,<script>alert(1)</script>'),
      source('file', 'file:///etc/passwd'),
      source('ok', 'https://safe.example.com'),
    ]);
    expect(out).toEqual([{ id: 'ok', title: 'safe.example.com', url: 'https://safe.example.com' }]);
  });

  it('ignores entries from non-URL tools', () => {
    expect(
      extractAgentSources([
        entry({ id: 'g', name: 'grep', argsBuffer: JSON.stringify({ url: 'https://x.com' }) }),
      ])
    ).toEqual([]);
  });

  it('does not present an unsuccessful fetch as a visited source', () => {
    expect(
      extractAgentSources([
        entry({
          id: 'failed',
          name: 'web_fetch',
          status: 'error',
          argsBuffer: JSON.stringify({ url: 'https://unreachable.example.com' }),
        }),
      ])
    ).toEqual([]);
  });
});