Skip to content

Self-hosted AI workspace.

Python
9.2 KB
Raw Blame
import json
from types import SimpleNamespace

import jsonschema
import pytest

from src.clean_agent_preview import compact_schemas, conversation, stream_preview
from src.tool_schemas import FUNCTION_TOOL_SCHEMAS
from src.tool_policy import ToolPolicy
from src.turn_contract import resolve_full_inventory_contract


def test_explicit_thinking_off_wins_without_tools(monkeypatch):
    import src.clean_agent_preview as module
    monkeypatch.setattr(module, 'uses_odysseus_progressive_thinking', lambda model: True)
    assert module.progressive_thinking_for_turn('Ajax', []) is True
    assert module.progressive_thinking_for_turn('Ajax', [], 'off') is False


def test_fetch_requires_a_page_in_full_and_compact_contracts():
    schema = next(s for s in FUNCTION_TOOL_SCHEMAS if s['function']['name'] == 'web_fetch')
    # The preview validates calls against the compact contract, which requires
    # url or urls. The full schema is sent to providers whose top-level
    # parameter contract rejects anyOf (test_model_tool_modes), so it accepts
    # both forms without the combinator.
    compact = compact_schemas([schema])[0]['function']['parameters']
    with pytest.raises(jsonschema.ValidationError):
        jsonschema.validate({'query': 'compose a letter'}, compact)
    for params in [schema['function']['parameters'], compact]:
        jsonschema.validate({'url': 'https://example.com', 'query': 'details'}, params)
        jsonschema.validate({'urls': ['https://example.com']}, params)


@pytest.mark.parametrize('reply', ['you suck', 'thanks', 'a complaint', 'friendly'])
def test_clarification_history_keeps_task_recipient_and_question(reply):
    history = SimpleNamespace(history=[
        {'role': 'user', 'content': 'write email'},
        {'role': 'assistant', 'content': 'Who is it for?'},
        {'role': 'user', 'content': 'Jordan'},
        {'role': 'assistant', 'content': 'What do you want to say to Jordan?'},
    ])
    result = conversation(history, [{'role': 'user', 'content': reply}])
    assert [m['content'] for m in result] == [
        'write email', 'Who is it for?', 'Jordan',
        'What do you want to say to Jordan?', reply,
    ]


@pytest.mark.asyncio
async def test_bad_fetch_gets_correction_not_permission_refusal(monkeypatch):
    import src.clean_agent_preview as module
    requests = []
    responses = iter([
        {'tool_calls': [{'index': 0, 'id': 'bad-fetch', 'type': 'function',
                         'function': {'name': 'web_fetch', 'arguments': '{"query":"compose a letter"}'}}]},
        {'content': 'Dear Jordan,\nGreetings from under the sea!'},
    ])

    class Response:
        def __init__(self, delta): self.delta = delta
        async def __aenter__(self): return self
        async def __aexit__(self, *args): pass
        def raise_for_status(self): pass
        async def aiter_lines(self):
            yield 'data: ' + json.dumps({'choices': [{'delta': self.delta}]})
            yield 'data: [DONE]'

    class Client:
        def __init__(self, **kwargs): pass
        async def __aenter__(self): return self
        async def __aexit__(self, *args): pass
        def stream(self, *args, **kwargs):
            requests.append(json.loads(json.dumps(kwargs['json'])))
            return Response(next(responses))

    async def execute(*args, **kwargs):
        pytest.fail('Malformed fetch must never execute')

    monkeypatch.setattr(module.httpx, 'AsyncClient', Client)
    monkeypatch.setattr(module, 'execute_tool_block', execute)
    schemas = [s for s in FUNCTION_TOOL_SCHEMAS if s['function']['name'] == 'web_fetch']
    policy = ToolPolicy()
    contract = resolve_full_inventory_contract(schemas=schemas, policy=policy)
    events = [json.loads(chunk[6:]) async for chunk in stream_preview(
        endpoint_url='http://test', model='test',
        messages=[{'role': 'user', 'content': 'write an email as a fish to Jordan'}],
        headers={}, turn_contract=contract, session_id='test', owner='test',
        disabled_tools=set(), tool_policy=policy,
    ) if '[DONE]' not in chunk]
    assert len(requests) == 2
    correction = requests[1]['messages']
    assert any('requires url or urls' in str(m.get('content')) for m in correction)
    assert any('not a permission denial' in str(m.get('content')) for m in correction)
    metrics = next(e['data'] for e in events if e.get('type') == 'metrics')
    assert metrics['clean_v3_turn'][-1]['content'].startswith('Dear Jordan')
    assert metrics['tool_schema_names'] == ['web_fetch']
    assert metrics['tool_schema_count'] == len(metrics['tool_schema_names'])


@pytest.mark.asyncio
async def test_ajax_complete_draft_has_no_lookup_or_delivery_tools(monkeypatch):
    import src.clean_agent_preview as module
    import src.email_task_intent as intent_module
    requests = []
    classifications = []
    async def classify(*args, **kwargs):
        classifications.append(kwargs)
        kwargs['accounting'].update(input_tokens=100, output_tokens=20, usage_source='real')
        return intent_module.EmailTaskIntent('draft', (), 'Thank Jon for lunch')
    class Response:
        async def __aenter__(self): return self
        async def __aexit__(self, *args): pass
        def raise_for_status(self): pass
        async def aiter_lines(self):
            yield 'data: ' + json.dumps({'choices': [{'delta': {'content': 'Hi Jon, thanks for lunch!'}}]})
            yield 'data: ' + json.dumps({'choices': [], 'usage': {'prompt_tokens': 200, 'completion_tokens': 30}})
            yield 'data: [DONE]'
    class Client:
        def __init__(self, **kwargs): pass
        async def __aenter__(self): return self
        async def __aexit__(self, *args): pass
        def stream(self, *args, **kwargs):
            requests.append(kwargs['json'])
            return Response()
    async def execute(*args, **kwargs):
        pytest.fail('A fully supplied chat draft needs no tool execution')
    monkeypatch.setattr(intent_module, 'classify_email_task', classify)
    monkeypatch.setattr(module.httpx, 'AsyncClient', Client)
    monkeypatch.setattr(module, 'execute_tool_block', execute)
    policy = ToolPolicy()
    schemas = [s for s in FUNCTION_TOOL_SCHEMAS if s['function']['name'] in {'web_fetch', 'ask_user', 'read_email'}]
    contract = resolve_full_inventory_contract(schemas=schemas, policy=policy)
    events = [json.loads(chunk[6:]) async for chunk in stream_preview(
        endpoint_url='http://test', model='Ajax', headers={}, turn_contract=contract,
        messages=[{'role': 'user', 'content': 'Draft an email thanking Jon for lunch'}],
        session_id='test', owner='test', disabled_tools=set(), tool_policy=policy,
        thinking_mode='off',
    ) if '[DONE]' not in chunk]
    assert len(requests) == 1
    assert len(classifications) == 1
    assert not requests[0].get('tools')
    assert any(e.get('stage') == 'email_task_scope' for e in events)
    metrics = next(e['data'] for e in events if e.get('type') == 'metrics')
    assert metrics['input_tokens'] == 300
    assert metrics['output_tokens'] == 50
    assert metrics['injected_tokens'] == metrics['last_request_tokens'] == 200


@pytest.mark.asyncio
@pytest.mark.parametrize('failure', ['timeout', 'invalid_json'])
async def test_ajax_scope_failure_finishes_protocol_without_execution(monkeypatch, failure):
    import httpx
    import src.clean_agent_preview as module
    import src.email_task_intent as intent_module

    async def classify(*args, **kwargs):
        if failure == 'timeout':
            raise httpx.ReadTimeout('fixture')
        kwargs['accounting'].update(input_tokens=101, output_tokens=12, usage_source='real')
        raise ValueError('Malformed classifier output')

    class Client:
        def __init__(self, **kwargs): pass
        async def __aenter__(self): return self
        async def __aexit__(self, *args): pass
        def stream(self, *args, **kwargs):
            pytest.fail('Unknown scope must not generate with broad tools')

    async def execute(*args, **kwargs):
        pytest.fail('Unknown scope must not execute')

    monkeypatch.setattr(intent_module, 'classify_email_task', classify)
    monkeypatch.setattr(module.httpx, 'AsyncClient', Client)
    monkeypatch.setattr(module, 'execute_tool_block', execute)
    policy = ToolPolicy()
    schemas = [s for s in FUNCTION_TOOL_SCHEMAS if s['function']['name'] in {'web_fetch', 'read_email'}]
    contract = resolve_full_inventory_contract(schemas=schemas, policy=policy)
    chunks = [c async for c in stream_preview(
        endpoint_url='http://test', model='Ajax', headers={}, turn_contract=contract,
        messages=[{'role': 'user', 'content': 'Draft an email thanking Jon for lunch'}],
        session_id='test', owner='test', disabled_tools=set(), tool_policy=policy)]
    events = [json.loads(c[6:]) for c in chunks if c.startswith('data: ') and '[DONE]' not in c]
    metrics = next(e['data'] for e in events if e.get('type') == 'metrics')
    assert metrics['email_task_scope']['failed'] is True
    assert metrics['agent_rounds'] == metrics['tool_calls'] == 0
    assert metrics['total_tokens'] == (113 if failure == 'invalid_json' else 0)
    assert 'No action was taken' in metrics['clean_v3_turn'][-1]['content']
    assert any(e.get('type') == 'final_response' for e in events)
    assert not any(e.get('delta') for e in events)
    assert chunks[-1] == 'data: [DONE]\n\n'