Skip to content

Agent Zero AI framework

19.8 KB
Raw Blame
from copy import deepcopy
import json

import pytest

from helpers import responses_history as history
from helpers.llm_result import LLMResult, ResponseItem
from helpers.litellm_transport import LiteLLMTransport, clear_transport_capability_cache


TOOLS = [{"type": "function", "name": "lookup", "parameters": {"type": "object", "properties": {}}}]
BASE = [{"role": "system", "content": "Rules"}, {"role": "user", "content": "Question"}]
COMMENTARY = {"type": "message", "id": "msg_progress", "role": "assistant",
              "phase": "commentary", "status": "completed",
              "content": [{"type": "output_text", "text": "The result points to a parsing issue."}]}


@pytest.fixture(autouse=True)
def clear_capabilities():
    clear_transport_capability_cache()


def render(record):
    value = record["content"]
    return json.dumps(value, separators=(",", ":")) if isinstance(value, dict) else value


def transport(prompt, prefix, records=(), **kwargs):
    return LiteLLMTransport(model=kwargs.pop("model", "openai/test"), messages=prompt, kwargs={
        "a0_api_mode": "responses", "responses_state": "local",
        "a0_responses_function_tools": TOOLS,
        "responses_history_context": {
            "prompt": prompt, "prefix": prefix,
            "groups": history.prepare_groups(list(records), render, lambda text: text),
        }, **kwargs,
    })


def completed_call(source, call_id="call_1", name="lookup"):
    request = source._responses_request(stream=False)
    return source._llm_result_from_response({"id": "resp_1", "output": [
        {"type": "reasoning", "id": "rs_" + call_id, "encrypted_content": "opaque", "summary": []},
        {"type": "function_call", "id": "fc_" + call_id, "call_id": call_id, "name": name, "arguments": '{"q":"a0"}'},
    ]}, request)


def records_for(result):
    return [
        {"ai": True, "content": result.function_calls_text(), "metadata": result.metadata()},
        {"ai": False, "content": {"tool_name": "lookup", "tool_result": "Found", "file": "/visible/file.txt"}, "metadata": {
            "responses": {"input_items": [{"type": "function_call_output", "call_id": result.function_calls[0].call_id, "output": "DO NOT REPLAY RAW OUTPUT"}]},
        }},
    ]


def prompt_for(records, base=None):
    return [*(base or BASE), {"role": "assistant", "content": render(records[0])},
            {"role": "user", "content": render(records[1]) + "\nCurrent extras"}]


def test_native_replay_uses_visible_results_and_retains_current_extras():
    source = transport([BASE[0], {"role": "user", "content": "Question\nOld extras"}], BASE)
    result = completed_call(source)
    records = records_for(result)
    original = deepcopy(records)
    prompt = prompt_for(records)
    replay = transport(prompt, prompt, records)
    request = replay._responses_request(stream=False)
    assert request["input"][:2] == source.messages
    assert request["input"][2:4] == [item.to_dict() for item in result.output_items]
    assert request["input"][4] == {"type": "function_call_output", "call_id": "call_1", "output": render(records[1])}
    assert request["input"][5] == {"role": "user", "content": "Current extras"}
    assert "DO NOT REPLAY RAW OUTPUT" not in json.dumps(request)
    assert result.history_extras == "Old extras"
    assert "responses_history_context" not in request
    assert records == original
    assert prompt == prompt_for(records)


def test_commentary_replay_preserves_phase_native_items_and_generic_history():
    result = completed_call(transport(BASE, BASE))
    result.output_items.insert(1, ResponseItem.from_any(deepcopy(COMMENTARY)))
    records = records_for(result)
    original = deepcopy(records)
    prompt = prompt_for(records)
    replay = transport(prompt, prompt, records)
    request = replay._responses_request(stream=False)
    assert request["input"][2:5] == [item.to_dict() for item in result.output_items]
    assert request["input"][5]["call_id"] == "call_1"
    assert request["input"][5]["output"] == render(records[1])
    assert request["input"][6] == {"role": "user", "content": "Current extras"}
    assert records == original and prompt == prompt_for(records)
    assert replay._chat_request(stream=False)["messages"] == prompt


@pytest.mark.parametrize("change", ["unmarked", "final", "role", "incomplete", "empty", "refusal", "secret"])
def test_ineligible_commentary_keeps_prepared_history(change):
    result = completed_call(transport(BASE, BASE))
    item = deepcopy(COMMENTARY)
    mask = lambda text: text
    if change == "unmarked":
        item.pop("phase")
    elif change == "final":
        item["phase"] = "final_answer"
    elif change == "role":
        item["role"] = "user"
    elif change == "incomplete":
        item["status"] = "incomplete"
    elif change == "empty":
        item["content"] = []
    elif change == "refusal":
        item["content"].append({"type": "refusal", "refusal": "Not a progress update"})
    elif change == "secret":
        mask = lambda text: text.replace("parsing", "MASKED")
    result.output_items.insert(1, ResponseItem.from_any(item))
    records = records_for(result)
    assert history.prepare_groups(records, render, mask) == []


def test_prepared_context_lifecycle_and_post_hook_changes(monkeypatch):
    from types import SimpleNamespace
    from langchain_core.messages import SystemMessage, HumanMessage
    from langchain_core.prompts import ChatPromptTemplate
    from agent import Agent, LoopData
    from models import LiteLLMChatWrapper
    from helpers import secrets

    agent = object.__new__(Agent)
    agent.context = SimpleNamespace()
    agent.loop_data = LoopData()
    model = LiteLLMChatWrapper(model="test", provider="openai", model_config=None, a0_api_mode="responses")
    prompt = [SystemMessage("Rules"), HumanMessage("Question")]
    monkeypatch.setattr(secrets, "get_secrets_manager", lambda _: SimpleNamespace(mask_values=lambda text: text))
    history.remember_prompt(agent.loop_data, ChatPromptTemplate.from_messages(prompt).format(), prompt[0], [])
    call = {"model": model, "messages": prompt, "responses_prompt_replacements": {"Rules": "Override"}}
    prepared = history.prepare_call(agent, call)
    assert prepared["responses_history_context"]["prompt"] == BASE
    assert prepared["responses_prompt_replacements"] == {"Rules": "Override"}
    model.kwargs["a0_api_mode"] = "chat"
    with monkeypatch.context() as chat:
        chat.setattr(model, "_convert_messages", lambda _: pytest.fail("Chat must not prepare replay input"))
        chat.setattr(secrets, "get_secrets_manager", lambda _: pytest.fail("Chat must not prepare replay masking"))
        assert "responses_history_context" not in history.prepare_call(agent, call)
    model.kwargs["a0_api_mode"] = "responses"
    for changed in (
        [SystemMessage("Changed by hook"), prompt[1]],
        [SystemMessage("Rules", additional_kwargs={"provider_control": True}), prompt[1]],
    ):
        assert "responses_history_context" not in history.prepare_call(agent, {**call, "messages": changed})
    agent.loop_data.params_temporary["responses_prompt_replacements"] = {"stale": "value"}
    history.start_prompt(agent.loop_data)
    assert history.prepare_call(agent, {"model": model, "messages": prompt}) == {"responses_prompt_replacements": {}}


def test_current_and_retained_extras_use_the_same_secret_mask(monkeypatch):
    from types import SimpleNamespace
    from langchain_core.messages import SystemMessage, HumanMessage
    from langchain_core.prompts import ChatPromptTemplate
    from agent import LoopData
    from models import LiteLLMChatWrapper
    from helpers import secrets

    agent = SimpleNamespace(context=SimpleNamespace(), loop_data=LoopData())
    agent.loop_data.history_output = [{"ai": False, "content": "Question"}]
    model = LiteLLMChatWrapper(model="test", provider="openai", model_config=None, a0_api_mode="responses")
    prompt = [SystemMessage("Rules"), HumanMessage("Question\nProvider: private-value")]
    mask = lambda text: text.replace("private-value", "MASKED_KEY")
    monkeypatch.setattr(secrets, "get_secrets_manager", lambda _: SimpleNamespace(mask_values=mask))
    history.remember_prompt(agent.loop_data, ChatPromptTemplate.from_messages(prompt).format(), prompt[0], [])
    prepared = history.prepare_call(agent, {"model": model, "messages": prompt})
    original = prepared["responses_history_context"]["prompt"]
    source = transport(original, BASE)
    source.kwargs.update(prepared)
    result = completed_call(source)
    assert result.history_extras == "Provider: MASKED_KEY"
    assert result.input_items[-1]["content"] == "Question\nProvider: MASKED_KEY"
    records = records_for(result)
    replay_prompt = prompt_for(records)
    replay = transport(replay_prompt, replay_prompt)
    replay.kwargs["responses_history_context"]["groups"] = history.prepare_groups(records, render, mask)
    assert replay._responses_request(stream=False)["input"][:len(result.input_items)] == result.input_items
    assert source._chat_request(stream=False)["messages"] == original


@pytest.mark.parametrize("change", ["summary", "arguments", "missing_output", "wrong_id", "attachments", "commentary", "unencrypted", "legacy", "secret"])
def test_ineligible_history_stays_in_prepared_text(change):
    result = completed_call(transport(BASE, BASE))
    records = records_for(result)
    base = deepcopy(BASE)
    mask = lambda text: text
    if change == "summary":
        base[1]["content"] = "Compressed question"
    elif change == "arguments":
        records[0]["content"] = records[0]["content"].replace('a0', 'MASKED')
    elif change == "missing_output":
        records[1]["metadata"] = {}
    elif change == "wrong_id":
        records[1]["metadata"]["responses"]["input_items"][0]["call_id"] = "other"
    elif change == "attachments":
        records[1]["content"] = [{"type": "input_image", "image_url": "data:image/png;base64,AAAA"}]
    elif change == "commentary":
        records[0]["metadata"]["responses"]["output_items"].append({"type": "message", "content": [{"type": "output_text", "text": "second tool call"}]})
    elif change == "unencrypted":
        records[0]["metadata"]["responses"]["output_items"][0].pop("encrypted_content")
    elif change == "legacy":
        records[0]["metadata"]["responses"]["capability"].pop(history.PREFIX_HASH)
    elif change == "secret":
        mask = lambda text: text.replace("opaque", "MASKED")
    prompt = [*base, {"role": "assistant", "content": render(records[0])}, {"role": "user", "content": render(records[1])}]
    groups = history.prepare_groups(records, render, mask)
    replay = transport(prompt, prompt)
    replay.kwargs["responses_history_context"]["groups"] = groups
    assert replay._responses_request(stream=False)["input"] == prompt


@pytest.mark.parametrize("kwargs", [
    {"model": "openai/other"}, {"api_base": "https://other.invalid"},
    {"a0_responses_function_tools": [{**TOOLS[0], "description": "new policy"}]},
])
def test_replay_is_bound_to_current_model_endpoint_and_tool_scope(kwargs):
    records = records_for(completed_call(transport(BASE, BASE)))
    prompt = prompt_for(records)
    replay = transport(prompt, prompt, records, **kwargs)
    assert replay._responses_request(stream=False)["input"] == prompt


def test_final_response_call_without_result_is_never_fabricated():
    result = completed_call(transport(BASE, BASE), name="response")
    assert history.prepare_groups(records_for(result)[:1], render, lambda text: text) == []


def test_chat_request_drops_replay_control_and_keeps_text_examples():
    records = records_for(completed_call(transport(BASE, BASE)))
    prompt = prompt_for(records)
    replay = transport(prompt, prompt, records, a0_api_mode="chat")
    request = replay._chat_request(stream=False)
    assert request["messages"] == prompt
    assert "responses_history_context" not in request
    assert "tools" not in request
    assert history.PREFIX_HASH not in replay._capability_metadata()


def test_changed_selected_input_cannot_acquire_a_history_digest():
    replay = transport(BASE, BASE, responses_local_input_items=[{"role": "user", "content": "override"}])
    assert replay._responses_request(stream=False)["input"] == [{"role": "user", "content": "override"}]
    assert history.PREFIX_HASH not in replay._capability_metadata()


@pytest.mark.parametrize("duplicate", [False, True])
def test_multiple_groups_preserve_history_order_without_duplicate_native_ids(duplicate):
    first = records_for(completed_call(transport(BASE, BASE)))
    second_prompt = prompt_for(first)
    second_prefix = deepcopy(second_prompt)
    second_prefix[-1]["content"] = render(first[1])
    second = records_for(completed_call(transport(second_prompt, second_prefix, first), call_id="call_1" if duplicate else "call_2"))
    records = first + second
    prompt = [*second_prefix, *prompt_for(second)[len(BASE):]]
    replay = transport(prompt, prompt, records)
    request = replay._responses_request(stream=False)
    calls = [item for item in request["input"] if item.get("type") == "function_call"]
    assert [item["call_id"] for item in calls] == (["call_1"] if duplicate else ["call_1", "call_2"])
    assert request["input"][-1]["content"].endswith("Current extras")


def test_masking_failure_disables_native_projection():
    records = records_for(completed_call(transport(BASE, BASE)))
    def unavailable(text):
        raise OSError("masking unavailable")
    assert history.prepare_groups(records, render, unavailable) == []


@pytest.mark.asyncio
async def test_automatic_chat_fallback_keeps_text_and_drops_history_controls(monkeypatch):
    from helpers import litellm_transport
    records = records_for(completed_call(transport(BASE, BASE)))
    prompt = prompt_for(records)
    calls = []
    class BadRequestError(RuntimeError):
        pass
    async def responses(**kwargs):
        calls.append("responses")
        assert any(item.get("type") == "function_call" for item in kwargs["input"])
        assert "responses_history_context" not in kwargs
        raise BadRequestError("Cannot determine type of item")
    async def chat(**kwargs):
        calls.append("chat")
        assert kwargs["messages"] == prompt
        assert "responses_history_context" not in kwargs
        assert "tools" not in kwargs
        return {"choices": [{"message": {"content": "fallback"}}]}
    monkeypatch.setattr(litellm_transport, "aresponses", responses)
    monkeypatch.setattr(litellm_transport, "acompletion", chat)
    replay = transport(prompt, prompt, records)
    await replay.acomplete()
    assert calls == ["responses", "chat"]
    assert history.PREFIX_HASH not in replay.last_result.capability
    assert replay.last_result.mode == "chat_completions"


def test_tool_result_summary_is_replayed_as_visible_summary_only():
    records = records_for(completed_call(transport(BASE, BASE)))
    records[1]["content"] = "Condensed tool result"
    prompt = prompt_for(records)
    replay = transport(prompt, prompt, records)
    request = replay._responses_request(stream=False)
    output = next(item for item in request["input"] if item.get("type") == "function_call_output")
    assert output["output"] == "Condensed tool result"
    assert "DO NOT REPLAY RAW OUTPUT" not in json.dumps(request)


def test_growing_requests_keep_extras_across_tools_reload_and_user_followups():
    prefix = deepcopy(BASE)
    records = []
    previous = []
    for turn in range(4):
        prompt = deepcopy(prefix)
        prompt[-1]["content"] += f"\nExtras {turn}"
        source = transport(prompt, prefix, records)
        result = completed_call(source, call_id=f"call_{turn}", name="response" if turn == 2 else "lookup")
        assert result.input_items[:len(previous)] == previous
        for old in range(turn + 1):
            assert json.dumps(result.input_items).count(f"Extras {old}") == 1
        previous = deepcopy(result.input_items)
        persisted = json.loads(json.dumps(result.metadata()))["responses"]
        assert "input_items" not in persisted and persisted["history_extras"] == f"Extras {turn}"
        result = LLMResult.from_dict(persisted)
        added = records_for(result)
        if turn == 2:
            added[1] = {"ai": False, "content": "Here is a real user follow-up."}
        records.extend(added)
        prefix.extend([
            {"role": "assistant", "content": render(added[0])},
            {"role": "user", "content": render(added[1])},
        ])
    # A terminating response call has no result; never invent an acknowledgement.
    assert not any(item.get("call_id") == "call_2" for item in previous)


@pytest.mark.parametrize("tail", [
    {"role": "user", "content": "Question"},
    {"role": "user", "content": [{"type": "input_text", "text": "Question"},
                                   {"type": "input_image", "image_url": "data:image/png;base64,AAAA"}]},
    {"role": "assistant", "content": "Prior response"},
])
def test_extras_snapshot_preserves_prepared_message_boundaries(tail):
    prefix = [BASE[0], deepcopy(tail)]
    prompt = deepcopy(prefix)
    if tail["role"] == "assistant":
        prompt.append({"role": "user", "content": "Old extras"})
    elif isinstance(tail["content"], str):
        prompt[-1]["content"] += "\nOld extras"
    else:
        prompt[-1]["content"].append({"type": "input_text", "text": "Old extras"})
    result = completed_call(transport(prompt, prefix))
    assert result.history_extras == "Old extras"
    records = records_for(result)
    replay_prompt = prompt_for(records, prefix)
    request = transport(replay_prompt, replay_prompt, records)._responses_request(stream=False)
    assert request["input"][:len(prompt)] == prompt
    assert prefix[-1] == tail


@pytest.mark.parametrize("change", ["summary", "arguments", "secret", "mask_error", "tools"])
def test_extras_replay_respects_history_and_masking_changes(change):
    prompt = [BASE[0], {"role": "user", "content": "Question\nOld extras"}]
    records = records_for(completed_call(transport(prompt, BASE)))
    prefix = deepcopy(BASE)
    mask = lambda text: text
    kwargs = {}
    if change == "summary":
        prefix[-1]["content"] = "Summarized question"
    elif change == "arguments":
        records[0]["content"] = "Summarized answer"
    elif change == "secret":
        mask = lambda text: text.replace("Old extras", "MASKED")
    elif change == "mask_error":
        def mask(text):
            raise OSError("masking unavailable")
    else:
        kwargs["a0_responses_function_tools"] = [{**TOOLS[0], "description": "changed"}]
    prompt = prompt_for(records, prefix)
    source = transport(prompt, prompt, **kwargs)
    source.kwargs["responses_history_context"]["groups"] = history.prepare_groups(records, render, mask)
    assert source._responses_request(stream=False)["input"] == prompt


def test_retained_extras_count_toward_responses_compression_only(monkeypatch):
    from types import SimpleNamespace
    from helpers import history as stored_history, tokens

    model = SimpleNamespace(kwargs={"a0_api_mode": "responses"})
    agent = SimpleNamespace(get_chat_model=lambda: model)
    hist = stored_history.History(agent)
    result = completed_call(transport([BASE[0], {"role": "user", "content": "Question\n" + "large extras " * 100}], BASE))
    msg = hist.add_message(True, result.function_calls_text(), metadata=result.metadata())
    expected = msg.get_tokens() + tokens.approximate_tokens(result.history_extras)
    assert hist.get_tokens() == expected
    restored = stored_history.deserialize_history(hist.serialize(), agent)
    assert restored.get_tokens() == expected
    monkeypatch.setattr(hist, "_get_ctx_size_for_history", lambda: expected - 1)
    assert hist.is_over_limit()
    model.kwargs["a0_api_mode"] = "chat"
    assert hist.get_tokens() == msg.get_tokens() and not hist.is_over_limit()
    model.kwargs["a0_api_mode"] = "responses"
    msg.set_summary("Condensed answer")
    assert hist.get_tokens() == msg.get_tokens()
    restored.current.summary = "Condensed topic"
    assert restored.get_tokens() == tokens.approximate_tokens("Condensed topic")