Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

7.3 KB
Raw Blame
from __future__ import annotations

import copy
import json
from typing import Any

import pytest

from xagent.core.agent import ExecutionContext, PatternRuntime
from xagent.core.agent.pattern.dag.dag import DAGPattern, _DAGStepRuntime
from xagent.core.agent.prompt_prefix import (
    _MAX_BASELINES,
    PROMPT_PREFIX_KEY,
    PromptPrefixObserver,
)


def observe(observer, messages, tools=None, **kwargs):
    return observer.observe(
        messages,
        tools,
        scope=kwargs.get("scope", ("task", "model")),
        call_kind=kwargs.get("call_kind", "main"),
        phase=kwargs.get("phase", ""),
    )


def test_append_only_then_in_place_history_edit():
    observer = PromptPrefixObserver()
    messages = [
        {"role": "system", "content": "rules"},
        {"role": "user", "content": "task"},
    ]
    assert observe(observer, messages) == {"change": "initial"}
    messages.append({"role": "assistant", "content": "working"})
    result = observe(observer, messages)
    assert result["change"] == "append_only"
    assert result["common_prefix_messages"] == 2
    assert result["first_changed_message_index"] is None
    messages[0]["content"] = "new rules"
    result = observe(observer, messages)
    assert result["change"] == "messages_changed"
    assert result["first_changed_message_index"] == 0
    assert result["common_prefix_messages"] == 0


def test_removed_messages_and_identical_requests():
    observer = PromptPrefixObserver()
    messages = [
        {"role": "user", "content": "one"},
        {"role": "assistant", "content": "two"},
    ]
    observe(observer, messages)
    assert observe(observer, copy.deepcopy(messages))["change"] == "identical"
    result = observe(observer, messages[:1])
    assert result["change"] == "messages_removed"
    assert result["common_prefix_messages"] == 1


def test_tool_schema_and_order_changes_do_not_mutate_payloads():
    observer = PromptPrefixObserver()
    tools = [{"function": {"name": "one"}}, {"function": {"name": "two"}}]
    original = copy.deepcopy(tools)
    observe(observer, [], tools)
    assert observe(observer, [], copy.deepcopy(tools))["tools_changed"] is False
    assert observe(observer, [], list(reversed(tools)))["change"] == "tools_changed"
    tools[0]["function"]["description"] = "changed"
    assert observe(observer, [], tools)["tools_changed"] is True
    assert "description" not in original[0]["function"]
    assert tools[0]["function"]["name"] == "one"


def test_auxiliary_call_does_not_overwrite_main_baseline():
    observer = PromptPrefixObserver()
    original = [{"role": "user", "content": "main"}]
    observe(observer, original)
    observe(
        observer, [{"role": "user", "content": "summarize"}], call_kind="compaction"
    )
    assert observe(observer, original)["change"] == "identical"
    assert observe(observer, original, scope=("different-task", "model")) == {
        "change": "initial"
    }


def test_alternating_scopes_and_phases_preserve_their_own_baselines():
    observer = PromptPrefixObserver()
    original = [{"role": "user", "content": "A"}]
    other = [{"role": "user", "content": "B"}]
    first = {"scope": ("task", "step-a", "model"), "phase": "assessment"}
    second = {"scope": ("task", "step-b", "model"), "phase": "assessment"}
    third = {**first, "phase": "delivery"}
    assert observe(observer, original, **first)["change"] == "initial"
    assert observe(observer, other, **second)["change"] == "initial"
    assert observe(observer, other, **third)["change"] == "initial"
    assert observe(observer, original + other, **first)["change"] == "append_only"
    assert observe(observer, other, **second)["change"] == "identical"
    assert observe(observer, other, **third)["change"] == "identical"


def test_observer_evicts_the_least_recently_used_baseline():
    observer = PromptPrefixObserver()
    scopes = [(str(i), "model") for i in range(_MAX_BASELINES + 1)]
    for scope in scopes[:-1]:
        assert observe(observer, [], scope=scope)["change"] == "initial"
    assert observe(observer, [], scope=scopes[0])["change"] == "identical"
    assert observe(observer, [], scope=scopes[-1])["change"] == "initial"
    assert observe(observer, [], scope=scopes[0])["change"] == "identical"
    assert observe(observer, [], scope=scopes[1])["change"] == "initial"


def test_non_json_observation_does_not_break_calls_or_reuse_old_baseline():
    observer = PromptPrefixObserver()
    observe(observer, [])
    assert observe(observer, [{"content": object()}]) == {"change": "unavailable"}
    assert observe(observer, []) == {"change": "initial"}
    content: list[Any] = []
    content.append(content)
    assert observe(observer, [{"content": content}]) == {"change": "unavailable"}
    assert observe(observer, []) == {"change": "initial"}


def test_snapshot_and_telemetry_do_not_retain_content():
    observer = PromptPrefixObserver()
    secret = "private invoice contents"
    result = observe(observer, [{"role": "user", "content": secret}])
    assert secret not in repr(observer._previous)
    assert secret not in json.dumps(result)
    _, fingerprints = next(iter(observer._previous.values()))
    assert all(
        isinstance(digest, bytes) and len(digest) == 32 for digest in fingerprints
    )


class Trace:
    def __init__(self):
        self.events: list[dict[str, Any]] = []

    async def trace_event(self, event_type, **kwargs):
        self.events.append(kwargs["data"])


@pytest.mark.asyncio
async def test_runtime_trace_pairs_start_and_end_observation_by_default():
    trace = Trace()
    runtime = PatternRuntime(tracer=trace)
    context = ExecutionContext()
    metadata: dict[str, Any] = {}
    await runtime.on_llm_start(context=context, messages=[], metadata=metadata)
    await runtime.on_llm_end(
        context=context,
        response={"usage": {"prompt_tokens": 2, "completion_tokens": 1}},
        metadata=metadata,
    )
    assert trace.events[0][PROMPT_PREFIX_KEY] == {"change": "initial"}
    assert trace.events[1][PROMPT_PREFIX_KEY] == trace.events[0][PROMPT_PREFIX_KEY]
    metadata = {}
    await runtime.on_llm_start(context=context, messages=[], metadata=metadata)
    assert metadata[PROMPT_PREFIX_KEY]["change"] == "identical"


@pytest.mark.asyncio
@pytest.mark.parametrize("tracer", [None, object()])
async def test_runtime_without_trace_events_skips_observation(tracer):
    runtime = PatternRuntime(tracer=tracer)
    await runtime.on_llm_start(
        context=ExecutionContext(), messages=[{"content": object()}]
    )
    assert runtime._prompt_prefix_observer is None


@pytest.mark.asyncio
@pytest.mark.parametrize("failed", [False, True])
async def test_dag_step_keeps_prefix_observation_on_terminal_event(failed):
    trace = Trace()
    context = ExecutionContext()
    runtime = _DAGStepRuntime(
        parent=PatternRuntime(tracer=trace),
        dag_pattern=DAGPattern(lambda **_: None),
        root_context=context,
        step_id="step-a",
    )
    metadata = {}
    await runtime.on_llm_start(context=context, messages=[], metadata=metadata)
    if failed:
        await runtime.on_llm_error(
            context=context, error=ValueError("failed"), metadata=metadata
        )
    else:
        await runtime.on_llm_end(context=context, response={}, metadata=metadata)
    assert [event[PROMPT_PREFIX_KEY] for event in trace.events] == [
        {"change": "initial"},
        {"change": "initial"},
    ]