Skip to content

Self-hosted AI workspace.

Python
8.9 KB
Raw Blame
from types import SimpleNamespace

from scripts.eval_odysseus_tool_use import (
    _hard_turn_timeout,
    _command_contract_ok,
    _reported_model,
    _sse_events,
    _stream_exception_if_empty,
    _summary_exit_code,
    _tool_sequence_matches,
    run_case,
)
from scripts.eval_odysseus_crud import build_summary
from scripts.eval_qwen35_tool_router_extended import cleanup_notes


def test_shared_eval_runner_accepts_extended_evaluator_timeout_name() -> None:
    assert _hard_turn_timeout(SimpleNamespace(hard_case_timeout=30)) == 30.0


def test_shared_eval_runner_defaults_without_timeout_attribute() -> None:
    assert _hard_turn_timeout(SimpleNamespace()) == 0.0


def test_eval_summary_reports_selected_model_over_discovered_model() -> None:
    args = SimpleNamespace(selected_model="router/v12", model="router/v13")
    assert _reported_model(args) == "router/v12"


def test_crud_summary_survives_partial_or_infrastructure_records() -> None:
    summary = build_summary(
        [{
            "workflow": "memory",
            "turns": [{
                "native_call_ok": None,
                "first_action_ok": None,
                "tool_count_ok": None,
                "exact_args_ok": None,
                "execution_ok": None,
                "duplicate_textual_call": None,
                "stream_errors": [{"type": "infra_exception"}],
            }],
        }],
        "router",
        "fixture",
    )
    assert summary["turns"] == 1
    assert summary["native_success"] == 0
    assert summary["stream_errors"] == 1


def test_eval_sse_parser_preserves_standard_event_name() -> None:
    class Response:
        def iter_lines(self):
            return iter((
                "event: error",
                'data: {"error":"upstream unavailable","status":502}',
                "",
            ))

    assert list(_sse_events(Response())) == [
        {
            "type": "error",
            "error": "upstream unavailable",
            "status": 502,
        }
    ]


def test_empty_sse_stream_is_not_treated_as_a_valid_turn() -> None:
    assert _stream_exception_if_empty([], [], None) == "empty SSE stream"
    assert _stream_exception_if_empty([], [], "upstream error") == "upstream error"


def test_eval_summary_exit_code_rejects_failed_execution() -> None:
    failed = {"execution_ok": False, "response_quality_ok": True, "duplicate_textual_call": False}
    passed = {"execution_ok": True, "response_quality_ok": True, "duplicate_textual_call": False}
    assert _summary_exit_code([failed]) == 1
    assert _summary_exit_code([passed]) == 0


def test_eval_tool_contract_accepts_ordered_coding_sequence() -> None:
    assert _tool_sequence_matches(
        ["read_file", "edit_file"], "read_file->edit_file"
    )
    assert not _tool_sequence_matches(
        ["edit_file", "read_file"], "read_file->edit_file"
    )


def test_notes_search_contract_rejects_broad_list_with_ignored_search_key() -> None:
    assert not _command_contract_ok(
        "notes_search",
        [{
            "tool": "manage_notes",
            "command": '{"action":"list","search":"ODY-EVAL-TOOL-NOTES-SEARCH"}',
        }],
    )
    assert _command_contract_ok(
        "notes_search",
        [{
            "tool": "manage_notes",
            "command": '{"action":"search","text":"ODY-EVAL-TOOL-NOTES-SEARCH"}',
        }],
    )


def test_fixture_search_contracts_reject_broad_lists() -> None:
    assert not _command_contract_ok(
        "documents_search_fixture",
        [{"tool": "manage_documents", "command": '{"action":"list"}'}],
    )
    assert not _command_contract_ok(
        "documents_search_fixture",
        [{"tool": "manage_documents", "command": '{"action":"find","query":"ODY-EVAL-TOOL-DOCUMENT-SEARCH"}'}],
    )
    assert _command_contract_ok(
        "documents_search_fixture",
        [
            {"tool": "manage_documents", "command": '{"action":"find","query":"ODY-EVAL-TOOL-DOCUMENT-SEARCH"}'},
            {"tool": "manage_documents", "command": '{"action":"read","document_id":"doc-fixture"}'},
        ],
    )
    assert not _command_contract_ok(
        "tasks_search_fixture",
        [{"tool": "manage_tasks", "command": '{"action":"list"}'}],
    )
    assert _command_contract_ok(
        "tasks_search_fixture",
        [{"tool": "manage_tasks", "command": '{"action":"list","name":"ODY-EVAL-TOOL-TASK-SEARCH"}'}],
    )
    assert not _command_contract_ok(
        "calendar_search_fixture",
        [{"tool": "manage_calendar", "command": '{"action":"list_events","query":"ODY-EVAL-TOOL-CALENDAR-SEARCH"}'}],
    )
    assert _command_contract_ok(
        "calendar_search_fixture",
        [{
            "tool": "manage_calendar",
            "command": (
                '{"action":"list_events","query":"ODY-EVAL-TOOL-CALENDAR-SEARCH",'
                '"start":"2026-08-21","end":"2026-08-23"}'
            ),
        }],
    )


def test_eval_local_runtime_context_sends_tui_workspace_form_fields(monkeypatch) -> None:
    captured = {}

    class Response:
        status_code = 200

        def json(self):
            return {"id": "session-1"}

        def raise_for_status(self):
            return None

        def iter_lines(self):
            return iter(("data: [DONE]", ""))

    class Stream:
        def __enter__(self):
            return Response()

        def __exit__(self, *_args):
            return False

    class Client:
        def post(self, url, data=None, **_kwargs):
            if url.endswith("/api/session"):
                return Response()
            captured.update(data or {})
            return Response()

        def stream(self, _method, _url, data=None, **_kwargs):
            captured.update(data or {})
            return Stream()

        def delete(self, *_args, **_kwargs):
            return Response()

    args = SimpleNamespace(
        base_url="http://backend",
        endpoint="http://model/v1",
        selected_endpoint_url="http://model/v1/chat/completions",
        endpoint_id="router-1",
        selected_model="router",
        model="router",
        prompt_mode="auto",
        client_runtime_context={
            "surface": "odysseus-tui",
            "session_cwd": "/workspace/project",
        },
        timeout=5,
        auto_approve=False,
    )

    monkeypatch.setattr(
        "scripts.eval_odysseus_tool_use._cookie",
        lambda _path: "cookie",
    )
    run_case(Client(), args, "local", "inspect this repo", "host_shell")

    assert captured["cwd"] == "/workspace/project"
    assert captured["workspace"] == "/workspace/project"


def test_eval_local_runtime_context_accepts_cwd_alias(monkeypatch) -> None:
    captured = {}

    class Response:
        status_code = 200

        def json(self):
            return {"id": "session-1"}

        def raise_for_status(self):
            return None

        def iter_lines(self):
            return iter(("data: [DONE]", ""))

    class Stream:
        def __enter__(self):
            return Response()

        def __exit__(self, *_args):
            return False

    class Client:
        def post(self, url, data=None, **_kwargs):
            if url.endswith("/api/session"):
                return Response()
            captured.update(data or {})
            return Response()

        def stream(self, _method, _url, data=None, **_kwargs):
            captured.update(data or {})
            return Stream()

        def delete(self, *_args, **_kwargs):
            return Response()

    args = SimpleNamespace(
        base_url="http://backend",
        endpoint="http://model/v1",
        selected_endpoint_url="http://model/v1/chat/completions",
        endpoint_id="router-1",
        selected_model="router",
        model="router",
        prompt_mode="auto",
        client_runtime_context={
            "surface": "odysseus-tui",
            "cwd": "/workspace/project",
        },
        timeout=5,
        auto_approve=False,
    )

    monkeypatch.setattr(
        "scripts.eval_odysseus_tool_use._cookie",
        lambda _path: "cookie",
    )
    run_case(Client(), args, "local", "inspect this repo", "host_shell")

    assert captured["cwd"] == "/workspace/project"
    assert captured["workspace"] == "/workspace/project"


def test_extended_eval_cleanup_only_removes_tagged_fixture_notes() -> None:
    class Response:
        def raise_for_status(self):
            return None

        def json(self):
            return {
                "notes": [
                    {"id": "fixture", "title": "ODY-EVAL-EXT-CREATE"},
                    {"id": "user", "title": "Keep this note"},
                    {"id": "", "title": "ODY-EVAL-EXT-NO-ID"},
                ]
            }

    class Client:
        def __init__(self):
            self.deleted = []

        def get(self, url, timeout):
            return Response()

        def delete(self, url, timeout):
            self.deleted.append(url)

    client = Client()
    cleanup_notes(client, "http://backend")

    assert client.deleted == ["http://backend/api/notes/fixture"]