Skip to content

Personal AI, On Personal Devices

Python
8.3 KB
Raw Blame
"""Tests for the ACE agent optimizer (mocked — no ace dep required)."""

from __future__ import annotations

from pathlib import Path
from unittest.mock import MagicMock, patch


class TestACEOptimizerConfig:
    def test_default_config(self) -> None:
        from openjarvis.core.config import ACEOptimizerConfig

        cfg = ACEOptimizerConfig()
        assert cfg.api_provider == "openai"
        assert cfg.num_epochs == 1
        assert cfg.max_num_rounds == 3
        assert cfg.playbook_token_budget == 80_000
        assert cfg.min_traces == 20

    def test_optimizer_init(self) -> None:
        from openjarvis.core.config import ACEOptimizerConfig
        from openjarvis.learning.agents.ace_optimizer import ACEAgentOptimizer

        cfg = ACEOptimizerConfig()
        optimizer = ACEAgentOptimizer(cfg)
        assert optimizer.config is cfg


class TestTraceDataProcessor:
    def test_answer_is_correct_substring_match(self) -> None:
        from openjarvis.learning.agents.ace_optimizer import _TraceDataProcessor

        assert _TraceDataProcessor.answer_is_correct("the answer is 42", "42")
        assert _TraceDataProcessor.answer_is_correct("FORTY-TWO is right", "forty-two")
        assert not _TraceDataProcessor.answer_is_correct("twelve", "42")

    def test_empty_ground_truth_is_never_correct(self) -> None:
        from openjarvis.learning.agents.ace_optimizer import _TraceDataProcessor

        assert not _TraceDataProcessor.answer_is_correct("anything", "")
        assert not _TraceDataProcessor.answer_is_correct("", "")

    def test_evaluate_accuracy_mean_of_correctness(self) -> None:
        from openjarvis.learning.agents.ace_optimizer import _TraceDataProcessor

        preds = ["the answer is 1", "wrong", "the answer is 3"]
        truths = ["1", "2", "3"]
        # 2 of 3 correct
        assert _TraceDataProcessor.evaluate_accuracy(preds, truths) == 2 / 3

    def test_evaluate_accuracy_empty_returns_zero(self) -> None:
        from openjarvis.learning.agents.ace_optimizer import _TraceDataProcessor

        assert _TraceDataProcessor.evaluate_accuracy([], []) == 0.0


class TestSplitSamples:
    def test_70_15_15_split(self) -> None:
        from openjarvis.learning.agents.ace_optimizer import _split_samples

        samples = [{"i": i} for i in range(100)]
        train, val, test = _split_samples(samples)
        assert len(train) == 70
        assert len(val) == 15
        assert len(test) == 15

    def test_split_is_order_preserving(self) -> None:
        from openjarvis.learning.agents.ace_optimizer import _split_samples

        samples = [{"i": i} for i in range(20)]
        train, val, test = _split_samples(samples)
        assert train[0]["i"] == 0
        assert val[0]["i"] == 14  # 20*0.70 == 14
        assert test[0]["i"] == 17  # 20*0.85 == 17


class TestTracesToSamples:
    def test_drops_empty_query_or_result(self) -> None:
        from openjarvis.learning.agents.ace_optimizer import _traces_to_samples

        traces = [
            MagicMock(query="What is 2+2?", result="4"),
            MagicMock(query="", result="something"),
            MagicMock(query="Q", result=""),
            MagicMock(query=None, result="r"),
            MagicMock(query="Q2", result="A2"),
        ]
        samples = _traces_to_samples(traces)
        assert len(samples) == 2
        assert samples[0] == {"question": "What is 2+2?", "ground_truth_answer": "4"}
        assert samples[1] == {"question": "Q2", "ground_truth_answer": "A2"}


class TestACEOptimizerOptimize:
    def _store_with(self, n: int):
        store = MagicMock()
        store.list_traces.return_value = [
            MagicMock(query=f"Q{i}", result=f"A{i}") for i in range(n)
        ]
        return store

    def test_too_few_traces_skipped(self) -> None:
        from openjarvis.core.config import ACEOptimizerConfig
        from openjarvis.learning.agents.ace_optimizer import ACEAgentOptimizer

        cfg = ACEOptimizerConfig(min_traces=20)
        result = ACEAgentOptimizer(cfg).optimize(self._store_with(5))
        assert result["status"] == "skipped"
        assert "5 traces" in result["reason"]

    def test_missing_ace_dep_returns_error(self) -> None:
        from openjarvis.core.config import ACEOptimizerConfig
        from openjarvis.learning.agents import ace_optimizer

        with patch.object(ace_optimizer, "HAS_ACE", False):
            cfg = ACEOptimizerConfig(min_traces=5)
            result = ace_optimizer.ACEAgentOptimizer(cfg).optimize(self._store_with(20))
        assert result["status"] == "error"
        assert "learning-ace" in result["reason"]

    def test_filters_to_usable_samples(self) -> None:
        """If filtering trims samples below min_traces, skip cleanly."""
        from openjarvis.core.config import ACEOptimizerConfig
        from openjarvis.learning.agents import ace_optimizer

        # 20 traces but only 3 have both query+result populated
        store = MagicMock()
        store.list_traces.return_value = [
            MagicMock(query=f"Q{i}" if i < 3 else "", result=f"A{i}" if i < 3 else "")
            for i in range(20)
        ]
        cfg = ACEOptimizerConfig(min_traces=10)
        with patch.object(ace_optimizer, "HAS_ACE", True):
            result = ace_optimizer.ACEAgentOptimizer(cfg).optimize(store)
        assert result["status"] == "skipped"
        assert "3 usable samples" in result["reason"]

    def test_successful_run_returns_playbook_metadata(self, tmp_path: Path) -> None:
        from openjarvis.core.config import ACEOptimizerConfig
        from openjarvis.learning.agents import ace_optimizer

        # Stub ACE: a class whose .run() writes final_playbook.txt.
        playbook_text = "## STRATEGIES\n[str-00001] helpful=5 :: be concise"

        class _FakeACE:
            def __init__(self, **kwargs):
                self.kwargs = kwargs

            def run(self, **kwargs):
                save_dir = Path(kwargs["config"]["save_dir"])
                (save_dir / "final_playbook.txt").write_text(playbook_text)

        with (
            patch.object(ace_optimizer, "HAS_ACE", True),
            patch.object(ace_optimizer, "ace", MagicMock(ACE=_FakeACE)),
        ):
            cfg = ACEOptimizerConfig(
                min_traces=5,
                save_dir=str(tmp_path),
                task_name="unittest",
            )
            result = ace_optimizer.ACEAgentOptimizer(cfg).optimize(self._store_with(20))

        assert result["status"] == "completed"
        assert result["traces_used"] == 20
        assert result["samples_used"] == 20
        assert result["playbook_path"].endswith("final_playbook.txt")
        assert result["playbook_chars"] == len(playbook_text)

    def test_ace_completes_without_playbook_surfaces_error(
        self, tmp_path: Path
    ) -> None:
        """ACE returns but doesn't write final_playbook.txt → surface error."""
        from openjarvis.core.config import ACEOptimizerConfig
        from openjarvis.learning.agents import ace_optimizer

        class _SilentACE:
            def __init__(self, **kwargs):
                pass

            def run(self, **kwargs):  # writes nothing
                pass

        with (
            patch.object(ace_optimizer, "HAS_ACE", True),
            patch.object(ace_optimizer, "ace", MagicMock(ACE=_SilentACE)),
        ):
            cfg = ACEOptimizerConfig(
                min_traces=5,
                save_dir=str(tmp_path),
            )
            result = ace_optimizer.ACEAgentOptimizer(cfg).optimize(self._store_with(20))

        assert result["status"] == "error"
        assert "without writing" in result["reason"]

    def test_ace_raises_returns_error(self, tmp_path: Path) -> None:
        from openjarvis.core.config import ACEOptimizerConfig
        from openjarvis.learning.agents import ace_optimizer

        class _BoomACE:
            def __init__(self, **kwargs):
                pass

            def run(self, **kwargs):
                raise RuntimeError("API key invalid")

        with (
            patch.object(ace_optimizer, "HAS_ACE", True),
            patch.object(ace_optimizer, "ace", MagicMock(ACE=_BoomACE)),
        ):
            cfg = ACEOptimizerConfig(
                min_traces=5,
                save_dir=str(tmp_path),
            )
            result = ace_optimizer.ACEAgentOptimizer(cfg).optimize(self._store_with(20))

        assert result["status"] == "error"
        assert "API key invalid" in result["reason"]