Skip to content

Autonomous research system for measurable, computer-executable research.

Python
129 KB
Raw Blame
from __future__ import annotations

import asyncio
import json
import os
import tempfile
import unittest
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch

from praxist.core.credentials import CredentialRef
from praxist.core.protocol import (
    AgentEvent,
    AgentRunRequest,
    AgentRunResult,
    BudgetDecision,
    BudgetGrant,
    BudgetRequest,
    CachePolicy,
    EnvPolicy,
    ModelCallSpec,
    ModelProfile,
    ModelResult,
    ToolCallRecord,
    ToolCallResult,
    ToolPermissionSet,
    ToolServerRef,
)
from praxist.plugins.graph_maintainers.finding_graph_mvp import engine, viz
from praxist.plugins.workflow_stages.research_loop.backend.frontier import (
    FrontierStore,
    _walk_for_metric,
    annotate_findings_with_diversity_overlap,
    compute_dimension_overlap,
)
from praxist.plugins.workflow_stages.research_loop.backend.generation_loop import (
    GenerationLoop,
)
from praxist.plugins.workflow_stages.research_loop.backend.prompt_strategy import (
    _build_axis_assignment_block,
    _build_diversity_penalty_block,
    _generate_variant_hint,
)
from praxist.plugins.workflow_stages.research_loop.backend.tools import local_store
from praxist.task_spec import (
    _normalize_anchor_metrics,
    _normalize_diversity_dimensions,
    _normalize_frontier_lanes,
    _normalize_must_explore_axes,
    load_task_spec,
)


class ProtocolCoverageContractsTest(unittest.TestCase):
    def test_runtime_protocol_serializes_nested_redacted_refs(self) -> None:
        credential = CredentialRef(
            scope="model_provider",
            provider="fake_provider",
            target_ref="model_provider:fake_provider",
            key_id="fake:key",
            source="runtime_profile",
        )
        profile = ModelProfile(
            profile_id="cheap",
            provider_ref="model_provider:fake_provider",
            model="fake-model",
            api_format="fake",
            capability_tags=["tool_use"],
            cost_tier="cheap",
            default_parameters={"temperature": 0},
        )
        call = ModelCallSpec(
            profile_id=profile.profile_id,
            provider_ref=profile.provider_ref,
            api_format=profile.api_format,
            model=profile.model,
            parameters=profile.default_parameters,
            credential_ref=credential,
        )
        tool_permissions = ToolPermissionSet(allowed_tools=["Read"], denied_tools=["Bash"])
        tool_server = ToolServerRef(
            ref="tool_server:evaluation_tools",
            server_name="evaluation-tools",
            tool_names=["share_finding"],
            credential_ref=credential,
            metadata={"mode": "offline"},
        )
        env_policy = EnvPolicy(
            exposed_env_keys=["PYTHONPATH"],
            scoped_credential_refs=[credential],
        )
        cache_policy = CachePolicy(
            mode="runtime_managed",
            frozen_prefix_hash="abc123",
            cache_breakpoints=["frozen"],
            runtime_cache_strategy="stable_prefix",
            provider_cache_strategy="automatic",
        )
        request = AgentRunRequest(
            request_id="req",
            run_id="run",
            stage_id="research_loop",
            role_ref="task_role:peer",
            agent_runtime_ref="agent_runtime:fake_runtime",
            prompt_ref={"path": "prompt.txt"},
            system_prompt_ref=None,
            cwd="/tmp",
            model_profile_ref=profile.profile_id,
            model_call=call,
            tool_permissions=tool_permissions,
            tool_servers=[tool_server.to_dict()],
            env_policy=env_policy,
            credential_ref=credential,
            credential_mode="single",
            budget_grant_id="grant",
            artifact_scope="peer",
            timeout_seconds=30,
            cache_policy=cache_policy,
            runtime_options={"stream": False},
        )
        event = AgentEvent(
            event_id="evt",
            run_id="run",
            agent_run_id="agent",
            stage_id="research_loop",
            type="message",
            payload={"text": "ok"},
            artifact_refs=[{"path": "out.txt"}],
            credential_refs=[credential],
            timestamp_ms=1,
        )
        tool_use = ToolCallRecord(
            tool_call_id="tool",
            server_name="evaluation-tools",
            tool_name="share_finding",
            started_at_ms=1,
            finished_at_ms=2,
            success=True,
            artifact_refs=[],
            failover_reason=None,
        )
        result = AgentRunResult(
            success=True,
            events=[event],
            text_output_refs=[{"path": "out.txt"}],
            tool_uses=[tool_use],
            error=None,
            failover_reason=None,
            credential_ref=credential,
        )

        self.assertEqual(profile.to_dict()["model"], "fake-model")
        self.assertEqual(
            ModelResult(True, "p", "m", "text", {"tokens": 1}, None, None).to_dict()["usage"][
                "tokens"
            ],
            1,
        )
        legacy_result = AgentRunResult(
            True,
            [],
            [],
            [],
            None,
            None,
            None,
            {"tokens": 1},
            "completed",
            False,
            False,
        )
        self.assertEqual(legacy_result.terminal_status, "completed")
        self.assertEqual(legacy_result.cost, {})
        self.assertEqual(
            ToolCallResult("s", "t", True, {"ok": True}).to_dict()["output"]["ok"], True
        )
        self.assertEqual(request.to_dict()["model_call"]["credential_ref"]["key_id"], "fake:key")
        self.assertEqual(
            request.to_dict()["env_policy"]["scoped_credential_refs"][0]["provider"],
            "fake_provider",
        )
        self.assertEqual(
            result.to_dict()["events"][0]["credential_refs"][0]["source"], "runtime_profile"
        )

    def test_budget_protocol_serializes_grant_and_review_decision(self) -> None:
        from praxist.core.budget import _invalid_budget_units, policy_for_ref
        from praxist.plugins.budget_policies.default_basic.policy import (
            DefaultBasicBudgetPolicy,
            create_policy,
        )

        request = BudgetRequest(
            request_id="req",
            requester_id="peer",
            experiment_id="exp",
            model_profile_ref=None,
            requested={"tokens": 10},
            expected_value={"confidence": "medium"},
            evidence_refs=["finding:a"],
            cheaper_alternatives=["downscope"],
            abort_conditions=["leak"],
        )
        grant = BudgetGrant(
            grant_id="grant",
            approved={"tokens": 5},
            conditions=["record_usage"],
            expires_at_generation=2,
        )
        decision = BudgetDecision(
            decision="downscope",
            reason_codes=["over_budget"],
            grant=grant,
            model_profile_override="cheap",
            review_target="chair",
        )

        self.assertEqual(request.requested["tokens"], 10)
        self.assertEqual(decision.to_dict()["grant"]["approved"]["tokens"], 5)
        self.assertIsNone(BudgetDecision("deny", ["risk"], None).to_dict()["grant"])
        self.assertEqual(
            _invalid_budget_units(
                {
                    "unknown": 1,
                    "tokens": "bad",
                    "wall_clock_seconds": float("inf"),
                    "gpu_hours": -1,
                }
            ),
            ["unknown", "tokens", "wall_clock_seconds", "gpu_hours"],
        )

        policy = create_policy()
        self.assertIsInstance(policy, DefaultBasicBudgetPolicy)
        impossible = policy.decide(replace(request, expected_value={"impossible": True}))
        self.assertEqual(impossible.decision, "deny")
        cheap = policy.decide(
            replace(
                request,
                request_id="cheap",
                requested={"tokens": 100, "wall_clock_seconds": 30, "gpu_hours": 0},
                expected_value={"confidence": "weak"},
                evidence_refs=[],
            )
        )
        self.assertEqual(cheap.decision, "grant")
        review = policy.decide(
            replace(
                request,
                request_id="review",
                requested={"tokens": 10_000, "wall_clock_seconds": 300, "gpu_hours": 1},
                expected_value={"confidence": "medium"},
                evidence_refs=[],
            )
        )
        self.assertEqual(review.decision, "require_review")
        with self.assertRaises(ValueError):
            policy_for_ref("agent_runtime:not_budget")


class TaskSpecCoverageContractsTest(unittest.TestCase):
    def test_task_spec_normalizers_are_tolerant_and_typed(self) -> None:
        from praxist import task_spec

        class BadString:
            def __str__(self) -> str:
                raise TypeError("bad string")

        self.assertEqual(_normalize_diversity_dimensions(None), [])
        self.assertEqual(_normalize_diversity_dimensions({"bad": True}), [])
        self.assertEqual(
            _normalize_diversity_dimensions(
                [
                    "mechanism",
                    {"name": "cost", "description": "runtime"},
                    {"name": BadString()},
                    {"bad": True},
                    3,
                ]
            ),
            [
                {"name": "mechanism", "description": "", "examples": ""},
                {"name": "cost", "description": "runtime", "examples": ""},
            ],
        )
        self.assertEqual(_normalize_must_explore_axes({"bad": True}), [])
        self.assertEqual(
            _normalize_must_explore_axes(
                ["axis A", {"name": "axis B", "description": None}, {"name": BadString()}, {}]
            ),
            [{"name": "axis A", "description": ""}, {"name": "axis B", "description": ""}],
        )
        self.assertEqual(_normalize_anchor_metrics({"bad": True}), [])
        self.assertEqual(
            _normalize_anchor_metrics(
                [
                    {"name": "acc", "direction": "maximize"},
                    ["gap", "minimize"],
                    {"name": "bad", "direction": "asc"},
                    {"name": ""},
                    [BadString(), "maximize"],
                    ["malformed"],
                ]
            ),
            [("acc", "maximize"), ("gap", "minimize")],
        )
        self.assertEqual(task_spec._optional_float("bad", field_name="x"), None)
        self.assertEqual(task_spec._normalize_str_list([None, BadString(), " ok "]), ["None", "ok"])
        self.assertEqual(
            task_spec._normalize_metric_bounds(
                {"": 1, "bad": "nan?", "ok": "2.5"},
                field_name="min_metrics",
            ),
            {"ok": 2.5},
        )
        self.assertEqual(_normalize_frontier_lanes({"bad": True}), [])
        self.assertEqual(
            _normalize_frontier_lanes(
                [
                    1,
                    {"name": ""},
                    {
                        "name": "candidate",
                        "k": "bad",
                        "cumulative_cap": "bad",
                        "include_lanes": "candidate",
                        "exclude_roles": ["theorist"],
                        "require_metrics": ["task_score"],
                        "require_truthy_metrics": ["promotion_eligible"],
                        "require_falsey_metrics": ["risk_repair_required"],
                        "min_metrics": {"task_score": "0.0"},
                        "max_metrics": {"resource_cost": 10},
                        "allow_risk_violating": "false",
                        "allow_lower_tier": "true",
                        "allow_non_promotable": "true",
                        "allow_missing_tier": "false",
                        "admit_new_high": "false",
                        "axes": [{"metric": "task_score", "direction": "maximize"}],
                        "optional_axes": [{"metric": "future_fitness", "direction": "maximize"}],
                    },
                    {"bad": True},
                ]
            ),
            [
                {
                    "name": "candidate",
                    "k": 1,
                    "axes": [("task_score", "maximize")],
                    "optional_axes": [("future_fitness", "maximize")],
                    "include_lanes": ["candidate"],
                    "exclude_lanes": [],
                    "include_families": [],
                    "exclude_families": [],
                    "include_tags": [],
                    "exclude_tags": [],
                    "include_roles": [],
                    "exclude_roles": ["theorist"],
                    "require_metrics": ["task_score"],
                    "require_truthy_metrics": ["promotion_eligible"],
                    "require_falsey_metrics": ["risk_repair_required"],
                    "min_metrics": {"task_score": 0.0},
                    "max_metrics": {"resource_cost": 10.0},
                    "allow_risk_violating": False,
                    "allow_lower_tier": True,
                    "parent_eligible": False,
                    "allow_non_promotable": True,
                    "allow_missing_tier": False,
                    "admit_new_high": False,
                    "description": "",
                }
            ],
        )
        string_axis_lane = _normalize_frontier_lanes(
            [
                {
                    "name": "novelty",
                    "direction": "minimize",
                    "axes": ["novelty_score"],
                    "optional_axes": ["runtime_cost"],
                }
            ]
        )[0]
        self.assertEqual(string_axis_lane["axes"], [("novelty_score", "minimize")])
        self.assertEqual(string_axis_lane["optional_axes"], [("runtime_cost", "minimize")])

    def test_task_spec_loads_defensive_config_edges(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / "description.md").write_text("desc", encoding="utf-8")
            prompt_base = root / "base.jinja2"
            prompt_base.write_text("base", encoding="utf-8")
            spec_path = root / "task.yaml"
            spec_path.write_text(
                """
task_id: t
task_name: Test Task
description_file: description.md
evaluation:
  requires_tier: "false"
  frontier_lanes:
    - name: capped
      cumulative_cap: 0
      max_metrics: bad
gems:
  result_cell_metric_derivations:
    - name: validation_metric
      source_keys: [score]
      validation_only: "false"
generation_policy:
  max_generations: 3
  cohort_size: 2
  per_generation_hours: 2
run_lifecycle: bad
multi_pi: bad
research_memory: bad
prompt_layout: bad
synthesis_trigger:
  max_interval_minutes: 110
  adaptive: bad
praxist_plugins: bad
""",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

            self.assertEqual(loaded.run_lifecycle.max_wall_clock_hours, None)
            self.assertEqual(loaded.multi_pi.n_rounds, 2)
            self.assertFalse(loaded.research_memory.enabled)
            self.assertEqual(loaded.prompt_layout.base_template, "")
            self.assertEqual(loaded.gems.max_gems_total, 4)
            self.assertEqual(loaded.panel_topology_ref, "panel_topology:legacy_multi_pi_two_round")
            self.assertFalse(loaded.evaluation.requires_tier)
            self.assertEqual(loaded.evaluation.frontier_lanes[0]["cumulative_cap"], 1)
            self.assertFalse(loaded.gems.result_cell_metric_derivations[0]["validation_only"])
            self.assertEqual(loaded.get_prompt_base_path(prompt_base), prompt_base)

            dir_base = root / "base_dir"
            dir_base.mkdir()
            dir_spec_path = root / "task_with_dir_base.yaml"
            dir_spec_path.write_text(
                f"prompt_layout:\n  base_template: {dir_base.name}\n",
                encoding="utf-8",
            )
            dir_loaded = load_task_spec(dir_spec_path)
            with self.assertRaises(ValueError):
                dir_loaded.get_prompt_base_path(prompt_base)

            fatal_spec_path = root / "task_with_tiny_cap.yaml"
            fatal_spec_path.write_text(
                """
generation_policy:
  per_generation_hours: 0.1
synthesis_trigger:
  max_interval_minutes: 20
""",
                encoding="utf-8",
            )
            with self.assertRaises(ValueError):
                load_task_spec(fatal_spec_path)

            adaptive_spec_path = root / "task_with_adaptive_clamp.yaml"
            adaptive_spec_path.write_text(
                """
generation_policy:
  per_generation_hours: 2
synthesis_trigger:
  max_interval_minutes: 60
  adaptive:
    enabled: true
    max_interval_ceiling_minutes: 180
gems:
  reset_interval_generations: bad
  max_gems_total: 99
  prompt_max_gems: 99
""",
                encoding="utf-8",
            )
            adaptive_loaded = load_task_spec(adaptive_spec_path)
            self.assertEqual(adaptive_loaded.synthesis_trigger.max_interval_minutes, 60)
            self.assertEqual(
                adaptive_loaded.synthesis_trigger.adaptive["max_interval_ceiling_minutes"],
                90,
            )
            self.assertEqual(adaptive_loaded.gems.reset_interval_generations, 6)
            self.assertEqual(adaptive_loaded.gems.max_gems_total, 99)

    def test_load_task_spec_applies_cross_field_safety_defaults(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / "description.md").write_text("desc", encoding="utf-8")
            (root / "prompt_task.jinja2").write_text("prompt", encoding="utf-8")
            (root / "eval.py").write_text("print('ok')\n", encoding="utf-8")
            spec_path = root / "task.yaml"
            spec_path.write_text(
                """
task_id: t
task_name: Test Task
description_file: description.md
research_direction: fallback
evaluation:
  primary_metric: loss
  direction: minimize
  aux_metrics: [gap]
  anchor_metrics:
    - {name: cost, direction: minimize}
  frontier_lanes:
    - name: alpha
      k: 2
      cumulative_cap: 10
      include_lanes: [alpha]
      allow_lower_tier: true
      allow_non_promotable: true
      require_truthy_metrics: [promotion_eligible]
      require_falsey_metrics: [risk_repair_required]
      min_metrics: {active_alpha: 0.0}
      axes:
        - {name: active_alpha, direction: maximize}
      optional_axes:
        - {name: future_fitness, direction: maximize}
  diversity_dimensions:
    - name: mechanism
  must_explore_axes:
    - name: axis-a
  requires_tier: true
compute_budget:
  per_experiment_gpu_hours: 1.5
  max_parallel_runs_per_peer: 3
  peer_gpu_memory_gb: 12
  peer_gpu_util_pct: 40
  peer_cpu_cores: 2
generation_policy:
  max_generations: 2
  cohort_size: 4
  per_generation_hours: 2
synthesis_trigger:
  enabled: false
  max_interval_minutes: 60
  adaptive:
    enabled: true
    min_evidence_units: 3
    min_formal_result_peers: 2
pi_agent:
  enabled: false
multi_pi:
  enabled: true
  panel_mode_default: invalid
  chair_peer_budget: 2
  n_rounds: 7
research_memory:
  enabled: false
  rollout_phase: 1
agent:
  premium_mode: true
tiered_eval:
  T1: {seeds: 1}
baselines:
  - {name: base, expected_acc: 0.7}
toolchain:
  framework: python
  entrypoint_template: train.py
  eval_entrypoint: eval.py
  benchmark_entrypoint: harness/benchmark.py
""",
                encoding="utf-8",
            )

            spec = load_task_spec(str(spec_path))

            self.assertEqual(spec.task_id, "t")
            self.assertEqual(spec.get_description(), "desc")
            self.assertEqual(spec.get_prompt_task_path(), root / "prompt_task.jinja2")
            self.assertEqual(spec.evaluation.anchor_metrics, [("cost", "minimize")])
            self.assertEqual(spec.evaluation.frontier_lanes[0]["name"], "alpha")
            self.assertEqual(spec.evaluation.frontier_lanes[0]["k"], 2)
            self.assertEqual(spec.evaluation.frontier_lanes[0]["cumulative_cap"], 10)
            self.assertEqual(
                spec.evaluation.frontier_lanes[0]["axes"], [("active_alpha", "maximize")]
            )
            self.assertEqual(
                spec.evaluation.frontier_lanes[0]["optional_axes"],
                [("future_fitness", "maximize")],
            )
            self.assertTrue(spec.evaluation.frontier_lanes[0]["allow_lower_tier"])
            self.assertTrue(spec.evaluation.frontier_lanes[0]["allow_non_promotable"])
            self.assertEqual(
                spec.evaluation.frontier_lanes[0]["require_truthy_metrics"],
                ["promotion_eligible"],
            )
            self.assertEqual(
                spec.evaluation.frontier_lanes[0]["require_falsey_metrics"],
                ["risk_repair_required"],
            )
            self.assertEqual(spec.evaluation.direction, "minimize")
            self.assertTrue(spec.evaluation.requires_tier)
            self.assertEqual(spec.compute_budget.peer_gpu_util_pct, 40)
            self.assertEqual(spec.multi_pi.panel_mode_default, "full")
            self.assertEqual(spec.multi_pi.n_rounds, 2)
            self.assertEqual(spec.multi_pi.chair_peer_budget, 4)
            self.assertEqual(spec.synthesis_trigger.adaptive["min_evidence_units"], 3)
            self.assertEqual(spec.synthesis_trigger.adaptive["min_formal_result_peers"], 2)
            self.assertTrue(spec.research_memory.enabled)
            self.assertEqual(spec.research_memory.rollout_phase, 2)
            self.assertTrue(spec.agent.premium_mode)
            self.assertEqual(spec.tiered_eval["T1"]["seeds"], 1)
            self.assertEqual(spec.toolchain.eval_entrypoint, "eval.py")
            self.assertEqual(spec.toolchain.benchmark_entrypoint, "harness/benchmark.py")
            self.assertEqual(spec.baselines[0].metric_value, 0.7)
            self.assertEqual(spec.baselines[0].expected_acc, 0.7)

    def test_load_task_spec_parses_research_loop_control_fields(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / "description.md").write_text("desc", encoding="utf-8")
            (root / "prompt_task.jinja2").write_text("prompt", encoding="utf-8")
            spec_path = root / "task.yaml"
            spec_path.write_text(
                """
task_id: generic
task_name: Generic Task
description_file: description.md
evaluation:
  primary_metric: score
  direction: maximize
  maturity_policy:
    min_effort_ratio: 0.6
    min_coverage_ratio: 0.7
    require_ratio_gate: true
  constructive_peer_mix_enabled: false
  constructive_target_ratio: 0.8
  launch_guard:
    enabled: true
    estimated_heavy_eval_minutes: 12
    estimated_close_grade_eval_minutes: 10
    safety_factor: 1.5
  frontier_lanes:
    - name: incubator
      admit_new_high: true
      axes:
        - {metric: score, direction: maximize}
        - {metric: risk_score, direction: minimize}
synthesis_trigger:
  mature_quorum_fraction: 0.4
  max_interval_minutes: 60
generation_policy:
  per_generation_hours: 2
""",
                encoding="utf-8",
            )

            spec = load_task_spec(spec_path)

        self.assertEqual(spec.evaluation.maturity_policy["min_effort_ratio"], 0.6)
        self.assertEqual(spec.evaluation.maturity_policy["min_coverage_ratio"], 0.7)
        self.assertTrue(spec.evaluation.maturity_policy["require_ratio_gate"])
        self.assertFalse(spec.evaluation.constructive_peer_mix_enabled)
        self.assertEqual(spec.evaluation.constructive_target_ratio, 0.8)
        self.assertTrue(spec.evaluation.launch_guard["enabled"])
        self.assertEqual(spec.evaluation.launch_guard["estimated_heavy_eval_minutes"], 12.0)
        self.assertEqual(
            spec.evaluation.launch_guard["estimated_close_grade_eval_minutes"],
            10.0,
        )
        self.assertEqual(spec.evaluation.launch_guard["safety_factor"], 1.5)
        self.assertTrue(spec.evaluation.frontier_lanes[0]["admit_new_high"])
        self.assertEqual(
            spec.evaluation.frontier_lanes[0]["axes"],
            [("score", "maximize"), ("risk_score", "minimize")],
        )
        self.assertEqual(spec.synthesis_trigger.mature_quorum_fraction, 0.4)

    def test_task_entrypoint_evaluation_command_falls_back_to_toolchain(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / "description.md").write_text("desc", encoding="utf-8")
            (root / "prompt_task.jinja2").write_text("prompt", encoding="utf-8")
            evaluator = root / "evaluations" / "public" / "run.py"
            evaluator.parent.mkdir(parents=True)
            evaluator.write_text("print('ok')\n", encoding="utf-8")
            spec_path = root / "task.yaml"
            spec_path.write_text(
                """
task_id: generic
task_name: Generic Task
description_file: description.md
task_entrypoints:
  evaluation:
    command: evaluations/public/run.py
""",
                encoding="utf-8",
            )
            spec = load_task_spec(spec_path)

        self.assertEqual(spec.toolchain.eval_entrypoint, "evaluations/public/run.py")

    def test_task_entrypoint_is_validated_relative_to_task_root(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            spec_path = root / "task.yaml"
            spec_path.write_text(
                "task_entrypoints:\n  evaluation:\n    command: evaluations/missing/run.py\n",
                encoding="utf-8",
            )

            with self.assertRaisesRegex(FileNotFoundError, "relative to task root"):
                load_task_spec(spec_path)

    def test_wrapped_task_entrypoints_are_validated_relative_to_task_root(self) -> None:
        from praxist.task_spec import load_task_spec

        commands = (
            "python3.8 evaluations/missing/run.py",
            "/usr/bin/python3 -u evaluations/missing/run.py",
            "python --check-hash-based-pycs always evaluations/missing/run.py",
            "env MODE=test python evaluations/missing/run.py",
            "bash -lc 'python -u evaluations/missing/run.py'",
            "bash -lc 'python evaluations/missing/run.py > result.log'",
            "bash -lc 'exec python evaluations/missing/run.py'",
            "bash -lc 'command -p python evaluations/missing/run.py'",
        )
        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            for command in commands:
                with self.subTest(command=command):
                    spec_path.write_text(
                        "task_entrypoints:\n  evaluation:\n    command: "
                        + json.dumps(command)
                        + "\n",
                        encoding="utf-8",
                    )
                    with self.assertRaisesRegex(FileNotFoundError, "relative to task root"):
                        load_task_spec(spec_path)

    def test_direct_task_entrypoint_with_arguments_is_still_validated(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            spec_path.write_text(
                "task_entrypoints:\n"
                "  evaluation:\n"
                "    command: evaluations/missing/run.py --mode full\n",
                encoding="utf-8",
            )

            with self.assertRaisesRegex(FileNotFoundError, "relative to task root"):
                load_task_spec(spec_path)

    def test_static_env_chdir_entrypoint_is_validated_from_task_root(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            evaluator = root / "evaluations" / "v2" / "run.py"
            evaluator.parent.mkdir(parents=True)
            evaluator.write_text("print('ok')\n", encoding="utf-8")
            spec_path = root / "task.yaml"
            command = "env -C evaluations/v2 python run.py"
            spec_path.write_text(
                "task_entrypoints:\n  evaluation:\n    command: " + json.dumps(command) + "\n",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

        self.assertEqual(loaded.toolchain.eval_entrypoint, command)

    def test_static_shell_cd_and_redirection_entrypoint_is_validated(self) -> None:
        from praxist.task_spec import (
            load_task_spec,
            resolve_declared_evaluation_entrypoint,
        )

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            evaluator = root / "evaluations" / "run.py"
            evaluator.parent.mkdir(parents=True)
            evaluator.write_text("print('ok')\n", encoding="utf-8")
            command = "bash -lc 'cd evaluations && python run.py 2>&1'"
            spec_path = root / "task.yaml"
            spec_path.write_text(
                "task_entrypoints:\n  evaluation:\n    command: " + json.dumps(command) + "\n",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)
            resolved = resolve_declared_evaluation_entrypoint(command, task_dir=root)

        self.assertEqual(loaded.toolchain.eval_entrypoint, command)
        self.assertEqual(resolved, evaluator.resolve())

    def test_static_env_chdir_missing_entrypoint_is_rejected(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / "evaluations" / "v2").mkdir(parents=True)
            spec_path = root / "task.yaml"
            spec_path.write_text(
                "task_entrypoints:\n"
                "  evaluation:\n"
                "    command: env --chdir=evaluations/v2 python missing.py\n",
                encoding="utf-8",
            )

            with self.assertRaisesRegex(FileNotFoundError, "relative to task root"):
                load_task_spec(spec_path)

    def test_static_env_chdir_is_validated_through_transparent_wrappers(self) -> None:
        from praxist.task_spec import load_task_spec

        commands = (
            "exec env --chdir evaluations/v2 python run.py",
            "bash -lc 'env -C evaluations/v2 python run.py'",
        )
        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            evaluator = root / "evaluations" / "v2" / "run.py"
            evaluator.parent.mkdir(parents=True)
            evaluator.write_text("print('ok')\n", encoding="utf-8")
            spec_path = root / "task.yaml"
            for command in commands:
                with self.subTest(command=command):
                    spec_path.write_text(
                        "task_entrypoints:\n"
                        "  evaluation:\n"
                        "    command: " + json.dumps(command) + "\n",
                        encoding="utf-8",
                    )

                    loaded = load_task_spec(spec_path)

                    self.assertEqual(loaded.toolchain.eval_entrypoint, command)

    def test_evaluator_chdir_argument_is_not_parsed_as_env_chdir(self) -> None:
        from praxist.task_spec import (
            declared_evaluation_entrypoint_chdir,
            load_task_spec,
        )

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            evaluator = root / "evaluate.py"
            evaluator.write_text("print('ok')\n", encoding="utf-8")
            command = "env MODE=test python evaluate.py --chdir outputs"
            spec_path = root / "task.yaml"
            spec_path.write_text(
                "task_entrypoints:\n  evaluation:\n    command: " + json.dumps(command) + "\n",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

        self.assertEqual(loaded.toolchain.eval_entrypoint, command)
        self.assertEqual(declared_evaluation_entrypoint_chdir(command), "")

    def test_failed_cd_alternative_does_not_rebase_static_evaluator(self) -> None:
        from praxist.task_spec import (
            load_task_spec,
            resolve_declared_evaluation_entrypoint,
        )

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            evaluator = root / "evaluations" / "run.py"
            evaluator.parent.mkdir(parents=True)
            evaluator.write_text("print('ok')\n", encoding="utf-8")
            command = "bash -lc 'cd missing || python evaluations/run.py'"
            spec_path = root / "task.yaml"
            spec_path.write_text(
                "task_entrypoints:\n  evaluation:\n    command: " + json.dumps(command) + "\n",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)
            resolved = resolve_declared_evaluation_entrypoint(command, task_dir=root)

        self.assertEqual(loaded.toolchain.eval_entrypoint, command)
        self.assertEqual(resolved, evaluator.resolve())

    def test_compound_entrypoint_selects_last_path_launch(self) -> None:
        from praxist.task_spec import resolve_declared_evaluation_entrypoint

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            prepare = root / "prepare.py"
            evaluator = root / "evaluations" / "run.py"
            prepare.write_text("print('prepare')\n", encoding="utf-8")
            evaluator.parent.mkdir(parents=True)
            evaluator.write_text("print('evaluate')\n", encoding="utf-8")
            command = "bash -lc 'python prepare.py && python evaluations/run.py'"

            resolved = resolve_declared_evaluation_entrypoint(command, task_dir=root)

        self.assertEqual(resolved, evaluator.resolve())

    def test_compound_entrypoint_may_generate_evaluator_at_runtime(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / "build.py").write_text("print('build')\n", encoding="utf-8")
            command = "bash -lc 'python build.py && python generated/evaluate.py'"
            spec_path = root / "task.yaml"
            spec_path.write_text(
                "task_entrypoints:\n  evaluation:\n    command: " + json.dumps(command) + "\n",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

        self.assertEqual(loaded.toolchain.eval_entrypoint, command)

    def test_dynamic_evaluator_commands_remain_runtime_resolved(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            spec_path.write_text(
                "task_entrypoints:\n"
                "  evaluation:\n"
                "    command: \"bash -lc 'python $TASK_EVALUATOR --mode full'\"\n",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

        self.assertIn("$TASK_EVALUATOR", loaded.toolchain.eval_entrypoint)

    def test_shell_entrypoint_with_attached_operator_is_left_to_runtime(self) -> None:
        from praxist.task_spec import load_task_spec

        commands = (
            "bash -lc 'python evaluations/missing/run.py>result.log'",
            "bash -lc 'python evaluations/missing/run.py # operator note'",
        )
        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            for command in commands:
                with self.subTest(command=command):
                    spec_path.write_text(
                        "task_entrypoints:\n  evaluation:\n    command: "
                        + json.dumps(command)
                        + "\n",
                        encoding="utf-8",
                    )
                    loaded = load_task_spec(spec_path)
                    self.assertEqual(loaded.toolchain.eval_entrypoint, command)

    def test_bare_external_evaluator_command_remains_supported(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            spec_path.write_text(
                "task_entrypoints:\n  evaluation:\n    command: external-evaluator\n",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

        self.assertEqual(loaded.toolchain.eval_entrypoint, "external-evaluator")

    def test_evaluator_entrypoint_can_be_relative_to_configured_runtime_cwd(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            evaluations = root / "evaluations"
            evaluations.mkdir()
            (evaluations / "run.py").write_text("print('ok')\n", encoding="utf-8")
            spec_path = root / "task.yaml"
            spec_path.write_text(
                "runtime_environment:\n"
                "  cwd: evaluations\n"
                "task_entrypoints:\n"
                "  evaluation:\n"
                "    command: python run.py\n",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

        self.assertEqual(loaded.toolchain.eval_entrypoint, "python run.py")

    def test_mature_evaluator_must_fit_generation_close_horizon(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            spec_path = root / "task.yaml"
            spec_path.write_text(
                """
evaluation:
  launch_guard:
    enabled: true
    estimated_heavy_eval_minutes: 330
    estimated_close_grade_eval_minutes: 330
    safety_factor: 1.1
generation_policy:
  per_generation_hours: 7
synthesis_trigger:
  max_interval_minutes: 350
  mature_quorum_fraction: 0.25
""",
                encoding="utf-8",
            )

            with self.assertRaisesRegex(ValueError, "evaluation timing is unreachable"):
                load_task_spec(spec_path)

            spec_path.write_text(
                spec_path.read_text(encoding="utf-8").replace(
                    "max_interval_minutes: 350", "max_interval_minutes: 420"
                ),
                encoding="utf-8",
            )
            loaded = load_task_spec(spec_path)

        self.assertEqual(loaded.synthesis_trigger.max_interval_minutes, 420)

    def test_legacy_heavy_estimate_warns_without_rejecting_an_unknown_close_protocol(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            spec_path.write_text(
                """
evaluation:
  launch_guard:
    enabled: true
    estimated_heavy_eval_minutes: 330
    safety_factor: 1.1
generation_policy:
  per_generation_hours: 7
synthesis_trigger:
  max_interval_minutes: 350
  mature_quorum_fraction: 0.25
""",
                encoding="utf-8",
            )

            with self.assertLogs("praxist.task_spec", level="WARNING") as captured:
                legacy = load_task_spec(spec_path)
            self.assertTrue(
                any("legacy task timing may be unreachable" in line for line in captured.output)
            )
            spec_path.write_text(
                spec_path.read_text(encoding="utf-8").replace(
                    "estimated_heavy_eval_minutes: 330",
                    "estimated_heavy_eval_minutes: 330\n"
                    "    estimated_close_grade_eval_minutes: 180",
                ),
                encoding="utf-8",
            )
            loaded = load_task_spec(spec_path)

        self.assertEqual(legacy.evaluation.launch_guard["estimated_heavy_eval_minutes"], 330.0)
        self.assertEqual(loaded.evaluation.launch_guard["estimated_heavy_eval_minutes"], 330.0)
        self.assertEqual(
            loaded.evaluation.launch_guard["estimated_close_grade_eval_minutes"],
            180.0,
        )

    def test_nonpositive_close_grade_estimate_keeps_legacy_task_runnable(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            for invalid_estimate in ("0", "not-a-number"):
                with self.subTest(invalid_estimate=invalid_estimate):
                    spec_path.write_text(
                        "evaluation:\n"
                        "  launch_guard:\n"
                        "    estimated_heavy_eval_minutes: 330\n"
                        f"    estimated_close_grade_eval_minutes: {invalid_estimate}\n"
                        "    safety_factor: 1.1\n"
                        "generation_policy:\n"
                        "  per_generation_hours: 7\n"
                        "synthesis_trigger:\n"
                        "  max_interval_minutes: 350\n"
                        "  mature_quorum_fraction: 0.25\n",
                        encoding="utf-8",
                    )

                    with self.assertLogs("praxist.task_spec", level="WARNING") as captured:
                        loaded = load_task_spec(spec_path)
                    self.assertTrue(
                        any(
                            "legacy task timing may be unreachable" in line
                            for line in captured.output
                        )
                    )
                    self.assertEqual(
                        loaded.evaluation.launch_guard["estimated_heavy_eval_minutes"],
                        330.0,
                    )

    def test_explicit_late_signal_workflow_can_disable_timing_gate(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            spec_path.write_text(
                """
evaluation:
  launch_guard:
    enabled: true
    estimated_heavy_eval_minutes: 330
    safety_factor: 1.1
generation_policy:
  per_generation_hours: 7
synthesis_trigger:
  max_interval_minutes: 350
  mature_quorum_fraction: 0
""",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

        self.assertTrue(loaded.evaluation.launch_guard["enabled"])

    def test_optional_adaptive_formal_target_does_not_override_information_density_close(
        self,
    ) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            spec_path.write_text(
                """
evaluation:
  launch_guard:
    enabled: true
    estimated_close_grade_eval_minutes: 330
    safety_factor: 1.1
generation_policy:
  per_generation_hours: 7
synthesis_trigger:
  max_interval_minutes: 350
  mature_quorum_fraction: 0
  adaptive:
    enabled: true
    min_formal_result_peers: 2
""",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

        self.assertEqual(loaded.synthesis_trigger.mature_quorum_fraction, 0.0)

    def test_user_authorized_close_grade_protocol_can_fit_below_optional_heavy_eval(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            spec_path.write_text(
                """
evaluation:
  launch_guard:
    enabled: true
    estimated_heavy_eval_minutes: 330
    estimated_close_grade_eval_minutes: 180
    safety_factor: 1.1
generation_policy:
  per_generation_hours: 7
synthesis_trigger:
  max_interval_minutes: 350
  mature_quorum_fraction: 0.25
""",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

        self.assertEqual(
            loaded.evaluation.launch_guard["estimated_close_grade_eval_minutes"],
            180.0,
        )

    def test_disabled_adaptive_ceiling_does_not_extend_close_horizon(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            spec_path.write_text(
                """
evaluation:
  launch_guard:
    enabled: true
    estimated_close_grade_eval_minutes: 330
    safety_factor: 1.1
generation_policy:
  per_generation_hours: 10
synthesis_trigger:
  max_interval_minutes: 350
  mature_quorum_fraction: 0.25
  adaptive:
    enabled: false
    max_interval_ceiling_minutes: 500
""",
                encoding="utf-8",
            )

            with self.assertRaisesRegex(ValueError, "evaluation timing is unreachable"):
                load_task_spec(spec_path)

    def test_disabled_adaptive_ceiling_does_not_clamp_fixed_synthesis_horizon(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            spec_path.write_text(
                """
generation_policy:
  per_generation_hours: 7
synthesis_trigger:
  max_interval_minutes: 350
  adaptive:
    enabled: false
    max_interval_ceiling_minutes: 500
""",
                encoding="utf-8",
            )

            loaded = load_task_spec(spec_path)

        self.assertEqual(loaded.synthesis_trigger.max_interval_minutes, 350)
        self.assertEqual(
            loaded.synthesis_trigger.adaptive["max_interval_ceiling_minutes"],
            500,
        )

    def test_disabled_launch_guard_still_validates_required_close_evidence(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            spec_path = Path(tmp) / "task.yaml"
            spec_path.write_text(
                """
evaluation:
  launch_guard:
    enabled: false
    estimated_close_grade_eval_minutes: 330
    safety_factor: 1.1
generation_policy:
  per_generation_hours: 7
synthesis_trigger:
  max_interval_minutes: 350
  mature_quorum_fraction: 0.25
""",
                encoding="utf-8",
            )

            with self.assertRaisesRegex(ValueError, "evaluation timing is unreachable"):
                load_task_spec(spec_path)

            spec_path.write_text(
                spec_path.read_text(encoding="utf-8").replace(
                    "mature_quorum_fraction: 0.25", "mature_quorum_fraction: 0"
                ),
                encoding="utf-8",
            )
            loaded = load_task_spec(spec_path)

        self.assertFalse(loaded.evaluation.launch_guard["enabled"])

    def test_load_task_spec_tolerates_malformed_synthesis_trigger_numbers(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / "description.md").write_text("desc", encoding="utf-8")
            (root / "prompt_task.jinja2").write_text("prompt", encoding="utf-8")
            spec_path = root / "task.yaml"
            spec_path.write_text(
                """
task_id: generic
task_name: Generic Task
description_file: description.md
evaluation:
  primary_metric: score
  direction: maximize
  maturity_policy:
    min_effort_ratio: .nan
    min_coverage_ratio: .inf
    require_ratio_gate: "false"
  constructive_peer_mix_enabled: "not-a-bool"
  constructive_target_ratio: .inf
  launch_guard:
    enabled: "false"
    estimated_heavy_eval_minutes: -.inf
    safety_factor: .nan
synthesis_trigger:
  enabled: "false"
  min_findings: .nan
  min_contributing_peers: .inf
  poll_interval_seconds: -.inf
  mature_quorum_fraction: .nan
  min_interval_minutes: -.inf
  max_interval_minutes: .inf
  adaptive:
    enabled: "false"
pi_agent:
  enabled: "false"
  strict: "false"
generation_policy:
  max_generations: .nan
  cohort_size: .inf
  per_generation_hours: .nan
  promote_top_k: -.inf
""",
                encoding="utf-8",
            )

            spec = load_task_spec(spec_path)

        self.assertEqual(spec.evaluation.maturity_policy["min_effort_ratio"], 0.75)
        self.assertEqual(spec.evaluation.maturity_policy["min_coverage_ratio"], 0.80)
        self.assertFalse(spec.evaluation.maturity_policy["require_ratio_gate"])
        self.assertTrue(spec.evaluation.constructive_peer_mix_enabled)
        self.assertEqual(spec.evaluation.constructive_target_ratio, 0.75)
        self.assertFalse(spec.evaluation.launch_guard["enabled"])
        self.assertEqual(spec.evaluation.launch_guard["estimated_heavy_eval_minutes"], 0.0)
        self.assertEqual(
            spec.evaluation.launch_guard["estimated_close_grade_eval_minutes"],
            0.0,
        )
        self.assertEqual(spec.evaluation.launch_guard["safety_factor"], 1.25)
        self.assertFalse(spec.synthesis_trigger.enabled)
        self.assertEqual(spec.synthesis_trigger.min_findings, 30)
        self.assertEqual(spec.synthesis_trigger.min_contributing_peers, 3)
        self.assertEqual(spec.synthesis_trigger.poll_interval_seconds, 30)
        self.assertEqual(spec.synthesis_trigger.mature_quorum_fraction, 0.0)
        self.assertEqual(spec.synthesis_trigger.min_interval_minutes, 120.0)
        self.assertEqual(spec.synthesis_trigger.max_interval_minutes, 240.0)
        self.assertFalse(spec.synthesis_trigger.adaptive.get("enabled"))
        self.assertFalse(spec.pi_agent.enabled)
        self.assertFalse(spec.pi_agent.strict)
        self.assertEqual(spec.generation_policy.max_generations, 8)
        self.assertEqual(spec.generation_policy.cohort_size, 5)
        self.assertEqual(spec.generation_policy.per_generation_hours, 5.0)
        self.assertEqual(spec.generation_policy.promote_top_k, 2)

    def test_load_task_spec_clamps_mature_quorum_fraction_to_valid_ratio(self) -> None:
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / "description.md").write_text("desc", encoding="utf-8")
            (root / "prompt_task.jinja2").write_text("prompt", encoding="utf-8")
            spec_path = root / "task.yaml"
            spec_path.write_text(
                """
task_id: generic
task_name: Generic Task
description_file: description.md
evaluation:
  primary_metric: score
  direction: maximize
synthesis_trigger:
  mature_quorum_fraction: 1.4
generation_policy:
  cohort_size: 4
""",
                encoding="utf-8",
            )

            spec = load_task_spec(spec_path)

        self.assertEqual(spec.synthesis_trigger.mature_quorum_fraction, 1.0)

    def test_tiered_stage_names_are_not_implicitly_all_mature(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend.evidence_maturity import (
            evidence_maturity_snapshot,
        )
        from praxist.task_spec import load_task_spec

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            (root / "description.md").write_text("desc", encoding="utf-8")
            (root / "prompt_task.jinja2").write_text("prompt", encoding="utf-8")
            spec_path = root / "task.yaml"
            spec_path.write_text(
                """
task_id: generic
task_name: Generic Task
description_file: description.md
evaluation:
  primary_metric: score
  direction: maximize
tiered_eval:
  triage: {effort: low}
  full: {effort: high}
""",
                encoding="utf-8",
            )

            spec = load_task_spec(spec_path)

        policy = spec.evaluation.maturity_policy
        self.assertEqual(policy["complete_stage_labels"], [])
        self.assertEqual(policy["preliminary_stage_labels"], [])
        self.assertIsNone(
            evidence_maturity_snapshot({"evidence_stage": "triage"}, policy)["mature_enough"]
        )
        self.assertIsNone(
            evidence_maturity_snapshot({"evidence_stage": "full"}, policy)["mature_enough"]
        )


class FrontierAndPromptCoverageContractsTest(unittest.TestCase):
    def test_frontier_promotes_primary_and_anchor_without_mutating_inputs(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            snapshot = Path(tmp) / "snapshot"
            snapshot.mkdir()
            (snapshot / "model.py").write_text("x = 1\n", encoding="utf-8")
            (snapshot / "__pycache__").mkdir()
            (snapshot / "__pycache__" / "skip.pyc").write_bytes(b"skip")
            findings = [
                {
                    "id": "f1",
                    "finding_id": "f1",
                    "finding_type": "result",
                    "variant_name": "A",
                    "metrics": {
                        "score": 0.5,
                        "cost": 5.0,
                        "tier": "T3",
                        "scored_complete": True,
                    },
                    "snapshot_local_path": str(snapshot),
                },
                {
                    "id": "f2",
                    "finding_id": "f2",
                    "finding_type": "result",
                    "variant_name": "B",
                    "metrics": {
                        "score": 0.9,
                        "cost": 10.0,
                        "tier": "T3",
                        "scored_complete": True,
                    },
                },
                {
                    "id": "f3",
                    "finding_id": "f3",
                    "finding_type": "insight",
                    "variant_name": "C",
                    "metrics": {
                        "score": 0.8,
                        "cost": 1.0,
                        "tier": "T3",
                        "scored_complete": True,
                    },
                },
                {
                    "id": "low",
                    "finding_id": "low",
                    "finding_type": "result",
                    "variant_name": "Low",
                    "metrics": {"score": 1.0, "tier": "T2", "scored_complete": True},
                },
                {
                    "id": "no",
                    "finding_id": "no",
                    "finding_type": "result",
                    "variant_name": "No",
                    "metrics": {
                        "score": 2.0,
                        "tier": "T3",
                        "promotion_eligible": "no",
                        "scored_complete": True,
                    },
                },
                {
                    "id": "theory",
                    "finding_id": "theory",
                    "finding_type": "result",
                    "variant_name": "Theory",
                    "metrics": {"score": 3.0, "tier": "T3", "scored_complete": True},
                },
            ]
            original_first_metrics = dict(findings[0]["metrics"])
            store = FrontierStore(
                Path(tmp) / "frontier",
                promote_top_k=1,
                primary_metric="score",
                metric_direction="maximize",
                anchor_metrics=[("cost", "minimize")],
                require_tier=True,
            )

            promoted = store.promote(0, findings)

            self.assertEqual([p["finding_id"] for p in promoted], ["theory", "f3"])
            self.assertEqual(promoted[1]["promoted_for_anchor"], "cost")
            self.assertEqual(findings[0]["metrics"], original_first_metrics)
            self.assertTrue((Path(tmp) / "frontier" / "frontier_manifest.json").exists())
            self.assertIsNone(promoted[0]["snapshot_path"])
            self.assertIsNone(promoted[1]["snapshot_path"])

            promoted_again = FrontierStore(
                Path(tmp) / "frontier_min",
                promote_top_k=1,
                primary_metric="score",
                metric_direction="minimize",
            ).promote(1, findings[:2])
            self.assertEqual(promoted_again[0]["finding_id"], "f1")

    def test_metric_walk_and_diversity_overlap_are_conservative(self) -> None:
        shared = {"score": 0.7}
        cyclic: dict[str, object] = {"details": {"nested": shared}}
        cyclic["self"] = cyclic
        self.assertEqual(_walk_for_metric(cyclic, "score"), 0.7)
        self.assertIsNone(_walk_for_metric({"score": True}, "score"))
        self.assertIsNone(
            _walk_for_metric({"details": {"score": 2.0}}, "score", _strict_canonical=True)
        )

        finding = {"design_dimensions": {"mechanism": "A", "cost": "low", "info": "grad"}}
        anchor = {
            "variant_name": "anchor",
            "metrics": {"design_dimensions": {"mechanism": "A", "cost": "high", "info": "grad"}},
        }
        overlap = compute_dimension_overlap(finding, anchor)
        self.assertEqual(overlap["overlap_count"], 2)
        annotated = annotate_findings_with_diversity_overlap(
            [finding, {"metrics": {}}], [anchor], 3
        )
        self.assertEqual(annotated[0]["metrics"]["diversity_overlap_status"], "narrow")
        self.assertEqual(annotated[1]["metrics"]["diversity_overlap_status"], "no_data")
        self.assertEqual(
            annotate_findings_with_diversity_overlap([finding], [], 3)[0]["metrics"][
                "diversity_overlap_status"
            ],
            "no_anchors",
        )

    def test_prompt_strategy_covers_axis_diversity_and_strategy_modes(self) -> None:
        class Frontier:
            def __init__(self, entries):
                self._entries = entries

            def get_summary(self):
                return list(self._entries)

        axes = [{"name": "axis-a", "description": "try A"}, {"name": "axis-b"}]
        axis_block = _build_axis_assignment_block(0, 0, 3, axes)
        self.assertIn("axis-a", axis_block)
        self.assertIn("peers 2..2 are free-explore", axis_block.lower())
        self.assertEqual(_build_axis_assignment_block(0, 5, 3, axes), "")

        anchors = [
            {
                "finding_id": "f1",
                "variant_name": "A",
                "metric_value": 0.9,
                "design_dimensions": {"mechanism": "x"},
                "generation_id": 0,
            }
        ]
        diversity = _build_diversity_penalty_block(anchors, [{"name": "mechanism"}])
        self.assertIn("finding_id=f1", diversity)
        self.assertIn("mechanism", diversity)
        self.assertIn("task's", diversity)
        self.assertIn("frontier policy", diversity)
        self.assertNotIn("primary metric alone", diversity)
        self.assertEqual(_build_diversity_penalty_block([]), "")

        cold_hint = _generate_variant_hint(0, 0, 2, "explore", Frontier([]), must_explore_axes=axes)
        self.assertIn("axis-a", cold_hint)
        self.assertEqual(
            _generate_variant_hint(0, 0, 2, "mixed", Frontier([]), must_explore_axes=axes), ""
        )
        pi_hint = _generate_variant_hint(
            1,
            0,
            2,
            "pi_directed",
            Frontier(anchors),
            diversity_dimensions=[{"name": "mechanism"}],
        )
        self.assertIn("authoritative directive", pi_hint)
        self.assertIn("DIVERSITY GUIDANCE", pi_hint)
        self.assertIn("design_dimensions", pi_hint)
        self.assertIn("planned_dimensions", pi_hint)
        self.assertIn("actually implemented and evaluated", pi_hint)
        self.assertIn("finding_id=f1", pi_hint)
        self.assertIn(
            "DIFFERENT direction", _generate_variant_hint(1, 0, 2, "explore", Frontier(anchors))
        )
        self.assertIn(
            "Download its snapshot", _generate_variant_hint(1, 1, 2, "exploit", Frontier(anchors))
        )
        self.assertIn(
            "DIFFERENT direction", _generate_variant_hint(1, 0, 4, "mixed", Frontier(anchors))
        )
        self.assertIn(
            "Download its snapshot", _generate_variant_hint(1, 3, 4, "mixed", Frontier(anchors))
        )

    def test_generic_boundary_helpers_preserve_neutral_fallbacks(self) -> None:
        from praxist.core.cache import frozen_prefix_hash
        from praxist.plugins.workflow_stages.research_loop.backend.artifact_semantics import (
            is_committed_runtime_fact_source,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.multi_pi.role_bindings import (
            instantiate_pi_roles,
        )
        from praxist.task_spec_compat import migrate_legacy_gems_config

        self.assertTrue(frozen_prefix_hash({"scalar": 1}).startswith("sha256:"))
        self.assertFalse(
            is_committed_runtime_fact_source(
                {
                    "artifact_semantics": {
                        "role": "canonical_state",
                        "status": "committed",
                        "runtime_fact_source": False,
                    }
                }
            )
        )
        self.assertEqual(
            instantiate_pi_roles(
                ["unknown_role"],
                run_dir=Path("/tmp/run"),
                workspace=Path("/tmp/workspace"),
                model="model",
                max_runtime_minutes=1,
                mcp_servers=None,
                stop_check_fn=None,
                premium_mode=False,
            ),
            [],
        )
        self.assertEqual(migrate_legacy_gems_config(None), ({}, ()))


class GraphAndLocalStoreCoverageContractsTest(unittest.TestCase):
    def test_graph_rule_engine_builds_and_resolves_edges(self) -> None:
        older_uuid = "84d85198-d1b1-4290-a8c0-0b667a4ef593"
        newer_uuid = "94d85198-d1b1-4290-a8c0-0b667a4ef594"
        findings = [
            {
                "id": older_uuid,
                "timestamp": "2026-01-01T00:00:00",
                "peer_id": "gen0_peer0",
                "finding_type": "result",
                "title": "VARIANT-X baseline",
                "variant_name": "VARIANT-X alpha=0.1",
                "content": "original",
            },
            {
                "id": "fs_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
                "timestamp": "2026-01-01T00:01:00",
                "peer_id": "gen0_peer0",
                "finding_type": "insight",
                "title": "VARIANT-X insight",
                "variant_name": "VARIANT-X",
                "content": "confirmed and consistent with baseline",
            },
            {
                "id": newer_uuid,
                "timestamp": "2026-01-01T00:02:00",
                "peer_id": "gen0_peer1",
                "finding_type": "result",
                "title": "VARIANT-X followup",
                "variant_name": "VARIANT-X",
                "content": f"failed to reproduce {older_uuid}",
                "links": json.dumps(
                    [
                        {
                            "target_finding_id": older_uuid,
                            "edge_type": "supports",
                            "rationale": "agent intent wins",
                        },
                        {"target_finding_id": "missing", "edge_type": "junk"},
                    ]
                ),
            },
        ]
        builder = engine.FindingGraphBuilder(findings)

        self.assertEqual(builder._norm_variant("VARIANT-X alpha=0.3"), "variant-x")
        self.assertEqual(builder.chronological()[0]["id"], older_uuid)
        self.assertEqual(
            engine._extract_referenced_ids(
                f"see {older_uuid} and fs_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
            )[0],
            older_uuid,
        )
        self.assertTrue(engine._has_any("confirmed", ("confirmed",)))
        self.assertFalse(engine._has_any_non_negated("not consistent with x", ("consistent with",)))

        edges = builder.propose_edges_for(findings[-1])
        edge_types = {edge["edge_type"] for edge in edges}
        self.assertIn("supports", edge_types)
        self.assertIn("related_to", edge_types)
        self.assertTrue(all(edge["src_finding_id"] == newer_uuid for edge in edges))
        self.assertGreaterEqual(len(builder.build_all_edges()), len(edges))
        self.assertEqual(engine._previous_generation_peer_id("gen3_peer2"), "gen2_peer2")
        self.assertIsNone(engine._previous_generation_peer_id("gen0_peer2"))
        self.assertNotIn("\n", engine._snippet("a\n```b```", 20))
        self.assertGreater(
            engine._score_edge_pair({"edge_type": "supports", "confidence": 0.8}, "p1", "p2"), 0.8
        )

    def test_local_store_graph_health_viz_and_maintainer_cycle(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            env = {"LOCAL_STORE_DIR": tmp}
            with patch.dict(os.environ, env, clear=False):
                local_store.init_db()
                older = local_store.insert_finding(
                    {
                        "id": "old",
                        "finding_type": "result",
                        "title": "VARIANT-Y old",
                        "content": "baseline",
                        "metrics": {
                            "score": 0.5,
                            "gap": 0.2,
                            "tier": "T3",
                            "promotion_eligible": True,
                            "clean_promotion_eligible": True,
                            "is_smoke_eval": False,
                        },
                        "variant_name": "VARIANT-Z",
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "timestamp": "2026-01-01T00:00:00",
                    }
                )
                newer = local_store.insert_finding(
                    {
                        "id": "new",
                        "finding_type": "result",
                        "title": "VARIANT-Y new",
                        "content": "confirmed improvement",
                        "metrics": {
                            "score": 0.8,
                            "gap": 0.1,
                            "tier": "T3",
                            "promotion_eligible": True,
                            "clean_promotion_eligible": True,
                            "is_smoke_eval": False,
                        },
                        "variant_name": "VARIANT-Y",
                        "peer_id": "gen1_peer0",
                        "generation_id": 1,
                        "timestamp": "2026-01-01T00:01:00",
                    }
                )
                metric_id = local_store.insert_metric({"run_id": "r", "metrics": {"score": 1}})
                self.assertGreater(metric_id, 0)
                self.assertEqual(local_store.count_findings(), 2)
                self.assertEqual(local_store.get_findings(peer_id="gen0_peer0")[0]["id"], older)

                edge_id = local_store.insert_edge(
                    {
                        "src_finding_id": newer,
                        "dst_finding_id": older,
                        "edge_type": "supports",
                        "confidence": 0.9,
                        "created_by": "test",
                        "provenance": {"rule": "manual"},
                    }
                )
                self.assertIsNotNone(edge_id)
                self.assertIsNone(
                    local_store.insert_edge(
                        {
                            "src_finding_id": newer,
                            "dst_finding_id": older,
                            "edge_type": "supports",
                            "confidence": 0.9,
                            "created_by": "test",
                        }
                    )
                )
                self.assertEqual(local_store.count_edges(), 1)
                self.assertEqual(local_store.edge_count_by_type()["supports"], 1)
                self.assertEqual(
                    local_store.get_edges_for_finding(newer, direction="out")[0]["dst_finding_id"],
                    older,
                )
                self.assertEqual(
                    local_store.get_subgraph(newer, max_depth=1)["nodes"][0]["graph_depth"], 0
                )
                self.assertEqual(
                    local_store.get_leaderboard(primary_metric="score")[0]["id"], newer
                )
                pareto = local_store.get_pareto_leaderboard(
                    "score",
                    "maximize",
                    [{"name": "gap", "direction": "minimize"}],
                    requires_tier=True,
                )
                self.assertEqual(pareto["n_total"], 2)
                self.assertGreaterEqual(pareto["n_pareto"], 1)

                engine.reset_graph_observability_state()
                maintainer = engine.FindingGraphMaintainer(Path(tmp), poll_interval=1)
                cycle = maintainer.sync_once()
                self.assertIn(cycle["status"], {"ok", "empty"})
                health = engine.write_graph_health(Path(tmp) / "graph")
                self.assertEqual(health["num_findings"], 2)
                context = engine.build_session_start_graph_context("gen1_peer0")
                self.assertIn("Graph-surfaced context", context)
                orientation = engine._render_orientation_context(2)
                self.assertIn("Graph-surfaced context", orientation)

                (Path(tmp) / "task_spec.yaml").write_text(
                    """
evaluation:
  primary_metric: score
  direction: maximize
  aux_metrics: [gap]
baselines:
  - {name: base, expected_acc: 0.5}
""",
                    encoding="utf-8",
                )
                payload = viz.build_viz_payload()
                self.assertEqual(payload["meta"]["num_findings"], 2)
                self.assertEqual(payload["meta"]["leaderboard"][0]["id"], newer)
                html_path = Path(tmp) / "graph" / "graph.html"
                viz.render_graph_html(html_path)
                self.assertIn("Finding Graph", html_path.read_text(encoding="utf-8"))

    def test_local_store_rejects_invalid_edges_and_batch_resolves_conflicts(self) -> None:
        with (
            tempfile.TemporaryDirectory() as tmp,
            patch.dict(os.environ, {"LOCAL_STORE_DIR": tmp}, clear=False),
        ):
            local_store.init_db()
            for fid in ("a", "b"):
                local_store.insert_finding(
                    {"id": fid, "title": fid, "timestamp": f"2026-01-01T00:00:0{fid == 'b'}"}
                )
            with self.assertRaises(ValueError):
                local_store.insert_edge(
                    {
                        "src_finding_id": "b",
                        "dst_finding_id": "a",
                        "edge_type": "bad",
                        "confidence": 0.5,
                        "created_by": "test",
                    }
                )
            with self.assertRaises(ValueError):
                local_store.insert_edge(
                    {
                        "src_finding_id": "b",
                        "dst_finding_id": "a",
                        "edge_type": "supports",
                        "confidence": 2.0,
                        "created_by": "test",
                    }
                )
            inserted = local_store.insert_edges_batch(
                [
                    {
                        "edge_id": "weak",
                        "src_finding_id": "b",
                        "dst_finding_id": "a",
                        "edge_type": "supports",
                        "confidence": 0.7,
                        "created_by": "rule_engine",
                    },
                    {
                        "edge_id": "strong",
                        "src_finding_id": "b",
                        "dst_finding_id": "a",
                        "edge_type": "derived_from",
                        "confidence": 0.6,
                        "created_by": "rule_engine",
                    },
                    {
                        "edge_id": "agent",
                        "src_finding_id": "b",
                        "dst_finding_id": "a",
                        "edge_type": "challenges",
                        "confidence": 0.55,
                        "created_by": "agent_declared",
                    },
                    {"edge_type": "bad"},
                ]
            )
            self.assertEqual(inserted, 3)
            strong = [
                edge
                for edge in local_store.get_edges_for_finding("b", direction="out")
                if edge["edge_type"] in {"derived_from", "supports", "challenges"}
            ]
            self.assertEqual(strong[0]["edge_type"], "challenges")


class EvaluationToolsCoverageContractsTest(unittest.TestCase):
    @staticmethod
    def _payload(response: dict[str, object]) -> dict[str, object]:
        content = response["content"]  # type: ignore[index]
        text = content[0]["text"]  # type: ignore[index]
        return json.loads(text)

    def test_evaluation_tool_handlers_persist_metrics_findings_and_leaderboards(self) -> None:
        from praxist.plugins.tools.evaluation_tools import adapter

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            logs_dir = root / "logs"
            findings_dir = root / "findings"
            env = {
                "LOCAL_MODE": "true",
                "LOCAL_STORE_DIR": str(root),
                "LOCAL_FINDINGS_DIR": str(findings_dir),
                "LOGS_DIR": str(logs_dir),
                "GENERATION_ID": "9",
                "PRIMARY_METRIC": "score",
                "METRIC_DIRECTION": "maximize",
                "ANCHOR_METRICS": json.dumps([{"name": "gap", "direction": "minimize"}]),
                "REQUIRES_TIER": "true",
            }
            with patch.dict(os.environ, env, clear=False):
                metrics = asyncio.run(
                    adapter._handle_log_experiment_metrics(
                        {
                            "run_id": "run",
                            "variant_name": "A",
                            "metrics": '{"score": 0.4}',
                            "notes": "note",
                            "step": 3,
                            "peer_id": "Gen2_Peer3",
                        }
                    )
                )
                self.assertEqual(self._payload(metrics)["status"], "recorded")
                self.assertTrue((logs_dir / "metrics_log.jsonl").exists())

                invalid_metrics = asyncio.run(
                    adapter._handle_log_experiment_metrics(
                        {
                            "run_id": "run",
                            "variant_name": "A",
                            "metrics": "{bad",
                            "peer_id": "gen0_peer0",
                        }
                    )
                )
                self.assertTrue(invalid_metrics.get("is_error"))

                bad_type = asyncio.run(
                    adapter._handle_share_finding(
                        {
                            "finding_type": "bad",
                            "title": "bad",
                            "content": "bad",
                        }
                    )
                )
                self.assertTrue(bad_type.get("is_error"))

                first = asyncio.run(
                    adapter._handle_share_finding(
                        {
                            "finding_type": "result",
                            "title": "Variant A result",
                            "content": "scored result",
                            "metrics": json.dumps(
                                {
                                    "score": 0.5,
                                    "gap": 0.2,
                                    "tier": "T3",
                                    "promotion_eligible": True,
                                    "clean_promotion_eligible": True,
                                    "is_smoke_eval": False,
                                }
                            ),
                            "variant_name": "A",
                            "notes": "short",
                            "peer_id": "gen2_peer0",
                            "design_dimensions": json.dumps({"mechanism": "a"}),
                            "extra": json.dumps({"peer_role": "exploit"}),
                        }
                    )
                )
                first_id = self._payload(first)["finding_id"]
                second = asyncio.run(
                    adapter._handle_share_finding(
                        {
                            "finding_type": "insight",
                            "title": "Variant B insight",
                            "content": "confirmed and supports previous",
                            "metrics": json.dumps(
                                {
                                    "score": 0.7,
                                    "gap": 0.1,
                                    "tier": "T3",
                                    "promotion_eligible": "true",
                                    "clean_promotion_eligible": "true",
                                    "is_smoke_eval": False,
                                }
                            ),
                            "variant_name": "B",
                            "notes": "short",
                            "peer_id": "gen2_peer1",
                            "links": json.dumps(
                                [
                                    {
                                        "target_finding_id": first_id,
                                        "edge_type": "supports",
                                        "rationale": "replicated",
                                    }
                                ]
                            ),
                            "design_dimensions": "{bad",
                            "extra": "not-json",
                        }
                    )
                )
                self.assertEqual(self._payload(second)["type"], "insight")
                self.assertEqual(local_store.count_findings(), 2)
                self.assertGreaterEqual(local_store.count_edges(), 1)

                pareto = self._payload(
                    asyncio.run(adapter._handle_get_leaderboard({"generation": "bad", "top_k": -1}))
                )
                self.assertEqual(pareto["mode"], "pareto")
                self.assertEqual(pareto["n_total"], 2)

                with patch.dict(os.environ, {"ANCHOR_METRICS": ""}, clear=False):
                    single = self._payload(
                        asyncio.run(adapter._handle_get_leaderboard({"generation": 2, "top_k": 1}))
                    )
                self.assertEqual(single["mode"], "single_metric")
                self.assertEqual(len(single["entries"]), 1)

    def test_wait_for_file_and_filesystem_leaderboard_cover_degraded_paths(self) -> None:
        from praxist.plugins.tools.evaluation_tools import adapter

        with tempfile.TemporaryDirectory() as tmp:
            root = Path(tmp)
            ready = root / "ready.json"
            ready.write_text("prefix\nDONE\n", encoding="utf-8")
            findings = root / "findings"
            findings.mkdir()
            (findings / "f.json").write_text(
                json.dumps(
                    {
                        "finding_type": "result",
                        "title": "fallback",
                        "metrics": {"score": 0.9, "tier": "T3"},
                        "variant_name": "fallback",
                        "generation_id": 0,
                    }
                ),
                encoding="utf-8",
            )
            env = {
                "LOCAL_STORE_DIR": str(root),
                "LOCAL_FINDINGS_DIR": str(findings),
                "PRIMARY_METRIC": "score",
                "METRIC_DIRECTION": "maximize",
            }
            with patch.dict(os.environ, env, clear=False):
                missing = asyncio.run(adapter._handle_wait_for_file_impl({}))
                self.assertTrue(missing.get("is_error"))

                unsafe = asyncio.run(
                    adapter._handle_wait_for_file_impl(
                        {"path": "/etc/passwd", "timeout_seconds": 1}
                    )
                )
                self.assertTrue(unsafe.get("is_error"))

                ready_payload = self._payload(
                    asyncio.run(
                        adapter._handle_wait_for_file_impl(
                            {
                                "path": f"{ready},{ready}",
                                "timeout_seconds": 1,
                                "poll_interval_seconds": 2,
                                "min_bytes": 1,
                                "contains_text": "DONE",
                                "mode": "all",
                            }
                        )
                    )
                )
                self.assertEqual(ready_payload["status"], "ready")
                self.assertEqual(ready_payload["deduped_count"], 1)

                too_many = asyncio.run(
                    adapter._handle_wait_for_file_impl(
                        {
                            "path": ",".join(str(root / f"f{i}") for i in range(33)),
                            "timeout_seconds": 1,
                        }
                    )
                )
                self.assertTrue(too_many.get("is_error"))

                self.assertEqual(adapter._parse_anchor_metrics_env(), [])
                with patch.dict(
                    os.environ,
                    {"ANCHOR_METRICS": '[{"name":"gap","direction":"bad"}]'},
                    clear=False,
                ):
                    self.assertEqual(adapter._parse_anchor_metrics_env(), [])
                fallback = json.loads(adapter._filesystem_leaderboard(0, 5))
                self.assertEqual(fallback["mode"], "filesystem_fallback")
                self.assertEqual(fallback["entries"][0]["variant_name"], "fallback")
                plugin = adapter.create_tool_plugin()
                self.assertIn("wait_for_file", plugin["tool_names"])
                with (
                    patch.object(adapter, "create_sdk_mcp_server", None),
                    patch.object(adapter, "tool", None),
                    self.assertRaises(ImportError),
                ):
                    adapter.create_evaluation_tools_server()


class ResearchMemoryCoverageContractsTest(unittest.TestCase):
    def test_ledgers_cards_memory_tools_and_evidence_pack_are_queryable(self) -> None:
        from praxist.plugins.tools.memory_tools import adapter as memory_tools
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.card_builder import (
            _detect_negative,
            _evidence_id,
            _safe_get_metric,
            build_card_from_finding,
            build_cards_from_db,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.context_auditor import (
            _has_overclaim_language,
            _has_source_id,
            audit_agenda,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.evidence_pack_builder import (
            _role_filter,
            _sanitize_value,
            build_evidence_pack,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.ledgers._ledger_base import (
            LedgerEntry,
            LedgerStore,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.ledgers.claim_ledger import (
            ClaimLedger,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.ledgers.coverage_matrix import (
            CoverageMatrix,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.ledgers.dissent_ledger import (
            DissentLedger,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.ledgers.frontier_delta_ledger import (
            FrontierDeltaLedger,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.ledgers.negative_evidence_ledger import (
            NegativeEvidenceLedger,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.ledgers.retired_claim_ledger import (
            RetiredClaimLedger,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.research_memory.ledgers.role_roi_ledger import (
            RoleROILedger,
        )

        with tempfile.TemporaryDirectory() as tmp:
            run_dir = Path(tmp)
            with patch.dict(os.environ, {"LOCAL_STORE_DIR": str(run_dir)}, clear=False):
                local_store.init_db()
                local_store.insert_finding(
                    {
                        "id": "finding_a",
                        "finding_type": "result",
                        "title": "Variant A champion",
                        "content": "positive",
                        "metrics": {
                            "mean_test_accuracy": 0.8,
                            "mean_train_test_gap": 0.1,
                            "promotion_eligible": True,
                            "tier": "T3",
                            "seed_count": 5,
                        },
                        "variant_name": "A",
                        "notes": "scaling synergy",
                        "peer_id": "gen1_peer0",
                        "generation_id": 1,
                        "timestamp": "2026-01-01T00:00:00",
                        "extra": {"peer_role": "exploit"},
                    }
                )
                local_store.insert_finding(
                    {
                        "id": "finding_b",
                        "finding_type": "challenge",
                        "title": "Variant B failed",
                        "content": "negative",
                        "metrics": {
                            "mean_test_accuracy": 0.4,
                            "promotion_eligible": False,
                            "tier": "T3",
                        },
                        "variant_name": "B",
                        "notes": "did not improve baseline",
                        "peer_id": "gen1_peer1",
                        "generation_id": 1,
                        "timestamp": "2026-01-01T00:01:00",
                        "extra": {"is_negative": True, "peer_role": "falsifier"},
                    }
                )

                raw_card = build_card_from_finding(local_store.get_findings()[0], run_dir)
                self.assertTrue(raw_card["evidence_id"].startswith("E_"))
                self.assertEqual(_safe_get_metric({"x": {"mean": "0.5"}}, "x"), 0.5)
                self.assertIsNone(_safe_get_metric({"x": float("nan")}, "x"))
                self.assertTrue(_detect_negative({"title": "kill result", "extra": "{}"}))
                self.assertNotEqual(
                    _evidence_id("", 0, "gen0_peer0", content_seed="x"),
                    _evidence_id("", 0, "gen0_peer0", content_seed="y"),
                )
                self.assertEqual(len(build_cards_from_db(run_dir, only_gen=1)), 2)

                store = LedgerStore(
                    run_dir / "research_memory" / "ledgers" / "custom.yaml", "custom"
                )
                created = store.upsert("entry", {"a": 1}, created_by="test")
                updated = store.upsert("entry", {"b": 2}, created_by="test", action="update")
                self.assertEqual(created.id, updated.id)
                self.assertEqual(store.get("entry").data["b"], 2)
                appended = store.append_only("append", {"x": True})
                self.assertEqual(LedgerEntry.from_dict(appended.to_dict()).id, "append")
                with self.assertRaises(ValueError):
                    store.append_only("append", {})
                self.assertEqual(len(store.filter(lambda entry: entry.id.startswith("e"))), 1)
                self.assertEqual(len(store), 2)

                claims = ClaimLedger(run_dir)
                claims.upsert_claim(
                    "C1",
                    "Universal champion claim",
                    "active",
                    0.8,
                    supports=[raw_card["evidence_id"]],
                    missing_tests=["control"],
                )
                claims.upsert_claim(
                    "C2",
                    "Retired claim",
                    "retired",
                    0.2,
                    boundary="only small tasks",
                    revive_if=["new seed"],
                )
                self.assertEqual(len(claims.list_active()), 1)
                self.assertEqual(len(claims.list_recently_killed()), 1)
                with self.assertRaises(ValueError):
                    claims.upsert_claim("bad", "bad", "unknown", 0.1)
                with self.assertRaises(ValueError):
                    claims.upsert_claim("bad", "bad", "active", 1.5)

                coverage = CoverageMatrix(run_dir)
                coverage.record_grid_point("A", "rho", 0.1, seed_count=2, source_evidence_id="E1")
                coverage.record_grid_point("A", "rho", 0.2, seed_count=3, source_evidence_id="E2")
                coverage.record_bridge_point("A", "B", "mix", "mid", source_evidence_id="E3")
                self.assertTrue(coverage.query_grid("A", "rho")["values_tested"])
                self.assertTrue(coverage.is_bridge_covered("B", "A", "mix"))
                self.assertEqual(coverage.query_bridge("A", "B", "mix")["variant_pair"], ["A", "B"])

                dissent = DissentLedger(run_dir)
                dissent.add(
                    "D1",
                    "C1",
                    {"skeptic": "needs control"},
                    resolving_experiment="run control",
                    decision_rule={"metric": "score"},
                )
                dissent.update_status("D1", "experiment_assigned")
                self.assertEqual(len(dissent.list_open()), 1)
                neg = NegativeEvidenceLedger(run_dir)
                neg.add("N1", "failed", "control", summary="control failed")
                retired = RetiredClaimLedger(run_dir)
                retired.retire("C2", "Retired claim", "obsolete", "small only", ["new evidence"])
                frontier_delta = FrontierDeltaLedger(run_dir)
                frontier_delta.record_promote(
                    generation_id=1,
                    axis="mean_test_accuracy",
                    current_anchor={"variant": "A", "value": 0.8},
                    previous_anchor={"variant": "B", "value": 0.6},
                )
                role_roi = RoleROILedger(run_dir)
                role_roi.record_gen_summary(1, {"exploit": {"accepted": 1}})

                self.assertEqual(memory_tools.list_active_claims(run_dir)[0]["id"], "C1")
                self.assertEqual(memory_tools.list_open_objections(run_dir)[0]["id"], "D1")
                self.assertTrue(
                    memory_tools.query_coverage_matrix(
                        run_dir, variant_family="A", parameter="rho"
                    )["covered"]
                )
                self.assertTrue(
                    memory_tools.query_coverage_matrix(
                        run_dir,
                        bridge_pair=["A", "B"],
                        bridge_dimension="mix",
                    )["covered"]
                )
                self.assertIn("error", memory_tools.query_coverage_matrix(run_dir))
                self.assertEqual(
                    memory_tools.get_ledger_entry(run_dir, "claim_ledger", "C1")["id"],
                    "C1",
                )
                self.assertIn("error", memory_tools.get_ledger_entry(run_dir, "bad", "x"))
                self.assertIn(
                    "error", memory_tools.get_ledger_entry(run_dir, "claim_ledger", "missing")
                )
                cards = memory_tools.query_evidence_cards(
                    run_dir,
                    mechanism="champion",
                    peer_id="gen1_peer0",
                    generation_id=1,
                    is_negative=False,
                )
                self.assertEqual(cards[0]["peer_id"], "gen1_peer0")
                self.assertIn(
                    "error",
                    memory_tools.get_evidence_card(run_dir, "missing"),
                )

                pack = build_evidence_pack(
                    run_dir,
                    panel_mode="high_stakes",
                    current_gen_id=1,
                    target_decisions=["next"],
                    pi_roles=["builder", "skeptic", "portfolio", "external_validity"],
                    max_cards_total=4,
                    max_cards_per_pack=3,
                    findings_summary={"count": 2},
                )
                self.assertTrue(pack.pack_id.startswith("EP::"))
                self.assertIn("builder", pack.private_packs)
                self.assertIsNone(_sanitize_value(float("inf")))
                self.assertIn("{\u200b{", _sanitize_value("{{ x }}"))
                self.assertGreater(_role_filter("skeptic", raw_card), 0)

                agenda = {
                    "consensus_actions": [{"id": "A1", "claim_or_hypothesis": "universal win"}],
                    "retired_claims": [{"id": "C2", "boundary": "", "revive_if": []}],
                    "peer_contracts": {"gen2_peer0": {"role": "Bridge"}},
                }
                report = audit_agenda(
                    agenda,
                    {"private_packs": {"skeptic": pack.all_cards}},
                    {"builder": {}},
                    completed_gen_id=1,
                )
                self.assertFalse(report.pass_)
                self.assertFalse(_has_source_id({"supports": [""]}))
                self.assertTrue(_has_source_id({"supports": ["E1"]}))
                self.assertIn("universal", _has_overclaim_language("Universal result"))
                plugin = memory_tools.create_tool_plugin()
                self.assertEqual(plugin["server_name"], "memory-tools")
                with (
                    patch.object(memory_tools, "create_sdk_mcp_server", None),
                    patch.object(memory_tools, "tool", None),
                    self.assertRaises(ImportError),
                ):
                    memory_tools.create_memory_tools_server(run_dir)


class PIAgentCoverageContractsTest(unittest.TestCase):
    @staticmethod
    def _agenda(next_gen_id: int, cohort_size: int = 5) -> dict[str, object]:
        roles = ["exploit", "falsifier", "bridge", "anti_mainline"]
        return {
            "generation": next_gen_id,
            "mainline_observation": {"main_risk": "control"},
            "cross_peer_hypotheses": [
                {
                    "id": f"H_g{next_gen_id}_01",
                    "claim": "testable claim",
                    "minimal_test": "run control",
                    "kill_condition": "control wins",
                    "promote_condition": "claim survives",
                    "source_findings": ["finding_a"],
                }
            ],
            "bridge_hypothesis": {"id": f"B_g{next_gen_id}_01"},
            "anti_mainline_contract": {"target_axes": ["non-mainline"]},
            "falsification_contract": {"target_hypothesis": f"H_g{next_gen_id}_01"},
            "peer_contracts": {
                f"gen{next_gen_id}_peer{i}": {
                    "role": roles[i % len(roles)],
                    "target_hypothesis": f"H_g{next_gen_id}_01",
                    "success_signal": "valid result",
                }
                for i in range(cohort_size)
            },
        }

    def test_pi_agent_loads_state_validates_agenda_and_runs_with_fake_agent(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend import pi_agent

        with tempfile.TemporaryDirectory() as tmp:
            run_dir = Path(tmp) / "run"
            run_dir.mkdir()
            template = Path(tmp) / "pi_template.jinja2"
            template.write_text(
                "completed={{ completed_gen_id }} next={{ next_gen_id }} "
                "findings={{ n_findings }} edges={{ n_edges }} output={{ agenda_output_path }}",
                encoding="utf-8",
            )
            with patch.dict(os.environ, {"LOCAL_STORE_DIR": str(run_dir)}, clear=False):
                local_store.init_db()
                local_store.insert_finding(
                    {
                        "id": "finding_a",
                        "finding_type": "result",
                        "title": "result",
                        "content": "content",
                        "metrics": {"score": 1.0},
                        "variant_name": "A",
                        "peer_id": "gen1_peer0",
                        "generation_id": 1,
                        "timestamp": "2026-01-01T00:00:00",
                    }
                )
                local_store.insert_finding(
                    {
                        "id": "finding_old",
                        "finding_type": "insight",
                        "title": "old",
                        "content": "content",
                        "metrics": {"score": 0.5},
                        "variant_name": "Old",
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "timestamp": "2026-01-01T00:00:00",
                    }
                )
                local_store.insert_edge(
                    {
                        "src_finding_id": "finding_a",
                        "dst_finding_id": "finding_old",
                        "edge_type": "supports",
                        "confidence": 0.8,
                        "created_by": "test",
                        "rationale": "supports",
                    }
                )

                frontier_dir = run_dir / "frontier"
                frontier_dir.mkdir()
                (frontier_dir / "frontier_manifest.json").write_text(
                    json.dumps(
                        {
                            "generations": {"0": [{"finding_id": "finding_old"}]},
                            "cumulative_top": [
                                {
                                    "finding_id": "finding_a",
                                    "variant_name": "A",
                                    "metrics": {"bad": float("nan"), "ok": 1.0},
                                }
                            ],
                        }
                    ),
                    encoding="utf-8",
                )
                agendas = run_dir / pi_agent.AGENDAS_DIRNAME
                agendas.mkdir()
                (agendas / pi_agent.AGENDA_FILE_PATTERN.format(1)).write_text(
                    "```yaml\n" + yaml_dump(self._agenda(1)) + "```\n",
                    encoding="utf-8",
                )

                pi = pi_agent.PIAgent(
                    run_dir=run_dir,
                    workspace=Path(tmp),
                    cohort_size=5,
                    model="fake-model",
                    max_runtime_minutes=1,
                    prompt_template_path=template,
                    mcp_servers={"evaluation-tools": object()},
                )
                self.assertEqual(pi.expected_peer_ids(2)[0], "gen2_peer0")
                self.assertEqual(pi._normalize_role("Anti-Mainline"), "anti_mainline")
                self.assertEqual(len(pi._load_gen_findings(1)), 1)
                self.assertEqual(len(pi._load_gen_edges(1)), 1)
                self.assertIsNone(pi._sanitize_json_value(float("nan")))
                self.assertEqual(
                    pi._trim_prior_metrics({"score": 1, "tier": "T3"}),
                    {"score": 1, "tier": "T3"},
                )
                self.assertEqual(pi._load_prior_agenda(1)["generation"], 1)
                self.assertEqual(pi._load_prior_agendas_summary(2)[0]["generation"], 1)
                self.assertTrue(pi._load_prior_findings_summary(1))
                self.assertEqual(
                    pi._build_findings_summary_for_panel(1)["total_since_last_synthesis"], 1
                )

                valid = self._agenda(2)
                self.assertIsNone(pi.validate_agenda(valid, 2))
                self.assertEqual(valid["cross_peer_hypotheses"][0]["id"], "H_g2_01")
                self.assertIn("missing top-level", pi.validate_agenda({}, 2))
                self.assertIn(
                    "cannot be parsed",
                    pi.validate_agenda({**self._agenda(2), "generation": "bad"}, 2),
                )
                self.assertIn(
                    "exactly cohort_size",
                    pi.validate_agenda({**self._agenda(2), "peer_contracts": {}}, 2),
                )
                placeholder = self._agenda(2)
                placeholder["cross_peer_hypotheses"][0]["claim"] = "<one paragraph>"
                self.assertIn("placeholder", pi.validate_agenda(placeholder, 2))

                class FakeBaseAgent:
                    def __init__(self, **kwargs):
                        self.kwargs = kwargs

                    async def execute(self, task: str):
                        out_path = run_dir / "agendas" / pi_agent.AGENDA_FILE_PATTERN.format(2)
                        out_path = out_path.with_suffix(out_path.suffix + ".candidate")
                        out_path = (
                            run_dir / "peer_workspaces" / self.kwargs["request_id"] / out_path.name
                        )
                        out_path.parent.mkdir(parents=True, exist_ok=True)
                        out_path.write_text(
                            yaml_dump(PIAgentCoverageContractsTest._agenda(2)), encoding="utf-8"
                        )
                        return SimpleNamespace(
                            success=True,
                            error=None,
                            request_id=self.kwargs["request_id"],
                        )

                with patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.agent.BaseAgent",
                    FakeBaseAgent,
                ):
                    result = asyncio.run(pi.run(1))

                self.assertTrue(result.success)
                self.assertEqual(result.next_gen_id, 2)
                self.assertTrue(result.agenda_path and result.agenda_path.exists())
                self.assertTrue((agendas / "pi_prompt_for_gen2.md").exists())

    def test_pi_agent_handles_invalid_single_pi_and_multi_pi_fallback_paths(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend import pi_agent
        from praxist.plugins.workflow_stages.research_loop.backend.multi_pi import PanelResult

        with tempfile.TemporaryDirectory() as tmp:
            run_dir = Path(tmp) / "run"
            run_dir.mkdir()
            template = Path(tmp) / "pi_template.jinja2"
            template.write_text("output={{ agenda_output_path }}", encoding="utf-8")

            class BadAgendaAgent:
                def __init__(self, **kwargs):
                    self.kwargs = kwargs

                async def execute(self, task: str):
                    out_path = run_dir / "agendas" / pi_agent.AGENDA_FILE_PATTERN.format(1)
                    out_path = out_path.with_suffix(out_path.suffix + ".candidate")
                    out_path = (
                        run_dir / "peer_workspaces" / self.kwargs["request_id"] / out_path.name
                    )
                    out_path.parent.mkdir(parents=True, exist_ok=True)
                    out_path.write_text(
                        "generation: 99\ncross_peer_hypotheses: []\npeer_contracts: {}\n",
                        encoding="utf-8",
                    )
                    return SimpleNamespace(
                        success=True,
                        error=None,
                        request_id=self.kwargs["request_id"],
                    )

            pi = pi_agent.PIAgent(
                run_dir=run_dir,
                workspace=Path(tmp),
                cohort_size=5,
                model="fake",
                max_runtime_minutes=1,
                prompt_template_path=template,
            )
            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend.agent.BaseAgent",
                BadAgendaAgent,
            ):
                invalid = asyncio.run(pi.run(0))
            self.assertFalse(invalid.success)
            self.assertTrue((run_dir / "agendas" / "research_agenda_gen1.yaml.rejected").exists())

            class MultiPiConfig:
                panel_mode_default = "full"
                auto_escalate_to_high_stakes = False
                pi_max_runtime_minutes = 1
                chair_max_runtime_minutes = 1
                n_rounds = 1
                round2_max_runtime_minutes = 1
                fallback_to_single_pi_on_panel_failure = False

            async def fake_panel_success(**kwargs):
                return PanelResult(success=True, panel_mode="full", agenda=self._agenda(1))

            async def fake_panel_failure(**kwargs):
                return PanelResult(success=False, panel_mode="full", error="panel failed")

            pi_panel = pi_agent.PIAgent(
                run_dir=run_dir,
                workspace=Path(tmp),
                cohort_size=5,
                model="fake",
                use_multi_pi_panel=True,
                multi_pi_config=MultiPiConfig(),
            )
            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend.multi_pi.run_panel",
                fake_panel_success,
            ):
                panel_result = asyncio.run(pi_panel.run(0))
            self.assertTrue(panel_result.success)

            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend.multi_pi.run_panel",
                fake_panel_failure,
            ):
                panel_failure = asyncio.run(pi_panel.run(0))
            self.assertFalse(panel_failure.success)
            self.assertIn("panel failed", panel_failure.error or "")
            self.assertFalse(
                (run_dir / "agendas" / pi_agent.AGENDA_FILE_PATTERN.format(1)).exists()
            )

            bad_agenda_path = run_dir / "agendas" / pi_agent.AGENDA_FILE_PATTERN.format(3)
            bad_agenda_path.write_text("peer_contracts: []\n", encoding="utf-8")
            self.assertIsNone(pi_agent.load_agenda_for_gen(run_dir, 3))
            good_agenda_path = run_dir / "agendas" / pi_agent.AGENDA_FILE_PATTERN.format(4)
            good_agenda_path.write_text(yaml_dump(self._agenda(4)), encoding="utf-8")
            self.assertEqual(pi_agent.load_agenda_for_gen(run_dir, 4)["generation"], 4)


def yaml_dump(data: object) -> str:
    import yaml

    return yaml.safe_dump(data, sort_keys=False, allow_unicode=True)


class GenerationLoopCoverageContractsTest(unittest.TestCase):
    def test_generation_loop_run_success_and_error_paths_are_offline(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend.resume_state import (
            write_boundary_marker,
        )

        async def fake_complete(loop, *, gen_id, pi_agent, pi_cfg, generation_results):
            self.assertEqual(generation_results, [{"generation": gen_id}])
            loop.frontier.promote(
                gen_id,
                [
                    {
                        "id": f"f{gen_id}",
                        "finding_id": f"f{gen_id}",
                        "finding_type": "result",
                        "variant_name": f"v{gen_id}",
                        "metrics": {
                            "score": 0.5 + gen_id,
                            "tier": "T3",
                            "scored_complete": True,
                            "effort_ratio": 1.0,
                            "coverage_ratio": 1.0,
                        },
                    }
                ],
            )
            write_boundary_marker(
                loop.run_dir,
                gen_id=gen_id,
                promoted_count=1,
                pi_status="test_committed",
            )

        async def fake_generation(loop, gen_id):
            return [{"generation": gen_id}]

        with tempfile.TemporaryDirectory() as tmp:
            spec = load_task_spec("templates/tasks/toy_math/task.yaml")
            spec = replace(
                spec,
                evaluation=replace(
                    spec.evaluation,
                    primary_metric="score",
                    direction="maximize",
                    frontier_lanes=[],
                ),
                generation_policy=replace(spec.generation_policy, max_generations=2, cohort_size=1),
                pi_agent=replace(spec.pi_agent, enabled=False),
                synthesis_trigger=replace(spec.synthesis_trigger, max_interval_minutes=1),
            )
            run_dir = Path(tmp) / "run"
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.run_generation_cohort",
                    fake_generation,
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.complete_generation_boundary",
                    fake_complete,
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.configure_runtime_environment"
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.initialize_local_store_if_needed"
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.validate_baseline_cache_for_run"
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.start_sidecars"
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.sidecars.stop_sidecars"
                ),
            ):
                loop = GenerationLoop(
                    task_spec=spec,
                    workspace=Path(tmp),
                    run_dir=run_dir,
                    local_mode=True,
                    tool_server_refs=[],
                )
                self.assertEqual(loop._strategy_for_gen(0), "explore")
                self.assertEqual(loop._strategy_for_gen(1), "pi_directed")
                summary = asyncio.run(loop.run())

            self.assertEqual(summary["generations_completed"], 2)
            self.assertEqual(summary["exit_condition"], "max_generations")
            self.assertEqual(summary["status"], "succeeded")
            self.assertEqual(summary["exit_code"], 0)
            self.assertEqual(summary["stop_reason"], "max_generations")
            self.assertFalse((run_dir / "orchestrator.lock").exists())
            self.assertTrue((run_dir / "run_summary.json").exists())
            self.assertTrue((run_dir / "run_stop_report.json").exists())
            self.assertGreaterEqual(len(summary["frontier_summary"]), 1)

        async def boom_generation(loop, gen_id):
            raise RuntimeError("boom")

        with tempfile.TemporaryDirectory() as tmp:
            spec = load_task_spec("templates/tasks/toy_math/task.yaml")
            spec = replace(
                spec,
                evaluation=replace(spec.evaluation, primary_metric="score"),
                generation_policy=replace(spec.generation_policy, max_generations=1, cohort_size=1),
                pi_agent=replace(spec.pi_agent, enabled=False),
            )
            run_dir = Path(tmp) / "run_error"
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.run_generation_cohort",
                    boom_generation,
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.configure_runtime_environment"
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.initialize_local_store_if_needed"
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.validate_baseline_cache_for_run"
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.generation_loop.start_sidecars"
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend.sidecars.stop_sidecars"
                ),
            ):
                loop = GenerationLoop(
                    task_spec=spec,
                    workspace=Path(tmp),
                    run_dir=run_dir,
                    local_mode=True,
                    tool_server_refs=[],
                )
                with self.assertRaises(RuntimeError):
                    asyncio.run(loop.run())
            error_summary = json.loads((run_dir / "run_summary.json").read_text(encoding="utf-8"))
            self.assertEqual(error_summary["exit_condition"], "error")
            self.assertEqual(error_summary["error_type"], "RuntimeError")


class MultiPIPanelCoverageContractsTest(unittest.TestCase):
    def test_base_pi_loads_kb_runs_round1_and_round2_with_tolerant_yaml(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend.multi_pi.pi_roles import (
            _base_pi,
        )

        class TestPI(_base_pi.BasePI):
            role_name = "tester"
            prompt_template_name = "base.jinja2"

            def fixed_questions(self) -> list[str]:
                return ["What is the strongest claim?"]

        class FakeAgentResult:
            success = True
            error = None

            def __init__(self, text: str):
                self.output = {"text_outputs": [text]}

        class FakeBaseAgent:
            calls: list[dict[str, object]] = []

            def __init__(self, **kwargs):
                self.kwargs = kwargs
                FakeBaseAgent.calls.append(kwargs)

            async def execute(self, task: str) -> FakeAgentResult:
                self.task = task
                if self.kwargs["name"].endswith("round2"):
                    return FakeAgentResult(
                        "preamble\n```yaml\n"
                        "role: tester\n"
                        "round: 2\n"
                        "strongest_agreement:\n"
                        '  peer_label: "PI #A"\n'
                        "  claim_id: peer\n"
                        "  why: useful\n"
                        "strongest_objection:\n"
                        '  peer_label: "PI #A"\n'
                        "  claim_id: peer\n"
                        "  objection: weak evidence\n"
                        "  proposed_kill_test: rerun\n"
                        "missing_experiment:\n"
                        "  description: ablation\n"
                        "  why_critical: isolate mechanism\n"
                        "private_kb_revealed_blind_spot:\n"
                        "  triggered: false\n"
                        "  peer_label: null\n"
                        "  blind_spot: none\n"
                        "claim_that_should_be_downgraded:\n"
                        "  claim_id: c1\n"
                        "  current_language: useful\n"
                        "  recommended_language: bounded useful\n"
                        "  reason: needs validation\n"
                        "singleton_high_upside_idea_to_preserve:\n"
                        "  source: self\n"
                        "  peer_label: null\n"
                        "  idea_summary: preserve weak signal\n"
                        "  protected_budget_recommendation: 1 peer\n"
                        "own_revisions:\n"
                        "  - claim_id: c1\n    revision: keep\n```"
                    )
                return FakeAgentResult(
                    "Now the memo follows\nrole: tester\n"
                    "top_claims:\n  - id: c1\n    claim: useful\n"
                    "private_knowledge_used:\n  - kb1\n"
                )

        with tempfile.TemporaryDirectory() as tmp:
            workspace = Path(tmp)
            kb_dir = workspace / "kb"
            kb_dir.mkdir()
            (kb_dir / "kb1.md").write_text(
                "accuracy generalization optimizer evidence", encoding="utf-8"
            )
            (kb_dir / "ignored.txt").write_text("ignored", encoding="utf-8")
            pi = TestPI(
                run_dir=workspace / "run",
                workspace=workspace,
                model="fake-model",
                mcp_servers={"memory-tools": object()},
                premium_mode=True,
                reasoning_effort="high",
            )
            pi._private_kb_path = lambda: kb_dir  # type: ignore[method-assign]

            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend.agent.BaseAgent",
                FakeBaseAgent,
            ):
                memo = asyncio.run(
                    pi.run(
                        shared_core={"claim": "accuracy evidence"},
                        private_pack=[{"id": "p1"}],
                        target_decisions=["choose next experiment"],
                    )
                )
                round2 = asyncio.run(
                    pi.run_cross_review(
                        own_memo=memo.parsed,
                        anon_peers={"PI #A": {"top_claims": [{"id": "peer"}]}},
                        round2_max_runtime_minutes=1,
                    )
                )

        self.assertTrue(memo.success)
        self.assertEqual(memo.parsed["role"], "tester")
        self.assertEqual(memo.private_kb_used, ["kb1"])
        self.assertTrue(round2.success)
        self.assertEqual(round2.parsed["round"], 2)
        self.assertEqual(FakeBaseAgent.calls[0]["reasoning_effort"], "high")
        self.assertEqual(FakeBaseAgent.calls[1]["reasoning_effort"], "high")
        self.assertIn(
            "mcp__memory-tools__query_evidence_cards",
            FakeBaseAgent.calls[0]["allowed_tools"],
        )
        for raw_tool in ("Read", "Bash", "Glob", "Grep"):
            self.assertNotIn(raw_tool, FakeBaseAgent.calls[0]["allowed_tools"])
            self.assertNotIn(raw_tool, FakeBaseAgent.calls[1]["allowed_tools"])

    def test_base_pi_failure_paths_return_structured_unavailable_memos(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend.multi_pi.pi_roles import (
            _base_pi,
        )

        class TestPI(_base_pi.BasePI):
            role_name = "tester"

        class FailedAgentResult:
            success = False
            error = "model failed"
            output: dict[str, object] = {}

        class FailedBaseAgent:
            def __init__(self, **_kwargs):
                pass

            async def execute(self, task: str) -> FailedAgentResult:
                return FailedAgentResult()

        with tempfile.TemporaryDirectory() as tmp:
            pi = TestPI(run_dir=Path(tmp) / "run", workspace=Path(tmp), model="fake-model")
            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend.agent.BaseAgent",
                FailedBaseAgent,
            ):
                memo = asyncio.run(pi.run({}, [], []))
                round2 = asyncio.run(pi.run_cross_review({}, {}, round2_max_runtime_minutes=1))

        self.assertFalse(memo.success)
        self.assertEqual(memo.parsed["error"], "model failed")
        self.assertFalse(round2.success)
        self.assertTrue(round2.parsed["_round2_failed"])

    def test_two_round_panel_persists_evidence_memos_agenda_and_audit(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend.multi_pi import (
            legacy_two_round_executor as panel,
        )
        from praxist.plugins.workflow_stages.research_loop.backend.multi_pi.pi_roles import (
            _base_pi,
        )

        class FakePI:
            def __init__(self, role_name: str):
                self.role_name = role_name

            async def run(self, shared_core, private_pack, target_decisions):
                if self.role_name == "external_validity":
                    raise RuntimeError("offline PI unavailable")
                parsed = {
                    "role": self.role_name,
                    "top_claims": [{"id": f"{self.role_name}_claim", "claim": "works"}],
                    "objections_or_warnings": [],
                    "proposed_experiments": [],
                    "proposed_peer_contracts": [],
                    "private_knowledge_used": [],
                }
                if self.role_name == "skeptic":
                    parsed["objections_or_warnings"] = [
                        {
                            "severity": "blocking",
                            "target_claim": "builder_claim",
                            "objection": "needs replication",
                            "resolving_experiment": "replicate",
                        }
                    ]
                return _base_pi.PIMemo(
                    role=self.role_name,
                    raw_text="raw",
                    parsed=parsed,
                    private_kb_used=[],
                    success=True,
                )

            async def run_cross_review(self, own_memo, anon_peers, round2_max_runtime_minutes):
                return _base_pi.PIMemo(
                    role=self.role_name,
                    raw_text="round2",
                    parsed={
                        "own_revisions": [
                            {"claim_id": own_memo["top_claims"][0]["id"], "revision": "keep"},
                            {"claim_id": "hallucinated", "revision": "drop"},
                        ]
                    },
                    private_kb_used=[],
                    success=True,
                )

        class FakeChair:
            calls: list[dict[str, object]] = []
            init_calls: list[dict[str, object]] = []

            def __init__(self, **kwargs):
                self.kwargs = kwargs
                self.__class__.init_calls.append(dict(kwargs))

            async def run(self, **kwargs):
                self.kwargs.update(kwargs)
                self.__class__.calls.append(dict(kwargs))
                attempt = len(self.__class__.calls)
                return SimpleNamespace(
                    success=True,
                    agenda={
                        "agenda_version": "2.0",
                        "attempt": attempt,
                        "peer_contracts": [],
                    },
                    raw_text=f"agenda yaml {attempt}",
                    error=None,
                )

        pack = SimpleNamespace(
            shared_core={"shared_core_id": "abcdef12", "claim": "dominates prior result"},
            all_cards=[{"id": "card1"}],
            audit={"negative_evidence_ratio_global": 0.25},
        )
        validation_missing_role = SimpleNamespace(
            valid=False,
            blocking_issues=["peer_contracts missing roles: ['peer_generalist'] in full panel"],
            warnings=[],
        )
        validation = SimpleNamespace(valid=True, blocking_issues=[], warnings=["soft"])
        audit = SimpleNamespace(
            pass_=True,
            blocking_issues=[],
            warnings=["audit warning"],
            metrics={"citation_coverage": 0.8},
        )
        specs = [
            SimpleNamespace(legacy_role_id=r) for r in ("builder", "skeptic", "external_validity")
        ]

        def fake_instantiate_pi_roles(
            roles,
            *,
            run_dir,
            workspace,
            model,
            max_runtime_minutes,
            mcp_servers,
            stop_check_fn,
            premium_mode,
            reasoning_effort,
            prompts_dir=None,
            task_project_path=None,
            plugin_registry=None,
        ):
            self.assertEqual(
                [role.legacy_role_id for role in roles], [role.legacy_role_id for role in specs]
            )
            self.assertEqual(reasoning_effort, "high")
            return [FakePI(role.legacy_role_id) for role in roles]

        with tempfile.TemporaryDirectory() as tmp:
            run_dir = Path(tmp) / "run"
            workspace = Path(tmp) / "workspace"
            workspace.mkdir()
            with (
                patch.object(panel, "_select_pi_roles", return_value=["builder", "skeptic"]),
                patch.object(panel, "_select_pi_role_specs", return_value=specs),
                patch.object(panel, "_has_high_stakes_signal", return_value=True),
                patch.object(panel, "build_evidence_pack", return_value=pack),
                patch.object(
                    panel,
                    "fit_pack_to_budget",
                    return_value={
                        "shared_core": pack.shared_core,
                        "private_packs": {
                            "builder": [{"id": "b"}],
                            "skeptic": [{"id": "s"}],
                        },
                    },
                ),
                patch.object(
                    panel,
                    "instantiate_pi_roles",
                    side_effect=fake_instantiate_pi_roles,
                ),
                patch.object(panel, "ChairArbiter", FakeChair),
                patch.object(
                    panel,
                    "validate_agenda_v2",
                    side_effect=[validation_missing_role, validation],
                ),
                patch.object(panel, "audit_agenda", return_value=audit),
                patch.object(panel, "log_synthesis_metrics") as log_metrics,
            ):
                result = asyncio.run(
                    panel.run_panel(
                        run_dir=run_dir,
                        workspace=workspace,
                        model="fake-model",
                        completed_gen_id=0,
                        panel_mode="mini",
                        cohort_size=3,
                        n_rounds=3,
                        round2_max_runtime_minutes=1,
                        reasoning_effort="high",
                    )
                )

            out_dir = run_dir / "research_memory" / "synth_gen0_to_1"
            self.assertTrue((out_dir / "evidence_pack.json").exists())
            self.assertTrue((out_dir / "memo_external_validity.yaml").exists())
            self.assertTrue((out_dir / "round2_label_maps.yaml").exists())
            self.assertTrue((out_dir / "final_agenda.yaml").exists())
            self.assertTrue((out_dir / "audit.json").exists())
            self.assertTrue(result.success)
            self.assertEqual(result.panel_mode, "high_stakes")
            self.assertEqual(
                result.cross_reviews["builder"]["own_revisions"][0]["claim_id"],
                "builder_claim",
            )
            self.assertEqual(len(result.cross_reviews["builder"]["own_revisions"]), 1)
            self.assertEqual(len(FakeChair.calls), 2)
            self.assertEqual(FakeChair.init_calls[0]["reasoning_effort"], "high")
            self.assertEqual(
                FakeChair.calls[1]["validation_feedback"],
                tuple(validation_missing_role.blocking_issues),
            )
            self.assertEqual(FakeChair.calls[1]["validation_candidate"]["attempt"], 1)
            self.assertEqual((out_dir / "chair_raw_failed.txt").read_text(), "agenda yaml 1")
            self.assertEqual((out_dir / "chair_raw.txt").read_text(), "agenda yaml 2")
            log_metrics.assert_called_once()

    def test_panel_chair_failure_preserves_raw_output(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend.multi_pi import (
            legacy_two_round_executor as panel,
        )

        class FakePI:
            role_name = "builder"

            async def run(self, shared_core, private_pack, target_decisions):
                return SimpleNamespace(
                    parsed={
                        "role": "builder",
                        "top_claims": [{"id": "c1"}],
                        "objections_or_warnings": [],
                        "claim_boundaries": [],
                        "private_knowledge_used": [],
                    }
                )

        class FailingChair:
            def __init__(self, **_kwargs):
                pass

            async def run(self, **_kwargs):
                return SimpleNamespace(
                    success=False,
                    agenda={},
                    raw_text="bad yaml",
                    error="parse failed",
                )

        pack = SimpleNamespace(shared_core={"shared_core_id": "not-hex"}, all_cards=[], audit={})
        specs = [SimpleNamespace(legacy_role_id="builder")]

        with tempfile.TemporaryDirectory() as tmp:
            run_dir = Path(tmp) / "run"
            with (
                patch.object(panel, "_select_pi_roles", return_value=["builder"]),
                patch.object(panel, "_select_pi_role_specs", return_value=specs),
                patch.object(panel, "build_evidence_pack", return_value=pack),
                patch.object(
                    panel,
                    "fit_pack_to_budget",
                    return_value={"shared_core": pack.shared_core, "private_packs": {}},
                ),
                patch.object(panel, "instantiate_pi_roles", return_value=[FakePI()]),
                patch.object(panel, "ChairArbiter", FailingChair),
            ):
                result = asyncio.run(
                    panel.run_panel(
                        run_dir=run_dir,
                        workspace=Path(tmp),
                        model="fake-model",
                        completed_gen_id=2,
                        panel_mode="mini",
                        cohort_size=1,
                        n_rounds=1,
                    )
                )

            out_dir = run_dir / "research_memory" / "synth_gen2_to_3"
            self.assertFalse(result.success)
            self.assertEqual(result.error, "chair: parse failed")
            self.assertEqual((out_dir / "chair_raw_failed.txt").read_text(), "bad yaml")
            self.assertEqual((out_dir / "chair_error.txt").read_text(), "parse failed")


if __name__ == "__main__":
    unittest.main()