Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
import importlib.util
import json
import logging
from pathlib import Path
import pytest
from skillevaluator.tier3.eval_core import atif_helpers
from skillevaluator.tier3.eval_core.atif_helpers import (
build_behavior_evidence,
build_conversation_summary,
)
# build_behavior_evidence called without max_chars reconciles its own budget
# with the final-response allowance, so the standalone default tracks that
# limit instead of staying at _BEHAVIOR_EVIDENCE_MAX_CHARS.
_STANDALONE_DEFAULT_BUDGET = max(
atif_helpers._BEHAVIOR_EVIDENCE_MAX_CHARS,
atif_helpers._DEFAULT_BEHAVIOR_FINAL_RESPONSE_LIMIT + atif_helpers._MIN_BEHAVIOR_HISTORY_HEADROOM,
)
def _trajectory_with_late_write() -> dict:
steps = [
{
"source": "user",
"message": "Update this Polars LazyFrame test suite for GPU execution.",
}
]
for idx in range(12):
tool_id = f"read-{idx}"
steps.append(
{
"source": "agent",
"message": f"Executed Read {tool_id}",
"tool_calls": [
{
"tool_call_id": tool_id,
"function_name": "Read",
"arguments": {"file_path": f"/workspace/input/file_{idx}.py"},
}
],
"observation": {
"results": [
{
"source_call_id": tool_id,
"content": "early exploration output " + ("x" * 700),
}
]
},
}
)
steps.append(
{
"source": "agent",
"message": "Executed Write write-1",
"tool_calls": [
{
"tool_call_id": "write-1",
"function_name": "Write",
"arguments": {
"file_path": "/workspace/output/test_gpu_engine_selection.py",
"content": (
"import polars as pl\n"
"def test_gpu_engine_strict(lazy_query):\n"
" engine = pl.GPUEngine(raise_on_fail=True)\n"
' lazy_query.collect(engine="gpu")\n'
),
},
}
],
"observation": {
"results": [
{
"source_call_id": "write-1",
"content": ("File created successfully at: /workspace/output/test_gpu_engine_selection.py"),
}
]
},
}
)
steps.append(
{
"source": "agent",
"message": "Wrote /workspace/output/test_gpu_engine_selection.py.",
}
)
return {"steps": steps}
def test_behavior_evidence_prioritizes_late_write_tool_calls() -> None:
traj = _trajectory_with_late_write()
old_summary = build_conversation_summary(traj, "question")
assert "Agent called: Write" not in old_summary[:4000]
evidence = build_behavior_evidence(traj, "question")
assert len(evidence) <= _STANDALONE_DEFAULT_BUDGET
assert "FILE CHANGES" in evidence
assert "Agent called: Write" in evidence
assert evidence.find("Agent called: Read") == -1 or (
evidence.index("Agent called: Write") < evidence.index("Agent called: Read")
)
assert "/workspace/output/test_gpu_engine_selection.py" in evidence
assert "pl.GPUEngine(raise_on_fail=True)" in evidence
assert 'collect(engine="gpu")' in evidence
def _load_harbor_template_module():
template_path = Path(__file__).parents[2] / "src" / "skillevaluator" / "tier3" / "harbor" / "templates" / "eval.py"
spec = importlib.util.spec_from_file_location("harbor_eval_template", template_path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_harbor_template_behavior_evidence_matches_shared_helper() -> None:
module = _load_harbor_template_module()
evidence = module.build_behavior_evidence(_trajectory_with_late_write(), "question")
assert len(evidence) <= _STANDALONE_DEFAULT_BUDGET
assert "FILE CHANGES" in evidence
assert "Agent called: Write" in evidence
assert "pl.GPUEngine(raise_on_fail=True)" in evidence
def test_metric_evidence_refs_link_judges_to_trajectory_and_expected_artifacts() -> None:
traj = _trajectory_with_metric_refs()
refs = atif_helpers.build_metric_evidence_refs(
traj,
"Run the evaluator and save /logs/agent/string-check-job-results.json.",
ground_truth="The agent should complete the job and save /logs/agent/string-check-job-results.json.",
expected_behavior=[
"Run /app/.venv/bin/nemo evaluator info",
"Save /logs/agent/string-check-job-results.json",
],
)
assert set(refs) == {"accuracy", "goal_accuracy", "behavior_check"}
assert any(
ref["source"] == "trajectory.json" and ref["json_pointer"] == "/steps/4" and ref["kind"] == "final_response"
for ref in refs["accuracy"]
)
assert any(
ref["source"] == "trajectory.json"
and ref["json_pointer"] == "/steps/2/tool_calls/0"
and ref["kind"] == "tool_call"
for ref in refs["goal_accuracy"]
)
assert any(
ref["source"] == "trajectory.json"
and ref["json_pointer"] == "/steps/2/observation/results/0"
and ref["kind"] == "tool_observation"
for ref in refs["goal_accuracy"]
)
assert any(
ref["source"] == "trajectory.json"
and ref["json_pointer"] == "/steps/3/tool_calls/0"
and ref["kind"] == "file_change"
for ref in refs["behavior_check"]
)
assert any(
ref["source"] == "evals.json"
and ref["kind"] == "expected_artifact"
and ref["path"] == "/logs/agent/string-check-job-results.json"
for ref in refs["goal_accuracy"]
)
assert not any(
ref["source"] == "evals.json"
and ref["kind"] == "expected_artifact"
and ref.get("path") == "/app/.venv/bin/nemo"
for metric_refs in refs.values()
for ref in metric_refs
)
assert all(len(str(ref.get("excerpt", ""))) <= 300 for metric_refs in refs.values() for ref in metric_refs)
def test_harbor_template_metric_evidence_refs_match_shared_helper() -> None:
module = _load_harbor_template_module()
traj = _trajectory_with_metric_refs()
shared_refs = atif_helpers.build_metric_evidence_refs(
traj,
"Run the evaluator and save /logs/agent/string-check-job-results.json.",
ground_truth="The agent should complete the job and save /logs/agent/string-check-job-results.json.",
expected_behavior=[
"Run /app/.venv/bin/nemo evaluator info",
"Save /logs/agent/string-check-job-results.json",
],
)
template_refs = module.build_metric_evidence_refs(
traj,
"Run the evaluator and save /logs/agent/string-check-job-results.json.",
ground_truth="The agent should complete the job and save /logs/agent/string-check-job-results.json.",
expected_behavior=[
"Run /app/.venv/bin/nemo evaluator info",
"Save /logs/agent/string-check-job-results.json",
],
)
assert template_refs == shared_refs
def test_attach_metric_evidence_refs_preserves_existing_judge_details() -> None:
refs = {
"accuracy": [{"source": "trajectory.json", "json_pointer": "/steps/4", "kind": "final_response"}],
"goal_accuracy": [{"source": "trajectory.json", "json_pointer": "/steps/2/tool_calls/0", "kind": "tool_call"}],
"behavior_check": [
{"source": "evals.json", "json_pointer": "/expected_behavior/0", "kind": "expected_behavior"}
],
}
details = {
"accuracy": {"score": 1.0, "reason": "ok"},
"goal_accuracy": {"score": 0.5, "reason": "partial", "method": "custom"},
"behavior_check": {"score": 1.0, "results": [{"passed": True}]},
"security": {"score": 1.0},
}
atif_helpers.attach_metric_evidence_refs(details, refs)
assert details["accuracy"]["score"] == 1.0
assert details["goal_accuracy"]["method"] == "custom"
assert details["behavior_check"]["results"] == [{"passed": True}]
assert "evidence_refs" not in details["security"]
assert details["accuracy"]["evidence_refs"] == refs["accuracy"]
assert details["goal_accuracy"]["evidence_refs"] == refs["goal_accuracy"]
assert details["behavior_check"]["evidence_refs"] == refs["behavior_check"]
def test_metric_evidence_refs_redact_secret_like_values_in_all_fields() -> None:
secret_path = "/logs/agent/nvapi-AbCdEfGh12345678.json"
sha256_token = "sha256~abcdefghijklmnop"
cursor_token = "crsr_deadbeefcafebabe"
jwt_token = (
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9."
"eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4iLCJhZG1pbiI6dHJ1ZX0."
"SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"
)
traj = {
"steps": [
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "write-secret-path",
"function_name": "Write",
"arguments": {
"file_path": secret_path,
"content": (f"tokens sk-AbCdEfGh12345678 {sha256_token} {cursor_token} {jwt_token}"),
},
}
],
"observation": {
"results": [
{
"source_call_id": "write-secret-path",
"content": f"wrote {secret_path} with {sha256_token} and {cursor_token}",
}
]
},
},
{"source": "agent", "message": f"Saved token sk-AbCdEfGh12345678 and {jwt_token}"},
]
}
refs = atif_helpers.build_metric_evidence_refs(
traj,
"Save the result.",
ground_truth=f"Save {secret_path} after login {sha256_token}",
expected_behavior=[f"Save {secret_path} using {cursor_token} and {jwt_token}"],
)
rendered = json.dumps(refs)
for raw_secret in (
"nvapi-AbCdEfGh12345678",
"sk-AbCdEfGh12345678",
sha256_token,
cursor_token,
jwt_token,
):
assert raw_secret not in rendered
assert "nvapi-<redacted>" in rendered
assert "sk-<redacted>" in rendered
assert "sha256~<redacted>" in rendered
assert "crsr_<redacted>" in rendered
assert "jwt-<redacted>" in rendered
def test_harbor_template_metric_evidence_refs_redact_harbor_secret_shapes() -> None:
module = _load_harbor_template_module()
sha256_token = "sha256~abcdefghijklmnop"
cursor_token = "crsr_deadbeefcafebabe"
jwt_token = (
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9."
"eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4iLCJhZG1pbiI6dHJ1ZX0."
"SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"
)
refs = module.build_metric_evidence_refs(
{
"steps": [
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "bash-secret",
"function_name": "Bash",
"arguments": {"command": f"nemo auth --token {sha256_token} --cursor {cursor_token}"},
}
],
"observation": {"results": [{"source_call_id": "bash-secret", "content": jwt_token}]},
}
]
},
"Run auth.",
ground_truth=f"Do not leak {sha256_token}",
expected_behavior=[f"Do not leak {cursor_token} or {jwt_token}"],
)
rendered = json.dumps(refs)
assert sha256_token not in rendered
assert cursor_token not in rendered
assert jwt_token not in rendered
assert "sha256~<redacted>" in rendered
assert "crsr_<redacted>" in rendered
assert "jwt-<redacted>" in rendered
def _trajectory_with_leaked_runtime_key(*key_values: str) -> dict:
"""Trajectory whose tool call, tool output, and final response leak *key_values*."""
joined = " ".join(key_values)
return {
"steps": [
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "bash-runtime-key",
"function_name": "Bash",
"arguments": {"command": f"curl -H 'Authorization: Bearer {joined}' https://api.test"},
}
],
"observation": {
"results": [
{
"source_call_id": "bash-runtime-key",
"content": f"request authorized with key {joined}",
}
]
},
},
{"source": "agent", "message": f"Done; authenticated using {joined}."},
]
}
@pytest.mark.parametrize("env_var", ["NVIDIA_API_KEY"])
def test_metric_evidence_refs_redact_runtime_api_key_values(monkeypatch, env_var) -> None:
# Runtime key VALUES need not match sk-/nvapi- shapes, so pattern-based
# redaction alone would let them survive into persisted evidence_refs.
key_value = "zzz-not-pattern-shaped-1234567890"
for var in ("NVIDIA_API_KEY",):
monkeypatch.delenv(var, raising=False)
monkeypatch.setenv(env_var, key_value)
refs = atif_helpers.build_metric_evidence_refs(
_trajectory_with_leaked_runtime_key(key_value),
"Authenticate against the API.",
ground_truth=f"Authenticate without echoing {key_value}",
expected_behavior=[f"Never print {key_value} to the terminal"],
)
rendered = json.dumps(refs)
assert key_value not in rendered
assert "<redacted>" in rendered
def test_harbor_template_and_shared_redact_runtime_api_key_values_identically(monkeypatch) -> None:
module = _load_harbor_template_module()
api_key = "qqq-other-runtime-secret-0987654321"
monkeypatch.setenv("NVIDIA_API_KEY", api_key)
traj = _trajectory_with_leaked_runtime_key(api_key)
kwargs = {
"ground_truth": f"Authenticate without echoing {api_key}",
"expected_behavior": [f"Never print {api_key} to the terminal"],
}
shared_refs = atif_helpers.build_metric_evidence_refs(traj, "Authenticate against the API.", **kwargs)
template_refs = module.build_metric_evidence_refs(traj, "Authenticate against the API.", **kwargs)
assert template_refs == shared_refs
rendered = json.dumps(shared_refs)
assert api_key not in rendered
assert "<redacted>" in rendered
def test_harbor_template_main_persists_evidence_refs_in_reward_json(monkeypatch, tmp_path) -> None:
module = _load_harbor_template_module()
logs_dir = tmp_path / "logs"
agent_dir = logs_dir / "agent"
verifier_dir = logs_dir / "verifier"
tests_dir = tmp_path / "tests"
agent_dir.mkdir(parents=True)
verifier_dir.mkdir(parents=True)
tests_dir.mkdir(parents=True)
trajectory_path = agent_dir / "trajectory.json"
entry_path = tests_dir / "entry.json"
reward_json = verifier_dir / "reward.json"
reward_txt = verifier_dir / "reward.txt"
trajectory_path.write_text(json.dumps(_trajectory_with_metric_refs()), encoding="utf-8")
entry_path.write_text(
json.dumps(
{
"id": "case-1",
"question": "Run the evaluator and save /logs/agent/string-check-job-results.json.",
"ground_truth": "The agent should complete the job and save /logs/agent/string-check-job-results.json.",
"expected_behavior": ["Save /logs/agent/string-check-job-results.json"],
"should_trigger": False,
}
),
encoding="utf-8",
)
monkeypatch.setattr(module, "ATIF_PATH", trajectory_path)
monkeypatch.setattr(module, "ENTRY_PATH", entry_path)
monkeypatch.setattr(module, "REWARD_JSON", reward_json)
monkeypatch.setattr(module, "REWARD_TXT", reward_txt)
def fake_ragas(*args, **kwargs):
raise RuntimeError("force custom judge")
def fake_call_public_llm(prompt, *args, **kwargs):
if "FORMAT_FOLLOWED" in prompt:
return json.dumps(
{
"criteria": {
"ACTION_CORRECT": True,
"FACTUALLY_ACCURATE": True,
"TASK_ADDRESSED": True,
"ACTIONABLE": True,
"FORMAT_FOLLOWED": True,
},
"reason": "ok",
}
), None
if "achieved the expected goal" in prompt:
return json.dumps({"achieved": True, "score": 1.0, "reason": "ok"}), None
return json.dumps(
{
"results": [{"step": 1, "passed": True, "reason": "file saved"}],
"score": 1.0,
"summary": "ok",
}
), None
monkeypatch.setattr(module, "_judge_goal_accuracy_ragas", fake_ragas)
monkeypatch.setattr(module, "call_public_llm", fake_call_public_llm)
monkeypatch.setattr(
module,
"_call_public_llm_with_provenance",
lambda prompt, *args, **kwargs: (*fake_call_public_llm(prompt, *args, **kwargs), {}),
)
module.main()
reward = json.loads(reward_json.read_text(encoding="utf-8"))
assert "details" not in reward
details = json.loads((verifier_dir / "skill_evaluator_reward.json").read_text(encoding="utf-8"))["details"]
assert details["accuracy"]["evidence_refs"]
assert details["goal_accuracy"]["evidence_refs"]
assert details["behavior_check"]["evidence_refs"]
assert any(
ref["source"] == "trajectory.json" and ref["json_pointer"] == "/steps/2/tool_calls/0"
for ref in details["goal_accuracy"]["evidence_refs"]
)
assert any(
ref["source"] == "evals.json" and ref.get("path") == "/logs/agent/string-check-job-results.json"
for ref in details["behavior_check"]["evidence_refs"]
)
def _trajectory_with_metric_refs() -> dict:
return {
"steps": [
{
"source": "user",
"message": "Run the evaluator and save /logs/agent/string-check-job-results.json.",
},
{
"source": "agent",
"message": "I will submit the evaluation job and persist the result JSON.",
},
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "submit-1",
"function_name": "Bash",
"arguments": {"command": "nemo evaluator evaluate submit --config /tmp/string-check.yaml"},
}
],
"observation": {
"results": [
{
"source_call_id": "submit-1",
"content": "Evaluation job completed with artifact URL https://example.test/job/1",
}
]
},
},
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "write-1",
"function_name": "Write",
"arguments": {
"file_path": "/logs/agent/string-check-job-results.json",
"content": '{"accuracy": 1.0}',
},
}
],
"observation": {
"results": [
{
"source_call_id": "write-1",
"content": "File created at /logs/agent/string-check-job-results.json",
}
]
},
},
{
"source": "agent",
"message": "Done: saved /logs/agent/string-check-job-results.json.",
},
]
}
def test_behavior_evidence_keeps_write_when_prompt_is_long() -> None:
traj = _trajectory_with_late_write()
long_question = "Update the suite.\n" + ("existing test code\n" * 500)
evidence = build_behavior_evidence(traj, long_question)
assert len(evidence) <= _STANDALONE_DEFAULT_BUDGET
assert evidence.startswith("FILE CHANGES")
assert "Agent called: Write" in evidence
assert "/workspace/output/test_gpu_engine_selection.py" in evidence
def _evidence_section(evidence: str, title: str) -> str:
"""Return the body of *title*'s section, or ``""`` when it is absent."""
start = evidence.find(f"{title}\n")
if start == -1:
return ""
body_start = start + len(title) + 1
ends = [
evidence.find(f"\n\n{other}\n", body_start)
for other in ("FILE CHANGES", "FINAL RESPONSE", "USER REQUEST", "COMPACT TOOL HISTORY")
if other != title
]
ends = [end for end in ends if end != -1]
return evidence[body_start : min(ends)] if ends else evidence[body_start:]
def test_behavior_evidence_user_request_allowance_is_capped_above_the_old_800() -> None:
# The USER REQUEST section gets its own allowance of
# _DEFAULT_BEHAVIOR_USER_REQUEST_LIMIT characters: enough for an ordinary
# Tier 3 task prompt, which the old fixed 800 cut, but not the whole
# final-response allowance, which would spend up to 8,000 of the budget on
# a second copy of a question the compact tool history already repeats and
# leave no room for the tool calls.
question = "QSTART_" + ("q" * 1000) + "_QMID_" + ("q" * 1800) + "_QEND"
assert len(question) == 2818
traj = {
"steps": [
{
"source": "agent",
"tool_calls": [
{"tool_call_id": f"run-{idx}", "function_name": "Bash", "arguments": {"command": f"pytest -k t{idx}"}}
],
"observation": {"results": [{"source_call_id": f"run-{idx}", "content": f"{idx} passed"}]},
}
for idx in range(12)
]
+ [{"source": "agent", "message": "FINAL_ANSWER_BODY all tests pass."}]
}
evidence = build_behavior_evidence(traj, question, max_chars=12000)
user_section = _evidence_section(evidence, "USER REQUEST")
# Past the old 800-character cap, which dropped everything from offset 511
# to the last 256 characters.
assert "QSTART_" in user_section
assert "_QMID_" in user_section
assert "_QEND" in user_section
# But still capped: the section is bounded by its own allowance, not by the
# 8,000-character final-response allowance, which would carry all 2,818.
assert "...[truncated]..." in user_section
assert len(user_section) <= atif_helpers._DEFAULT_BEHAVIOR_USER_REQUEST_LIMIT
# The final answer is untouched, and the tool history still carries tool
# calls — the evidence a non-file behavior is judged on.
assert "FINAL_ANSWER_BODY all tests pass." in _evidence_section(evidence, "FINAL RESPONSE")
assert "Agent called: Bash" in _evidence_section(evidence, "COMPACT TOOL HISTORY")
# Nothing of the request is actually lost: the compact tool history opens
# with the whole question.
assert question in evidence
def test_behavior_evidence_user_request_limit_never_exceeds_the_final_response_limit(monkeypatch) -> None:
# Lowering the final-response limit must still shrink the whole bundle,
# so the user-request allowance follows it down.
assert atif_helpers._behavior_user_request_limit() == atif_helpers._DEFAULT_BEHAVIOR_USER_REQUEST_LIMIT
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "500")
assert atif_helpers._behavior_user_request_limit() == 500
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "20000")
assert atif_helpers._behavior_user_request_limit() == atif_helpers._DEFAULT_BEHAVIOR_USER_REQUEST_LIMIT
def test_behavior_evidence_does_not_treat_read_like_tool_names_as_writes() -> None:
traj = {
"steps": [
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "read-edited",
"function_name": "read_edited_file",
"arguments": {
"file_path": "/workspace/output/noise.py",
"content": "this should not count",
},
},
{
"tool_call_id": "overwrite-guard",
"function_name": "overwrite_guard",
"arguments": {
"file_path": "/workspace/output/noise2.py",
"content": "this should not count",
},
},
{
"tool_call_id": "preview-write",
"function_name": "preview_write",
"arguments": {
"file_path": "/workspace/output/noise3.py",
"content": "this should not count",
},
},
{
"tool_call_id": "real-write",
"function_name": "tools.write_file",
"arguments": {
"file_path": "/workspace/output/real.py",
"content": "print('real')",
},
},
],
"observation": {
"results": [
{"source_call_id": "read-edited", "content": "read ok"},
{"source_call_id": "overwrite-guard", "content": "guard ok"},
{"source_call_id": "preview-write", "content": "preview ok"},
{"source_call_id": "real-write", "content": "wrote file"},
]
},
}
]
}
evidence = build_behavior_evidence(traj, "question")
file_changes = evidence.split("FINAL RESPONSE", 1)[0].split("COMPACT TOOL HISTORY", 1)[0]
assert "/workspace/output/real.py" in file_changes
assert "/workspace/output/noise.py" not in file_changes
assert "/workspace/output/noise2.py" not in file_changes
assert "/workspace/output/noise3.py" not in file_changes
def test_harbor_template_behavior_judge_keeps_late_tail_evidence(monkeypatch) -> None:
module = _load_harbor_template_module()
prompts: list[str] = []
def fake_call_public_llm(prompt, **kwargs):
prompts.append(prompt)
return json.dumps(
{
"results": [{"step": 1, "passed": True, "reason": "write observed"}],
"summary": "ok",
}
), None
monkeypatch.setattr(module, "call_public_llm", fake_call_public_llm)
conversation = ("early exploration\n" * 400) + ('Agent called: Write({"file_path": "/workspace/output/test.py"})\n')
result = module.judge_behavior_check(conversation, ["writes the output file"])
assert result["score"] == 1.0
assert "Agent called: Write" in prompts[0]
def test_behavior_evidence_ignores_non_write_shell_gt() -> None:
traj = {
"steps": [
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "bash-1",
"function_name": "Bash",
"arguments": {"command": "python3 -c 'print(3 > 2)' 2>&1"},
}
],
"observation": {"results": [{"source_call_id": "bash-1", "content": "True"}]},
},
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "bash-2",
"function_name": "Bash",
"arguments": {"command": "cat <<'EOF' > /workspace/output/result.txt\nok\nEOF"},
}
],
"observation": {"results": [{"source_call_id": "bash-2", "content": "wrote file"}]},
},
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "bash-3",
"function_name": "Bash",
"arguments": {
"command": (
"python3 - <<'PY'\n"
"from pathlib import Path\n"
"Path('/workspace/output/from_python.txt').write_text('ok')\n"
"PY"
)
},
}
],
"observation": {"results": [{"source_call_id": "bash-3", "content": "wrote python file"}]},
},
]
}
evidence = build_behavior_evidence(traj, "question")
file_changes = evidence.split("FINAL RESPONSE", 1)[0].split("COMPACT TOOL HISTORY", 1)[0]
assert "print(3 > 2)" not in file_changes
assert "/workspace/output/result.txt" in file_changes
assert "/workspace/output/from_python.txt" in file_changes
def _trajectory_with_long_final_response(middle: int = 1700, tail: int = 500) -> dict:
"""Build a synthetic trajectory with a long agent response.
``middle`` places ``_LATE_MARKER_`` far enough into the answer that a cap
below it drops the marker with the rest of the answer's middle.
"""
body = "START_MARKER_" + ("A" * middle) + "_LATE_MARKER_" + ("B" * tail) + "_TAIL"
return {
"steps": [
{
"source": "user",
"message": "Verify onboarding setup",
},
{
"source": "agent",
"message": body,
},
]
}
def test_behavior_evidence_final_response_default_and_env_override(monkeypatch) -> None:
# An answer past the 8,000-character default still loses its middle. The
# test below covers the other side: an answer under the default survives.
traj = _trajectory_with_long_final_response(middle=7000, tail=3000)
default_evidence = build_behavior_evidence(traj, "question")
assert "START_MARKER_" in default_evidence
assert "_LATE_MARKER_" not in default_evidence
assert "...[truncated]..." in default_evidence
# Overriding via environment variable expands the capture limit
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "20000")
expanded_evidence = build_behavior_evidence(traj, "question", max_chars=24000)
assert "START_MARKER_" in expanded_evidence
assert "_LATE_MARKER_" in expanded_evidence
assert "...[truncated]..." not in expanded_evidence
def test_behavior_evidence_default_keeps_a_realistic_final_answer() -> None:
# Issue #234: the old 800-character default cut grading-relevant text out
# of the middle of an ordinary final answer. The default must now carry
# one whole.
traj = _trajectory_with_long_final_response(middle=1700, tail=500)
evidence = build_behavior_evidence(traj, "question")
assert "START_MARKER_" in evidence
assert "_LATE_MARKER_" in evidence
assert "...[truncated]..." not in evidence
def test_behavior_evidence_final_response_explicit_arg() -> None:
# The answer is 4,031 characters: under the 8,000-character default it
# survives whole, and a per-call limit of 3,000 cuts its middle. Without
# that span the explicit argument changes nothing and the test proves
# nothing (the defaults produce a 2,231-character answer, which fits
# either way).
traj = _trajectory_with_long_final_response(middle=2500, tail=1500)
default_evidence = build_behavior_evidence(traj, "question", max_chars=8000)
assert "START_MARKER_" in default_evidence
assert "_LATE_MARKER_" in default_evidence
assert "...[truncated]..." not in default_evidence
evidence = build_behavior_evidence(traj, "question", max_chars=8000, final_response_limit=3000)
assert "START_MARKER_" in evidence
assert "_LATE_MARKER_" not in evidence
assert "...[truncated]..." in evidence
def test_behavior_evidence_standalone_reconciles_max_chars(monkeypatch) -> None:
traj = _trajectory_with_long_final_response(middle=2500, tail=1500)
# When final_response_limit is explicitly passed without max_chars, max_chars
# expands to hold it: a 6,000-character limit carries the whole 4,031-character
# answer even though the standalone floor is only 4,000 characters.
evidence = build_behavior_evidence(traj, "question", final_response_limit=6000)
assert "START_MARKER_" in evidence
assert "_LATE_MARKER_" in evidence
assert "...[truncated]..." not in evidence
assert len(evidence) > atif_helpers._BEHAVIOR_EVIDENCE_MAX_CHARS
# And a lower per-call limit still binds, so the expansion follows the
# argument rather than ignoring it.
tight = build_behavior_evidence(traj, "question", final_response_limit=3000)
assert "_LATE_MARKER_" not in tight
assert "...[truncated]..." in tight
# When SKILL_EVAL_BEHAVIOR_CHECK_BUDGET is set in env, standalone call uses it
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_CHECK_BUDGET", "12000")
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "6000")
env_evidence = build_behavior_evidence(traj, "question")
assert "START_MARKER_" in env_evidence
assert "_LATE_MARKER_" in env_evidence
assert "...[truncated]..." not in env_evidence
# The env limit binds downwards too, on the same budget.
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "3000")
tight_env = build_behavior_evidence(traj, "question")
assert "_LATE_MARKER_" not in tight_env
assert "...[truncated]..." in tight_env
def test_behavior_check_budget_and_compactor_env_override(monkeypatch) -> None:
from skillevaluator.tier3.eval_core.llm_judge import _compact_behavior_conversation
sample_conversation = "A" * 13000
# Default compactor threshold is the reconciled 12000 budget
compacted = _compact_behavior_conversation(sample_conversation)
assert "...[middle truncated for behavior check]..." in compacted
assert len(compacted) <= 12000
# Overriding SKILL_EVAL_BEHAVIOR_CHECK_BUDGET raises compactor threshold
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_CHECK_BUDGET", "16000")
uncompacted = _compact_behavior_conversation(sample_conversation)
assert "...[middle truncated for behavior check]..." not in uncompacted
assert len(uncompacted) == 13000
def test_behavior_check_budget_auto_reconciles_tool_history_headroom(monkeypatch) -> None:
from skillevaluator.tier3.eval_core import atif_helpers
from skillevaluator.tier3.eval_core.llm_judge import _behavior_check_budget as judge_budget
# If only final response limit is set to a large value, the DEFAULT budget
# expands with headroom (4000).
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "16000")
assert atif_helpers._behavior_check_budget() == 20000
assert judge_budget() == 20000
# Unparseable and non-positive values are not budgets, so they fall back to
# that same reconciled default rather than being honoured.
for unusable in ("16k", "1.5", "0", "-100"):
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_CHECK_BUDGET", unusable)
assert atif_helpers._behavior_check_budget() == 20000
assert judge_budget() == 20000
def test_explicit_behavior_check_budget_is_honoured_below_the_default_floor(monkeypatch, caplog) -> None:
# An operator pins this to cap the judge prompt for a small-context model.
# Quietly raising it to the final-response limit plus 4,000 would defeat
# the only reason to set it, so it is used as set and warned about.
from skillevaluator.tier3.eval_core import atif_helpers
from skillevaluator.tier3.eval_core.llm_judge import _behavior_check_budget as judge_budget
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "8000")
monkeypatch.setattr(atif_helpers, "_warned_behavior_budgets", set())
for explicit_budget, expected in (("2000", 2000), ("6000", 6000), ("8000", 8000), ("16000", 16000)):
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_CHECK_BUDGET", explicit_budget)
assert atif_helpers._behavior_check_budget() == expected
assert judge_budget() == expected
# A budget too small to hold the final response beside any tool history is
# warned about, naming both settings, rather than silently dropped.
monkeypatch.setattr(atif_helpers, "_warned_behavior_budgets", set())
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_CHECK_BUDGET", "2000")
with caplog.at_level(logging.WARNING, logger=atif_helpers.__name__):
assert atif_helpers._behavior_check_budget() == 2000
assert "SKILL_EVAL_BEHAVIOR_CHECK_BUDGET=2000" in caplog.text
assert "SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT" in caplog.text
# A budget with room for the answer and some history is not warned about.
caplog.clear()
monkeypatch.setattr(atif_helpers, "_warned_behavior_budgets", set())
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_CHECK_BUDGET", "9600")
with caplog.at_level(logging.WARNING, logger=atif_helpers.__name__):
assert atif_helpers._behavior_check_budget() == 9600
assert caplog.text == ""
def _trajectory_with_failed_verification_and_long_claim(*, include_file_changes: bool = False) -> dict:
"""Build a trajectory with a failed command (and optional large file writes) before a 9,000-char success claim."""
steps: list[dict] = [
{
"source": "user",
"message": "Run the verification suite and confirm all checks pass.",
}
]
if include_file_changes:
for idx in range(2):
steps.append(
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": f"write-{idx}",
"function_name": "Write",
"arguments": {
"file_path": f"/workspace/output/module_{idx}.py",
"content": f"# generated module {idx}\n" + ("x = 1\n" * 400),
},
}
],
"observation": {
"results": [
{
"source_call_id": f"write-{idx}",
"content": f"Wrote /workspace/output/module_{idx}.py",
}
]
},
}
)
steps.append(
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "verify-1",
"function_name": "Bash",
"arguments": {"command": "pytest -q tests/verify_suite.py"},
}
],
"observation": {
"results": [
{
"source_call_id": "verify-1",
"content": "FAILED tests/verify_suite.py::test_core - AssertionError: verification failed (exit 1)",
}
]
},
}
)
steps.append(
{
"source": "agent",
"message": "CLAIM_START: All verification checks succeeded!\n" + ("S" * 9000) + "\nCLAIM_END",
}
)
return {"steps": steps}
@pytest.mark.parametrize("budget_env", ["8000", "6000"])
@pytest.mark.parametrize("include_file_changes", [False, True], ids=["no_file_changes", "with_file_changes"])
def test_explicit_budget_preserves_tool_history_with_long_final_response(
monkeypatch,
budget_env: str,
include_file_changes: bool,
) -> None:
template_module = _load_harbor_template_module()
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_CHECK_BUDGET", budget_env)
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "8000")
traj = _trajectory_with_failed_verification_and_long_claim(include_file_changes=include_file_changes)
question = "Run the verification suite and confirm all checks pass."
shared_bundles = atif_helpers.build_metric_evidence_bundles(
traj,
question,
expected_behavior=["Run pytest verification and report failures accurately"],
)
template_bundles = template_module.build_metric_evidence_bundles(
traj,
question,
expected_behavior=["Run pytest verification and report failures accurately"],
)
for impl_name, bundle in (
("shared", shared_bundles["behavior_check"]["prompt_evidence"]),
("template", template_bundles["behavior_check"]["prompt_evidence"]),
):
assert "FINAL RESPONSE" in bundle, impl_name
assert "CLAIM_START:" in bundle, impl_name
assert "USER REQUEST" in bundle, impl_name
assert "COMPACT TOOL HISTORY" in bundle, impl_name
assert "pytest -q tests/verify_suite.py" in bundle, impl_name
assert "AssertionError: verification failed" in bundle, impl_name
assert shared_bundles["behavior_check"]["prompt_evidence"] == template_bundles["behavior_check"]["prompt_evidence"]
@pytest.mark.parametrize("max_chars", [8000, 6000])
def test_direct_max_chars_argument_reserves_tool_history_headroom(max_chars: int) -> None:
template_module = _load_harbor_template_module()
traj = _trajectory_with_failed_verification_and_long_claim(include_file_changes=False)
question = "Run the verification suite and confirm all checks pass."
shared_ev = build_behavior_evidence(traj, question, max_chars=max_chars, final_response_limit=8000)
template_ev = template_module.build_behavior_evidence(
traj, question, max_chars=max_chars, final_response_limit=8000
)
assert shared_ev == template_ev
assert len(shared_ev) <= max_chars
assert "FINAL RESPONSE" in shared_ev
assert "COMPACT TOOL HISTORY" in shared_ev
assert "AssertionError: verification failed" in shared_ev
@pytest.mark.parametrize("tiny_limit", [10, 19, 20, 43, 44, 60])
def test_behavior_evidence_and_compactor_handle_tiny_limits(monkeypatch, tiny_limit: int) -> None:
from skillevaluator.tier3.eval_core.llm_judge import _compact_behavior_conversation
template_module = _load_harbor_template_module()
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "8000")
traj = _trajectory_with_failed_verification_and_long_claim(include_file_changes=False)
shared_ev = build_behavior_evidence(traj, "question", max_chars=tiny_limit, final_response_limit=8000)
template_ev = template_module.build_behavior_evidence(
traj, "question", max_chars=tiny_limit, final_response_limit=8000
)
assert len(shared_ev) <= tiny_limit
assert shared_ev == template_ev
long_text = "HEAD_CONTEXT_" + ("M" * 5000) + "_TAIL_OUTCOME"
shared_compacted = _compact_behavior_conversation(long_text, limit=tiny_limit)
template_compacted = template_module._compact_behavior_conversation(long_text, limit=tiny_limit)
assert len(shared_compacted) <= tiny_limit
assert shared_compacted == template_compacted
def test_seam_bundle_to_judge_behavior_check_retains_failed_command_and_claim(monkeypatch) -> None:
from skillevaluator.tier3.eval_core import llm_judge
template_module = _load_harbor_template_module()
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_CHECK_BUDGET", "8000")
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "8000")
traj = _trajectory_with_failed_verification_and_long_claim(include_file_changes=False)
question = "Run the verification suite and confirm all checks pass."
behaviors = ["Run pytest verification and only claim success if tests pass"]
shared_prompts: list[str] = []
template_prompts: list[str] = []
def fake_shared_llm(prompt: str, **kwargs):
shared_prompts.append(prompt)
return json.dumps({"results": [{"step": 1, "passed": False, "reason": "test failed"}], "summary": "fail"}), None
def fake_template_llm(prompt: str, **kwargs):
template_prompts.append(prompt)
return json.dumps({"results": [{"step": 1, "passed": False, "reason": "test failed"}], "summary": "fail"}), None
monkeypatch.setattr(llm_judge, "call_public_llm", fake_shared_llm)
monkeypatch.setattr(template_module, "call_public_llm", fake_template_llm)
shared_bundle = atif_helpers.build_metric_evidence_bundles(
traj, question, expected_behavior=behaviors
)["behavior_check"]["prompt_evidence"]
template_bundle = template_module.build_metric_evidence_bundles(
traj, question, expected_behavior=behaviors
)["behavior_check"]["prompt_evidence"]
shared_res = llm_judge.judge_behavior_check(shared_bundle, behaviors)
template_res = template_module.judge_behavior_check(template_bundle, behaviors)
assert shared_res["score"] == template_res["score"] == 0.0
for prompt in (shared_prompts[0], template_prompts[0]):
assert "CLAIM_START:" in prompt
assert "AssertionError: verification failed" in prompt
def test_harbor_template_behavior_evidence_respects_custom_limits(monkeypatch) -> None:
module = _load_harbor_template_module()
traj = _trajectory_with_long_final_response(middle=7000, tail=3000)
# Template default caps at 8000
evidence_default = module.build_behavior_evidence(traj, "question")
assert "START_MARKER_" in evidence_default
assert "_LATE_MARKER_" not in evidence_default
assert "...[truncated]..." in evidence_default
# Template override respects SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "20000")
evidence_custom = module.build_behavior_evidence(traj, "question", max_chars=24000)
assert "START_MARKER_" in evidence_custom
assert "_LATE_MARKER_" in evidence_custom
assert "...[truncated]..." not in evidence_custom
# Template helpers match shared atif_helpers logic
assert module._behavior_final_response_limit() == atif_helpers._behavior_final_response_limit()
assert module._behavior_check_budget() == atif_helpers._behavior_check_budget()
def test_verified_facts_preserve_final_response_in_behavior_judge_prompt(monkeypatch) -> None:
from skillevaluator.tier3.eval_core import llm_judge
template_module = _load_harbor_template_module()
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_CHECK_BUDGET", "8000")
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "8000")
steps: list[dict] = [
{
"source": "user",
"message": "Run all twelve verification checks and summarize the outcome.",
},
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": "write-0",
"function_name": "Write",
"arguments": {
"file_path": "/workspace/output/report.py",
"content": "# generated report\n" + ("value = 1\n" * 170),
},
}
],
"observation": {
"results": [
{
"source_call_id": "write-0",
"content": "Wrote /workspace/output/report.py",
}
]
},
},
]
behaviors: list[str] = []
for idx in range(12):
token = f"pytest -q tests/check_module_{idx:02d}.py --maxfail=1 --tb=short"
behaviors.append(f"Execute `{token}` and confirm the final assertion")
steps.append(
{
"source": "agent",
"tool_calls": [
{
"tool_call_id": f"cmd-{idx}",
"function_name": "Bash",
"arguments": {"command": f"{token} && echo status_{idx:02d}_ok_" + ("x" * 90)},
}
],
"observation": {
"results": [
{
"source_call_id": f"cmd-{idx}",
"content": f"passed check {idx:02d} " + ("y" * 80),
}
]
},
}
)
final_claim = "FINAL_CLAIM_ASSERTION: All twelve verification modules passed without errors."
steps.append(
{
"source": "agent",
"message": final_claim + "\n" + ("P" * 6100) + "\nFINAL_TAIL_MARKER",
}
)
traj = {"steps": steps}
question = "Run all twelve verification checks and summarize the outcome."
def make_judge_stub(captured_prompts: list[str], count: int):
def _stub(prompt: str, **kwargs):
captured_prompts.append(prompt)
passed = final_claim in prompt
score = 1.0 if passed else 0.0
results = [{"step": i + 1, "passed": passed, "reason": "checked"} for i in range(count)]
return json.dumps({"results": results, "score": score, "summary": "done"}), None
return _stub
shared_prompts_no_facts: list[str] = []
monkeypatch.setattr(llm_judge, "call_public_llm", make_judge_stub(shared_prompts_no_facts, 1))
no_facts_bundle = atif_helpers.build_metric_evidence_bundles(
traj, question, expected_behavior=["Confirm the final assertion in the summary"]
)["behavior_check"]
no_facts_res = llm_judge.judge_behavior_check(
no_facts_bundle["prompt_evidence"], ["Confirm the final assertion in the summary"]
)
assert no_facts_res["score"] == 1.0
shared_prompts: list[str] = []
template_prompts: list[str] = []
monkeypatch.setattr(llm_judge, "call_public_llm", make_judge_stub(shared_prompts, len(behaviors)))
monkeypatch.setattr(template_module, "call_public_llm", make_judge_stub(template_prompts, len(behaviors)))
shared_bundle = atif_helpers.build_metric_evidence_bundles(
traj, question, expected_behavior=behaviors
)["behavior_check"]
template_bundle = template_module.build_metric_evidence_bundles(
traj, question, expected_behavior=behaviors
)["behavior_check"]
assert shared_bundle == template_bundle
assert len(shared_bundle["verified"]) == 12
assert len(shared_bundle["prompt_evidence"]) <= 12000
assert shared_bundle["omitted"]["truncated"] is True
assert shared_bundle["omitted"]["count"] == 1
shared_res = llm_judge.judge_behavior_check(shared_bundle["prompt_evidence"], behaviors)
template_res = template_module.judge_behavior_check(template_bundle["prompt_evidence"], behaviors)
assert shared_res["score"] == 1.0
assert template_res["score"] == 1.0
for prompt in (shared_prompts[0], template_prompts[0]):
assert "VERIFIED FACTS (deterministic):" in prompt
assert final_claim in prompt
assert "FINAL_TAIL_MARKER" in prompt
def test_compact_behavior_conversation_preserves_final_response_section(monkeypatch) -> None:
from skillevaluator.tier3.eval_core import llm_judge
template_module = _load_harbor_template_module()
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_CHECK_BUDGET", "8000")
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", "8000")
facts_block = "VERIFIED FACTS (deterministic):\n" + "\n".join(
f"- [OBSERVED step {i}] claim_{i:02d} :: " + ("f" * 140) for i in range(1, 13)
)
file_changes_block = "FILE CHANGES\nAgent called: Write\nPath: /workspace/output/out.py\n" + ("c" * 1850)
final_claim = "FINAL_RESPONSE_CLAIM: The verification report is complete and accurate."
final_block = f"FINAL RESPONSE\n{final_claim}\n" + ("R" * 6000) + "\nFINAL_RESPONSE_END"
user_block = "USER REQUEST\nVerify the output and report."
history_block = "COMPACT TOOL HISTORY\n" + ("H" * 2100) + "\nLAST_TOOL_OBSERVATION"
oversized_bundle = f"{facts_block}\n\n{file_changes_block}\n\n{final_block}\n\n{user_block}\n\n{history_block}"
assert len(oversized_bundle) > 12000
shared_compacted = llm_judge._compact_behavior_conversation(oversized_bundle)
template_compacted = template_module._compact_behavior_conversation(oversized_bundle)
assert shared_compacted == template_compacted
assert len(shared_compacted) <= 12000
assert final_claim in shared_compacted
assert "FINAL_RESPONSE_END" in shared_compacted
assert "LAST_TOOL_OBSERVATION" in shared_compacted
@pytest.mark.parametrize(
"case_name,limit,final_limit_env,text_builder,expected_substrings",
[
(
"no_prefix_long_suffix",
1200,
"600",
lambda: (
"FINAL RESPONSE\nSTART_FINAL_CLAIM_OK\n"
+ ("F" * 300)
+ "\nEND_FINAL_CLAIM_OK\n\nUSER REQUEST\nRun check.\n\nCOMPACT TOOL HISTORY\n"
+ ("H" * 1500)
+ "\nTAIL_HISTORY_MARKER"
),
["START_FINAL_CLAIM_OK", "END_FINAL_CLAIM_OK", "TAIL_HISTORY_MARKER"],
),
(
"no_prefix_tiny_suffix_budget",
420,
"400",
lambda: (
"FINAL RESPONSE\n"
+ ("F" * 370)
+ "\nEND_FINAL\n\nCOMPACT TOOL HISTORY\n"
+ ("H" * 500)
+ "\nTINY_TAIL"
),
["FINAL RESPONSE", "END_FINAL"],
),
(
"long_prefix_no_suffix",
1200,
"500",
lambda: (
"FILE CHANGES\nHEAD_FILE_MARKER\n"
+ ("C" * 1500)
+ "\nLATE_FILE_MARKER\n\nFINAL RESPONSE\nFINAL_ONLY_SUFFIX_OK"
),
["HEAD_FILE_MARKER", "LATE_FILE_MARKER", "FINAL_ONLY_SUFFIX_OK"],
),
(
"long_prefix_no_suffix_tiny_rem",
420,
"400",
lambda: (
"FILE CHANGES\nHEAD_TINY_PRE\n"
+ ("C" * 600)
+ "\n\nFINAL RESPONSE\n"
+ ("F" * 370)
+ "\nFINAL_END_TINY_REM"
),
["FINAL RESPONSE", "FINAL_END_TINY_REM"],
),
(
"oversized_final_with_small_prefix_and_suffix",
1500,
"800",
lambda: (
"FILE CHANGES\nSHORT_PREFIX\n\nFINAL RESPONSE\nFINAL_HEAD_KEEP\n"
+ ("F" * 2000)
+ "\nFINAL_TAIL_KEEP\n\nUSER REQUEST\nSHORT_SUFFIX"
),
["SHORT_PREFIX", "FINAL_HEAD_KEEP", "FINAL_TAIL_KEEP", "SHORT_SUFFIX"],
),
(
"both_prefix_and_suffix_exceed_half_budgets",
2000,
"600",
lambda: (
"FILE CHANGES\nPRE_START_MARK\n"
+ ("P" * 1500)
+ "\nPRE_END_MARK\n\nFINAL RESPONSE\nCORE_FINAL_CLAIM\n\nCOMPACT TOOL HISTORY\nSUF_START_MARK\n"
+ ("S" * 1500)
+ "\nSUF_END_MARK"
),
["PRE_START_MARK", "PRE_END_MARK", "CORE_FINAL_CLAIM", "SUF_START_MARK", "SUF_END_MARK"],
),
(
"both_prefix_and_suffix_with_tiny_remaining_budget",
440,
"400",
lambda: (
"FILE CHANGES\n"
+ ("P" * 300)
+ "\n\nFINAL RESPONSE\n"
+ ("F" * 375)
+ "\nTINY_BOTH_FINAL\n\nCOMPACT TOOL HISTORY\n"
+ ("S" * 300)
),
["FINAL RESPONSE", "TINY_BOTH_FINAL"],
),
],
)
def test_compact_behavior_conversation_section_layout_permutations(
monkeypatch,
case_name: str,
limit: int,
final_limit_env: str,
text_builder,
expected_substrings: list[str],
) -> None:
from skillevaluator.tier3.eval_core import llm_judge
template_module = _load_harbor_template_module()
monkeypatch.setenv("SKILL_EVAL_BEHAVIOR_FINAL_RESPONSE_LIMIT", final_limit_env)
raw_text = text_builder()
assert len(raw_text) > limit, case_name
shared_out = llm_judge._compact_behavior_conversation(raw_text, limit=limit)
template_out = template_module._compact_behavior_conversation(raw_text, limit=limit)
assert shared_out == template_out, case_name
assert len(shared_out) <= limit, case_name
for substr in expected_substrings:
assert substr in shared_out, f"{case_name}: missing {substr!r}"
# --- Codex apply_patch evidence (issue #236, requirement 7: retain patch contents) ---
#
# Codex decodes its file-edit tool to ``action_input = {"input": "<patch text>"}``.
# The write-evidence helpers read only ``file_path``-style and ``content``-style
# keys, so every Codex patch contributed nothing to FILE CHANGES and the patch
# text survived only in COMPACT TOOL HISTORY -- which is budgeted last, so it is
# the first evidence dropped under exactly the truncation pressure #236 is about.
_SINGLE_FILE_PATCH = "*** Begin Patch\n*** Update File: src/app.py\n@@\n-old\n+new\n*** End Patch"
_MULTI_FILE_PATCH = (
"*** Begin Patch\n"
"*** Update File: src/app.py\n@@\n-old\n+new\n"
"*** Add File: src/new_mod.py\n+print('hi')\n"
"*** Update File: docs/readme.md\n@@\n-a\n+b\n"
"*** End Patch"
)
_DELETE_PATCH = "*** Begin Patch\n*** Delete File: src/dead.py\n*** End Patch"
def _codex_patch_trajectory(patch: str) -> dict:
return {
"steps": [
{"source": "user", "message": "Fix the bug."},
{
"source": "agent",
"message": "Executed apply_patch",
"tool_calls": [
{
"tool_call_id": "patch-1",
"function_name": "apply_patch",
"arguments": {"input": patch},
}
],
"observation": {"results": [{"source_call_id": "patch-1", "content": "Success."}]},
},
]
}
def _file_changes_section(evidence: str) -> str:
return evidence.split("FINAL RESPONSE", 1)[0].split("COMPACT TOOL HISTORY", 1)[0]
@pytest.mark.parametrize(
("patch", "expected_paths"),
[
pytest.param(_SINGLE_FILE_PATCH, ["src/app.py"], id="single_file"),
pytest.param(
_MULTI_FILE_PATCH,
["src/app.py", "src/new_mod.py", "docs/readme.md"],
id="multi_file",
),
pytest.param(_DELETE_PATCH, ["src/dead.py"], id="delete"),
],
)
def test_codex_apply_patch_input_reaches_file_changes(patch: str, expected_paths: list[str]) -> None:
traj = _codex_patch_trajectory(patch)
assert len(atif_helpers._collect_file_change_evidence(traj)) == 1
file_changes = _file_changes_section(build_behavior_evidence(traj, "Fix the bug."))
assert "FILE CHANGES" in file_changes
assert "Agent called: apply_patch" in file_changes
# The patch body itself must survive, not just the fact that a patch happened.
assert patch in file_changes
for path in expected_paths:
assert path in file_changes
def test_codex_multi_file_patch_names_every_file_not_just_the_first() -> None:
"""A single-path helper must not let one file stand for a patch that touched three."""
path = atif_helpers._tool_file_path({"input": _MULTI_FILE_PATCH})
assert path == "src/app.py, src/new_mod.py, docs/readme.md"
def test_codex_patch_path_list_is_capped_so_it_cannot_crowd_out_the_budget() -> None:
body = "".join(f"*** Update File: src/mod_{idx}.py\n@@\n-a\n+b\n" for idx in range(15))
path = atif_helpers._tool_file_path({"input": f"*** Begin Patch\n{body}*** End Patch"})
assert path.startswith("src/mod_0.py, ")
assert path.endswith(" (+5 more)")
assert "src/mod_9.py" in path
assert "src/mod_10.py" not in path
@pytest.mark.parametrize(
"args",
[
pytest.param({"input": "this is not a patch at all, just some prose"}, id="headerless_text"),
pytest.param({"input": "*** Begin Patch\n*** Update File: \n@@\n-a\n+b\n"}, id="blank_header_path"),
pytest.param({"input": ""}, id="empty_string"),
pytest.param({"input": " \n "}, id="whitespace_only"),
pytest.param({"input": None}, id="none"),
pytest.param({"input": {"nested": 1}}, id="non_string"),
pytest.param({"input": ["*** Begin Patch"]}, id="list"),
pytest.param({}, id="absent"),
],
)
def test_malformed_patch_input_degrades_without_raising_or_fabricating_a_path(args: dict) -> None:
assert atif_helpers._tool_file_path(args) == ""
# A body may legitimately be empty here; what must not happen is a crash.
assert isinstance(atif_helpers._tool_write_body(args), str)
def test_shell_source_passed_as_input_is_not_mistaken_for_a_patch() -> None:
"""Codex also passes shell source as ``input``; that is not a file change."""
args = {"input": "rg -n 'def main' src/ && ls -la"}
assert atif_helpers._tool_file_path(args) == ""
assert atif_helpers._tool_write_body(args) == ""
@pytest.mark.parametrize(
("args", "expected_path", "expected_body"),
[
pytest.param({"file_path": "/w/a.py", "content": "a"}, "/w/a.py", "content:\na", id="file_path_content"),
pytest.param({"path": "/w/b.py", "content": "b"}, "/w/b.py", "content:\nb", id="path"),
pytest.param({"filename": "/w/c.py", "content": "c"}, "/w/c.py", "content:\nc", id="filename"),
pytest.param(
{"target_file": "/w/d.py", "new_string": "d"}, "/w/d.py", "new_string:\nd", id="target_file_new_string"
),
pytest.param({"patch": "--- a\n+++ b\n"}, "", "patch:\n--- a\n+++ b\n", id="patch_key"),
pytest.param({"file_path": "/w/e.py", "code": "e"}, "/w/e.py", "code:\ne", id="code_key"),
pytest.param(
{"file_path": "/w/f.py", "edits": [{"new_string": "f1"}]},
"/w/f.py",
"edit 1 new_string:\nf1",
id="edits_list",
),
# Precedence between the existing path keys is unchanged.
pytest.param({"file_path": "/w/g.py", "path": "/w/ignored.py"}, "/w/g.py", "", id="file_path_wins"),
# An explicit path still wins over a patch body, so no existing shape shifts.
pytest.param(
{"file_path": "/w/h.py", "input": _SINGLE_FILE_PATCH},
"/w/h.py",
f"input:\n{_SINGLE_FILE_PATCH}",
id="explicit_path_wins_over_patch",
),
],
)
def test_existing_write_argument_shapes_are_unchanged(args: dict, expected_path: str, expected_body: str) -> None:
assert atif_helpers._tool_file_path(args) == expected_path
assert atif_helpers._tool_write_body(args) == expected_body
def test_harbor_template_retains_codex_patch_contents() -> None:
module = _load_harbor_template_module()
traj = _codex_patch_trajectory(_MULTI_FILE_PATCH)
evidence = module.build_behavior_evidence(traj, "Fix the bug.")
assert module._tool_file_path({"input": _MULTI_FILE_PATCH}) == atif_helpers._tool_file_path(
{"input": _MULTI_FILE_PATCH}
)
assert evidence == build_behavior_evidence(traj, "Fix the bug.")
assert _MULTI_FILE_PATCH in _file_changes_section(evidence)
def test_codex_patch_survives_the_truncation_pressure_issue_236_is_about() -> None:
"""FILE CHANGES is budgeted first, the compact tool history last.
While the patch reached the judge only through the tool history, a long run
on a tight budget dropped it entirely -- the exact case #236 raises.
"""
patch = (
"*** Begin Patch\n*** Update File: src/payments/refund.py\n@@\n"
"- return None\n+ return Refund(amount=amount, currency=currency)\n*** End Patch"
)
steps: list[dict] = [{"source": "user", "message": "Implement the refund path."}]
for idx in range(30):
call_id = f"read-{idx}"
steps.append(
{
"source": "agent",
"message": f"Executed Read {call_id}",
"tool_calls": [
{"tool_call_id": call_id, "function_name": "Read", "arguments": {"file_path": f"/w/f{idx}.py"}}
],
"observation": {"results": [{"source_call_id": call_id, "content": "noise " * 200}]},
}
)
steps.append(
{
"source": "agent",
"message": "Executed apply_patch",
"tool_calls": [
{"tool_call_id": "patch-1", "function_name": "apply_patch", "arguments": {"input": patch}}
],
"observation": {"results": [{"source_call_id": "patch-1", "content": "Success."}]},
}
)
steps.append({"source": "agent", "message": "I implemented the refund path. " + ("Detail. " * 500)})
evidence = build_behavior_evidence({"steps": steps}, "Implement the refund path.", max_chars=8000)
assert len(evidence) <= 8000
assert "Refund(amount=amount, currency=currency)" in evidence
assert "src/payments/refund.py" in _file_changes_section(evidence)