Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
import json
import os
import re
import pytest
import yaml
from click.testing import CliRunner
from skillevaluator.cli import cli
from skillevaluator.tier3.dataset_utils import load_dataset_entries_with_format
from skillevaluator.tier3.evals_config import (
EvalsConfigError,
load_evals_config,
parse_environment_kwarg_overrides,
validate_environment_kwargs,
)
from skillevaluator.tier3.evals_spec import validate_skillevaluators as validate_skill_evals
from skillevaluator.tier3.evals_spec import validate_tier3_source
def _skill_with_config(tmp_path, text):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_text(text, encoding="utf-8")
return skill
def _link_or_skip(link, target, *, hard=False, target_is_directory=False):
try:
if hard:
link.hardlink_to(target)
else:
link.symlink_to(target, target_is_directory=target_is_directory)
except OSError as exc:
pytest.skip(f"link creation is unavailable: {exc}")
def test_load_evals_config_valid_harbor_policy(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_text(
"""\
schema_version: 1
harbor:
task_source: native_harbor
custom_dockerfile_mode: preserve
n_attempts: 3
pass_threshold: 0.60
n_concurrent: 4
max_agents: 2
timeout_multiplier: 2.0
agent_workdir: /app
resources:
cpus: 4
memory_mb: 8192
storage_mb: 4096
agents:
claude-code:
model: aws/anthropic/bedrock-claude-opus-4-6
skill_workspace:
mode: group
include:
- ../helper-skill
grading:
mode: default_plus_custom
""",
encoding="utf-8",
)
config, path = load_evals_config(skill)
assert path == skill / "evals" / "config.yml"
assert config["harbor"]["task_source"] == "native_harbor"
assert config["harbor"]["custom_dockerfile_mode"] == "preserve"
assert config["harbor"]["n_attempts"] == 3
assert config["harbor"]["pass_threshold"] == 0.60
assert config["harbor"]["agent_workdir"] == "/app"
assert config["harbor"]["resources"] == {
"cpus": 4,
"memory_mb": 8192,
"storage_mb": 4096,
}
assert config["harbor"]["agents"]["claude-code"]["model"].endswith("bedrock-claude-opus-4-6")
assert config["skill_workspace"]["mode"] == "group"
assert config["skill_workspace"]["include"] == ["../helper-skill"]
assert config["grading"]["mode"] == "default_plus_custom"
@pytest.mark.parametrize("value", [".nan", ".inf", "-.inf"])
def test_load_evals_config_rejects_nonfinite_timeout_multiplier(tmp_path, value):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_text(
f"schema_version: 1\nharbor:\n timeout_multiplier: {value}\n",
encoding="utf-8",
)
with pytest.raises(EvalsConfigError, match=r"harbor\.timeout_multiplier.*finite"):
load_evals_config(skill)
def test_load_evals_config_rejects_overflowing_timeout_multiplier(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_text(
f"schema_version: 1\nharbor:\n timeout_multiplier: {10**1000}\n",
encoding="utf-8",
)
with pytest.raises(EvalsConfigError, match=r"harbor\.timeout_multiplier.*finite"):
load_evals_config(skill)
def test_load_evals_config_rejects_finite_multiplier_that_overflows_default_timeouts(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_text(
"schema_version: 1\nharbor:\n timeout_multiplier: 1.0e+308\n",
encoding="utf-8",
)
with pytest.raises(EvalsConfigError, match=r"harbor\.timeout_multiplier.*finite Harbor timeouts"):
load_evals_config(skill)
def test_load_evals_config_rejects_overflowing_pass_threshold(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_text(
f"schema_version: 1\nharbor:\n pass_threshold: {10**1000}\n",
encoding="utf-8",
)
with pytest.raises(EvalsConfigError, match=r"harbor\.pass_threshold.*between"):
load_evals_config(skill)
def test_load_evals_config_missing_is_empty(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
config, path = load_evals_config(skill)
assert config == {}
assert path is None
def test_load_evals_config_rejects_unknown_keys(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_text(
"""\
schema_version: 1
harbor:
surprise: true
""",
encoding="utf-8",
)
with pytest.raises(EvalsConfigError, match="unknown harbor key"):
load_evals_config(skill)
@pytest.mark.parametrize(
"document",
[
pytest.param("schema_version: 1\nschema_version: 1\n", id="top-level"),
pytest.param("schema_version: 1\nharbor:\n n_attempts: 1\n n_attempts: 2\n", id="harbor"),
pytest.param("schema_version: 1\nharbor:\n resources:\n cpus: 1\n cpus: 2\n", id="nested"),
pytest.param(
"schema_version: 1\nharbor:\n <<: {stop_on_pass: true, stop_on_pass: false}\n", id="merge-mapping"
),
pytest.param("schema_version: 1\nharbor:\n <<: [{n_attempts: 1, n_attempts: 2}]\n", id="merge-list"),
],
)
def test_load_evals_config_rejects_duplicate_keys(tmp_path, document):
skill = _skill_with_config(tmp_path, document)
with pytest.raises(
EvalsConfigError,
match=r"invalid YAML: line \d+, column \d+: while constructing a mapping; "
r"line \d+, column \d+: duplicate mapping key$",
):
load_evals_config(skill)
def test_load_evals_config_names_the_position_of_a_duplicate_key(tmp_path):
skill = _skill_with_config(tmp_path, "schema_version: 1\nharbor:\n n_attempts: 1\n n_attempts: 2\n")
with pytest.raises(EvalsConfigError) as caught:
load_evals_config(skill)
assert str(caught.value).endswith(
"config.yml: invalid YAML: line 3, column 3: while constructing a mapping; "
"line 4, column 3: duplicate mapping key"
)
def test_load_evals_config_keeps_yaml_merge_overrides(tmp_path):
skill = _skill_with_config(
tmp_path,
"schema_version: 1\nharbor:\n agents:\n codex: &defaults\n model: model-one\n"
" claude-code:\n <<: *defaults\n model: model-two\n",
)
config, _ = load_evals_config(skill)
assert config["harbor"]["agents"] == {"codex": {"model": "model-one"}, "claude-code": {"model": "model-two"}}
def test_load_evals_config_rejects_cyclic_yaml_merge(tmp_path):
skill = _skill_with_config(tmp_path, "schema_version: 1\nharbor:\n agents: &agents\n <<: *agents\n")
with pytest.raises(EvalsConfigError, match=r"invalid YAML: .*cyclic YAML merge"):
load_evals_config(skill)
def test_load_evals_config_yaml_errors_give_position_without_source_text(tmp_path):
skill = _skill_with_config(tmp_path, 'schema_version: 1\nharbor:\n agent_workdir: "do-not-echo-this-value\n')
with pytest.raises(EvalsConfigError) as caught:
load_evals_config(skill)
message = str(caught.value)
assert message.endswith(
"config.yml: invalid YAML: line 3, column 18: while scanning a quoted scalar; "
"line 4, column 1: found unexpected end of stream"
)
assert "do-not-echo-this-value" not in message
assert "agent_workdir" not in message
assert isinstance(caught.value.__cause__, yaml.YAMLError)
@pytest.mark.parametrize(
("document", "detail"),
[
pytest.param(
"schema_version: 1\nharbor:\n agents:\n codex: &a {model: model-one}\n"
" claude-code: &a {model: model-two}\n",
"line 4, column 12: found duplicate anchor 'a'; first occurrence; line 5, column 18: second occurrence",
id="duplicate-anchor",
),
pytest.param(
"schema_version: 1\n---\nschema_version: 1\n",
"line 1, column 1: expected a single document in the stream; line 2, column 1: but found another document",
id="second-document",
),
],
)
def test_load_evals_config_yaml_errors_keep_the_pyyaml_context(tmp_path, document, detail):
skill = _skill_with_config(tmp_path, document)
with pytest.raises(EvalsConfigError) as caught:
load_evals_config(skill)
assert str(caught.value).endswith(f"config.yml: invalid YAML: {detail}")
assert "model-one" not in str(caught.value)
@pytest.mark.parametrize(
"value",
[pytest.param("!!int not-a-number", id="tagged-int"), pytest.param("2026-13-01", id="invalid-date")],
)
def test_load_evals_config_reports_unconstructible_value_without_echoing_it(tmp_path, value):
skill = _skill_with_config(tmp_path, f"schema_version: 1\nharbor:\n agent_workdir: {value}\n")
with pytest.raises(EvalsConfigError, match=r"invalid YAML: cannot parse a value$") as caught:
load_evals_config(skill)
assert value.removeprefix("!!int ") not in str(caught.value)
def _include_list(count):
return "schema_version: 1\nskill_workspace:\n include:\n" + "".join(f" - f{i}.md\n" for i in range(count))
@pytest.mark.parametrize(
("document", "limit"),
[
pytest.param(
"schema_version: 1\nharbor:\n agent_workdir: " + "[" * 101 + "]" * 101 + "\n",
"nesting depth exceeds 100",
id="nesting",
),
pytest.param(_include_list(1025), "sequence size exceeds 1024", id="list-entries"),
pytest.param(
"schema_version: 1\nharbor:\n agents:\n codex:\n model: " + "m" * 65_537 + "\n",
"scalar length exceeds 65536",
id="value-length",
),
],
)
def test_load_evals_config_rejects_documents_over_the_yaml_complexity_limits(tmp_path, document, limit):
skill = _skill_with_config(tmp_path, document)
with pytest.raises(EvalsConfigError, match=rf"invalid YAML: .*{limit}$") as caught:
load_evals_config(skill)
assert "mmmm" not in str(caught.value)
def test_load_evals_config_accepts_a_list_at_the_entry_limit(tmp_path):
skill = _skill_with_config(tmp_path, _include_list(1024))
config, _ = load_evals_config(skill)
assert len(config["skill_workspace"]["include"]) == 1024
@pytest.mark.parametrize(
("document", "field"),
[
pytest.param("schema_version: 1\n1: invalid\n", "top-level", id="top-level-int"),
pytest.param("schema_version: 1\nyes: invalid\n", "top-level", id="top-level-bool"),
pytest.param("schema_version: 1\n~: invalid\n", "top-level", id="top-level-null"),
pytest.param("schema_version: 1\nharbor:\n 1: invalid\n", "harbor", id="harbor"),
pytest.param(
"schema_version: 1\nharbor:\n resources:\n 1: invalid\n", "harbor.resources", id="harbor-resources"
),
pytest.param(
"schema_version: 1\nharbor:\n agents:\n codex:\n 1: invalid\n",
"harbor.agents.codex",
id="harbor-agent",
),
pytest.param("schema_version: 1\nskill_workspace:\n on: invalid\n", "skill_workspace", id="skill-workspace"),
pytest.param("schema_version: 1\ngrading:\n 2: invalid\n", "grading", id="grading"),
],
)
def test_load_evals_config_rejects_non_string_keys(tmp_path, document, field):
skill = _skill_with_config(tmp_path, document)
with pytest.raises(EvalsConfigError, match=rf": {re.escape(field)} keys must be strings$"):
load_evals_config(skill)
@pytest.mark.parametrize(
("content", "message"),
[
pytest.param(b"schema_version: 1\nharbor:\n yes: 2\n", "harbor keys must be strings", id="non-string-key"),
pytest.param(
b"schema_version: 1\n# caf\xe9\n", "cannot read config: not valid UTF-8 at byte 23", id="non-utf8"
),
],
)
def test_validate_tier3_source_reports_config_errors_instead_of_crashing(tmp_path, content, message):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_bytes(content)
source, results = validate_tier3_source(skill)
assert source == "invalid"
assert [(result.status, result.message.endswith(message)) for result in results] == [("error", True)]
def test_load_evals_config_rejects_symlinked_config(tmp_path):
outside = tmp_path / "outside.yml"
outside.write_text("schema_version: 1\nharbor:\n n_attempts: 9\n", encoding="utf-8")
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
_link_or_skip(skill / "evals" / "config.yml", outside)
with pytest.raises(EvalsConfigError, match=r"cannot read config: .*symlink"):
load_evals_config(skill)
def test_load_evals_config_rejects_broken_config_symlink(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
_link_or_skip(skill / "evals" / "config.yml", tmp_path / "missing.yml")
with pytest.raises(EvalsConfigError, match=r"cannot read config: .*symlink"):
load_evals_config(skill)
def test_tier3_validate_reports_broken_config_symlink(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "evals.json").write_text('[{"id": "case-1", "question": "q"}]\n', encoding="utf-8")
_link_or_skip(skill / "evals" / "config.yml", tmp_path / "missing.yml")
result = CliRunner().invoke(cli, ["tier3", "validate", str(skill), "--json"])
assert result.exit_code == 1, result.output
errors = [item for item in json.loads(result.output) if item["status"] == "error"]
assert len(errors) == 1
assert errors[0]["path"].endswith("config.yml")
assert re.search(r"Invalid evals config: .*cannot read config: .*symlink", errors[0]["message"])
def test_load_evals_config_rejects_config_in_symlinked_evals_directory(tmp_path):
outside = tmp_path / "outside-evals"
outside.mkdir()
(outside / "config.yml").write_text("schema_version: 1\n", encoding="utf-8")
skill = tmp_path / "skill"
skill.mkdir()
_link_or_skip(skill / "evals", outside, target_is_directory=True)
with pytest.raises(EvalsConfigError, match=r"cannot read config: .*symlink"):
load_evals_config(skill)
@pytest.mark.parametrize(
("link_target", "skill_path"),
[
pytest.param("real", "link", id="symlinked-skill"),
pytest.param(
"real/sub",
"link/..",
id="parent-of-link",
marks=pytest.mark.skipif(os.name == "nt", reason="Windows folds '..' before it follows a link"),
),
],
)
def test_load_evals_config_reads_config_through_links_in_the_skill_path(tmp_path, link_target, skill_path):
real = tmp_path / "real"
(real / "sub").mkdir(parents=True)
(real / "evals").mkdir()
(real / "evals" / "config.yml").write_text("schema_version: 1\nharbor:\n n_attempts: 3\n", encoding="utf-8")
_link_or_skip(tmp_path / "link", tmp_path / link_target, target_is_directory=True)
config, config_path = load_evals_config(tmp_path / skill_path)
assert config["harbor"]["n_attempts"] == 3
assert config_path is not None and config_path.name == "config.yml"
def test_load_evals_config_rejects_hard_linked_config(tmp_path):
outside = tmp_path / "outside.yml"
outside.write_text("schema_version: 1\n", encoding="utf-8")
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
_link_or_skip(skill / "evals" / "config.yml", outside, hard=True)
with pytest.raises(EvalsConfigError, match=r"cannot read config: .*hard-linked"):
load_evals_config(skill)
@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="FIFO creation is unavailable on this platform")
def test_load_evals_config_rejects_fifo_without_blocking(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
os.mkfifo(skill / "evals" / "config.yml")
with pytest.raises(EvalsConfigError, match=r"cannot read config: .*not a regular file"):
load_evals_config(skill)
def test_load_evals_config_rejects_config_over_the_size_limit(tmp_path):
limit = 4 * 1024 * 1024
skill = _skill_with_config(tmp_path, "schema_version: 1\n" + "#" * limit)
with pytest.raises(EvalsConfigError, match=rf"cannot read config: .*exceeds the {limit}-byte limit"):
load_evals_config(skill)
def test_load_evals_config_rejects_environment_kwargs_because_they_are_operator_only(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_text(
"schema_version: 1\nharbor:\n environment_kwargs:\n region: us-west-2\n",
encoding="utf-8",
)
with pytest.raises(EvalsConfigError, match=r"unknown harbor key.*environment_kwargs"):
load_evals_config(skill)
@pytest.mark.parametrize(
("entry", "secret"),
[
("api_key=do-not-render-this", "do-not-render-this"),
('headers={"Authorization":"do-not-render-this"}', "do-not-render-this"),
('headers={"X-API-Key":"do-not-render-this"}', "do-not-render-this"),
('metadata={"clientSecret":"do-not-render-this"}', "do-not-render-this"),
('nested={"sudoPassword":"do-not-render-this"}', "do-not-render-this"),
("endpoint=https://[email protected]/path", "bearer-token"),
("endpoint=https://user%[email protected]/path", "user%3Apassword"),
("endpoint=ssh://[email protected]/repo", "[email protected]"),
("proxy=https://user:[email protected]", "do-not-render-this"),
("missing-equals-do-not-render-this", "do-not-render-this"),
],
)
def test_cli_environment_kwargs_reject_secrets_without_echoing_values(entry, secret):
with pytest.raises(ValueError) as caught:
parse_environment_kwarg_overrides((entry,))
assert secret not in str(caught.value)
def test_cli_environment_kwargs_allow_non_secret_key_names() -> None:
assert parse_environment_kwarg_overrides(('key_name="evaluation-key"',)) == {"key_name": "evaluation-key"}
def test_cli_environment_kwargs_allow_kubernetes_secret_object_reference() -> None:
assert parse_environment_kwarg_overrides(
('image_pull_secret="registry-credentials"',),
env_mode="ack",
) == {"image_pull_secret": "registry-credentials"}
@pytest.mark.parametrize(
("env_mode", "reference"),
[
("modal", "registry-credentials"),
("daytona", "registry-credentials"),
("ack", "username:password"),
("ack", "UPPER_CASE"),
("ack", "contains spaces"),
],
)
def test_cli_environment_kwargs_reject_image_pull_secret_outside_ack_or_invalid_kubernetes_names(
env_mode: str,
reference: str,
) -> None:
with pytest.raises(ValueError, match="Invalid --environment-kwarg"):
parse_environment_kwarg_overrides(
(f"image_pull_secret={json.dumps(reference)}",),
env_mode=env_mode,
)
def test_environment_kwargs_reject_cycles_without_recursing() -> None:
cyclic: dict[str, object] = {}
cyclic["nested"] = cyclic
with pytest.raises(ValueError, match="cyclic"):
validate_environment_kwargs({"options": cyclic})
def test_environment_kwargs_reject_excessive_direct_api_nesting_without_recursing() -> None:
nested: object = "leaf"
for _ in range(40):
nested = [nested]
with pytest.raises(ValueError, match="nest at most"):
validate_environment_kwargs({"options": nested})
def test_cli_environment_kwargs_reject_extreme_json_nesting_without_traceback() -> None:
deeply_nested = "[" * 1200 + "0" + "]" * 1200
with pytest.raises(ValueError, match="nest"):
parse_environment_kwarg_overrides((f"options={deeply_nested}",))
@pytest.mark.parametrize(
("env_mode", "entry", "expected"),
[
(
"daytona",
'secrets={"TARGET_API_KEY":"organization-secret-name"}',
{"secrets": {"TARGET_API_KEY": "organization-secret-name"}},
),
(
"modal",
'secrets=["runtime-secret","telemetry-secret"]',
{"secrets": ["runtime-secret", "telemetry-secret"]},
),
(
"modal",
'registry_secret="private-registry-login"',
{"registry_secret": "private-registry-login"},
),
(
"skypilot",
'secrets=["cluster-secret"]',
{"secrets": ["cluster-secret"]},
),
(
"cwsandbox",
'secrets=[{"store":"team-store","name":"runtime-key","field":"value","env_var":"API_KEY"}]',
{"secrets": [{"store": "team-store", "name": "runtime-key", "field": "value", "env_var": "API_KEY"}]},
),
(
"wandb",
'secrets=[{"name":"runtime-key","env_var":"API_KEY"}]',
{"secrets": [{"name": "runtime-key", "env_var": "API_KEY"}]},
),
],
)
def test_cli_environment_kwargs_allow_provider_secret_references(env_mode, entry, expected) -> None:
assert parse_environment_kwarg_overrides((entry,), env_mode=env_mode) == expected
@pytest.mark.parametrize(
("env_mode", "entry"),
[
("e2b", 'secrets=["not-supported"]'),
("daytona", 'secrets=["wrong-shape"]'),
("modal", 'secrets={"wrong":"shape"}'),
("cwsandbox", 'secrets=[{"value":"plaintext-not-a-reference"}]'),
],
)
def test_cli_environment_kwargs_reject_secret_reference_fields_outside_exact_harbor_shapes(env_mode, entry) -> None:
with pytest.raises(ValueError, match="Invalid --environment-kwarg"):
parse_environment_kwarg_overrides((entry,), env_mode=env_mode)
def test_load_evals_config_validates_resource_shapes(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_text(
"""\
schema_version: 1
harbor:
resources:
memory_mb: 0
""",
encoding="utf-8",
)
with pytest.raises(EvalsConfigError, match=r"harbor\.resources\.memory_mb must be >= 1"):
load_evals_config(skill)
def test_load_evals_config_validates_ranges(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "config.yml").write_text(
"""\
schema_version: 1
harbor:
pass_threshold: 1.5
""",
encoding="utf-8",
)
with pytest.raises(EvalsConfigError, match="pass_threshold"):
load_evals_config(skill)
@pytest.mark.parametrize(
("entry", "secret"),
[
("custom_setting=sk-ant-api03-1234567890abcdefghijklmnop", "sk-ant-api03-1234567890abcdefghijklmnop"),
("custom_setting=ghp_1234567890abcdefghijklmnopqrstuvwxyz", "ghp_1234567890abcdefghijklmnopqrstuvwxyz"),
("custom_setting=AIzaSyA1234567890abcdefghijklmnopqrstuvw", "AIzaSyA1234567890abcdefghijklmnopqrstuvw"),
("custom_setting=-----BEGIN PRIVATE KEY-----", "-----BEGIN PRIVATE KEY-----"),
("service_account_key=my-secret-sa-key", "my-secret-sa-key"),
],
)
def test_cli_environment_kwargs_reject_secret_value_patterns_without_echoing(entry, secret):
with pytest.raises(ValueError, match="Sensitive key or value detected") as caught:
parse_environment_kwarg_overrides((entry,))
assert secret not in str(caught.value)
def test_validate_skill_evals_does_not_warn_expected_script_for_guide_only_skill(tmp_path):
skill = tmp_path / "guide-skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "evals.json").write_text(
'[{"id":"case-1","question":"Explain the guide."}]',
encoding="utf-8",
)
messages = [r.message for r in validate_skill_evals(skill)]
assert not any("expected_script" in msg for msg in messages)
def test_validate_skill_evals_warns_expected_script_when_scripts_exist(tmp_path):
skill = tmp_path / "script-skill"
(skill / "evals").mkdir(parents=True)
(skill / "scripts").mkdir()
(skill / "scripts" / "main.py").write_text("print('ok')\n", encoding="utf-8")
(skill / "evals" / "evals.json").write_text(
'[{"id":"case-1","question":"Run it."}]',
encoding="utf-8",
)
messages = [r.message for r in validate_skill_evals(skill)]
assert any("expected_script is missing" in msg for msg in messages)
def test_validate_skill_evals_accepts_placeholder_expected_script_as_explicit_no_script(tmp_path):
skill = tmp_path / "script-skill"
(skill / "evals").mkdir(parents=True)
(skill / "scripts").mkdir()
(skill / "scripts" / "main.py").write_text("print('ok')\n", encoding="utf-8")
(skill / "evals" / "evals.json").write_text(
'[{"id":"case-1","question":"Run it.","expected_script":"None"}]',
encoding="utf-8",
)
messages = [result.message for result in validate_skill_evals(skill)]
assert not any("expected_script is missing" in message for message in messages)
def test_agentskills_evals_json_is_accepted_without_deprecation_warning(tmp_path):
skill = tmp_path / "agent-skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "evals.json").write_text(
"""\
{
"skill_name": "agent-skill",
"evals": [
{
"id": 1,
"prompt": "Use the skill.",
"expected_output": "The skill returns a useful answer.",
"files": ["evals/files/input.txt"],
"assertions": ["The answer references the input."]
}
]
}
""",
encoding="utf-8",
)
entries, dataset_format = load_dataset_entries_with_format(skill / "evals" / "evals.json")
results = validate_skill_evals(skill)
messages = [r.message for r in results]
assert dataset_format == "agentskills"
assert entries[0]["question"] == "Use the skill."
assert entries[0]["ground_truth"] == "The skill returns a useful answer."
assert entries[0]["expected_behavior"] == ["The answer references the input."]
assert entries[0]["expected_skill"] == "agent-skill"
assert not any("Deprecated eval dataset format" in msg for msg in messages)
assert not any(r.status == "error" for r in results)
def test_agentskills_evals_json_requires_skill_name(tmp_path):
skill = tmp_path / "agent-skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "evals.json").write_text(
"""\
{
"evals": [
{"id": 1, "prompt": "Use the skill.", "expected_output": "The skill answers."}
]
}
""",
encoding="utf-8",
)
messages = [r.message for r in validate_skill_evals(skill)]
assert any("skill_name" in msg for msg in messages)
def test_agentskills_evals_json_rejects_non_object_items(tmp_path):
skill = tmp_path / "agent-skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "evals.json").write_text(
"""\
{
"skill_name": "agent-skill",
"evals": [
{"id": 1, "prompt": "Use the skill.", "expected_output": "The skill answers."},
"bad"
]
}
""",
encoding="utf-8",
)
messages = [r.message for r in validate_skill_evals(skill)]
assert any("evals[1]" in msg and "object" in msg for msg in messages)
def test_agentskills_evals_json_reports_authored_required_field_names(tmp_path):
skill = tmp_path / "agent-skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "evals.json").write_text(
"""\
{
"skill_name": "agent-skill",
"evals": [
{"id": 1, "prompt": "Use the skill."}
]
}
""",
encoding="utf-8",
)
messages = [r.message for r in validate_skill_evals(skill)]
assert any("expected_output" in msg for msg in messages)
assert not any("ground_truth" in msg for msg in messages)
def test_legacy_evals_json_is_accepted_with_deprecation_warning(tmp_path):
skill = tmp_path / "legacy-skill"
(skill / "evals").mkdir(parents=True)
(skill / "evals" / "evals.json").write_text(
'[{"id":"case-1","question":"Use the skill."}]',
encoding="utf-8",
)
results = validate_skill_evals(skill)
assert not any(r.status == "error" for r in results)
assert any("Deprecated eval dataset format" in r.message for r in results)
@pytest.mark.parametrize(
"infra_key",
[
"allow_workload_identity",
"autopilot",
"cluster_name",
"region",
"namespace",
"registry_location",
"registry_name",
"project_id",
"cloud_build_machine_type",
"cloud_build_disk_size_gb",
"memory_limit_multiplier",
],
)
def test_validate_environment_kwargs_gke_infrastructure_boundary(infra_key: str) -> None:
"""Reject GKE infrastructure keys when allow_gke_infrastructure_kwargs=False while allowing them on host CLI."""
from skillevaluator.tier3.evals_config import validate_environment_kwargs
with pytest.raises(ValueError, match=r"GKE infrastructure setting .* is not allowed in skill config"):
validate_environment_kwargs({infra_key: "test-val"}, allow_gke_infrastructure_kwargs=False)
assert validate_environment_kwargs({infra_key: "test-val"}, allow_gke_infrastructure_kwargs=True) == {
infra_key: "test-val"
}