Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): load evals/config.yml strictly before the Harbor run (#554)

Narendran Raghavane9ea45cf9750
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4a66fb4c..c084e81d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -813,6 +813,15 @@ All notable changes to SkillEvaluator are documented in this file.
1.75 points higher; the bundled `text-analyzer` example now scores 86.2
instead of 84.5.
+- Tier 3 `evals/config.yml` loading rejects a repeated key (also inside a YAML
+ merge source) or a cyclic merge instead of keeping the last value, applies
+ the shared YAML size and nesting limits, and reports a non-string key such
+ as `yes:` or a non-UTF-8 file as a config error instead of crashing
+ `validate`. Checks before the Harbor run now read the file as the run does,
+ rejecting a symlink (even a broken one), a hard link, a FIFO, a file over
+ 4 MiB or a symlinked `evals/` directory, and a YAML error no longer repeats
+ the source line.
+
## 0.9.0 - 2026-11-13
### Fixed
diff --git a/docs/eval-datasets.mdx b/docs/eval-datasets.mdx
index 07e26c34..91546c49 100644
--- a/docs/eval-datasets.mdx
+++ b/docs/eval-datasets.mdx
@@ -594,6 +594,27 @@ reserved.
section in [Providers & Credentials](/skills/skillevaluator/configuration).
</Note>
+Validation and evaluation load the config file strictly:
+
+- A repeated key is an error, including inside a YAML merge (`<<`) source. A
+ mapping's own keys still override the keys it merges. A cyclic merge is an
+ error.
+- Keys must be strings. An unquoted `1:`, `yes:`, `on:` or `~:` key is
+ rejected with the name of the block that holds it.
+- The file must be a regular UTF-8 file of at most 4 MiB in a real `evals/`
+ directory. A symlink (even a broken one), a hard link, a FIFO or a symlinked
+ `evals/` directory is rejected instead of being followed or skipped.
+- The document may have at most 100 nesting levels, 20,000 nodes, 1,024 alias
+ references, 1,024 entries in any list or mapping, 65,536 characters in any
+ value and 14,000 bits in any integer.
+- A YAML error never includes the text of the line. Most syntax errors, and
+ every repeated key, give the line, column and problem. When PyYAML also
+ reports a context, such as `found duplicate anchor 'a'; first occurrence`,
+ the context comes first, with its own line and column when they differ from
+ the problem's. An error for a cyclic merge
+ or for one of the limits above names it without a position. A value that
+ cannot be parsed, such as `!!int abc`, gives `cannot parse a value`.
+
## Fixtures and custom environments
Two optional directories shape the world the agent works in.
diff --git a/src/skillevaluator/tier3/evals_config.py b/src/skillevaluator/tier3/evals_config.py
index 34664c2c..a3e759d8 100644
--- a/src/skillevaluator/tier3/evals_config.py
+++ b/src/skillevaluator/tier3/evals_config.py
@@ -11,6 +11,7 @@ from __future__ import annotations
import json
import math
+import os
import re
import sys
from itertools import pairwise
@@ -19,9 +20,19 @@ from typing import Any
import yaml
+from skillevaluator.quality_config import _yaml_error_detail
from skillevaluator.tier3.harbor import canonical_agent_name
+from skillevaluator.utils.secure_fs import SecurePathError, SecureRoot
+from skillevaluator.utils.structured_data import (
+ StructuredDataLimitError,
+ StructuredDataSyntaxError,
+ _UniqueKeySafeLoader,
+ load_bounded_yaml,
+)
CONFIG_FILENAMES = ("config.yml", "config.yaml")
+# The one size limit for evals/config.yml, also used by the Harbor run's evaluator projection.
+MAX_EVALS_CONFIG_BYTES = 4 * 1024 * 1024
HARBOR_CUSTOM_DOCKERFILE_MODES = {"preserve", "rebase"}
HARBOR_BASE_IMAGE_MODES = {"reuse", "rebuild", "disabled"}
SKILL_WORKSPACE_MODES = {"isolated", "group"}
@@ -128,11 +139,15 @@ class EvalsConfigError(ValueError):
def find_evals_config(skill_path: Path) -> Path | None:
- """Return the first supported evals config file for a skill, if present."""
+ """Return the first supported evals config file for a skill, if present.
+
+ A broken symlink counts as present, so loading rejects it instead of
+ running as if the skill had no config.
+ """
evals_dir = skill_path / "evals"
for name in CONFIG_FILENAMES:
candidate = evals_dir / name
- if candidate.exists():
+ if os.path.lexists(candidate):
return candidate
return None
@@ -148,12 +163,7 @@ def load_evals_config(skill_path: Path) -> tuple[dict[str, Any], Path | None]:
if config_path is None:
return {}, None
- try:
- raw = yaml.safe_load(config_path.read_text(encoding="utf-8"))
- except yaml.YAMLError as e:
- raise EvalsConfigError(f"{config_path}: invalid YAML: {e}") from e
- except OSError as e:
- raise EvalsConfigError(f"{config_path}: cannot read config: {e}") from e
+ raw = _parse_config_text(_read_config_text(config_path), config_path)
if raw is None:
raise EvalsConfigError(f"{config_path}: config must not be empty")
@@ -163,8 +173,72 @@ def load_evals_config(skill_path: Path) -> tuple[dict[str, Any], Path | None]:
return _validate_config(raw, config_path), config_path
+def _read_config_text(config_path: Path) -> str:
+ """Read the config as one bounded, single-link regular UTF-8 file without following links.
+
+ As on the Harbor run, ``evals/`` and the config file must not be links;
+ links in the skill path itself are resolved.
+ """
+ try:
+ evals_dir = config_path.parent.parent.resolve() / config_path.parent.name
+ with SecureRoot(evals_dir) as root:
+ data, _ = root.read_bytes(Path(config_path.name), MAX_EVALS_CONFIG_BYTES)
+ except (OSError, SecurePathError) as e:
+ raise EvalsConfigError(f"{config_path}: cannot read config: {e}") from e
+ try:
+ return data.decode("utf-8")
+ except UnicodeDecodeError as e:
+ raise EvalsConfigError(f"{config_path}: cannot read config: not valid UTF-8 at byte {e.start}") from e
+
+
+def _parse_config_text(text: str, config_path: Path) -> Any:
+ """Parse one bounded YAML document whose mappings, merge sources included, repeat no key.
+
+ A YAML error never repeats the source line. A marked error, such as a syntax error or
+ a repeated key, gives the line, column and problem, after PyYAML's context and its position
+ when it has one; a cyclic merge or limit error names it.
+ """
+ try:
+ try:
+ return load_bounded_yaml(text)
+ except StructuredDataSyntaxError:
+ # The bounded loader reports no position; the same loader gives a line and column.
+ return yaml.load(text, Loader=_UniqueKeySafeLoader) # nosec B506
+ except StructuredDataLimitError as e:
+ raise EvalsConfigError(f"{config_path}: invalid YAML: {e}") from e
+ except yaml.YAMLError as e:
+ raise EvalsConfigError(f"{config_path}: invalid YAML: {_yaml_error_text(e)}") from e
+ except (ValueError, IndexError, KeyError, AttributeError, TypeError) as e:
+ # SafeLoader's scalar constructors raise these for values such as ``!!int abc`` or the date ``2026-13-01``.
+ raise EvalsConfigError(f"{config_path}: invalid YAML: cannot parse a value") from e
+
+
+def _yaml_error_text(exc: yaml.YAMLError) -> str:
+ """Return PyYAML's context and problem with their positions, but not the source snippet.
+
+ Some errors say what is wrong only in the context, as in a duplicate anchor:
+ ``line 4, column 12: found duplicate anchor 'a'; first occurrence; line 5, column 18: second occurrence``.
+ """
+ detail = _yaml_error_detail(exc)
+ context = getattr(exc, "context", None)
+ problem_mark = getattr(exc, "problem_mark", None)
+ if not context or problem_mark is None or getattr(exc, "problem", None) is None:
+ return detail
+ mark = getattr(exc, "context_mark", None)
+ if mark is None or (mark.line, mark.column) == (problem_mark.line, problem_mark.column):
+ return f"{context}; {detail}"
+ return f"line {mark.line + 1}, column {mark.column + 1}: {context}; {detail}"
+
+
+def _unknown_keys(mapping: dict[Any, Any], allowed: set[str], config_path: Path, field: str) -> set[str]:
+ """Return the keys outside ``allowed``; a non-string key such as ``1``, ``yes`` or ``~`` is an error."""
+ if any(not isinstance(key, str) for key in mapping):
+ raise EvalsConfigError(f"{config_path}: {field} keys must be strings")
+ return set(mapping) - allowed
+
+
def _validate_config(raw: dict[str, Any], config_path: Path) -> dict[str, Any]:
- unknown_top = set(raw) - _TOP_LEVEL_KEYS
+ unknown_top = _unknown_keys(raw, _TOP_LEVEL_KEYS, config_path, "top-level")
if unknown_top:
raise EvalsConfigError(f"{config_path}: unknown top-level key(s): {', '.join(sorted(unknown_top))}")
@@ -178,7 +252,7 @@ def _validate_config(raw: dict[str, Any], config_path: Path) -> dict[str, Any]:
if not isinstance(harbor_raw, dict):
raise EvalsConfigError(f"{config_path}: harbor must be a mapping")
- unknown_harbor = set(harbor_raw) - _HARBOR_KEYS
+ unknown_harbor = _unknown_keys(harbor_raw, _HARBOR_KEYS, config_path, "harbor")
if unknown_harbor:
raise EvalsConfigError(f"{config_path}: unknown harbor key(s): {', '.join(sorted(unknown_harbor))}")
@@ -380,7 +454,7 @@ def _agents(value: Any, config_path: Path) -> dict[str, dict[str, str]]:
f"refer to the same agent ({canonical_name}); use only {canonical_name}"
)
- unknown_agent = set(agent_cfg) - _AGENT_KEYS
+ unknown_agent = _unknown_keys(agent_cfg, _AGENT_KEYS, config_path, f"harbor.agents.{agent_name}")
if unknown_agent:
raise EvalsConfigError(
f"{config_path}: unknown key(s) under harbor.agents.{agent_name}: {', '.join(sorted(unknown_agent))}"
@@ -688,7 +762,7 @@ def _resources(value: Any, config_path: Path) -> dict[str, int]:
if not isinstance(value, dict):
raise EvalsConfigError(f"{config_path}: harbor.resources must be a mapping")
- unknown = set(value) - _RESOURCE_KEYS
+ unknown = _unknown_keys(value, _RESOURCE_KEYS, config_path, "harbor.resources")
if unknown:
raise EvalsConfigError(f"{config_path}: unknown key(s) under harbor.resources: {', '.join(sorted(unknown))}")
@@ -759,7 +833,7 @@ def _skill_workspace(value: Any, config_path: Path) -> dict[str, Any]:
if not isinstance(value, dict):
raise EvalsConfigError(f"{config_path}: skill_workspace must be a mapping")
- unknown = set(value) - _SKILL_WORKSPACE_KEYS
+ unknown = _unknown_keys(value, _SKILL_WORKSPACE_KEYS, config_path, "skill_workspace")
if unknown:
raise EvalsConfigError(f"{config_path}: unknown skill_workspace key(s): {', '.join(sorted(unknown))}")
@@ -779,7 +853,7 @@ def _grading(value: Any, config_path: Path) -> dict[str, str]:
if not isinstance(value, dict):
raise EvalsConfigError(f"{config_path}: grading must be a mapping")
- unknown = set(value) - _GRADING_KEYS
+ unknown = _unknown_keys(value, _GRADING_KEYS, config_path, "grading")
if unknown:
raise EvalsConfigError(f"{config_path}: unknown grading key(s): {', '.join(sorted(unknown))}")
diff --git a/src/skillevaluator/tier3/evals_spec.py b/src/skillevaluator/tier3/evals_spec.py
index c7e5ac90..0c6f505d 100644
--- a/src/skillevaluator/tier3/evals_spec.py
+++ b/src/skillevaluator/tier3/evals_spec.py
@@ -16,6 +16,7 @@ from __future__ import annotations
import functools
import logging
+import os
import re
import unicodedata
from collections.abc import Iterator, Mapping, Sequence
@@ -332,6 +333,7 @@ AGENTSKILLS_REQUIRED_FIELDS = {"id", "prompt", "expected_output"}
DATASET_RECOMMENDED_FIELDS = {"expected_skill", "ground_truth", "expected_behavior"}
DOCKERFILE_MAX_BYTES = 20_000
+_EVALS_CONFIG_REL_PATHS = ("evals/config.yml", "evals/config.yaml")
# ---------------------------------------------------------------------------
@@ -372,7 +374,8 @@ def validate_skillevaluators(skill_path: Path) -> list[CheckResult]:
if target.is_dir():
results.append(CheckResult(spec.rel_path, "ok", spec.description))
# Dirs are always optional — don't warn about absence
- elif target.exists():
+ elif target.exists() or (spec.rel_path in _EVALS_CONFIG_REL_PATHS and os.path.lexists(target)):
+ # A broken config symlink is checked too: loading rejects it, as the run does.
results.append(CheckResult(spec.rel_path, "ok", spec.description))
_run_file_checks(target, spec, results)
# Optional files — no warning for absence
@@ -1187,7 +1190,7 @@ def _run_file_checks(target: Path, spec: EntrySpec, results: list[CheckResult])
if "mcp_servers.toml" in spec.rel_path:
_validate_mcp_toml(target, results)
- if spec.rel_path in ("evals/config.yml", "evals/config.yaml"):
+ if spec.rel_path in _EVALS_CONFIG_REL_PATHS:
_validate_evals_config(target, results)
diff --git a/src/skillevaluator/tier3/harbor/adapter.py b/src/skillevaluator/tier3/harbor/adapter.py
index 6a778201..cdd020fe 100644
--- a/src/skillevaluator/tier3/harbor/adapter.py
+++ b/src/skillevaluator/tier3/harbor/adapter.py
@@ -101,7 +101,6 @@ _MAIN_CONTAINER_READINESS_HEALTHCHECK = 'HEALTHCHECK --interval=1s --timeout=5s
_MAX_REPO_CONTEXT_FILE_BYTES = 10 * 1024 * 1024
_MAX_REPO_CONTEXT_TOTAL_BYTES = 200 * 1024 * 1024
_MAX_EVALUATOR_FILE_BYTES = 64 * 1024 * 1024
-_MAX_EVALS_CONFIG_BYTES = 4 * 1024 * 1024
_MAX_MCP_CONFIG_BYTES = 1024 * 1024
_EVALUATOR_DATASET_FILENAMES = (
"evals.json",
@@ -6512,7 +6511,7 @@ def _evaluator_projection_selection(
import yaml
from skillevaluator.tier3.dataset_utils import normalize_dataset_entries
- from skillevaluator.tier3.evals_config import CONFIG_FILENAMES
+ from skillevaluator.tier3.evals_config import CONFIG_FILENAMES, MAX_EVALS_CONFIG_BYTES
evals_dir = skill_path / "evals"
config_path = next(
@@ -6526,7 +6525,7 @@ def _evaluator_projection_selection(
config_payload = _read_regular_evals_file(
config_path,
label="Eval configuration",
- max_bytes=_MAX_EVALS_CONFIG_BYTES,
+ max_bytes=MAX_EVALS_CONFIG_BYTES,
allowed_root=evals_dir,
)
config_digest = hashlib.sha256(config_payload).hexdigest()
diff --git a/tests/tier3/test_evals_config.py b/tests/tier3/test_evals_config.py
index 9dd093aa..9470b3d2 100644
--- a/tests/tier3/test_evals_config.py
+++ b/tests/tier3/test_evals_config.py
@@ -2,9 +2,14 @@
# SPDX-License-Identifier: Apache-2.0
import json
+import os
+import re
import pytest
+import yaml
+from click.testing import CliRunner
+from skillevaluator.cli import cli
from skillevaluator.tier3.dataset_utils import load_dataset_entries_with_format
from skillevaluator.tier3.evals_config import (
EvalsConfigError,
@@ -13,6 +18,24 @@ from skillevaluator.tier3.evals_config import (
validate_environment_kwargs,
)
from skillevaluator.tier3.evals_spec import validate_skillevaluators as validate_skill_evals
+from skillevaluator.tier3.evals_spec import validate_tier3_source
+
+
+def _skill_with_config(tmp_path, text):
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "config.yml").write_text(text, encoding="utf-8")
+ return skill
+
+
+def _link_or_skip(link, target, *, hard=False, target_is_directory=False):
+ try:
+ if hard:
+ link.hardlink_to(target)
+ else:
+ link.symlink_to(target, target_is_directory=target_is_directory)
+ except OSError as exc:
+ pytest.skip(f"link creation is unavailable: {exc}")
def test_load_evals_config_valid_harbor_policy(tmp_path):
@@ -144,6 +167,299 @@ harbor:
load_evals_config(skill)
+ "document",
+ [
+ pytest.param("schema_version: 1\nschema_version: 1\n", id="top-level"),
+ pytest.param("schema_version: 1\nharbor:\n n_attempts: 1\n n_attempts: 2\n", id="harbor"),
+ pytest.param("schema_version: 1\nharbor:\n resources:\n cpus: 1\n cpus: 2\n", id="nested"),
+ pytest.param(
+ "schema_version: 1\nharbor:\n <<: {stop_on_pass: true, stop_on_pass: false}\n", id="merge-mapping"
+ ),
+ pytest.param("schema_version: 1\nharbor:\n <<: [{n_attempts: 1, n_attempts: 2}]\n", id="merge-list"),
+ ],
+)
+def test_load_evals_config_rejects_duplicate_keys(tmp_path, document):
+ skill = _skill_with_config(tmp_path, document)
+
+ with pytest.raises(
+ EvalsConfigError,
+ match=r"invalid YAML: line \d+, column \d+: while constructing a mapping; "
+ r"line \d+, column \d+: duplicate mapping key$",
+ ):
+ load_evals_config(skill)
+
+
+def test_load_evals_config_names_the_position_of_a_duplicate_key(tmp_path):
+ skill = _skill_with_config(tmp_path, "schema_version: 1\nharbor:\n n_attempts: 1\n n_attempts: 2\n")
+
+ with pytest.raises(EvalsConfigError) as caught:
+ load_evals_config(skill)
+
+ assert str(caught.value).endswith(
+ "config.yml: invalid YAML: line 3, column 3: while constructing a mapping; "
+ "line 4, column 3: duplicate mapping key"
+ )
+
+
+def test_load_evals_config_keeps_yaml_merge_overrides(tmp_path):
+ skill = _skill_with_config(
+ tmp_path,
+ "schema_version: 1\nharbor:\n agents:\n codex: &defaults\n model: model-one\n"
+ " claude-code:\n <<: *defaults\n model: model-two\n",
+ )
+
+ config, _ = load_evals_config(skill)
+
+ assert config["harbor"]["agents"] == {"codex": {"model": "model-one"}, "claude-code": {"model": "model-two"}}
+
+
+def test_load_evals_config_rejects_cyclic_yaml_merge(tmp_path):
+ skill = _skill_with_config(tmp_path, "schema_version: 1\nharbor:\n agents: &agents\n <<: *agents\n")
+
+ with pytest.raises(EvalsConfigError, match=r"invalid YAML: .*cyclic YAML merge"):
+ load_evals_config(skill)
+
+
+def test_load_evals_config_yaml_errors_give_position_without_source_text(tmp_path):
+ skill = _skill_with_config(tmp_path, 'schema_version: 1\nharbor:\n agent_workdir: "do-not-echo-this-value\n')
+
+ with pytest.raises(EvalsConfigError) as caught:
+ load_evals_config(skill)
+
+ message = str(caught.value)
+ assert message.endswith(
+ "config.yml: invalid YAML: line 3, column 18: while scanning a quoted scalar; "
+ "line 4, column 1: found unexpected end of stream"
+ )
+ assert "do-not-echo-this-value" not in message
+ assert "agent_workdir" not in message
+ assert isinstance(caught.value.__cause__, yaml.YAMLError)
+
+
+ ("document", "detail"),
+ [
+ pytest.param(
+ "schema_version: 1\nharbor:\n agents:\n codex: &a {model: model-one}\n"
+ " claude-code: &a {model: model-two}\n",
+ "line 4, column 12: found duplicate anchor 'a'; first occurrence; line 5, column 18: second occurrence",
+ id="duplicate-anchor",
+ ),
+ pytest.param(
+ "schema_version: 1\n---\nschema_version: 1\n",
+ "line 1, column 1: expected a single document in the stream; line 2, column 1: but found another document",
+ id="second-document",
+ ),
+ ],
+)
+def test_load_evals_config_yaml_errors_keep_the_pyyaml_context(tmp_path, document, detail):
+ skill = _skill_with_config(tmp_path, document)
+
+ with pytest.raises(EvalsConfigError) as caught:
+ load_evals_config(skill)
+
+ assert str(caught.value).endswith(f"config.yml: invalid YAML: {detail}")
+ assert "model-one" not in str(caught.value)
+
+
+ "value",
+ [pytest.param("!!int not-a-number", id="tagged-int"), pytest.param("2026-13-01", id="invalid-date")],
+)
+def test_load_evals_config_reports_unconstructible_value_without_echoing_it(tmp_path, value):
+ skill = _skill_with_config(tmp_path, f"schema_version: 1\nharbor:\n agent_workdir: {value}\n")
+
+ with pytest.raises(EvalsConfigError, match=r"invalid YAML: cannot parse a value$") as caught:
+ load_evals_config(skill)
+
+ assert value.removeprefix("!!int ") not in str(caught.value)
+
+
+def _include_list(count):
+ return "schema_version: 1\nskill_workspace:\n include:\n" + "".join(f" - f{i}.md\n" for i in range(count))
+
+
+ ("document", "limit"),
+ [
+ pytest.param(
+ "schema_version: 1\nharbor:\n agent_workdir: " + "[" * 101 + "]" * 101 + "\n",
+ "nesting depth exceeds 100",
+ id="nesting",
+ ),
+ pytest.param(_include_list(1025), "sequence size exceeds 1024", id="list-entries"),
+ pytest.param(
+ "schema_version: 1\nharbor:\n agents:\n codex:\n model: " + "m" * 65_537 + "\n",
+ "scalar length exceeds 65536",
+ id="value-length",
+ ),
+ ],
+)
+def test_load_evals_config_rejects_documents_over_the_yaml_complexity_limits(tmp_path, document, limit):
+ skill = _skill_with_config(tmp_path, document)
+
+ with pytest.raises(EvalsConfigError, match=rf"invalid YAML: .*{limit}$") as caught:
+ load_evals_config(skill)
+
+ assert "mmmm" not in str(caught.value)
+
+
+def test_load_evals_config_accepts_a_list_at_the_entry_limit(tmp_path):
+ skill = _skill_with_config(tmp_path, _include_list(1024))
+
+ config, _ = load_evals_config(skill)
+
+ assert len(config["skill_workspace"]["include"]) == 1024
+
+
+ ("document", "field"),
+ [
+ pytest.param("schema_version: 1\n1: invalid\n", "top-level", id="top-level-int"),
+ pytest.param("schema_version: 1\nyes: invalid\n", "top-level", id="top-level-bool"),
+ pytest.param("schema_version: 1\n~: invalid\n", "top-level", id="top-level-null"),
+ pytest.param("schema_version: 1\nharbor:\n 1: invalid\n", "harbor", id="harbor"),
+ pytest.param(
+ "schema_version: 1\nharbor:\n resources:\n 1: invalid\n", "harbor.resources", id="harbor-resources"
+ ),
+ pytest.param(
+ "schema_version: 1\nharbor:\n agents:\n codex:\n 1: invalid\n",
+ "harbor.agents.codex",
+ id="harbor-agent",
+ ),
+ pytest.param("schema_version: 1\nskill_workspace:\n on: invalid\n", "skill_workspace", id="skill-workspace"),
+ pytest.param("schema_version: 1\ngrading:\n 2: invalid\n", "grading", id="grading"),
+ ],
+)
+def test_load_evals_config_rejects_non_string_keys(tmp_path, document, field):
+ skill = _skill_with_config(tmp_path, document)
+
+ with pytest.raises(EvalsConfigError, match=rf": {re.escape(field)} keys must be strings$"):
+ load_evals_config(skill)
+
+
+ ("content", "message"),
+ [
+ pytest.param(b"schema_version: 1\nharbor:\n yes: 2\n", "harbor keys must be strings", id="non-string-key"),
+ pytest.param(
+ b"schema_version: 1\n# caf\xe9\n", "cannot read config: not valid UTF-8 at byte 23", id="non-utf8"
+ ),
+ ],
+)
+def test_validate_tier3_source_reports_config_errors_instead_of_crashing(tmp_path, content, message):
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "config.yml").write_bytes(content)
+
+ source, results = validate_tier3_source(skill)
+
+ assert source == "invalid"
+ assert [(result.status, result.message.endswith(message)) for result in results] == [("error", True)]
+
+
+def test_load_evals_config_rejects_symlinked_config(tmp_path):
+ outside = tmp_path / "outside.yml"
+ outside.write_text("schema_version: 1\nharbor:\n n_attempts: 9\n", encoding="utf-8")
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ _link_or_skip(skill / "evals" / "config.yml", outside)
+
+ with pytest.raises(EvalsConfigError, match=r"cannot read config: .*symlink"):
+ load_evals_config(skill)
+
+
+def test_load_evals_config_rejects_broken_config_symlink(tmp_path):
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ _link_or_skip(skill / "evals" / "config.yml", tmp_path / "missing.yml")
+
+ with pytest.raises(EvalsConfigError, match=r"cannot read config: .*symlink"):
+ load_evals_config(skill)
+
+
+def test_tier3_validate_reports_broken_config_symlink(tmp_path):
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ (skill / "evals" / "evals.json").write_text('[{"id": "case-1", "question": "q"}]\n', encoding="utf-8")
+ _link_or_skip(skill / "evals" / "config.yml", tmp_path / "missing.yml")
+
+ result = CliRunner().invoke(cli, ["tier3", "validate", str(skill), "--json"])
+
+ assert result.exit_code == 1, result.output
+ errors = [item for item in json.loads(result.output) if item["status"] == "error"]
+ assert len(errors) == 1
+ assert errors[0]["path"].endswith("config.yml")
+ assert re.search(r"Invalid evals config: .*cannot read config: .*symlink", errors[0]["message"])
+
+
+def test_load_evals_config_rejects_config_in_symlinked_evals_directory(tmp_path):
+ outside = tmp_path / "outside-evals"
+ outside.mkdir()
+ (outside / "config.yml").write_text("schema_version: 1\n", encoding="utf-8")
+ skill = tmp_path / "skill"
+ skill.mkdir()
+ _link_or_skip(skill / "evals", outside, target_is_directory=True)
+
+ with pytest.raises(EvalsConfigError, match=r"cannot read config: .*symlink"):
+ load_evals_config(skill)
+
+
+ ("link_target", "skill_path"),
+ [
+ pytest.param("real", "link", id="symlinked-skill"),
+ pytest.param(
+ "real/sub",
+ "link/..",
+ id="parent-of-link",
+ marks=pytest.mark.skipif(os.name == "nt", reason="Windows folds '..' before it follows a link"),
+ ),
+ ],
+)
+def test_load_evals_config_reads_config_through_links_in_the_skill_path(tmp_path, link_target, skill_path):
+ real = tmp_path / "real"
+ (real / "sub").mkdir(parents=True)
+ (real / "evals").mkdir()
+ (real / "evals" / "config.yml").write_text("schema_version: 1\nharbor:\n n_attempts: 3\n", encoding="utf-8")
+ _link_or_skip(tmp_path / "link", tmp_path / link_target, target_is_directory=True)
+
+ config, config_path = load_evals_config(tmp_path / skill_path)
+
+ assert config["harbor"]["n_attempts"] == 3
+ assert config_path is not None and config_path.name == "config.yml"
+
+
+def test_load_evals_config_rejects_hard_linked_config(tmp_path):
+ outside = tmp_path / "outside.yml"
+ outside.write_text("schema_version: 1\n", encoding="utf-8")
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ _link_or_skip(skill / "evals" / "config.yml", outside, hard=True)
+
+ with pytest.raises(EvalsConfigError, match=r"cannot read config: .*hard-linked"):
+ load_evals_config(skill)
+
+
[email protected](not hasattr(os, "mkfifo"), reason="FIFO creation is unavailable on this platform")
+def test_load_evals_config_rejects_fifo_without_blocking(tmp_path):
+ skill = tmp_path / "skill"
+ (skill / "evals").mkdir(parents=True)
+ os.mkfifo(skill / "evals" / "config.yml")
+
+ with pytest.raises(EvalsConfigError, match=r"cannot read config: .*not a regular file"):
+ load_evals_config(skill)
+
+
+def test_load_evals_config_rejects_config_over_the_size_limit(tmp_path):
+ limit = 4 * 1024 * 1024
+ skill = _skill_with_config(tmp_path, "schema_version: 1\n" + "#" * limit)
+
+ with pytest.raises(EvalsConfigError, match=rf"cannot read config: .*exceeds the {limit}-byte limit"):
+ load_evals_config(skill)
+
+
def test_load_evals_config_rejects_environment_kwargs_because_they_are_operator_only(tmp_path):
skill = tmp_path / "skill"
(skill / "evals").mkdir(parents=True)