Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Incomplete SkillSpector scans name parser_diagnostic spans, causes, and checks.
SkillSpector 2.13.4 records ``static_parse_limit`` on some JavaScript helpers
with ``start_line`` empty and the span, cause and check only in
``parser_diagnostic``. The security check stays ``INCOMPLETE``; these tests
cover the labels, the runtime-dependent hint, and the JSON projection.
"""
import json
from pathlib import Path
from unittest.mock import patch
import pytest
from tests.validators.test_security import (
_REGULAR_ANALYZERS,
_ledger_exception,
_skillspector_partial_report,
_validate_skillspector_payload,
)
from skillevaluator.reporting.json_reporter import JSONReporter
from skillevaluator.validators.security import (
_SKILLSPECTOR_PARSE_LIMIT_ADVICE,
_SKILLSPECTOR_REFERENCE_REJECTION_HINT,
_skillspector_incomplete_analysis_error,
_skillspector_ledger_exception_label,
)
_FIXTURE = Path(__file__).parents[1] / "fixtures" / "skillspector-2.13.4-mjs-runtime-parse-limit-no-llm.json"
_RUNTIME_DIAGNOSTIC = {
"cause": "runtime_dependent",
"check": "command_string_runtime",
"start_line": 74,
"end_line": 170,
"unchecked_scope": "remaining_input",
"full_input_checked": False,
"source_start_offset": 3958,
"source_end_offset": 8650,
"location_precision": "source_span",
}
_BOUNDARY_DIAGNOSTIC = {
"cause": "boundary_unresolved",
"check": "command_word_boundary",
"start_line": 11,
"end_line": 11,
"unchecked_scope": "command_string",
"full_input_checked": False,
}
def _parse_limit_row(**overrides) -> dict:
row = {
**_ledger_exception("scripts/helper.mjs", "static_parse_limit"),
"parser_diagnostic": dict(_RUNTIME_DIAGNOSTIC),
}
row.update(overrides)
return row
def test_captured_mjs_rows_use_the_parser_span_and_runtime_dependent_context() -> None:
report = json.loads(_FIXTURE.read_text(encoding="utf-8"))
rows = report["analysis_completeness"]["ledger_exceptions"]
labels = [_skillspector_ledger_exception_label(row) for row in rows]
assert labels == [
"SKILL.md:150 [reference_resolution; reference_missing]",
"scripts/admission.mjs:74-170 [static; static_parse_limit; runtime_dependent command_string_runtime; "
"analyzers=static_patterns_tool_misuse]",
"scripts/machine-reading.mjs:104-236 [static; static_parse_limit; runtime_dependent command_string_runtime; "
"analyzers=static_patterns_tool_misuse]",
]
error = _skillspector_incomplete_analysis_error("partial", rows)
assert error.startswith(
"skillspector JSON field 'analysis_completeness' reports incomplete analysis (status 'partial'): "
)
assert labels[0] in error and labels[1] in error and labels[2] in error
assert (
"(for example a process spawn whose arguments come from variables) in "
"scripts/admission.mjs:74-170 and scripts/machine-reading.mjs:104-236. "
) in error
assert error.endswith(_SKILLSPECTOR_PARSE_LIMIT_ADVICE)
assert _SKILLSPECTOR_REFERENCE_REJECTION_HINT not in error
def test_parser_span_uses_a_range_or_a_single_line() -> None:
range_label = _skillspector_ledger_exception_label(_parse_limit_row())
assert range_label.startswith("scripts/helper.mjs:74-170 [")
single = _parse_limit_row(parser_diagnostic=dict(_BOUNDARY_DIAGNOSTIC))
assert _skillspector_ledger_exception_label(single).startswith("scripts/helper.mjs:11 [")
assert "11-11" not in _skillspector_ledger_exception_label(single)
def test_row_start_line_is_unchanged_when_parser_diagnostic_also_has_a_span() -> None:
row = _parse_limit_row(start_line=12)
assert _skillspector_ledger_exception_label(row) == (
f"scripts/helper.mjs:12 [static; static_parse_limit; runtime_dependent command_string_runtime; "
f"analyzers={','.join(_REGULAR_ANALYZERS)}]"
)
@pytest.mark.parametrize(
"diagnostic",
[
"not-a-dict",
["runtime_dependent"],
None,
{"cause": "runtime_dependent", "check": "command_string_runtime", "start_line": "74", "end_line": "170"},
{"cause": "runtime_dependent", "check": "command_string_runtime", "start_line": True, "end_line": True},
{"cause": "runtime_dependent", "check": "command_string_runtime", "start_line": -1, "end_line": 10},
{"cause": "runtime_dependent", "check": "command_string_runtime", "start_line": 170, "end_line": 74},
{"cause": "runtime_dependent", "check": "command_string_runtime", "start_line": 74, "end_line": False},
],
ids=[
"string",
"list",
"none",
"string-lines",
"bool-lines",
"negative",
"end-before-start",
"bool-end",
],
)
def test_malformed_parser_diagnostic_does_not_supply_a_location(diagnostic: object) -> None:
row = _parse_limit_row(parser_diagnostic=diagnostic)
label = _skillspector_ledger_exception_label(row)
assert label.startswith("scripts/helper.mjs [static; static_parse_limit")
assert ":74" not in label
assert ":170" not in label
def test_cause_and_check_are_sanitized_and_truncated() -> None:
row = _parse_limit_row(
parser_diagnostic={
"cause": "runtime\ndependent-" + "x" * 400,
"check": "command\tstring-" + "y" * 400,
"start_line": 74,
"end_line": 170,
}
)
label = _skillspector_ledger_exception_label(row)
assert "\n" not in label and "\t" not in label
assert "runtime dependent-" + "x" * 179 + "..." in label
assert "x" * 180 not in label
assert "command string-" + "y" * 182 + "..." in label
assert "y" * 183 not in label
other_reason = _ledger_exception("scripts/helper.mjs", "opaque_content")
other_reason["parser_diagnostic"] = dict(_RUNTIME_DIAGNOSTIC)
assert "runtime_dependent" not in _skillspector_ledger_exception_label(other_reason)
def test_runtime_dependent_hint_requires_every_parse_limit_row() -> None:
runtime = _parse_limit_row()
other = _parse_limit_row(path="scripts/other.mjs", parser_diagnostic=dict(_BOUNDARY_DIAGNOSTIC))
missing = _ledger_exception("scripts/plain.mjs", "static_parse_limit")
assert _SKILLSPECTOR_PARSE_LIMIT_ADVICE in _skillspector_incomplete_analysis_error("partial", [runtime])
mixed = _skillspector_incomplete_analysis_error("partial", [runtime, other])
assert _SKILLSPECTOR_PARSE_LIMIT_ADVICE not in mixed
assert "scripts/other.mjs:11 [" in mixed
assert _SKILLSPECTOR_PARSE_LIMIT_ADVICE not in _skillspector_incomplete_analysis_error("partial", [missing])
assert _SKILLSPECTOR_PARSE_LIMIT_ADVICE not in _skillspector_incomplete_analysis_error(
"partial", [_ledger_exception("scripts/render.py", "static_parse_limit") | {"start_line": 12}]
)
def test_parse_limit_hint_combines_with_runtime_limit_and_not_with_reference_rejection() -> None:
runtime_limit = _ledger_exception("references/sect-1.html")
parse_limit = _parse_limit_row()
reference = {
**_ledger_exception("SKILL.md", "reference_missing", analyzers=(), phase="reference_resolution"),
"start_line": 150,
}
combined = _skillspector_incomplete_analysis_error("partial", [runtime_limit, parse_limit])
assert "SkillSpector reached a runtime limit before it finished 1 file(s)." in combined
assert _SKILLSPECTOR_PARSE_LIMIT_ADVICE in combined
assert _SKILLSPECTOR_REFERENCE_REJECTION_HINT not in combined
with_reference = _skillspector_incomplete_analysis_error("partial", [reference, parse_limit])
assert "SKILL.md:150 [reference_resolution; reference_missing]" in with_reference
assert _SKILLSPECTOR_PARSE_LIMIT_ADVICE in with_reference
assert _SKILLSPECTOR_REFERENCE_REJECTION_HINT not in with_reference
only_reference = _skillspector_incomplete_analysis_error("partial", [reference])
assert only_reference.endswith(_SKILLSPECTOR_REFERENCE_REJECTION_HINT)
assert _SKILLSPECTOR_PARSE_LIMIT_ADVICE not in only_reference
def _reference_row(line: int) -> dict:
return {
**_ledger_exception("SKILL.md", "reference_missing", analyzers=(), phase="reference_resolution"),
"start_line": line,
}
def test_parse_limit_hint_names_locations_hidden_behind_earlier_rows() -> None:
rows = [_reference_row(10), _reference_row(20), _reference_row(30), _parse_limit_row()]
error = _skillspector_incomplete_analysis_error("partial", rows)
# The shown examples are the three reference rows; the helper's span is in "and 1 more".
assert "scripts/helper.mjs:74-170 [" not in error
assert "; and 1 more " in error
assert (
"(for example a process spawn whose arguments come from variables) in scripts/helper.mjs:74-170. "
f"{_SKILLSPECTOR_PARSE_LIMIT_ADVICE}"
) in error
def test_parse_limit_hint_caps_named_locations_and_points_to_the_json_report() -> None:
rows = [
_parse_limit_row(path=f"scripts/helper{index}.mjs", parser_diagnostic=dict(_RUNTIME_DIAGNOSTIC))
for index in range(5)
]
rows.append(_parse_limit_row(path="scripts/helper0.mjs"))
error = _skillspector_incomplete_analysis_error("partial", rows)
assert (
"in scripts/helper0.mjs:74-170, scripts/helper1.mjs:74-170 and scripts/helper2.mjs:74-170, "
"and 2 more listed in parser_diagnostic on the skillspector_completeness rows of the JSON report. "
) in error
def test_parse_limit_hint_without_a_span_names_the_path_or_the_json_report() -> None:
no_span = {"cause": "runtime_dependent", "check": "command_string_runtime"}
path_only = _skillspector_incomplete_analysis_error("partial", [_parse_limit_row(parser_diagnostic=no_span)])
assert "come from variables) in scripts/helper.mjs. " in path_only
nothing = _parse_limit_row(parser_diagnostic=no_span)
del nothing["path"]
unknown = _skillspector_incomplete_analysis_error("partial", [nothing])
assert (
"come from variables) at the locations in parser_diagnostic on the skillspector_completeness rows "
"of the JSON report. "
) in unknown
assert unknown.endswith(_SKILLSPECTOR_PARSE_LIMIT_ADVICE)
@patch("skillevaluator.validators.security.Tools")
def test_partial_scan_stays_incomplete_and_projects_parser_diagnostic(mock_tools, sample_skill_dir: Path) -> None:
payload = _skillspector_partial_report([_parse_limit_row(analyzers=["static_patterns_tool_misuse"])])
result = _validate_skillspector_payload(mock_tools, sample_skill_dir, payload)
assert result.status == "incomplete"
assert not result.passed
assert result.incomplete_scans == ["skillspector"]
assert (
"scripts/helper.mjs:74-170 [static; static_parse_limit; runtime_dependent command_string_runtime; "
"analyzers=static_patterns_tool_misuse]"
) in result.errors[0]
assert result.errors[0].endswith(_SKILLSPECTOR_PARSE_LIMIT_ADVICE)
record = json.loads(JSONReporter().render(result))["skillspector_completeness"]["records"][0]
diagnostic = record["ledger_exceptions"][0]["parser_diagnostic"]
assert diagnostic == {
"cause": "runtime_dependent",
"check": "command_string_runtime",
"start_line": 74,
"end_line": 170,
"unchecked_scope": "remaining_input",
"full_input_checked": False,
}
encoded = json.dumps(record)
assert "source_start_offset" not in encoded
assert "source_end_offset" not in encoded
assert "location_precision" not in encoded