Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier2): report context-deduplication work limits as incomplete (#280)

Narendran Raghavand596ec3b8df1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a45ec6d0..9c7f0a83 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -242,6 +242,19 @@ All notable changes to SkillEvaluator are documented in this file.
addresses on the same line and domains that merely resemble placeholders remain detectable
([#244](https://github.com/NVIDIA/SkillEvaluator/issues/244)).
+- A skill that exceeds a Tier 2 context-deduplication work limit (more than
+ 512 content chunks, too much pairwise comparison work, or an LLM-review limit:
+ more than 50 candidate clusters, more than 64 members in one cluster, or an
+ oversized per-cluster or total review prompt) is now reported as
+ **INCOMPLETE** instead of failing with a CRITICAL `chunk_count_limit`,
+ `scalar_comparison_limit`, `llm_cluster_count_limit`,
+ `llm_cluster_member_limit`, `llm_prompt_size_limit`, or
+ `llm_total_prompt_size_limit` finding. Those findings read as duplicate
+ content in the skill and showed `FAILED` with "Publication blocked" in
+ BENCHMARK.md, and a policy severity override could downgrade them into a pass
+ with no duplicate analysis. Exit codes are otherwise unchanged. An execution
+ error names the limit, and the JSON report records it under `work_limit`.
+
- Tier 3 security grading preserves ordinary nested `/tmp` cleanup, including
child operands with trailing slashes, unless the current or an earlier
execution action contains recognized symlink-creation syntax (`ln -s`,
diff --git a/docs/tier2-deduplication.mdx b/docs/tier2-deduplication.mdx
index 8e72f9ea..a3e0adb9 100644
--- a/docs/tier2-deduplication.mdx
+++ b/docs/tier2-deduplication.mdx
@@ -153,6 +153,15 @@ If chat-LLM analysis fails for a candidate cluster, Tier 2 reports the
incomplete analysis as a blocking error rather than treating the content as
clean.
+The same applies when a skill exceeds a work limit on content that was
+already collected: the content-chunk or comparison-work limit, or a limit on
+the LLM review (candidate clusters, members in one cluster, or prompt size per
+cluster or in total). Tier 2 stops and reports the scan as **INCOMPLETE**, not
+as a duplicate-content finding, with an execution error that names the limit;
+the JSON report records it under `work_limit`. A policy severity override
+cannot turn the incomplete scan into a pass. Split the skill or remove repeated
+material, then rerun.
+
## Compare skills across a collection
`similarity-check` recursively discovers skill manifests, embeds each skill's
@@ -345,6 +354,7 @@ handling terms before sending confidential skill content to a hosted endpoint.
| Skill carries large generated snapshots (`evals`, `.evals`, `results`, `.results`, `versions`, `.versions`) | Excluded directories are pruned during traversal, so their contents never count toward the discovered-path limit |
| A directory is a Windows junction or other reparse point | Rejected during the walk, like symbolic links |
| Embedding or chat-LLM analysis is incomplete | Reports an error; does not claim a clean result |
+ | Skill exceeds the content-chunk, comparison-work, or LLM-review limit (candidate clusters, cluster members, or prompt size) | Stops before the remaining analysis and reports the scan as incomplete; no duplicate-content finding is added |
</Accordion>
<Accordion title="Inter-skill analysis">
| Scenario | Behavior |
diff --git a/src/skillevaluator/deduplication/intra_skill/intra_skill_validator.py b/src/skillevaluator/deduplication/intra_skill/intra_skill_validator.py
index cbcb1675..a4d6c275 100644
--- a/src/skillevaluator/deduplication/intra_skill/intra_skill_validator.py
+++ b/src/skillevaluator/deduplication/intra_skill/intra_skill_validator.py
@@ -50,6 +50,9 @@ CONTENT_DEDUP_MAX_CLUSTER_MEMBERS = 64
CONTENT_DEDUP_MAX_LLM_PROMPT_CHARS = 64_000
CONTENT_DEDUP_MAX_TOTAL_LLM_PROMPT_CHARS = 1_000_000
+# Scan label recorded when a work limit stops duplicate analysis early.
+_WORK_LIMIT_SCAN = "context-deduplication"
+
def _is_comment_or_config_line(line: str) -> bool:
"""Return True for comment lines or simple ``key=value``/``key: value`` config lines."""
@@ -93,6 +96,25 @@ def _is_trivial_intra_file_duplicate(cluster: ContentCluster) -> bool:
return all(_chunk_is_short_config(member) for member in cluster.members)
+def _mark_work_limit_incomplete(
+ result: ValidationResult,
+ check_name: str,
+ message: str,
+ suggestion: str,
+ **details: object,
+) -> None:
+ """Record a tripped work limit as incomplete coverage, not a duplicate finding.
+
+ The scan stopped before duplicate analysis finished, so it can neither
+ report nor rule out duplicated content. A finding would misreport the skill
+ and could be waived by a policy severity override; the incomplete status
+ cannot. ``metadata["work_limit"]`` keeps the limit for JSON reports.
+ """
+ result.mark_scan_incomplete(_WORK_LIMIT_SCAN)
+ result.metadata["work_limit"] = {"check_name": check_name, **details}
+ result.add_error(f"Tier 2 scan incomplete: {message} {suggestion}")
+
+
class IntraSkillValidator(ValidatorBase):
"""Detect redundant content within a skill directory."""
@@ -134,7 +156,6 @@ class IntraSkillValidator(ValidatorBase):
def validate(self, skill_path: Path, *, _collected_files: list[CollectedFile] | None = None) -> ValidationResult:
"""Run context deduplication on a single skill directory."""
- report_path = skill_path.name or "."
result = ValidationResult(
validator_name=self.name,
validator_description=self.description,
@@ -175,16 +196,15 @@ class IntraSkillValidator(ValidatorBase):
file_chunks = chunk_file(f)
prospective_count = len(all_chunks) + len(file_chunks)
if prospective_count > CONTENT_DEDUP_MAX_CHUNKS:
- result.add_finding(
- Finding(
- category="CONTENT_DEDUP",
- severity=Severity.CRITICAL,
- check_name="chunk_count_limit",
- message=f"Tier 2 produced more than {CONTENT_DEDUP_MAX_CHUNKS} content chunks.",
- file_path=f.rel_path,
- suggestion="Reduce or split the skill content before running Tier 2.",
- metadata={"actual": prospective_count, "limit": CONTENT_DEDUP_MAX_CHUNKS},
- )
+ _mark_work_limit_incomplete(
+ result,
+ "chunk_count_limit",
+ f"the skill produced more than {CONTENT_DEDUP_MAX_CHUNKS} content chunks "
+ f"(the limit was reached in {f.rel_path}).",
+ "Reduce or split the skill content before running Tier 2.",
+ file_path=f.rel_path,
+ actual=prospective_count,
+ limit=CONTENT_DEDUP_MAX_CHUNKS,
)
return result
all_chunks.extend(file_chunks)
@@ -222,21 +242,15 @@ class IntraSkillValidator(ValidatorBase):
# excess pairwise work before requesting another provider batch.
scalar_work = pair_count * (vector_dimension or 0)
if scalar_work > self._max_scalar_comparisons:
- result.add_finding(
- Finding(
- category="CONTENT_DEDUP",
- severity=Severity.CRITICAL,
- check_name="scalar_comparison_limit",
- message=f"Tier 2 scalar comparison work exceeds the configured limit ({self._max_scalar_comparisons}).",
- file_path=report_path,
- suggestion="Reduce or split the skill content before running Tier 2.",
- metadata={
- "pair_count": pair_count,
- "vector_dimension": vector_dimension,
- "scalar_work": scalar_work,
- "limit": self._max_scalar_comparisons,
- },
- )
+ _mark_work_limit_incomplete(
+ result,
+ "scalar_comparison_limit",
+ f"scalar comparison work of {scalar_work} exceeds the limit of {self._max_scalar_comparisons}.",
+ "Reduce or split the skill content before running Tier 2.",
+ pair_count=pair_count,
+ vector_dimension=vector_dimension,
+ scalar_work=scalar_work,
+ limit=self._max_scalar_comparisons,
)
return result
@@ -253,16 +267,14 @@ class IntraSkillValidator(ValidatorBase):
logger.info("Found %d cluster(s)", len(clusters))
if len(clusters) > self._max_llm_clusters:
- result.add_finding(
- Finding(
- category="CONTENT_DEDUP",
- severity=Severity.CRITICAL,
- check_name="llm_cluster_count_limit",
- message=f"Tier 2 found more than {self._max_llm_clusters} clusters requiring LLM review.",
- file_path=report_path,
- suggestion="Reduce duplicated content or split the skill before rerunning Tier 2.",
- metadata={"actual": len(clusters), "limit": self._max_llm_clusters},
- )
+ _mark_work_limit_incomplete(
+ result,
+ "llm_cluster_count_limit",
+ f"{len(clusters)} content clusters require LLM review, more than the limit of "
+ f"{self._max_llm_clusters}.",
+ "Reduce duplicated content or split the skill before rerunning Tier 2.",
+ actual=len(clusters),
+ limit=self._max_llm_clusters,
)
return result
@@ -278,21 +290,34 @@ class IntraSkillValidator(ValidatorBase):
prompt_chars = len(SYSTEM_PROMPT) + len(build_user_prompt(cluster))
total_prompt_chars += prompt_chars
limits = (
- (len(cluster.members), CONTENT_DEDUP_MAX_CLUSTER_MEMBERS, "llm_cluster_member_limit"),
- (prompt_chars, CONTENT_DEDUP_MAX_LLM_PROMPT_CHARS, "llm_prompt_size_limit"),
- (total_prompt_chars, self._max_total_prompt_chars, "llm_total_prompt_size_limit"),
+ (
+ len(cluster.members),
+ CONTENT_DEDUP_MAX_CLUSTER_MEMBERS,
+ "llm_cluster_member_limit",
+ "a content cluster has {actual} members, more than the limit of {limit} for LLM review.",
+ ),
+ (
+ prompt_chars,
+ CONTENT_DEDUP_MAX_LLM_PROMPT_CHARS,
+ "llm_prompt_size_limit",
+ "an LLM review prompt of {actual} characters exceeds the limit of {limit}.",
+ ),
+ (
+ total_prompt_chars,
+ self._max_total_prompt_chars,
+ "llm_total_prompt_size_limit",
+ "LLM review prompts total {actual} characters, more than the limit of {limit}.",
+ ),
)
- for actual, limit, check in limits:
+ for actual, limit, check, message in limits:
if actual > limit:
- result.add_finding(
- Finding(
- "CONTENT_DEDUP",
- Severity.CRITICAL,
- check,
- "Tier 2 LLM review exceeds a bounded work limit.",
- file_path=report_path,
- metadata={"actual": actual, "limit": limit},
- )
+ _mark_work_limit_incomplete(
+ result,
+ check,
+ message.format(actual=actual, limit=limit),
+ "Reduce duplicated content or split the skill before rerunning Tier 2.",
+ actual=actual,
+ limit=limit,
)
return result
diff --git a/src/skillevaluator/reporting/json_reporter.py b/src/skillevaluator/reporting/json_reporter.py
index cdc8113a..b4780f3a 100644
--- a/src/skillevaluator/reporting/json_reporter.py
+++ b/src/skillevaluator/reporting/json_reporter.py
@@ -363,6 +363,10 @@ class JSONReporter(ReporterBase):
llm_analysis = result.metadata.get("llm_analysis")
if isinstance(llm_analysis, dict):
data["llm_analysis"] = llm_analysis
+ # A Tier 2 work limit makes the scan incomplete; record which limit stopped it.
+ work_limit = result.metadata.get("work_limit")
+ if isinstance(work_limit, dict):
+ data["work_limit"] = work_limit
return data
diff --git a/tests/deduplication/intra_skill/test_intra_skill_validator.py b/tests/deduplication/intra_skill/test_intra_skill_validator.py
index 3afd87fd..5593269b 100644
--- a/tests/deduplication/intra_skill/test_intra_skill_validator.py
+++ b/tests/deduplication/intra_skill/test_intra_skill_validator.py
@@ -10,8 +10,11 @@ import math
from pathlib import Path
from unittest.mock import patch
+import pytest
+
from skillevaluator.deduplication.intra_skill import intra_skill_validator
from skillevaluator.deduplication.intra_skill.intra_skill_validator import IntraSkillValidator
+from skillevaluator.deduplication.intra_skill.llm_analyzer import SYSTEM_PROMPT, build_user_prompt
from skillevaluator.deduplication.intra_skill.semantic_clustering import ContentCluster
from skillevaluator.deduplication.utils.chunker import ContentChunk
from skillevaluator.embedding.client import SimilarityConfigError
@@ -97,8 +100,20 @@ class TestIntraSkillValidatorValidate:
assert finding.suggestion is not None
assert "replace" in finding.suggestion.lower()
+ @staticmethod
+ def _assert_work_limit_incomplete(result, work_limit: dict) -> None:
+ """A tripped work limit is missing coverage, never a duplicate-content finding."""
+ assert result.passed is False
+ assert result.status == "incomplete"
+ assert result.incomplete_scans == ["context-deduplication"]
+ assert result.findings == []
+ assert result.summary.critical_count == 0
+ assert result.metadata["work_limit"] == work_limit
+ assert len(result.errors) == 1
+ assert result.errors[0].startswith("Tier 2 scan incomplete: ")
+
@patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.EmbeddingClient")
- def test_chunk_limit_fails_before_embedding(self, mock_embed, tmp_path: Path, monkeypatch) -> None:
+ def test_chunk_limit_is_incomplete_before_embedding(self, mock_embed, tmp_path: Path, monkeypatch) -> None:
monkeypatch.setattr(intra_skill_validator, "CONTENT_DEDUP_MAX_CHUNKS", 1)
skill_dir = tmp_path / "skill"
skill_dir.mkdir()
@@ -107,14 +122,18 @@ class TestIntraSkillValidatorValidate:
result = IntraSkillValidator().validate(skill_dir)
- assert result.passed is False
- assert result.findings[0].check_name == "chunk_count_limit"
- assert result.findings[0].metadata == {"actual": 2, "limit": 1}
+ self._assert_work_limit_incomplete(
+ result,
+ {"check_name": "chunk_count_limit", "file_path": "SKILL.md", "actual": 2, "limit": 1},
+ )
+ assert "more than 1 content chunks" in result.errors[0]
+ assert "reached in SKILL.md" in result.errors[0]
+ assert "Reduce or split the skill content" in result.errors[0]
mock_embed.assert_not_called()
@patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.build_clusters")
@patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.EmbeddingClient")
- def test_scalar_work_limit_fails_before_clustering(
+ def test_scalar_work_limit_is_incomplete_before_clustering(
self,
mock_embed,
mock_build_clusters,
@@ -130,22 +149,24 @@ class TestIntraSkillValidatorValidate:
result = IntraSkillValidator().validate(skill_dir)
- assert result.passed is False
- finding = result.findings[0]
- assert finding.check_name == "scalar_comparison_limit"
- assert finding.file_path == skill_dir.name
- assert finding.metadata == {
- "pair_count": 1,
- "vector_dimension": 2,
- "scalar_work": 2,
- "limit": 1,
- }
+ self._assert_work_limit_incomplete(
+ result,
+ {
+ "check_name": "scalar_comparison_limit",
+ "pair_count": 1,
+ "vector_dimension": 2,
+ "scalar_work": 2,
+ "limit": 1,
+ },
+ )
+ assert "scalar comparison work of 2 exceeds the limit of 1" in result.errors[0]
+ assert str(tmp_path) not in result.errors[0]
mock_build_clusters.assert_not_called()
@patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.LLMClient")
@patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.build_clusters")
@patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.EmbeddingClient")
- def test_llm_cluster_limit_fails_before_llm_analysis(
+ def test_llm_cluster_limit_is_incomplete_before_llm_analysis(
self, mock_embed, mock_build_clusters, mock_llm, tmp_path: Path, monkeypatch
) -> None:
monkeypatch.setattr(intra_skill_validator, "CONTENT_DEDUP_MAX_LLM_CLUSTERS", 1)
@@ -162,12 +183,92 @@ class TestIntraSkillValidatorValidate:
result = IntraSkillValidator().validate(skill_dir)
- assert result.passed is False
- assert result.findings[0].check_name == "llm_cluster_count_limit"
- assert result.findings[0].metadata == {"actual": 2, "limit": 1}
- assert result.findings[0].file_path == skill_dir.name
+ self._assert_work_limit_incomplete(
+ result,
+ {"check_name": "llm_cluster_count_limit", "actual": 2, "limit": 1},
+ )
+ assert "2 content clusters require LLM review, more than the limit of 1" in result.errors[0]
+ mock_llm.assert_not_called()
+
+ @pytest.mark.parametrize(
+ ("limit_name", "check_name", "expected_message"),
+ [
+ (
+ "CONTENT_DEDUP_MAX_CLUSTER_MEMBERS",
+ "llm_cluster_member_limit",
+ "a content cluster has 2 members, more than the limit of 1 for LLM review.",
+ ),
+ (
+ "CONTENT_DEDUP_MAX_LLM_PROMPT_CHARS",
+ "llm_prompt_size_limit",
+ "an LLM review prompt of {actual} characters exceeds the limit of 1.",
+ ),
+ (
+ "CONTENT_DEDUP_MAX_TOTAL_LLM_PROMPT_CHARS",
+ "llm_total_prompt_size_limit",
+ "LLM review prompts total {actual} characters, more than the limit of 1.",
+ ),
+ ],
+ )
+ @patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.LLMClient")
+ @patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.build_clusters")
+ @patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.EmbeddingClient")
+ def test_llm_review_limits_are_incomplete_before_llm_analysis(
+ self,
+ mock_embed,
+ mock_build_clusters,
+ mock_llm,
+ limit_name: str,
+ check_name: str,
+ expected_message: str,
+ tmp_path: Path,
+ monkeypatch,
+ ) -> None:
+ monkeypatch.setattr(intra_skill_validator, limit_name, 1)
+ skill_dir = tmp_path / "skill"
+ skill_dir.mkdir()
+ (skill_dir / "SKILL.md").write_text("## Section A\n" + "a" * 200 + "\n## Section B\n" + "a" * 200)
+ mock_embed.return_value.embed.return_value = [[1.0, 0.0], [1.0, 0.0]]
+ members = [
+ ContentChunk("SKILL.md", "## A", 1, 2, "a" * 100, "markdown"),
+ ContentChunk("SKILL.md", "## B", 3, 4, "a" * 100, "markdown"),
+ ]
+ cluster = ContentCluster(members, 1.0, 1.0, False, {"markdown"})
+ mock_build_clusters.return_value = [cluster]
+ actual = 2 if check_name == "llm_cluster_member_limit" else len(SYSTEM_PROMPT) + len(build_user_prompt(cluster))
+
+ result = IntraSkillValidator().validate(skill_dir)
+
+ self._assert_work_limit_incomplete(result, {"check_name": check_name, "actual": actual, "limit": 1})
+ assert expected_message.format(actual=actual) in result.errors[0]
+ assert "Reduce duplicated content or split the skill" in result.errors[0]
+ assert str(tmp_path) not in result.errors[0]
mock_llm.assert_not_called()
+ @patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.EmbeddingClient")
+ def test_policy_override_cannot_waive_work_limit(self, mock_embed, tmp_path: Path, monkeypatch) -> None:
+ """Severity overrides remap findings; they must not turn missing analysis into a pass."""
+ from skillevaluator.validators.policy import ValidationPolicy, apply_policy
+
+ monkeypatch.setattr(intra_skill_validator, "CONTENT_DEDUP_MAX_CHUNKS", 1)
+ skill_dir = tmp_path / "skill"
+ skill_dir.mkdir()
+ (skill_dir / "SKILL.md").write_text("## Section A\n" + "a" * 200 + "\n## Section B\n" + "b" * 200)
+ policy = ValidationPolicy(
+ severity_overrides={
+ "CONTENT_DEDUP.*": Severity.LOW,
+ "CONTENT_DEDUP.chunk_count_limit": Severity.LOW,
+ }
+ )
+
+ (result,) = apply_policy([IntraSkillValidator().validate(skill_dir)], policy)
+
+ assert result.passed is False
+ assert result.status == "incomplete"
+ assert result.findings == []
+ assert result.errors[0].startswith("Tier 2 scan incomplete: ")
+ mock_embed.assert_not_called()
+
@patch("skillevaluator.deduplication.intra_skill.intra_skill_validator.EmbeddingClient")
def test_embedding_error_is_incomplete_without_skill_finding(self, mock_embed, tmp_path: Path) -> None:
skill_dir = tmp_path / "skill"
diff --git a/tests/deduplication/plugin/test_private_dedup.py b/tests/deduplication/plugin/test_private_dedup.py
index ade0938c..e2b37b69 100644
--- a/tests/deduplication/plugin/test_private_dedup.py
+++ b/tests/deduplication/plugin/test_private_dedup.py
@@ -356,7 +356,9 @@ def test_context_scalar_limit_stops_after_first_provider_batch(tmp_path, monkeyp
monkeypatch.setattr(core, "CONTENT_DEDUP_EMBEDDING_BATCH_SIZE", 2)
monkeypatch.setattr(core, "CONTENT_DEDUP_MAX_SCALAR_COMPARISONS", 1)
result = core.IntraSkillValidator().validate(root)
- assert result.findings[0].check_name == "scalar_comparison_limit"
+ assert result.status == "incomplete"
+ assert result.findings == []
+ assert result.metadata["work_limit"]["check_name"] == "scalar_comparison_limit"
assert len(client.calls) == 1
@@ -410,7 +412,9 @@ def test_private_cluster_allowance_keeps_default_skill_behavior(tmp_path, monkey
default = core.IntraSkillValidator().validate(root)
constrained = core.IntraSkillValidator(max_llm_clusters=1).validate(root)
assert default.passed
- assert constrained.findings[0].check_name == "llm_cluster_count_limit"
+ assert constrained.status == "incomplete"
+ assert constrained.findings == []
+ assert constrained.metadata["work_limit"] == {"check_name": "llm_cluster_count_limit", "actual": 2, "limit": 1}
def test_context_prompt_limit_refuses_llm_before_external_work(tmp_path, monkeypatch):
@@ -427,7 +431,9 @@ def test_context_prompt_limit_refuses_llm_before_external_work(tmp_path, monkeyp
monkeypatch.setattr(core, "LLMClient", forbidden)
result = core.IntraSkillValidator().validate(root)
- assert result.findings[0].check_name == "llm_prompt_size_limit"
+ assert result.status == "incomplete"
+ assert result.findings == []
+ assert result.metadata["work_limit"]["check_name"] == "llm_prompt_size_limit"
def test_clustering_normalizes_once_and_checks_aggregate_scalar_work(monkeypatch):
diff --git a/tests/test_tier2_command_failures.py b/tests/test_tier2_command_failures.py
index 45ee6050..657c7c26 100644
--- a/tests/test_tier2_command_failures.py
+++ b/tests/test_tier2_command_failures.py
@@ -111,6 +111,47 @@ def test_detected_duplicate_remains_failed_not_incomplete(tmp_path: Path, monkey
assert actual.findings[0].category == "DUPLICATE"
+def test_work_limit_is_incomplete_in_cli_and_reports(tmp_path: Path, monkeypatch) -> None:
+ """A skill over a Tier 2 work limit reads INCOMPLETE, never a failed duplicate scan."""
+ from skillevaluator.deduplication.intra_skill import intra_skill_validator
+ from skillevaluator.reporting import BenchmarkReporter
+
+ monkeypatch.setattr(intra_skill_validator, "CONTENT_DEDUP_MAX_CHUNKS", 1)
+ skill = tmp_path / "skill"
+ skill.mkdir()
+ (skill / "SKILL.md").write_text("## Section A\n" + "a" * 200 + "\n## Section B\n" + "b" * 200)
+ reports = tmp_path / "reports"
+ for purpose in ("LLM", "EMBEDDING"):
+ # The limit trips before any provider request; port 9 is never contacted.
+ monkeypatch.setenv(f"SKILL_EVAL_{purpose}_PROVIDER", "openai-compatible")
+ monkeypatch.setenv(f"SKILL_EVAL_{purpose}_BASE_URL", "http://127.0.0.1:9/v1")
+ monkeypatch.setenv(f"SKILL_EVAL_{purpose}_API_KEY", "local-test-key")
+ monkeypatch.setenv(f"SKILL_EVAL_{purpose}_MODEL", "local-test-model")
+
+ invocation = CliRunner().invoke(cli, ["tier2", str(skill), "-r", "cli,json,markdown", "-o", str(reports)])
+
+ assert invocation.exit_code == 1, invocation.output
+ assert "context-deduplication did not complete" in invocation.output
+ payload = json.loads((reports / "skillevaluator-tier2.json").read_text())
+ assert payload["overall_status"] == "incomplete"
+ result = payload["results"][0]
+ assert result["status"] == "incomplete"
+ assert result["incomplete_scans"] == ["context-deduplication"]
+ assert result["findings"] == []
+ assert result["summary"]["critical_count"] == 0
+ assert result["work_limit"] == {"check_name": "chunk_count_limit", "file_path": "SKILL.md", "actual": 2, "limit": 1}
+ markdown = (reports / "skillevaluator-tier2.md").read_text()
+ assert "**Status:** ⚠️ INCOMPLETE" in markdown
+ assert "Tier 2 scan incomplete: the skill produced more than 1 content chunks" in markdown
+
+ benchmark = BenchmarkReporter(include_timestamp=False).render_all(commands.run_context_optimization_check(skill))
+ assert (
+ "| Tier 2 | Semantic deduplication | **INCOMPLETE** | Missing trustworthy evidence from context-deduplication |"
+ in benchmark
+ )
+ assert "**FAILED**" not in benchmark
+
+
def test_http_410_is_incomplete_in_real_cli_and_reports(tmp_path: Path, monkeypatch) -> None:
"""Exercise the actual SDK against a local HTTP service, without external calls."""
requests = []