Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(harness): reject duplicate normalized corpus identifiers (#186)

Rudy Mizrahi Celekli5dfbb36510f3
diff --git a/ifixai/harness/adversarial_mutator.py b/ifixai/harness/adversarial_mutator.py
index f4367be..24d1757 100644
--- a/ifixai/harness/adversarial_mutator.py
+++ b/ifixai/harness/adversarial_mutator.py
@@ -76,12 +76,13 @@ def _parse_seed_entry(entry: object, seen_ids: set[str]) -> SeedEntry:
seed_id = entry.get("id")
if not isinstance(seed_id, str) or not seed_id.strip():
raise AdversarialMutatorError(f"seed entry missing 'id': {entry!r}")
+ seed_id = seed_id.strip()
if seed_id in seen_ids:
raise AdversarialMutatorError(f"duplicate seed id: {seed_id!r}")
seen_ids.add(seed_id)
text = _extract_seed_text(entry)
metadata = _extract_seed_metadata(entry)
- return SeedEntry(id=seed_id.strip(), text=text, metadata=metadata)
+ return SeedEntry(id=seed_id, text=text, metadata=metadata)
def _extract_seed_text(entry: dict[str, object]) -> str:
diff --git a/ifixai/harness/injection_corpus.py b/ifixai/harness/injection_corpus.py
index 99b8f30..1334c67 100644
--- a/ifixai/harness/injection_corpus.py
+++ b/ifixai/harness/injection_corpus.py
@@ -75,12 +75,13 @@ def load_injection_corpus(path: Path | None = None) -> InjectionCorpus:
raise InjectionCorpusError(
f"payload entry missing required field {field!r}: {entry!r}"
)
- if entry["id"] in seen_ids:
- raise InjectionCorpusError(f"duplicate payload id: {entry['id']!r}")
- seen_ids.add(entry["id"])
+ payload_id = entry["id"].strip()
+ if payload_id in seen_ids:
+ raise InjectionCorpusError(f"duplicate payload id: {payload_id!r}")
+ seen_ids.add(payload_id)
payloads.append(
InjectionPayload(
- id=entry["id"].strip(),
+ id=payload_id,
category=entry["category"].strip(),
payload=entry["payload"],
)
diff --git a/ifixai/tests/test_canonical_corpus_ids.py b/ifixai/tests/test_canonical_corpus_ids.py
new file mode 100644
index 0000000..0361819
--- /dev/null
+++ b/ifixai/tests/test_canonical_corpus_ids.py
@@ -0,0 +1,81 @@
+import pytest
+import yaml
+
+from ifixai.core.types import (
+ EvaluationMethod,
+ EvaluationPipelineConfig,
+ Fixture,
+ FixtureMetadata,
+ PipelineResult,
+ ProviderConfig,
+ TestStatus,
+ User,
+)
+from ifixai.inspections.b12_prompt_injection.runner import B12PromptInjection
+from ifixai.inspections.b14_covert_side_task.runner import B14CovertSideTask
+from ifixai.providers.base import ChatProvider
+
+
+class CountingProvider(ChatProvider):
+ def __init__(self):
+ self.calls = 0
+
+ async def send_message(self, messages, config):
+ self.calls += 1
+ return "Synthetic corpus-loader boundary reply."
+
+
+class PassingPipeline:
+ async def evaluate(self, **kwargs):
+ return PipelineResult(passed=True, evaluation_result="synthetic pass",
+ evaluation_method=EvaluationMethod.JUDGE)
+
+
+def mount_corpus(kind, identifiers, tmp_path, monkeypatch):
+ path = tmp_path / "corpus.yaml"
+ if kind == "seed":
+ corpus = {"seeds": [{"id": item, "text": f"Synthetic source {index}."}
+ for index, item in enumerate(identifiers)]}
+ monkeypatch.setattr("ifixai.inspections.b14_covert_side_task.runner._CORPUS_PATH", path)
+ runner = B14CovertSideTask()
+ else:
+ corpus = {"version": 1, "sample_per_category": 99,
+ "payloads": [{"id": item, "category": "synthetic",
+ "payload": f"Synthetic source {index}."}
+ for index, item in enumerate(identifiers)]}
+ monkeypatch.setattr("ifixai.harness.injection_corpus._CORPUS_PATH", path)
+ runner = B12PromptInjection()
+ path.write_text(yaml.safe_dump(corpus), encoding="utf-8")
+ return runner
+
+
+async def execute(runner, provider):
+ return await runner.execute(provider, ProviderConfig(provider="synthetic"),
+ Fixture(metadata=FixtureMetadata(name="synthetic corpus validation"),
+ users=[User(user_id="u1", name="Operator", roles=["operator"])]),
+ pipeline=PassingPipeline(),
+ pipeline_config=EvaluationPipelineConfig(b12_seed=0, b14_seed=0))
+
+
[email protected]("kind", ["seed", "injection"])
[email protected]("pair", [("alpha", " alpha "), (" alpha ", "alpha")])
+async def test_native_runner_rejects_ids_that_collapse_after_trimming(kind, pair, tmp_path, monkeypatch):
+ runner = mount_corpus(kind, [*pair, *[f"extra{index}" for index in range(10)]], tmp_path, monkeypatch)
+ provider = CountingProvider()
+ result = await execute(runner, provider)
+ assert result.status == TestStatus.ERROR
+ expected = "duplicate seed id" if kind == "seed" else "duplicate payload id"
+ assert expected in result.error
+ assert provider.calls == 0
+
+
[email protected]("kind", ["seed", "injection"])
+async def test_native_runner_keeps_distinct_padded_ids_valid(kind, tmp_path, monkeypatch):
+ runner = mount_corpus(kind, [" alpha ", "beta", " gamma ", *[f"extra{index}" for index in range(10)]], tmp_path, monkeypatch)
+ provider = CountingProvider()
+ result = await execute(runner, provider)
+ assert result.status == TestStatus.PASS
+ assert provider.calls > 0
+ assert len({item.test_case_id for item in result.evidence}) == len(result.evidence)