Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(inspections): record B12 and B30 effective replay seeds (#177)

Rudy Mizrahi Celeklic76cb5af2054
diff --git a/ifixai/inspections/b12_prompt_injection/runner.py b/ifixai/inspections/b12_prompt_injection/runner.py
index 8ced2d3..4c5b761 100644
--- a/ifixai/inspections/b12_prompt_injection/runner.py
+++ b/ifixai/inspections/b12_prompt_injection/runner.py
@@ -44,6 +44,7 @@ class B12PromptInjection(BaseTest):
def __init__(self) -> None:
super().__init__(SPEC)
+ self.last_variant_seed: int | None = None
async def run(
self,
@@ -58,6 +59,7 @@ class B12PromptInjection(BaseTest):
if self.pipeline_config is not None
else secrets.randbelow(2**31)
)
+ self.last_variant_seed = seed
payloads = sample_corpus(corpus, seed=seed)
rubric = await load_analytic_rubric(self.spec.test_id, "refuse")
semaphore = asyncio.Semaphore(_MAX_CONCURRENT_PAYLOADS)
@@ -69,6 +71,12 @@ class B12PromptInjection(BaseTest):
)
return list(evidence)
+ def get_variant_seed(self) -> int | None:
+ return self.last_variant_seed
+
+ def get_variant_seed_pinned(self) -> bool:
+ return self.pipeline_config is not None and self.pipeline_config.b12_seed_pinned
+
def compute_score(self, evidence: list[EvidenceItem]) -> float:
"""Binary fraction of payloads BLOCKED, matching SPEC '% Injections Blocked'.
diff --git a/ifixai/inspections/b30_malicious_deployer_rules/runner.py b/ifixai/inspections/b30_malicious_deployer_rules/runner.py
index c510cf2..7caaadb 100644
--- a/ifixai/inspections/b30_malicious_deployer_rules/runner.py
+++ b/ifixai/inspections/b30_malicious_deployer_rules/runner.py
@@ -48,6 +48,7 @@ class B30MaliciousDeployerRules(BaseTest):
def __init__(self) -> None:
super().__init__(SPEC)
+ self.last_variant_seed: int | None = None
async def run(
self,
@@ -57,10 +58,12 @@ class B30MaliciousDeployerRules(BaseTest):
) -> list[EvidenceItem]:
del fixture
seeds = load_seeds(_CORPUS_PATH)
+ effective_seed = self._effective_b30_seed()
+ self.last_variant_seed = effective_seed
variants = expand_seeds(
seeds,
_SYNONYMS_PATH,
- seed=self._effective_b30_seed(),
+ seed=effective_seed,
variants_per_seed=_VARIANTS_PER_SEED,
)
semaphore = asyncio.Semaphore(_MAX_CONCURRENT_VARIANTS)
@@ -79,6 +82,12 @@ class B30MaliciousDeployerRules(BaseTest):
)
return list(evidence)
+ def get_variant_seed(self) -> int | None:
+ return self.last_variant_seed
+
+ def get_variant_seed_pinned(self) -> bool:
+ return self.pipeline_config is not None and self.pipeline_config.b30_seed_pinned
+
def _effective_b30_seed(self) -> int:
if self.pipeline_config is not None:
return self.pipeline_config.b30_seed
diff --git a/ifixai/tests/test_b12_replay_seed.py b/ifixai/tests/test_b12_replay_seed.py
new file mode 100644
index 0000000..ee2defa
--- /dev/null
+++ b/ifixai/tests/test_b12_replay_seed.py
@@ -0,0 +1,61 @@
+import pytest
+
+from ifixai.core.types import (
+ EvaluationMethod,
+ EvaluationPipelineConfig,
+ Fixture,
+ FixtureMetadata,
+ PipelineResult,
+ ProviderConfig,
+ TestRunResult,
+ TestStatus,
+)
+from ifixai.harness.injection_corpus import load_injection_corpus, sample_corpus
+from ifixai.inspections.b12_prompt_injection.runner import B12PromptInjection
+from ifixai.providers.base import ChatProvider
+from ifixai.reporting.scorecard import generate_markdown_report
+
+
+class SyntheticProvider(ChatProvider):
+ async def send_message(self, messages, config):
+ return "Synthetic refusal used only to exercise the replay contract."
+
+
+class PassingPipeline:
+ async def evaluate(self, **kwargs):
+ return PipelineResult(passed=True, evaluation_result="synthetic pass",
+ evaluation_method=EvaluationMethod.JUDGE)
+
+
[email protected](("seed", "pinned"), [(0, True), (8675309, False)])
+async def test_native_b12_preserves_effective_seed_and_pinning(seed, pinned):
+ runner = B12PromptInjection()
+ result = await runner.execute(
+ SyntheticProvider(), ProviderConfig(provider="synthetic"),
+ Fixture(metadata=FixtureMetadata(name="synthetic B12 replay test")),
+ pipeline_config=EvaluationPipelineConfig(b12_seed=seed, b12_seed_pinned=pinned),
+ pipeline=PassingPipeline())
+ assert result.status == TestStatus.PASS
+ expected = sample_corpus(load_injection_corpus(), seed=seed)
+ assert [item.details["payload_id"] for item in result.evidence] == [item.id for item in expected]
+ assert result.variant_seed == seed
+ assert result.variant_seed_pinned is pinned
+ label = "pinned" if pinned else "random"
+ assert f"**B12** seed: {seed} ({label})" in generate_markdown_report(
+ TestRunResult(test_results=[result]))
+
+
+async def test_native_b12_without_pipeline_config_records_drawn_seed(monkeypatch):
+ monkeypatch.setattr("ifixai.inspections.b12_prompt_injection.runner.secrets.randbelow",
+ lambda upper: 123456)
+ result = await B12PromptInjection().execute(
+ SyntheticProvider(), ProviderConfig(provider="synthetic"),
+ Fixture(metadata=FixtureMetadata(name="synthetic B12 random replay test")),
+ pipeline=PassingPipeline())
+ assert result.status == TestStatus.PASS
+ expected = sample_corpus(load_injection_corpus(), seed=123456)
+ assert [item.details["payload_id"] for item in result.evidence] == [item.id for item in expected]
+ assert result.variant_seed == 123456
+ assert result.variant_seed_pinned is False
diff --git a/ifixai/tests/test_b30_replay_seed.py b/ifixai/tests/test_b30_replay_seed.py
new file mode 100644
index 0000000..88d9019
--- /dev/null
+++ b/ifixai/tests/test_b30_replay_seed.py
@@ -0,0 +1,60 @@
+import pytest
+
+from ifixai.core.types import (
+ EvaluationPipelineConfig,
+ Fixture,
+ FixtureMetadata,
+ ProviderConfig,
+ TestRunResult,
+ TestStatus,
+)
+from ifixai.evaluation.response_classifier import ResponseClass
+from ifixai.harness.adversarial_mutator import expand_seeds, load_seeds
+from ifixai.inspections.b30_malicious_deployer_rules import runner as b30
+from ifixai.providers.base import ChatProvider
+from ifixai.reporting.scorecard import generate_markdown_report
+
+
+class SyntheticProvider(ChatProvider):
+ async def send_message(self, messages, config):
+ return "Synthetic B30 refusal for replay contract test."
+
+
+class RefusalPipeline:
+ async def classify(self, response, query):
+ return ResponseClass.REFUSAL
+
+
[email protected](("seed", "pinned"), [(0, True), (8675309, False)])
+async def test_native_b30_records_actual_expansion_seed(seed, pinned):
+ result = await b30.B30MaliciousDeployerRules().execute(
+ SyntheticProvider(), ProviderConfig(provider="synthetic"),
+ Fixture(metadata=FixtureMetadata(name="synthetic B30 replay")),
+ pipeline=RefusalPipeline(),
+ pipeline_config=EvaluationPipelineConfig(b30_seed=seed, b30_seed_pinned=pinned))
+ assert result.status == TestStatus.PASS
+ expected = expand_seeds(load_seeds(b30._CORPUS_PATH), b30._SYNONYMS_PATH,
+ seed=seed, variants_per_seed=b30._VARIANTS_PER_SEED)
+ assert [item.prompt_sent for item in result.evidence] == [item.mutated_text for item in expected]
+ assert result.variant_seed == seed
+ assert result.variant_seed_pinned is pinned
+ label = "pinned" if pinned else "random"
+ assert f"**B30** seed: {seed} ({label})" in generate_markdown_report(
+ TestRunResult(test_results=[result]))
+
+
+async def test_native_b30_unconfigured_seed_preserves_actual_draw(monkeypatch):
+ monkeypatch.setattr("ifixai.inspections.b30_malicious_deployer_rules.runner.secrets.randbelow",
+ lambda upper: 123456)
+ result = await b30.B30MaliciousDeployerRules().execute(
+ SyntheticProvider(), ProviderConfig(provider="synthetic"),
+ Fixture(metadata=FixtureMetadata(name="synthetic B30 random replay")),
+ pipeline=RefusalPipeline())
+ assert result.status == TestStatus.PASS
+ expected = expand_seeds(load_seeds(b30._CORPUS_PATH), b30._SYNONYMS_PATH,
+ seed=123456, variants_per_seed=b30._VARIANTS_PER_SEED)
+ assert [item.prompt_sent for item in result.evidence] == [item.mutated_text for item in expected]
+ assert result.variant_seed == 123456
+ assert result.variant_seed_pinned is False