Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(b26): identify each per-tool rate-limit probe (#330)
Rudy Mizrahi Celeklidfa4443b367a
diff --git a/ifixai/inspections/b26_rate_limiting/runner.py b/ifixai/inspections/b26_rate_limiting/runner.pyindex 1a8e81b..eaa302d 100644--- a/ifixai/inspections/b26_rate_limiting/runner.py+++ b/ifixai/inspections/b26_rate_limiting/runner.py@@ -2,6 +2,7 @@ import asyncioimport loggingimport timefrom typing import Optional, TypedDict+from urllib.parse import quotefrom ifixai.core.types import (ConversationPlan,@@ -109,7 +110,10 @@ class B26RateLimiting(BaseTest):provider,config,plan,- template_vars={"tool": tool.name},+ template_vars={+ "tool": tool.name,+ "case_id": f"tool={quote(tool.tool_id, safe='')}",+ },rubric_override=rubric,)tool_items = _apply_forbidden_keyword_check(plan, tool_items)diff --git a/ifixai/tests/test_b26_probe_identifiers.py b/ifixai/tests/test_b26_probe_identifiers.pynew file mode 100644index 0000000..a23d7dc--- /dev/null+++ b/ifixai/tests/test_b26_probe_identifiers.py@@ -0,0 +1,53 @@+import asyncio+import json+from pathlib import Path+from urllib.parse import quote++from ifixai.core.fixture_loader import load_fixture+from ifixai.core.types import (+ EvaluationMethod,+ EvaluationPipelineConfig,+ PipelineResult,+ ProviderConfig,+ TestRunResult,+)+from ifixai.inspections.b26_rate_limiting.runner import B26RateLimiting+from ifixai.providers.base import ChatProvider+from ifixai.reporting.scorecard import generate_json_report++_SMOKE_TINY = Path(__file__).resolve().parents[1] / "fixtures" / "smoke_tiny.yaml"+++class _Provider(ChatProvider):+ surfaces_rate_limit_errors = True++ async def send_message(self, messages, config):+ return "60 requests per minute; excess requests are throttled."+++class _PassingPipeline:+ async def evaluate(self, **kwargs):+ return PipelineResult(+ passed=True, evaluation_result="ok", evaluation_method=EvaluationMethod.JUDGE+ )+++def test_each_tool_probe_exports_its_own_case_id():+ fixture = load_fixture(str(_SMOKE_TINY))+ assert len(fixture.tools) > 1+ result = asyncio.run(+ B26RateLimiting().execute(+ _Provider(),+ ProviderConfig(provider="synthetic"),+ fixture,+ pipeline=_PassingPipeline(),+ pipeline_config=EvaluationPipelineConfig(),+ )+ )+ exported = json.loads(generate_json_report(TestRunResult(test_results=[result])))+ ids = [item["test_case_id"] for item in exported["test_results"][0]["evidence"]]+ tool_ids = [i for i in ids if i.startswith("B26_step")]+ assert len(tool_ids) == len(set(tool_ids))+ assert set(tool_ids) == {+ f"B26_step1_tool={quote(tool.tool_id, safe='')}" for tool in fixture.tools+ }