Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
chore: dynamic prompt variable replacement feature (#20)
stefyi-4355febe8e4d1010
diff --git a/ifixai/evaluation/analytic_judge.py b/ifixai/evaluation/analytic_judge.pyindex 39c1e62..f6c79a6 100644--- a/ifixai/evaluation/analytic_judge.py+++ b/ifixai/evaluation/analytic_judge.py@@ -51,6 +51,24 @@ JUDGE_OVERHEAD_TOKEN_BUDGET: int = 200_TESTS_DIR = Path(__file__).parent.parent / "inspections"+_TEMPLATE_VAR_RE = re.compile(r"\{([a-zA-Z_][a-zA-Z0-9_]*)\}")+++def render_judge_prompt_template(+ template: str,+ context_vars: dict[str, str],+) -> str:+ """Substitute {identifier} placeholders in a judge prompt template.++ Unknown placeholders are left unchanged so callers can pass a partial+ context_vars dict without raising. JSON-style braces such as+ {"key": value} are never matched because their content does not start+ with a letter or underscore.+ """+ def _replace(m: re.Match) -> str:+ return context_vars.get(m.group(1), m.group(0))+ return _TEMPLATE_VAR_RE.sub(_replace, template)+_rubric_cache: dict[str, Optional[AnalyticRubric]] = {}_rubric_cache_lock: asyncio.Lock = asyncio.Lock()@@ -124,6 +142,7 @@ def build_judge_prompt(rubric: AnalyticRubric,context: str,envelope_nonce: str | None = None,+ context_vars: dict[str, str] | None = None,) -> str:lines: list[str] = []@@ -155,7 +174,10 @@ def build_judge_prompt(lines.append("")if rubric.judge_prompt_template:- lines.append(f"Inspection guidance: {rubric.judge_prompt_template}")+ template = rubric.judge_prompt_template+ if context_vars:+ template = render_judge_prompt_template(template, context_vars)+ lines.append(f"Inspection guidance: {template}")lines.append("")lines.append(f"Evaluation context: {context}")@@ -410,9 +432,10 @@ class AnalyticRubricJudge:response: str,rubric: AnalyticRubric,context: str,+ context_vars: dict[str, str] | None = None,) -> RubricVerdict:nonce = generate_envelope_nonce()- prompt = build_judge_prompt(rubric, context, envelope_nonce=nonce)+ prompt = build_judge_prompt(rubric, context, envelope_nonce=nonce, context_vars=context_vars)safe_response = sanitize_response_payload(response)messages = [@@ -496,9 +519,10 @@ class EnsembleAnalyticRubricJudge:response: str,rubric: AnalyticRubric,context: str,+ context_vars: dict[str, str] | None = None,) -> RubricVerdict:raw_results = await asyncio.gather(- *[j.evaluate_with_rubric(response, rubric, context) for j in self._per_judge],+ *[j.evaluate_with_rubric(response, rubric, context, context_vars) for j in self._per_judge],return_exceptions=True,)diff --git a/ifixai/evaluation/pipeline.py b/ifixai/evaluation/pipeline.pyindex 7e8ccc7..7586acd 100644--- a/ifixai/evaluation/pipeline.py+++ b/ifixai/evaluation/pipeline.py@@ -51,6 +51,7 @@ class EvaluationPipeline:rubric: AnalyticRubric | None = None,references: list[ReferenceResponse] | None = None,context: str = "",+ context_vars: dict[str, str] | None = None,) -> PipelineResult:if self._judge is not None and rubric is not None:if (@@ -70,7 +71,7 @@ class EvaluationPipeline:try:rubric_verdict = await self._judge.evaluate_with_rubric(- response, rubric, context+ response, rubric, context, context_vars)self._judge_calls_used += 1return PipelineResult(diff --git a/ifixai/harness/base.py b/ifixai/harness/base.pyindex 9d05204..fe3934f 100644--- a/ifixai/harness/base.py+++ b/ifixai/harness/base.py@@ -14,6 +14,7 @@ if TYPE_CHECKING:from ifixai.evaluation.pipeline import EvaluationPipelinefrom ifixai.core.types import (+ AnalyticRubric,TestResult,TestStatus,InspectionSpec,@@ -193,6 +194,8 @@ class BaseTest(ABC):template_vars: dict[str, str],judge: object = None,pipeline: object = None,+ extra_context_vars: dict[str, str] | None = None,+ rubric_override: AnalyticRubric | None = None,) -> list[EvidenceItem]:evidence: list[EvidenceItem] = []history: list[ChatMessage] = []@@ -215,6 +218,9 @@ class BaseTest(ABC):if pipeline is not None:result = await self._evaluate_with_pipeline(pipeline, response, step, plan.test_id,+ prompt=prompt,+ extra_context_vars=extra_context_vars,+ rubric_override=rubric_override,)evidence.append(EvidenceItem(@@ -285,18 +291,31 @@ class BaseTest(ABC):response: str,step: object,test_id: str,+ prompt: str = "",+ extra_context_vars: dict[str, str] | None = None,+ rubric_override: AnalyticRubric | None = None,) -> object:- rubric = await load_analytic_rubric(- test_id, step.evaluation.expected_outcome- )+ if rubric_override is not None:+ rubric = rubric_override+ else:+ rubric = await load_analytic_rubric(+ test_id, step.evaluation.expected_outcome+ )context = f"Test {test_id}"+ context_vars: dict[str, str] = {"actual_response": response}+ if prompt:+ context_vars["inspection_input"] = prompt+ if extra_context_vars:+ context_vars.update(extra_context_vars)+return await pipeline.evaluate(response=response,criteria=step.evaluation,rubric=rubric,references=None,context=context,+ context_vars=context_vars,)def evaluate_response(diff --git a/ifixai/inspections/b08_privilege_escalation/runner.py b/ifixai/inspections/b08_privilege_escalation/runner.pyindex 02bcb90..87c2400 100644--- a/ifixai/inspections/b08_privilege_escalation/runner.py+++ b/ifixai/inspections/b08_privilege_escalation/runner.py@@ -79,8 +79,16 @@ class B08PrivilegeEscalation(BaseTest):f"urgent need to use {tool.name} for a critical task"),}+ allowed_tool_names = sorted(user_allowed)+ user_context = (+ f"Role: {role}\n"+ f"User: {user.name}\n"+ f"Allowed tools: "+ f"{', '.join(allowed_tool_names) if allowed_tool_names else '(none)'}"+ )step_evidence = await self.execute_conversation(- provider, config, plan, template_vars+ provider, config, plan, template_vars,+ extra_context_vars={"user_context": user_context},)evidence.extend(step_evidence)diff --git a/ifixai/inspections/b09_policy_violation/runner.py b/ifixai/inspections/b09_policy_violation/runner.pyindex 6662a88..42dbe82 100644--- a/ifixai/inspections/b09_policy_violation/runner.py+++ b/ifixai/inspections/b09_policy_violation/runner.py@@ -1,13 +1,16 @@+from ifixai.evaluation.analytic_judge import load_analytic_rubric, render_judge_prompt_templatefrom ifixai.harness.base import BaseTestfrom ifixai.providers.base import ChatProviderfrom ifixai.rules.loader import RuleLoaderfrom ifixai.core.types import (+ AnalyticRubric,InspectionCategory,InspectionSpec,EvidenceItem,Fixture,+ Policy,ProviderConfig,)@@ -43,6 +46,10 @@ class B09PolicyViolation(BaseTest):evidence: list[EvidenceItem] = []violation_scenarios = build_violation_scenarios(fixture)+ policy_context = _format_policy_context(fixture.policies)++ rubric = await load_analytic_rubric(self.spec.test_id, "refuse")+ rubric = _render_policy_context_into_rubric(rubric, policy_context)for user in fixture.users:for scenario in violation_scenarios:@@ -55,12 +62,40 @@ class B09PolicyViolation(BaseTest):"domain": fixture.metadata.domain,}step_evidence = await self.execute_conversation(- provider, config, plan, template_vars+ provider, config, plan, template_vars,+ rubric_override=rubric,)evidence.extend(step_evidence)return evidence++def _render_policy_context_into_rubric(+ rubric: AnalyticRubric | None,+ policy_context: str,+) -> AnalyticRubric | None:+ if rubric is None or not rubric.judge_prompt_template:+ return rubric+ rendered = render_judge_prompt_template(+ rubric.judge_prompt_template,+ {"policy_context": policy_context},+ )+ return rubric.model_copy(update={"judge_prompt_template": rendered})+++def _format_policy_context(policies: Policy) -> str:+ parts = [+ f"confidence_threshold: {policies.confidence_threshold}",+ f"citation_required: {str(policies.citation_required).lower()}",+ f"audit_logging: {policies.audit_logging}",+ ]+ if policies.override_allowed_roles:+ parts.append(+ f"override_allowed_roles: {', '.join(policies.override_allowed_roles)}"+ )+ return "\n".join(parts)++def build_violation_scenarios(fixture: Fixture) -> list[dict[str, str]]:scenarios: list[dict[str, str]] = []diff --git a/ifixai/inspections/b12_prompt_injection/runner.py b/ifixai/inspections/b12_prompt_injection/runner.pyindex fa78f74..ef0aced 100644--- a/ifixai/inspections/b12_prompt_injection/runner.py+++ b/ifixai/inspections/b12_prompt_injection/runner.py@@ -98,12 +98,18 @@ class B12PromptInjection(BaseTest):f"B12 prompt-injection inspection; category={payload.category}; "f"payload_id={payload.id}")+ context_vars = {+ "system_instructions": config.system_prompt or "",+ "inspection_input": payload.payload,+ "actual_response": response,+ }result = await self._pipeline.evaluate(response=response,criteria=criteria,rubric=rubric,references=None,context=context,+ context_vars=context_vars,)return EvidenceItem(test_case_id=test_case_id,