Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

chore: dynamic prompt variable replacement feature (#20)

stefyi-4355febe8e4d1010
diff --git a/ifixai/evaluation/analytic_judge.py b/ifixai/evaluation/analytic_judge.py
index 39c1e62..f6c79a6 100644
--- a/ifixai/evaluation/analytic_judge.py
+++ b/ifixai/evaluation/analytic_judge.py
@@ -51,6 +51,24 @@ JUDGE_OVERHEAD_TOKEN_BUDGET: int = 200
_TESTS_DIR = Path(__file__).parent.parent / "inspections"
+_TEMPLATE_VAR_RE = re.compile(r"\{([a-zA-Z_][a-zA-Z0-9_]*)\}")
+
+
+def render_judge_prompt_template(
+ template: str,
+ context_vars: dict[str, str],
+) -> str:
+ """Substitute {identifier} placeholders in a judge prompt template.
+
+ Unknown placeholders are left unchanged so callers can pass a partial
+ context_vars dict without raising. JSON-style braces such as
+ {"key": value} are never matched because their content does not start
+ with a letter or underscore.
+ """
+ def _replace(m: re.Match) -> str:
+ return context_vars.get(m.group(1), m.group(0))
+ return _TEMPLATE_VAR_RE.sub(_replace, template)
+
_rubric_cache: dict[str, Optional[AnalyticRubric]] = {}
_rubric_cache_lock: asyncio.Lock = asyncio.Lock()
@@ -124,6 +142,7 @@ def build_judge_prompt(
rubric: AnalyticRubric,
context: str,
envelope_nonce: str | None = None,
+ context_vars: dict[str, str] | None = None,
) -> str:
lines: list[str] = []
@@ -155,7 +174,10 @@ def build_judge_prompt(
lines.append("")
if rubric.judge_prompt_template:
- lines.append(f"Inspection guidance: {rubric.judge_prompt_template}")
+ template = rubric.judge_prompt_template
+ if context_vars:
+ template = render_judge_prompt_template(template, context_vars)
+ lines.append(f"Inspection guidance: {template}")
lines.append("")
lines.append(f"Evaluation context: {context}")
@@ -410,9 +432,10 @@ class AnalyticRubricJudge:
response: str,
rubric: AnalyticRubric,
context: str,
+ context_vars: dict[str, str] | None = None,
) -> RubricVerdict:
nonce = generate_envelope_nonce()
- prompt = build_judge_prompt(rubric, context, envelope_nonce=nonce)
+ prompt = build_judge_prompt(rubric, context, envelope_nonce=nonce, context_vars=context_vars)
safe_response = sanitize_response_payload(response)
messages = [
@@ -496,9 +519,10 @@ class EnsembleAnalyticRubricJudge:
response: str,
rubric: AnalyticRubric,
context: str,
+ context_vars: dict[str, str] | None = None,
) -> RubricVerdict:
raw_results = await asyncio.gather(
- *[j.evaluate_with_rubric(response, rubric, context) for j in self._per_judge],
+ *[j.evaluate_with_rubric(response, rubric, context, context_vars) for j in self._per_judge],
return_exceptions=True,
)
diff --git a/ifixai/evaluation/pipeline.py b/ifixai/evaluation/pipeline.py
index 7e8ccc7..7586acd 100644
--- a/ifixai/evaluation/pipeline.py
+++ b/ifixai/evaluation/pipeline.py
@@ -51,6 +51,7 @@ class EvaluationPipeline:
rubric: AnalyticRubric | None = None,
references: list[ReferenceResponse] | None = None,
context: str = "",
+ context_vars: dict[str, str] | None = None,
) -> PipelineResult:
if self._judge is not None and rubric is not None:
if (
@@ -70,7 +71,7 @@ class EvaluationPipeline:
try:
rubric_verdict = await self._judge.evaluate_with_rubric(
- response, rubric, context
+ response, rubric, context, context_vars
)
self._judge_calls_used += 1
return PipelineResult(
diff --git a/ifixai/harness/base.py b/ifixai/harness/base.py
index 9d05204..fe3934f 100644
--- a/ifixai/harness/base.py
+++ b/ifixai/harness/base.py
@@ -14,6 +14,7 @@ if TYPE_CHECKING:
from ifixai.evaluation.pipeline import EvaluationPipeline
from ifixai.core.types import (
+ AnalyticRubric,
TestResult,
TestStatus,
InspectionSpec,
@@ -193,6 +194,8 @@ class BaseTest(ABC):
template_vars: dict[str, str],
judge: object = None,
pipeline: object = None,
+ extra_context_vars: dict[str, str] | None = None,
+ rubric_override: AnalyticRubric | None = None,
) -> list[EvidenceItem]:
evidence: list[EvidenceItem] = []
history: list[ChatMessage] = []
@@ -215,6 +218,9 @@ class BaseTest(ABC):
if pipeline is not None:
result = await self._evaluate_with_pipeline(
pipeline, response, step, plan.test_id,
+ prompt=prompt,
+ extra_context_vars=extra_context_vars,
+ rubric_override=rubric_override,
)
evidence.append(
EvidenceItem(
@@ -285,18 +291,31 @@ class BaseTest(ABC):
response: str,
step: object,
test_id: str,
+ prompt: str = "",
+ extra_context_vars: dict[str, str] | None = None,
+ rubric_override: AnalyticRubric | None = None,
) -> object:
- rubric = await load_analytic_rubric(
- test_id, step.evaluation.expected_outcome
- )
+ if rubric_override is not None:
+ rubric = rubric_override
+ else:
+ rubric = await load_analytic_rubric(
+ test_id, step.evaluation.expected_outcome
+ )
context = f"Test {test_id}"
+ context_vars: dict[str, str] = {"actual_response": response}
+ if prompt:
+ context_vars["inspection_input"] = prompt
+ if extra_context_vars:
+ context_vars.update(extra_context_vars)
+
return await pipeline.evaluate(
response=response,
criteria=step.evaluation,
rubric=rubric,
references=None,
context=context,
+ context_vars=context_vars,
)
def evaluate_response(
diff --git a/ifixai/inspections/b08_privilege_escalation/runner.py b/ifixai/inspections/b08_privilege_escalation/runner.py
index 02bcb90..87c2400 100644
--- a/ifixai/inspections/b08_privilege_escalation/runner.py
+++ b/ifixai/inspections/b08_privilege_escalation/runner.py
@@ -79,8 +79,16 @@ class B08PrivilegeEscalation(BaseTest):
f"urgent need to use {tool.name} for a critical task"
),
}
+ allowed_tool_names = sorted(user_allowed)
+ user_context = (
+ f"Role: {role}\n"
+ f"User: {user.name}\n"
+ f"Allowed tools: "
+ f"{', '.join(allowed_tool_names) if allowed_tool_names else '(none)'}"
+ )
step_evidence = await self.execute_conversation(
- provider, config, plan, template_vars
+ provider, config, plan, template_vars,
+ extra_context_vars={"user_context": user_context},
)
evidence.extend(step_evidence)
diff --git a/ifixai/inspections/b09_policy_violation/runner.py b/ifixai/inspections/b09_policy_violation/runner.py
index 6662a88..42dbe82 100644
--- a/ifixai/inspections/b09_policy_violation/runner.py
+++ b/ifixai/inspections/b09_policy_violation/runner.py
@@ -1,13 +1,16 @@
+from ifixai.evaluation.analytic_judge import load_analytic_rubric, render_judge_prompt_template
from ifixai.harness.base import BaseTest
from ifixai.providers.base import ChatProvider
from ifixai.rules.loader import RuleLoader
from ifixai.core.types import (
+ AnalyticRubric,
InspectionCategory,
InspectionSpec,
EvidenceItem,
Fixture,
+ Policy,
ProviderConfig,
)
@@ -43,6 +46,10 @@ class B09PolicyViolation(BaseTest):
evidence: list[EvidenceItem] = []
violation_scenarios = build_violation_scenarios(fixture)
+ policy_context = _format_policy_context(fixture.policies)
+
+ rubric = await load_analytic_rubric(self.spec.test_id, "refuse")
+ rubric = _render_policy_context_into_rubric(rubric, policy_context)
for user in fixture.users:
for scenario in violation_scenarios:
@@ -55,12 +62,40 @@ class B09PolicyViolation(BaseTest):
"domain": fixture.metadata.domain,
}
step_evidence = await self.execute_conversation(
- provider, config, plan, template_vars
+ provider, config, plan, template_vars,
+ rubric_override=rubric,
)
evidence.extend(step_evidence)
return evidence
+
+def _render_policy_context_into_rubric(
+ rubric: AnalyticRubric | None,
+ policy_context: str,
+) -> AnalyticRubric | None:
+ if rubric is None or not rubric.judge_prompt_template:
+ return rubric
+ rendered = render_judge_prompt_template(
+ rubric.judge_prompt_template,
+ {"policy_context": policy_context},
+ )
+ return rubric.model_copy(update={"judge_prompt_template": rendered})
+
+
+def _format_policy_context(policies: Policy) -> str:
+ parts = [
+ f"confidence_threshold: {policies.confidence_threshold}",
+ f"citation_required: {str(policies.citation_required).lower()}",
+ f"audit_logging: {policies.audit_logging}",
+ ]
+ if policies.override_allowed_roles:
+ parts.append(
+ f"override_allowed_roles: {', '.join(policies.override_allowed_roles)}"
+ )
+ return "\n".join(parts)
+
+
def build_violation_scenarios(fixture: Fixture) -> list[dict[str, str]]:
scenarios: list[dict[str, str]] = []
diff --git a/ifixai/inspections/b12_prompt_injection/runner.py b/ifixai/inspections/b12_prompt_injection/runner.py
index fa78f74..ef0aced 100644
--- a/ifixai/inspections/b12_prompt_injection/runner.py
+++ b/ifixai/inspections/b12_prompt_injection/runner.py
@@ -98,12 +98,18 @@ class B12PromptInjection(BaseTest):
f"B12 prompt-injection inspection; category={payload.category}; "
f"payload_id={payload.id}"
)
+ context_vars = {
+ "system_instructions": config.system_prompt or "",
+ "inspection_input": payload.payload,
+ "actual_response": response,
+ }
result = await self._pipeline.evaluate(
response=response,
criteria=criteria,
rubric=rubric,
references=None,
context=context,
+ context_vars=context_vars,
)
return EvidenceItem(
test_case_id=test_case_id,