Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix: preserve responses with mismatched reasoning tags (#277)
Rudy Mizrahi Celekli83353ada8cbf
diff --git a/ifixai/evaluation/normalizer.py b/ifixai/evaluation/normalizer.pyindex 59c140d..a5d8d62 100644--- a/ifixai/evaluation/normalizer.py+++ b/ifixai/evaluation/normalizer.py@@ -9,7 +9,7 @@ NORMALIZER_VERSION = "1.0.0"_CODE_FENCE_RE = re.compile(r"```[a-zA-Z0-9_+-]*\n?(.*?)```", re.DOTALL)_REASONING_TAG_RE = re.compile(- r"<(?:thinking|reasoning|scratchpad)>.*?</(?:thinking|reasoning|scratchpad)>",+ r"<(thinking|reasoning|scratchpad)>.*?</\1>",re.DOTALL | re.IGNORECASE,)_REASONING_PREFIX_RE = re.compile(diff --git a/ifixai/tests/test_normalizer_resume_compatibility.py b/ifixai/tests/test_normalizer_resume_compatibility.pynew file mode 100644index 0000000..56ac0bc--- /dev/null+++ b/ifixai/tests/test_normalizer_resume_compatibility.py@@ -0,0 +1,29 @@+"""Changing unused normalization must not reject a completed native CLI resume."""+import importlib+import json++from click.testing import CliRunner++from ifixai.cli.main import ifixai_cli+++def test_completed_run_with_existing_normalizer_version_still_resumes(tmp_path, monkeypatch):+ run = importlib.import_module('ifixai.cli.run')+ current_version = run.NORMALIZER_VERSION+ monkeypatch.chdir(tmp_path)+ monkeypatch.setenv('HOME', str(tmp_path))+ monkeypatch.setenv('IFIXAI_TELEMETRY', '0')+ common = ['run', '--provider', 'mock', '--model', 'owned-model',+ '--test', 'B07', '--eval-mode', 'single', '--judge-provider', 'mock',+ '--output', str(tmp_path / 'reports'), '--reliability-out', str(tmp_path / 'runs'),+ '--no-telemetry', '--no-parallel']+ monkeypatch.setattr(run, 'NORMALIZER_VERSION', '1.0.0')+ first = CliRunner().invoke(ifixai_cli, common)+ manifests = list((tmp_path / 'runs').glob('*/manifest.json'))+ assert len(manifests) == 1, first.output+ manifest = json.loads(manifests[0].read_text())+ monkeypatch.setattr(run, 'NORMALIZER_VERSION', current_version)+ resumed = CliRunner().invoke(ifixai_cli, [*common, '--resume', manifest['run_id']])+ assert '[reused] B07' in resumed.output, resumed.output+ assert 'run configuration changed' not in resumed.output+ assert resumed.exit_code == first.exit_codediff --git a/ifixai/tests/test_normalizer_tag_pairs.py b/ifixai/tests/test_normalizer_tag_pairs.pynew file mode 100644index 0000000..a6e9555--- /dev/null+++ b/ifixai/tests/test_normalizer_tag_pairs.py@@ -0,0 +1,16 @@+import pytest++from ifixai.evaluation.normalizer import normalize++[email protected]("raw", [+ "<thinking>Access denied.</scratchpad>",+ "<reasoning>Access denied.</thinking>",+])+def test_mismatched_reasoning_tags_do_not_erase_the_response(raw):+ assert "Access denied." in normalize(raw).normalized++[email protected]("tag", ["thinking", "reasoning", "scratchpad"])+def test_matching_reasoning_tags_are_removed(tag):+ assert normalize(f"<{tag}>private chain</{tag}>Final answer").normalized == "Final answer"