Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

1.7 KB
Raw Blame
"""Damaged checkpoint data must never silently skip the wrong inspection."""

import json

import pytest

from ifixai.core.types import TestResult, TestStatus
from ifixai.evaluation.checkpoint import load_checkpoint


def _write_checkpoint(tmp_path, payload):
    run_dir = tmp_path / "run-1"
    run_dir.mkdir()
    (run_dir / "checkpoint.json").write_text(json.dumps(payload), encoding="utf-8")


def test_non_mapping_checkpoint_is_ignored(tmp_path):
    _write_checkpoint(tmp_path, [{"test_id": "B01"}])

    assert load_checkpoint(tmp_path, "run-1") == {}


def test_checkpoint_entry_with_mismatched_id_is_rerun(tmp_path):
    _write_checkpoint(tmp_path, {
        "B01": TestResult(test_id="B08", status=TestStatus.PASS).model_dump(mode="json")
    })

    assert load_checkpoint(tmp_path, "run-1") == {}


def test_error_status_is_rerun_even_without_error_message(tmp_path):
    _write_checkpoint(tmp_path, {
        "B01": TestResult(test_id="B01", status=TestStatus.ERROR).model_dump(mode="json")
    })

    assert load_checkpoint(tmp_path, "run-1") == {}


def test_valid_completed_entry_is_reused(tmp_path):
    _write_checkpoint(tmp_path, {
        "B01": TestResult(test_id="B01", status=TestStatus.PASS).model_dump(mode="json")
    })

    assert load_checkpoint(tmp_path, "run-1")["B01"].status == TestStatus.PASS


@pytest.mark.parametrize("damaged", [b"\xff", b'{"B01":"\xe2\x82'])
def test_unreadable_utf8_checkpoint_starts_fresh(tmp_path, caplog, damaged):
    run_dir = tmp_path / "run-1"
    run_dir.mkdir()
    checkpoint = run_dir / "checkpoint.json"
    checkpoint.write_bytes(damaged)

    assert load_checkpoint(tmp_path, "run-1") == {}
    assert "checkpoint unreadable" in caplog.text
    assert checkpoint.read_bytes() == damaged