Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

1.8 KB
Raw Blame
"""Each checkpoint save owns its staging file, including overlapping saves."""

from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from threading import Barrier

import pytest

from ifixai.core.types import TestResult
from ifixai.evaluation import checkpoint
from ifixai.evaluation.checkpoint import load_checkpoint, save_checkpoint


def test_overlapping_saves_publish_complete_checkpoints(tmp_path, monkeypatch):
    barrier = Barrier(2)
    dump = checkpoint.json.dump

    def synchronized_dump(payload, stream, **kwargs):
        dump(payload, stream, **kwargs)
        barrier.wait(timeout=5)

    monkeypatch.setattr(checkpoint.json, "dump", synchronized_dump)
    results = [{key: TestResult(test_id=key)} for key in ("B01", "B08")]
    with ThreadPoolExecutor(max_workers=2) as executor:
        futures = [executor.submit(save_checkpoint, tmp_path, "run-1", r) for r in results]
        for future in futures:
            assert future.result() == tmp_path / "run-1" / "checkpoint.json"

    assert set(load_checkpoint(tmp_path, "run-1")) in ({"B01"}, {"B08"})
    assert list((tmp_path / "run-1").iterdir()) == [tmp_path / "run-1" / "checkpoint.json"]


def test_failed_publication_preserves_previous_checkpoint(tmp_path, monkeypatch):
    checkpoint = save_checkpoint(tmp_path, "run-1", {"B01": TestResult(test_id="B01")})
    previous = checkpoint.read_bytes()

    def reject_replace(self, target):
        raise OSError("controlled publication failure")

    monkeypatch.setattr(Path, "replace", reject_replace)
    with pytest.raises(OSError, match="controlled publication failure"):
        save_checkpoint(tmp_path, "run-1", {"B08": TestResult(test_id="B08")})

    assert checkpoint.read_bytes() == previous
    assert list(checkpoint.parent.iterdir()) == [checkpoint]