Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix: use private staging files for checkpoint saves (#148)

Rudy Mizrahi Celekli7439900a3157
diff --git a/ifixai/evaluation/checkpoint.py b/ifixai/evaluation/checkpoint.py
index de697aa..a3a6d55 100644
--- a/ifixai/evaluation/checkpoint.py
+++ b/ifixai/evaluation/checkpoint.py
@@ -18,6 +18,7 @@ from __future__ import annotations
import json
import logging
from pathlib import Path
+from tempfile import NamedTemporaryFile
from typing import Optional
from ifixai.core.types import TestResult, TestStatus
@@ -71,20 +72,22 @@ def save_checkpoint(
run_id: str,
results: dict[str, TestResult],
) -> Path:
- """Persist atomically: writes to a sibling `.tmp` file then renames,
+ """Persist atomically: writes to a private sibling temporary file then renames,
so the checkpoint is never half-written if the process dies mid-write."""
path = _checkpoint_path(runs_root, run_id)
path.parent.mkdir(parents=True, exist_ok=True)
- tmp = path.with_suffix(".tmp")
payload = {
test_id: result.model_dump(mode="json")
for test_id, result in results.items()
}
- tmp.write_text(
- json.dumps(payload, indent=2, sort_keys=True, default=str),
- encoding="utf-8",
- )
- tmp.replace(path)
+ fh = NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, suffix=".tmp", delete=False)
+ tmp = Path(fh.name)
+ try:
+ with fh:
+ json.dump(payload, fh, indent=2, sort_keys=True, default=str)
+ tmp.replace(path)
+ finally:
+ tmp.unlink(missing_ok=True)
return path
diff --git a/ifixai/tests/evaluation/test_checkpoint_atomic_save.py b/ifixai/tests/evaluation/test_checkpoint_atomic_save.py
new file mode 100644
index 0000000..815acf9
--- /dev/null
+++ b/ifixai/tests/evaluation/test_checkpoint_atomic_save.py
@@ -0,0 +1,45 @@
+"""Each checkpoint save owns its staging file, including overlapping saves."""
+
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from threading import Barrier
+
+import pytest
+
+from ifixai.core.types import TestResult
+from ifixai.evaluation.checkpoint import load_checkpoint, save_checkpoint
+
+
+def test_overlapping_saves_publish_complete_checkpoints(tmp_path, monkeypatch):
+ barrier = Barrier(2)
+ replace = Path.replace
+
+ def synchronized_replace(self, target):
+ if Path(target).name == "checkpoint.json":
+ barrier.wait(timeout=5)
+ return replace(self, target)
+
+ monkeypatch.setattr(Path, "replace", synchronized_replace)
+ results = [{key: TestResult(test_id=key)} for key in ("B01", "B08")]
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ futures = [executor.submit(save_checkpoint, tmp_path, "run-1", r) for r in results]
+ for future in futures:
+ assert future.result() == tmp_path / "run-1" / "checkpoint.json"
+
+ assert set(load_checkpoint(tmp_path, "run-1")) in ({"B01"}, {"B08"})
+ assert list((tmp_path / "run-1").iterdir()) == [tmp_path / "run-1" / "checkpoint.json"]
+
+
+def test_failed_publication_preserves_previous_checkpoint(tmp_path, monkeypatch):
+ checkpoint = save_checkpoint(tmp_path, "run-1", {"B01": TestResult(test_id="B01")})
+ previous = checkpoint.read_bytes()
+
+ def reject_replace(self, target):
+ raise OSError("controlled publication failure")
+
+ monkeypatch.setattr(Path, "replace", reject_replace)
+ with pytest.raises(OSError, match="controlled publication failure"):
+ save_checkpoint(tmp_path, "run-1", {"B08": TestResult(test_id="B08")})
+
+ assert checkpoint.read_bytes() == previous
+ assert list(checkpoint.parent.iterdir()) == [checkpoint]