Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix: preserve run manifests after interrupted resume writes (#293)

Rudy Mizrahi Celekli87450c1aecbd
diff --git a/ifixai/evaluation/manifest.py b/ifixai/evaluation/manifest.py
index e95251d..fd3a8d3 100644
--- a/ifixai/evaluation/manifest.py
+++ b/ifixai/evaluation/manifest.py
@@ -3,12 +3,16 @@
import hashlib
import json
import logging
+import os
import re
import secrets
+import stat
import warnings
+from contextlib import suppress
from datetime import datetime
from pathlib import Path
from typing import Any
+from uuid import uuid4
from pydantic import BaseModel, Field, field_validator
@@ -313,11 +317,24 @@ def load_manifest(path: Path) -> RunManifest:
def write_manifest(manifest: RunManifest, base_dir: Path) -> Path:
+ """Replace atomically so an interrupted resume keeps its original identity."""
run_dir = base_dir / manifest.run_id
run_dir.mkdir(parents=True, exist_ok=True)
out_path = run_dir / "manifest.json"
- out_path.write_text(
- json.dumps(manifest.model_dump(mode="json"), indent=2, sort_keys=True),
- encoding="utf-8",
+ content = json.dumps(manifest.model_dump(mode="json"), indent=2, sort_keys=True)
+ mode = stat.S_IMODE(out_path.stat().st_mode) if out_path.exists() else None
+ temporary = run_dir / f".ifixai-manifest-{uuid4().hex}.tmp"
+ descriptor = os.open(
+ temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL,
+ 0o666 if mode is None else 0o600,
)
+ os.close(descriptor)
+ try:
+ temporary.write_text(content, encoding="utf-8")
+ if mode is not None:
+ temporary.chmod(mode)
+ os.replace(temporary, out_path)
+ finally:
+ with suppress(OSError):
+ temporary.unlink(missing_ok=True)
return out_path
diff --git a/ifixai/tests/test_manifest_atomic_write.py b/ifixai/tests/test_manifest_atomic_write.py
new file mode 100644
index 0000000..d87a8ef
--- /dev/null
+++ b/ifixai/tests/test_manifest_atomic_write.py
@@ -0,0 +1,70 @@
+import errno
+import json
+import stat
+from pathlib import Path
+
+import pytest
+
+from ifixai.core.types import RunMode, TestResult, TestStatus
+from ifixai.evaluation.checkpoint import load_checkpoint, save_checkpoint
+from ifixai.evaluation.manifest import (
+ build_manifest,
+ load_manifest,
+ verify_run_id,
+ write_manifest,
+)
+from ifixai.evaluation.types import ModelDescriptor
+
+
+def manifest(timestamp):
+ return build_manifest(
+ mode=RunMode.STANDARD,
+ model_under_test=ModelDescriptor(provider="mock", model_id="owned", version="1"),
+ judge_models=[], normalizer_version="1", test_versions={}, rubric_hashes={},
+ fixture_digest="a" * 64, run_nonce="b" * 16, timestamp=timestamp,
+ )
+
+
[email protected]("failure", [OSError(errno.ENOSPC, "owned disk-full injection"),
+ KeyboardInterrupt()])
+def test_failed_resume_manifest_write_preserves_existing_run(tmp_path, monkeypatch, failure):
+ original = manifest("2026-10-10T12:00:00Z")
+ path = write_manifest(original, tmp_path)
+ path.chmod(0o640)
+ before = path.read_bytes()
+ completed = TestResult(test_id="B01", score=1, passed=True, passing=True,
+ status=TestStatus.PASS)
+ save_checkpoint(tmp_path, original.run_id, {"B01": completed})
+ replacement = manifest("2026-10-10T13:00:00Z")
+ assert replacement.run_id == original.run_id
+
+ def interrupted_write(target, data, *args, **kwargs):
+ # Exercise a genuine truncating write before the injected interruption.
+ with target.open("w", encoding="utf-8") as handle:
+ handle.write(data[:10])
+ handle.flush()
+ raise failure
+
+ monkeypatch.setattr(Path, "write_text", interrupted_write)
+ with pytest.raises(type(failure)):
+ write_manifest(replacement, tmp_path)
+ assert path.read_bytes() == before
+ recovered = load_manifest(path)
+ assert verify_run_id(recovered)
+ assert load_checkpoint(tmp_path, recovered.run_id) == {"B01": completed}
+ assert stat.S_IMODE(path.stat().st_mode) == 0o640
+ assert sorted(p.name for p in path.parent.iterdir()) == ["checkpoint.json", "manifest.json"]
+
+
+def test_successful_resume_manifest_replacement_keeps_identity_and_mode(tmp_path):
+ original = manifest("2026-10-10T12:00:00Z")
+ path = write_manifest(original, tmp_path)
+ path.chmod(0o640)
+ replacement = manifest("2026-10-10T13:00:00Z")
+ assert write_manifest(replacement, tmp_path) == path
+ recovered = load_manifest(path)
+ assert recovered == replacement
+ assert verify_run_id(recovered)
+ assert stat.S_IMODE(path.stat().st_mode) == 0o640
+ assert [p.name for p in path.parent.iterdir()] == ["manifest.json"]
+ assert json.loads(path.read_text(encoding="utf-8"))["timestamp"] == replacement.timestamp