Skip to content

The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.

JavaScript

Fix Windows CI: serialize case.json reads against writes (#14)

OsintMeThat9bab937154a4
diff --git a/src/azimut/engine/media.py b/src/azimut/engine/media.py
index 3115939..f61548f 100644
--- a/src/azimut/engine/media.py
+++ b/src/azimut/engine/media.py
@@ -20,7 +20,7 @@ from typing import Any, BinaryIO
from PIL import Image
-from ..workspace import Case
+from ..workspace import Case, ensure_dir
from . import links as link_engine
THUMB_DIR = ".thumbs"
@@ -75,7 +75,7 @@ def ffmpeg_available() -> bool:
def make_thumbnail(media_path: Path, thumb_path: Path) -> bool:
"""Best-effort thumbnail. Images via Pillow, videos via ffmpeg if present."""
- thumb_path.parent.mkdir(exist_ok=True)
+ ensure_dir(thumb_path.parent)
kind = media_kind(media_path.name)
try:
if kind == "image":
@@ -290,7 +290,7 @@ def _register_downloaded_item(
its own yt-dlp path, minus the yt-dlp-specific extraction bits."""
media_dir = case.subdir("media")
tmp_dir = media_dir / ".dl" / uuid.uuid4().hex
- tmp_dir.mkdir(parents=True, exist_ok=True)
+ ensure_dir(tmp_dir)
try:
fname = safe_filename(filename)
tmp_path = tmp_dir / fname
@@ -459,7 +459,7 @@ def download_url(
# fires one per selected item) must not share a scratch dir, or the first
# one to finish rmtree()s it out from under the others still writing to it
tmp_dir = media_dir / ".dl" / uuid.uuid4().hex
- tmp_dir.mkdir(parents=True, exist_ok=True)
+ ensure_dir(tmp_dir)
extra_photos = _telegram_extra_photos(url) # [] — and free — for non-Telegram links
diff --git a/src/azimut/workspace.py b/src/azimut/workspace.py
index e187473..2d44233 100644
--- a/src/azimut/workspace.py
+++ b/src/azimut/workspace.py
@@ -19,6 +19,7 @@ import json
import re
import shutil
import threading
+import time
import uuid
from datetime import datetime, timezone
from pathlib import Path
@@ -33,16 +34,23 @@ CASE_SUBDIRS = ("media", "proofs", "exports", "inspect")
# read-modify-write of case.json — e.g. several media downloads from the
# multi-item picker finishing at once — silently drop each other's entity or
# crash on the tmp-file rename (spec §6 honest output requires none lost).
+#
+# Reentrant on purpose: every read() and every write goes through it, and a
+# mutator (add_entity, …) reads *inside* its own locked section, so the same
+# thread has to be able to re-take the lock it already holds. It also serialises
+# reads against writes — required on Windows, where os.replace() cannot rename
+# over a case.json another thread still has open, and open() cannot read one
+# mid-replace (both surface as PermissionError; POSIX has neither problem).
_case_locks_guard = threading.Lock()
-_case_locks: dict[str, threading.Lock] = {}
+_case_locks: dict[str, threading.RLock] = {}
-def _case_lock(path: Path) -> threading.Lock:
+def _case_lock(path: Path) -> threading.RLock:
key = str(path)
with _case_locks_guard:
lock = _case_locks.get(key)
if lock is None:
- lock = threading.Lock()
+ lock = threading.RLock()
_case_locks[key] = lock
return lock
@@ -69,6 +77,47 @@ def _new_id(prefix: str) -> str:
return f"{prefix}_{uuid.uuid4().hex[:10]}"
+def ensure_dir(path: Path) -> Path:
+ """``path.mkdir(parents=True, exist_ok=True)``, tolerant of a transient
+ ``PermissionError`` Windows can raise when several threads race to create
+ the very same directory for the first time (e.g. several concurrent
+ downloads all hitting a case's not-yet-created ``media/.dl`` or
+ ``media/.thumbs`` at once): CreateDirectory there occasionally answers
+ "access is denied" instead of "already exists" mid-race, which
+ ``exist_ok=True`` alone does not catch. Retried briefly — the directory
+ reliably exists by the next attempt, whoever won the race.
+ """
+ for attempt in range(20):
+ try:
+ path.mkdir(parents=True, exist_ok=True)
+ return path
+ except PermissionError:
+ if path.is_dir():
+ return path
+ if attempt == 19:
+ raise
+ time.sleep(0.01)
+ return path # pragma: no cover - loop always returns or raises above
+
+
+def _replace_with_retry(src: Path, dst: Path) -> None:
+ """``src.replace(dst)`` with a brief retry for a transient Windows
+ ``PermissionError``. The case lock already keeps our own threads off the
+ destination during a rename, but on Windows an external handle (a virus
+ scanner or the search indexer opening the freshly written file) can still
+ make os.replace() fail for a few milliseconds. POSIX rename is atomic and
+ never hits this, so the loop is a no-op there.
+ """
+ for attempt in range(20):
+ try:
+ src.replace(dst)
+ return
+ except PermissionError:
+ if attempt == 19:
+ raise
+ time.sleep(0.01)
+
+
class CaseError(Exception):
"""Raised for invalid case operations; maps to HTTP 4xx in the API layer."""
@@ -157,13 +206,22 @@ class Case:
return self.path / "case.json"
def read(self) -> dict[str, Any]:
- return json.loads(self.json_path.read_text(encoding="utf-8"))
+ # Under the lock so a concurrent write can't be replacing case.json
+ # while we open it — on Windows that open() raises PermissionError
+ # (the file is momentarily unopenable mid-rename).
+ with self._lock:
+ return json.loads(self.json_path.read_text(encoding="utf-8"))
def _write_json(self, data: dict[str, Any]) -> None:
- data["updated_at"] = _now()
- tmp = self.json_path.with_suffix(".json.tmp")
- tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
- tmp.replace(self.json_path)
+ # Under the lock so no reader has case.json open when we rename over it:
+ # Windows' os.replace() refuses a destination another handle holds open
+ # ("Access is denied"). A unique tmp name keeps two writers' scratch
+ # files apart even though the lock already serialises them.
+ with self._lock:
+ data["updated_at"] = _now()
+ tmp = self.json_path.with_suffix(f".{uuid.uuid4().hex}.tmp")
+ tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+ _replace_with_retry(tmp, self.json_path)
# -- notes -------------------------------------------------------------
@@ -419,9 +477,7 @@ class Case:
def subdir(self, name: str) -> Path:
if name not in CASE_SUBDIRS:
raise CaseError(f"unknown case subdir '{name}'")
- path = self.path / name
- path.mkdir(exist_ok=True)
- return path
+ return ensure_dir(self.path / name)
def resolve_inside(self, relative: str) -> Path:
"""Resolve a case-relative path, refusing traversal outside the case."""
diff --git a/tests/test_media_api.py b/tests/test_media_api.py
index 38db49d..4f96502 100644
--- a/tests/test_media_api.py
+++ b/tests/test_media_api.py
@@ -654,8 +654,10 @@ def test_concurrent_downloads_dont_lose_entities(client, monkeypatch):
def run(i):
try:
media_engine.download_url(case, "https://x.com/u/status/1", index=i, title=f"title {i}")
- except Exception as exc: # pragma: no cover - assertion below reports it
- errors.append(exc)
+ except Exception: # pragma: no cover - assertion below reports it
+ import traceback
+
+ errors.append(traceback.format_exc())
threads = [threading.Thread(target=run, args=(i,)) for i in range(1, n + 1)]
for t in threads:
@@ -663,7 +665,7 @@ def test_concurrent_downloads_dont_lose_entities(client, monkeypatch):
for t in threads:
t.join()
- assert errors == []
+ assert errors == [], "\n\n".join(errors)
assert len(case.read()["entities"]) == n
assert len(client.get(f"/api/cases/{cid}/media").json()) == n