The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.
JavaScript
"""The generic durable worker: one registry of kind -> handler, one thread, and
central settlement of every claimed job.
The queue is drained explicitly here (`start_workers = False`) so ordering is
deterministic and no background thread outlives a test.
"""
from __future__ import annotations
import threading
import time
import pytest
from azimut.engine import workqueue
from azimut.workspace import Case
@pytest.fixture()
def case(tmp_workspace, monkeypatch):
monkeypatch.setattr(workqueue, "start_workers", False)
monkeypatch.setattr(workqueue, "HANDLERS", dict(workqueue.HANDLERS))
return Case.create("Queue")
def test_drain_runs_the_handler_registered_for_each_kind(case):
seen: list[tuple[str, str]] = []
workqueue.register("alpha", lambda c, job: seen.append(("alpha", job["payload"]["n"])))
workqueue.register("beta", lambda c, job: seen.append(("beta", job["payload"]["n"])))
workqueue.enqueue(case, "alpha", payload={"n": "1"})
workqueue.enqueue(case, "beta", payload={"n": "2"})
assert workqueue.drain(case) == 2
assert seen == [("alpha", "1"), ("beta", "2")]
assert [j["state"] for j in case.list_jobs()] == ["ready", "ready"]
def test_a_handler_raising_job_cancelled_drops_the_job_without_retrying(case):
def handler(c, job):
raise workqueue.JobCancelled("nothing to do")
workqueue.register("alpha", handler)
workqueue.enqueue(case, "alpha", payload={})
assert workqueue.drain(case) == 1
job = case.list_jobs(kind="alpha")[0]
assert job["state"] == "cancelled"
assert job["attempts"] == 1
def test_a_failing_handler_retries_until_the_attempt_budget_is_spent(case):
attempts = []
def handler(c, job):
attempts.append(job["id"])
raise workqueue.JobFailed("boom")
workqueue.register("alpha", handler)
workqueue.enqueue(case, "alpha", payload={})
workqueue.drain(case) # claims, fails, requeues, up to the budget
job = case.list_jobs(kind="alpha")[0]
assert job["state"] == "failed"
assert job["error"] == "boom"
assert len(attempts) == job["max_attempts"]
def test_a_refused_job_fails_at_once(case):
"""No room for an export is no room on the next two attempts either."""
attempts = []
def handler(c, job):
attempts.append(job["id"])
raise workqueue.JobRefused("not enough free space to export this case")
workqueue.register("alpha", handler)
workqueue.enqueue(case, "alpha", payload={})
workqueue.drain(case)
job = case.list_jobs(kind="alpha")[0]
assert (job["state"], job["error"], len(attempts)) == (
"failed", "not enough free space to export this case", 1,
)
def test_an_unexpected_failure_is_recorded_as_a_sentence(case, tmp_path):
"""The job's error is what the analyst reads: no class name, no errno, no path."""
target = tmp_path / "Operation Heron" / "bundle.zip"
def handler(c, job):
raise OSError(28, "No space left on device", str(target))
workqueue.register("alpha", handler)
workqueue.enqueue(case, "alpha", payload={})
workqueue.drain(case)
job = case.list_jobs(kind="alpha")[0]
assert job["error"] == "the disk is full"
def test_a_job_of_an_unknown_kind_is_cancelled_not_left_running(case):
workqueue.enqueue(case, "gamma", payload={})
assert workqueue.drain(case) == 1
assert case.list_jobs(kind="gamma")[0]["state"] == "cancelled"
def test_has_queued_reports_work_of_any_kind(case):
workqueue.register("alpha", lambda c, job: None)
assert workqueue.has_queued(case) is False
workqueue.enqueue(case, "alpha", payload={})
assert workqueue.has_queued(case) is True
workqueue.drain(case)
assert workqueue.has_queued(case) is False
@pytest.mark.parametrize("operation", ["delete", "promote"])
def test_moving_a_case_directory_waits_for_the_background_worker(
tmp_workspace, monkeypatch, operation
):
"""A job writes inside the case folder, so moving that folder while the worker
is still in it is how ``Directory not empty`` happens on POSIX and an outright
refusal happens on Windows.
This one runs a real worker thread (no ``start_workers = False``): the race is
the point, and draining by hand would test nothing. Enrichment is what made the
window wide enough to hit — probing a video takes far longer than writing a
thumbnail — but the two operations that move a directory are the fix's place,
not one job kind.
"""
monkeypatch.setattr(workqueue, "HANDLERS", dict(workqueue.HANDLERS))
case = Case.create("Doomed", scratch=operation == "promote")
started = threading.Event()
def slow(c, job):
started.set()
time.sleep(0.3) # wide enough that an un-drained move lands mid-job
(c.path / "media").mkdir(parents=True, exist_ok=True)
(c.path / "media" / "late.json").write_text("{}", encoding="utf-8")
workqueue.register("slow", slow)
workqueue.enqueue(case, "slow", payload={})
workqueue.wake(case)
# A liveness budget, not the measurement: on an idle machine the worker is in
# the handler within milliseconds, and what is under test is the drain below.
# Two seconds is a scheduling guess, and a loaded Windows runner missed it.
assert started.wait(30), "the worker never picked the job up"
if operation == "delete":
case.delete()
assert not case.path.exists()
else:
moved = case.promote("Promoted")
# the file the worker wrote travelled with the case rather than being
# stranded in a half-moved directory
assert (moved.path / "media" / "late.json").exists()
# the operation returned only once the worker was out, so nothing is left
# writing into a path that no longer exists
assert workqueue.wait_until_idle(timeout=0) is True
def test_wait_until_idle_looks_once_more_after_the_deadline(case, monkeypatch):
"""A worker that finishes during the final sleep is idle. Reporting a timeout
for work that is done would have the caller refuse to move a case directory it
is now free to move."""
monkeypatch.setattr(workqueue, "_worker_running", False)
assert workqueue.wait_until_idle(timeout=0) is True
monkeypatch.setattr(workqueue, "_worker_running", True)
assert workqueue.wait_until_idle(timeout=0) is False
def test_a_worker_that_dies_hands_the_queue_back(case, monkeypatch):
"""A thread lost to something no `except Exception` catches must not leave the
flag set.
`wake` starts nothing while it believes a worker is draining, so a flag left
behind by a dead thread stalled every queued job in the process — a thumbnail
that never appeared, with nothing to say why, until a restart.
"""
def die(_case):
raise KeyboardInterrupt # not an Exception; the drain's own guard misses it
monkeypatch.setattr(workqueue, "drain", die)
monkeypatch.setitem(workqueue._pending, case.id, case)
monkeypatch.setattr(workqueue, "_worker_running", True)
with pytest.raises(KeyboardInterrupt):
workqueue._run_loop()
assert workqueue._worker_running is False
# and the case it never got to is still queued, for the worker after it
assert workqueue.wait_until_idle(timeout=0) is True
def test_a_long_job_lets_other_kinds_through_between_its_steps(case):
order: list[str] = []
def sweep(c, job):
for step in range(2):
order.append(f"sweep {job['payload']['n']}.{step}")
if step == 0:
workqueue.enqueue(c, "thumb", payload={})
workqueue.let_others_through(c, "sweep")
workqueue.register("sweep", sweep)
workqueue.register("thumb", lambda c, job: order.append("thumb"))
workqueue.enqueue(case, "sweep", payload={"n": "1"})
workqueue.enqueue(case, "sweep", payload={"n": "2"})
assert workqueue.drain(case) == 2
# the thumbnail never waits out a sweep, and the second sweep keeps its turn
assert order == ["sweep 1.0", "thumb", "sweep 1.1", "sweep 2.0", "thumb", "sweep 2.1"]
assert {j["state"] for j in case.list_jobs()} == {"ready"}
def test_letting_others_through_with_nothing_else_registered_claims_nothing(case):
workqueue.HANDLERS.clear()
nested: list[bool] = []
busy = [False]
def sweep(c, job):
nested.append(busy[0])
busy[0] = True
workqueue.let_others_through(c, "sweep")
busy[0] = False
workqueue.register("sweep", sweep)
workqueue.enqueue(case, "sweep", payload={})
workqueue.enqueue(case, "sweep", payload={})
# an empty kind list must not turn into "any kind", or the second sweep
# would run inside the first
assert workqueue.drain(case) == 2
assert nested == [False, False]
def test_a_keyed_job_enqueued_while_it_runs_runs_once_more(case):
"""A pass that started before a filing may have missed it, so the second
enqueue is kept: one more run at the end, with the newer payload, never two."""
seen: list[str] = []
def handler(c, job):
seen.append(job["payload"]["n"])
if len(seen) == 1:
workqueue.enqueue(c, "pass", key="case", payload={"n": "2"})
workqueue.enqueue(c, "pass", key="case", payload={"n": "3"})
workqueue.register("pass", handler)
workqueue.enqueue(case, "pass", key="case", payload={"n": "1"})
assert workqueue.drain(case) == 2
assert seen == ["1", "3"]
[job] = case.list_jobs()
assert job["state"] == "ready"
assert job["payload"] == {"n": "3"}