Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Fix cancellation and worker-exit races (#10) (#249)

Cinna Davis371ceb4ea035
diff --git a/src/dopplergeist/jobs.py b/src/dopplergeist/jobs.py
index 0af8249..7ef9699 100644
--- a/src/dopplergeist/jobs.py
+++ b/src/dopplergeist/jobs.py
@@ -1,5 +1,6 @@
"""Persistent local jobs; each analysis has its own cancellable process group."""
import argparse
+from contextlib import contextmanager
import fcntl
import hashlib
import json
@@ -39,6 +40,17 @@ def _evidence(value):
return {key: value[key] for key in ("recording", "events", "candidates", "coverage", "artifacts") if key in value}
+@contextmanager
+def _control_lock(directory):
+ """Serialize a local cancel request with the manager's final result."""
+ with (directory / ".control.lock").open("a") as lockfile:
+ fcntl.flock(lockfile, fcntl.LOCK_EX)
+ try:
+ yield
+ finally:
+ fcntl.flock(lockfile, fcntl.LOCK_UN)
+
+
def request_local_cancel(root, job_id):
"""Request cancellation through an active manager's cooperative marker."""
root = Path(root).resolve()
@@ -51,27 +63,28 @@ def request_local_cancel(root, job_id):
result_path = directory / "result.json"
if not result_path.is_file():
raise KeyError("Unknown job")
- current = read_json(result_path)
- if current["status"] in TERMINAL:
- return current
- lock_path = root / ".manager.lock"
- try:
- with lock_path.open("rb") as lockfile:
- try:
- fcntl.flock(lockfile.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
- except BlockingIOError:
- manager_active = True
- else:
- fcntl.flock(lockfile.fileno(), fcntl.LOCK_UN)
- manager_active = False
- except FileNotFoundError:
- manager_active = False
- if not manager_active:
- raise ValueError("No active job manager owns this job directory")
- (directory / "cancel").touch(mode=0o600, exist_ok=True)
- return {"job_id": job_id, "status": current["status"], "stage": current.get("stage"),
- "cancellation_requested": True,
- "message": "The job will stop at its next cancellation check."}
+ with _control_lock(directory):
+ current = read_json(result_path)
+ if current["status"] in TERMINAL:
+ return current
+ lock_path = root / ".manager.lock"
+ try:
+ with lock_path.open("rb") as lockfile:
+ try:
+ fcntl.flock(lockfile.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except BlockingIOError:
+ manager_active = True
+ else:
+ fcntl.flock(lockfile.fileno(), fcntl.LOCK_UN)
+ manager_active = False
+ except FileNotFoundError:
+ manager_active = False
+ if not manager_active:
+ raise ValueError("No active job manager owns this job directory")
+ (directory / "cancel").touch(mode=0o600, exist_ok=True)
+ return {"job_id": job_id, "status": current["status"], "stage": current.get("stage"),
+ "cancellation_requested": True,
+ "message": "The job will stop at its next cancellation check."}
class JobManager:
@@ -201,17 +214,29 @@ class JobManager:
break
if time.monotonic() >= deadline:
self._stop(process)
- with self._mutex:
- if self.get(directory.name)["status"] not in TERMINAL:
- result(directory, "inconclusive", reason_code="work_limit", message="Analysis exceeded the configured wall-time budget; inspect partial artifacts.", **_evidence(self.get(directory.name)))
+ with self._mutex, _control_lock(directory):
+ current = self.get(directory.name)
+ if current["status"] not in TERMINAL:
+ if (directory / "cancel").exists():
+ result(directory, "cancelled", message="Cancelled by the user; partial artifacts are preserved.", **_evidence(current))
+ else:
+ result(directory, "inconclusive", reason_code="work_limit", message="Analysis exceeded the configured wall-time budget; inspect partial artifacts.", **_evidence(current))
break
- with self._mutex:
- if self.get(directory.name)["status"] not in TERMINAL:
- result(directory, "failed", reason_code="worker_exit", message=f"Analysis process exited ({process.returncode}); inspect worker.log.", **_evidence(self.get(directory.name)))
+ with self._mutex, _control_lock(directory):
+ current = self.get(directory.name)
+ if current["status"] not in TERMINAL:
+ if (directory / "cancel").exists():
+ result(directory, "cancelled", message="Cancelled by the user; partial artifacts are preserved.", **_evidence(current))
+ else:
+ result(directory, "failed", reason_code="worker_exit", message=f"Analysis process exited ({process.returncode}); inspect worker.log.", **_evidence(current))
except Exception as exc:
- with self._mutex:
- if self.get(directory.name)["status"] not in TERMINAL:
- result(directory, "failed", reason_code="worker_launch", message=f"Could not run analysis: {exc}", **_evidence(self.get(directory.name)))
+ with self._mutex, _control_lock(directory):
+ current = self.get(directory.name)
+ if current["status"] not in TERMINAL:
+ if (directory / "cancel").exists():
+ result(directory, "cancelled", message="Cancelled by the user; partial artifacts are preserved.", **_evidence(current))
+ else:
+ result(directory, "failed", reason_code="worker_launch", message=f"Could not run analysis: {exc}", **_evidence(current))
finally:
with self._mutex:
self._processes.pop(directory.name, None)
diff --git a/tests/test_local_cancel.py b/tests/test_local_cancel.py
index c5301bf..164af7a 100644
--- a/tests/test_local_cancel.py
+++ b/tests/test_local_cancel.py
@@ -24,6 +24,19 @@ def use_idle_worker(monkeypatch):
monkeypatch.setattr(subprocess, "Popen", start_idle_worker)
+def use_controlled_worker(monkeypatch, release):
+ popen = subprocess.Popen
+ def start_controlled_worker(command, **kwargs):
+ child = popen([sys.executable, "-c",
+ "import pathlib,sys,time\nwhile not pathlib.Path(sys.argv[1]).exists(): time.sleep(.01)",
+ str(release)], **kwargs)
+ wait = child.wait
+ # Observe the child's exit rather than the manager's polling timeout.
+ child.wait = lambda timeout=None: wait(timeout=8 if timeout is not None else None)
+ return child
+ monkeypatch.setattr(subprocess, "Popen", start_controlled_worker)
+
+
def test_local_cancel_sets_marker_without_rewriting_job_result(tmp_path):
manager = JobManager(tmp_path)
try:
@@ -87,3 +100,105 @@ def test_time_limit_still_reports_inconclusive(tmp_path, monkeypatch):
assert not (directory / "cancel").exists()
finally:
manager.close()
+
+
[email protected]("worker_finished", [False, True])
+def test_cancel_racing_worker_exit_preserves_terminal_result(tmp_path, monkeypatch, worker_finished):
+ manager = JobManager(tmp_path, max_runtime_s=60)
+ job_id, directory = make_running_job(tmp_path)
+ release = directory / "release"
+ use_controlled_worker(monkeypatch, release)
+ thread = threading.Thread(target=manager._run, args=(directory,))
+ manager._threads[job_id] = thread
+ thread.start()
+ try:
+ deadline = time.monotonic() + 5
+ while job_id not in manager._processes and time.monotonic() < deadline:
+ time.sleep(0.01)
+ assert job_id in manager._processes
+ request_local_cancel(tmp_path, job_id)
+ if worker_finished:
+ expected = result(directory, "inconclusive", reason_code="no_events", message="No events.")
+ release.touch()
+ thread.join(timeout=8)
+ assert not thread.is_alive()
+ outcome = read_json(directory / "result.json")
+ if worker_finished:
+ assert outcome == expected
+ else:
+ assert outcome["status"] == "cancelled"
+ finally:
+ release.touch(exist_ok=True)
+ manager.close()
+ thread.join(timeout=8)
+
+
+def test_cancel_request_and_exit_do_not_leave_failed_result(tmp_path, monkeypatch):
+ manager = JobManager(tmp_path, max_runtime_s=60)
+ job_id, directory = make_running_job(tmp_path)
+ release = directory / "release"
+ use_controlled_worker(monkeypatch, release)
+ entering_touch, allow_touch = threading.Event(), threading.Event()
+ original_touch = type(release).touch
+
+ def delayed_touch(path, *args, **kwargs):
+ if path == directory / "cancel":
+ entering_touch.set()
+ assert allow_touch.wait(8)
+ return original_touch(path, *args, **kwargs)
+
+ monkeypatch.setattr(type(release), "touch", delayed_touch)
+ thread = threading.Thread(target=manager._run, args=(directory,))
+ manager._threads[job_id] = thread
+ thread.start()
+ requested = []
+ request_thread = None
+ try:
+ deadline = time.monotonic() + 5
+ while job_id not in manager._processes and time.monotonic() < deadline:
+ time.sleep(0.01)
+ assert job_id in manager._processes
+ request_thread = threading.Thread(target=lambda: requested.append(request_local_cancel(tmp_path, job_id)))
+ request_thread.start()
+ assert entering_touch.wait(5)
+ release.touch()
+ thread.join(timeout=1)
+ allow_touch.set()
+ request_thread.join(timeout=8)
+ thread.join(timeout=8)
+ assert requested[0]["cancellation_requested"]
+ assert read_json(directory / "result.json")["status"] == "cancelled"
+ finally:
+ allow_touch.set()
+ release.touch(exist_ok=True)
+ if request_thread is not None:
+ request_thread.join(timeout=8)
+ manager.close()
+ thread.join(timeout=8)
+
+
+def test_cancel_while_worker_launch_fails_is_cancelled(tmp_path, monkeypatch):
+ manager = JobManager(tmp_path, max_runtime_s=60)
+ job_id, directory = make_running_job(tmp_path)
+ launching, allow_failure = threading.Event(), threading.Event()
+
+ def fail_launch(*args, **kwargs):
+ launching.set()
+ assert allow_failure.wait(8)
+ raise OSError("simulated process launch failure")
+
+ monkeypatch.setattr(subprocess, "Popen", fail_launch)
+ thread = threading.Thread(target=manager._run, args=(directory,))
+ manager._threads[job_id] = thread
+ thread.start()
+ try:
+ assert launching.wait(5)
+ assert request_local_cancel(tmp_path, job_id)["cancellation_requested"]
+ allow_failure.set()
+ thread.join(timeout=8)
+ assert not thread.is_alive()
+ assert read_json(directory / "result.json")["status"] == "cancelled"
+ finally:
+ allow_failure.set()
+ manager.close()
+ thread.join(timeout=8)