Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Fix cancellation and worker-exit races (#10) (#249)
Cinna Davis371ceb4ea035
diff --git a/src/dopplergeist/jobs.py b/src/dopplergeist/jobs.pyindex 0af8249..7ef9699 100644--- a/src/dopplergeist/jobs.py+++ b/src/dopplergeist/jobs.py@@ -1,5 +1,6 @@"""Persistent local jobs; each analysis has its own cancellable process group."""import argparse+from contextlib import contextmanagerimport fcntlimport hashlibimport json@@ -39,6 +40,17 @@ def _evidence(value):return {key: value[key] for key in ("recording", "events", "candidates", "coverage", "artifacts") if key in value}+@contextmanager+def _control_lock(directory):+ """Serialize a local cancel request with the manager's final result."""+ with (directory / ".control.lock").open("a") as lockfile:+ fcntl.flock(lockfile, fcntl.LOCK_EX)+ try:+ yield+ finally:+ fcntl.flock(lockfile, fcntl.LOCK_UN)++def request_local_cancel(root, job_id):"""Request cancellation through an active manager's cooperative marker."""root = Path(root).resolve()@@ -51,27 +63,28 @@ def request_local_cancel(root, job_id):result_path = directory / "result.json"if not result_path.is_file():raise KeyError("Unknown job")- current = read_json(result_path)- if current["status"] in TERMINAL:- return current- lock_path = root / ".manager.lock"- try:- with lock_path.open("rb") as lockfile:- try:- fcntl.flock(lockfile.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)- except BlockingIOError:- manager_active = True- else:- fcntl.flock(lockfile.fileno(), fcntl.LOCK_UN)- manager_active = False- except FileNotFoundError:- manager_active = False- if not manager_active:- raise ValueError("No active job manager owns this job directory")- (directory / "cancel").touch(mode=0o600, exist_ok=True)- return {"job_id": job_id, "status": current["status"], "stage": current.get("stage"),- "cancellation_requested": True,- "message": "The job will stop at its next cancellation check."}+ with _control_lock(directory):+ current = read_json(result_path)+ if current["status"] in TERMINAL:+ return current+ lock_path = root / ".manager.lock"+ try:+ with lock_path.open("rb") as lockfile:+ try:+ fcntl.flock(lockfile.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)+ except BlockingIOError:+ manager_active = True+ else:+ fcntl.flock(lockfile.fileno(), fcntl.LOCK_UN)+ manager_active = False+ except FileNotFoundError:+ manager_active = False+ if not manager_active:+ raise ValueError("No active job manager owns this job directory")+ (directory / "cancel").touch(mode=0o600, exist_ok=True)+ return {"job_id": job_id, "status": current["status"], "stage": current.get("stage"),+ "cancellation_requested": True,+ "message": "The job will stop at its next cancellation check."}class JobManager:@@ -201,17 +214,29 @@ class JobManager:breakif time.monotonic() >= deadline:self._stop(process)- with self._mutex:- if self.get(directory.name)["status"] not in TERMINAL:- result(directory, "inconclusive", reason_code="work_limit", message="Analysis exceeded the configured wall-time budget; inspect partial artifacts.", **_evidence(self.get(directory.name)))+ with self._mutex, _control_lock(directory):+ current = self.get(directory.name)+ if current["status"] not in TERMINAL:+ if (directory / "cancel").exists():+ result(directory, "cancelled", message="Cancelled by the user; partial artifacts are preserved.", **_evidence(current))+ else:+ result(directory, "inconclusive", reason_code="work_limit", message="Analysis exceeded the configured wall-time budget; inspect partial artifacts.", **_evidence(current))break- with self._mutex:- if self.get(directory.name)["status"] not in TERMINAL:- result(directory, "failed", reason_code="worker_exit", message=f"Analysis process exited ({process.returncode}); inspect worker.log.", **_evidence(self.get(directory.name)))+ with self._mutex, _control_lock(directory):+ current = self.get(directory.name)+ if current["status"] not in TERMINAL:+ if (directory / "cancel").exists():+ result(directory, "cancelled", message="Cancelled by the user; partial artifacts are preserved.", **_evidence(current))+ else:+ result(directory, "failed", reason_code="worker_exit", message=f"Analysis process exited ({process.returncode}); inspect worker.log.", **_evidence(current))except Exception as exc:- with self._mutex:- if self.get(directory.name)["status"] not in TERMINAL:- result(directory, "failed", reason_code="worker_launch", message=f"Could not run analysis: {exc}", **_evidence(self.get(directory.name)))+ with self._mutex, _control_lock(directory):+ current = self.get(directory.name)+ if current["status"] not in TERMINAL:+ if (directory / "cancel").exists():+ result(directory, "cancelled", message="Cancelled by the user; partial artifacts are preserved.", **_evidence(current))+ else:+ result(directory, "failed", reason_code="worker_launch", message=f"Could not run analysis: {exc}", **_evidence(current))finally:with self._mutex:self._processes.pop(directory.name, None)diff --git a/tests/test_local_cancel.py b/tests/test_local_cancel.pyindex c5301bf..164af7a 100644--- a/tests/test_local_cancel.py+++ b/tests/test_local_cancel.py@@ -24,6 +24,19 @@ def use_idle_worker(monkeypatch):monkeypatch.setattr(subprocess, "Popen", start_idle_worker)+def use_controlled_worker(monkeypatch, release):+ popen = subprocess.Popen+ def start_controlled_worker(command, **kwargs):+ child = popen([sys.executable, "-c",+ "import pathlib,sys,time\nwhile not pathlib.Path(sys.argv[1]).exists(): time.sleep(.01)",+ str(release)], **kwargs)+ wait = child.wait+ # Observe the child's exit rather than the manager's polling timeout.+ child.wait = lambda timeout=None: wait(timeout=8 if timeout is not None else None)+ return child+ monkeypatch.setattr(subprocess, "Popen", start_controlled_worker)++def test_local_cancel_sets_marker_without_rewriting_job_result(tmp_path):manager = JobManager(tmp_path)try:@@ -87,3 +100,105 @@ def test_time_limit_still_reports_inconclusive(tmp_path, monkeypatch):assert not (directory / "cancel").exists()finally:manager.close()++[email protected]("worker_finished", [False, True])+def test_cancel_racing_worker_exit_preserves_terminal_result(tmp_path, monkeypatch, worker_finished):+ manager = JobManager(tmp_path, max_runtime_s=60)+ job_id, directory = make_running_job(tmp_path)+ release = directory / "release"+ use_controlled_worker(monkeypatch, release)+ thread = threading.Thread(target=manager._run, args=(directory,))+ manager._threads[job_id] = thread+ thread.start()+ try:+ deadline = time.monotonic() + 5+ while job_id not in manager._processes and time.monotonic() < deadline:+ time.sleep(0.01)+ assert job_id in manager._processes+ request_local_cancel(tmp_path, job_id)+ if worker_finished:+ expected = result(directory, "inconclusive", reason_code="no_events", message="No events.")+ release.touch()+ thread.join(timeout=8)+ assert not thread.is_alive()+ outcome = read_json(directory / "result.json")+ if worker_finished:+ assert outcome == expected+ else:+ assert outcome["status"] == "cancelled"+ finally:+ release.touch(exist_ok=True)+ manager.close()+ thread.join(timeout=8)+++def test_cancel_request_and_exit_do_not_leave_failed_result(tmp_path, monkeypatch):+ manager = JobManager(tmp_path, max_runtime_s=60)+ job_id, directory = make_running_job(tmp_path)+ release = directory / "release"+ use_controlled_worker(monkeypatch, release)+ entering_touch, allow_touch = threading.Event(), threading.Event()+ original_touch = type(release).touch++ def delayed_touch(path, *args, **kwargs):+ if path == directory / "cancel":+ entering_touch.set()+ assert allow_touch.wait(8)+ return original_touch(path, *args, **kwargs)++ monkeypatch.setattr(type(release), "touch", delayed_touch)+ thread = threading.Thread(target=manager._run, args=(directory,))+ manager._threads[job_id] = thread+ thread.start()+ requested = []+ request_thread = None+ try:+ deadline = time.monotonic() + 5+ while job_id not in manager._processes and time.monotonic() < deadline:+ time.sleep(0.01)+ assert job_id in manager._processes+ request_thread = threading.Thread(target=lambda: requested.append(request_local_cancel(tmp_path, job_id)))+ request_thread.start()+ assert entering_touch.wait(5)+ release.touch()+ thread.join(timeout=1)+ allow_touch.set()+ request_thread.join(timeout=8)+ thread.join(timeout=8)+ assert requested[0]["cancellation_requested"]+ assert read_json(directory / "result.json")["status"] == "cancelled"+ finally:+ allow_touch.set()+ release.touch(exist_ok=True)+ if request_thread is not None:+ request_thread.join(timeout=8)+ manager.close()+ thread.join(timeout=8)+++def test_cancel_while_worker_launch_fails_is_cancelled(tmp_path, monkeypatch):+ manager = JobManager(tmp_path, max_runtime_s=60)+ job_id, directory = make_running_job(tmp_path)+ launching, allow_failure = threading.Event(), threading.Event()++ def fail_launch(*args, **kwargs):+ launching.set()+ assert allow_failure.wait(8)+ raise OSError("simulated process launch failure")++ monkeypatch.setattr(subprocess, "Popen", fail_launch)+ thread = threading.Thread(target=manager._run, args=(directory,))+ manager._threads[job_id] = thread+ thread.start()+ try:+ assert launching.wait(5)+ assert request_local_cancel(tmp_path, job_id)["cancellation_requested"]+ allow_failure.set()+ thread.join(timeout=8)+ assert not thread.is_alive()+ assert read_json(directory / "result.json")["status"] == "cancelled"+ finally:+ allow_failure.set()+ manager.close()+ thread.join(timeout=8)