Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Stop local cancelled workers without waiting for search progress

Cinna Davis021c58fdea84
diff --git a/src/dopplergeist/jobs.py b/src/dopplergeist/jobs.py
index 30f0d94..0af8249 100644
--- a/src/dopplergeist/jobs.py
+++ b/src/dopplergeist/jobs.py
@@ -10,6 +10,7 @@ import signal
import subprocess
import sys
import threading
+import time
import uuid
from .contracts import validate_request, validate_result
@@ -185,13 +186,25 @@ class JobManager:
process = subprocess.Popen(command,
stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True)
self._processes[directory.name] = process
- try:
- process.wait(timeout=self.max_runtime_s)
- except subprocess.TimeoutExpired:
- self._stop(process)
- with self._mutex:
- if self.get(directory.name)["status"] not in TERMINAL:
- result(directory, "inconclusive", reason_code="work_limit", message="Analysis exceeded the configured wall-time budget; inspect partial artifacts.", **_evidence(self.get(directory.name)))
+ deadline = time.monotonic() + self.max_runtime_s
+ while True:
+ try:
+ process.wait(timeout=max(0, min(0.5, deadline - time.monotonic())))
+ break
+ except subprocess.TimeoutExpired:
+ if (directory / "cancel").exists():
+ with self._mutex:
+ self._stop(process)
+ current = self.get(directory.name)
+ if current["status"] not in TERMINAL:
+ result(directory, "cancelled", message="Cancelled by the user; partial artifacts are preserved.", **_evidence(current))
+ break
+ if time.monotonic() >= deadline:
+ self._stop(process)
+ with self._mutex:
+ if self.get(directory.name)["status"] not in TERMINAL:
+ result(directory, "inconclusive", reason_code="work_limit", message="Analysis exceeded the configured wall-time budget; inspect partial artifacts.", **_evidence(self.get(directory.name)))
+ break
with self._mutex:
if self.get(directory.name)["status"] not in TERMINAL:
result(directory, "failed", reason_code="worker_exit", message=f"Analysis process exited ({process.returncode}); inspect worker.log.", **_evidence(self.get(directory.name)))
diff --git a/tests/test_local_cancel.py b/tests/test_local_cancel.py
index be5e70b..c5301bf 100644
--- a/tests/test_local_cancel.py
+++ b/tests/test_local_cancel.py
@@ -1,4 +1,8 @@
import uuid
+import subprocess
+import sys
+import threading
+import time
import pytest
@@ -13,6 +17,13 @@ def make_running_job(root):
return job_id, directory
+def use_idle_worker(monkeypatch):
+ popen = subprocess.Popen
+ def start_idle_worker(command, **kwargs):
+ return popen([sys.executable, "-c", "import time; time.sleep(60)"], **kwargs)
+ monkeypatch.setattr(subprocess, "Popen", start_idle_worker)
+
+
def test_local_cancel_sets_marker_without_rewriting_job_result(tmp_path):
manager = JobManager(tmp_path)
try:
@@ -36,3 +47,43 @@ def test_local_cancel_requires_live_manager_and_preserves_terminal_jobs(tmp_path
previous = result(directory, "inconclusive", reason_code="no_events", message="No events.")
assert request_local_cancel(tmp_path, job_id) == previous
assert not (directory / "cancel").exists()
+
+
+def test_local_cancel_stops_unresponsive_worker_and_preserves_checkpoint(tmp_path, monkeypatch):
+ # Exercise the control plane with a real process that never reports progress;
+ # no recording or aircraft observations are generated.
+ manager = JobManager(tmp_path, max_runtime_s=60)
+ job_id, directory = make_running_job(tmp_path)
+ checkpoint = directory / "saved-checkpoint"
+ checkpoint.write_text("retained")
+ use_idle_worker(monkeypatch)
+ thread = threading.Thread(target=manager._run, args=(directory,))
+ manager._threads[job_id] = thread
+ thread.start()
+ try:
+ deadline = time.monotonic() + 5
+ while job_id not in manager._processes and time.monotonic() < deadline:
+ time.sleep(0.01)
+ assert job_id in manager._processes
+ request_local_cancel(tmp_path, job_id)
+ thread.join(timeout=3)
+ assert not thread.is_alive(), "manager did not stop the worker after cancellation"
+ assert read_json(directory / "result.json")["status"] == "cancelled"
+ assert checkpoint.read_text() == "retained"
+ finally:
+ manager.close()
+ thread.join(timeout=5)
+
+
+def test_time_limit_still_reports_inconclusive(tmp_path, monkeypatch):
+ manager = JobManager(tmp_path, max_runtime_s=0.2)
+ job_id, directory = make_running_job(tmp_path)
+ use_idle_worker(monkeypatch)
+ try:
+ manager._run(directory)
+ outcome = read_json(directory / "result.json")
+ assert outcome["status"] == "inconclusive"
+ assert outcome["reason_code"] == "work_limit"
+ assert not (directory / "cancel").exists()
+ finally:
+ manager.close()