Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Checkpoint completed worker tiles without blocking the queue
Cinna Davis0f25f9a40aff
diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.pyindex b86f6e7..cd3909f 100644--- a/src/dopplergeist/parallel_tiles.py+++ b/src/dopplergeist/parallel_tiles.py@@ -1,5 +1,4 @@"""Isolated coarse-tile workers with private retention ledgers."""-from collections import dequefrom pathlib import Pathimport tempfile@@ -93,25 +92,38 @@ def score_tile(stage, cells, score_floor=None):raise-def ordered_tile_futures(pool, jobs, workers):- """Prefetch at most two tiles per worker, yielding in search order.+def ordered_tile_results(pool, jobs, workers, finish):+ """Persist completed work promptly but yield lightweight handles in order."""+ from concurrent.futures import FIRST_COMPLETED, wait- Each job is ``(stage, cells, cached_payload, score_floor)``. Cached- payloads require no worker. For other jobs the caller receives a future,- imports its verified ledger, and checkpoints it before advancing.- """if type(workers) is not int or workers < 1:raise ValueError("workers must be a positive integer")- pending = deque()-- def enqueue(job):- stage, cells, cached, floor = job- future = None if cached is not None else pool.submit(score_tile, stage, cells, floor)- pending.append((stage, cells, cached, future))-- for job in jobs:- enqueue(job)- if len(pending) >= 2 * workers:- yield pending.popleft()- while pending:- yield pending.popleft()+ source = iter(jobs)+ pending, ready = {}, {}+ assigned = next_result = 0+ exhausted = False+ while not exhausted or pending or ready:+ while not exhausted and len(pending) < 2 * workers and next_result not in ready:+ try:+ stage, cells, cached, floor = next(source)+ except StopIteration:+ exhausted = True+ break+ index = assigned+ assigned += 1+ if cached is None:+ future = pool.submit(score_tile, stage, cells, floor)+ pending[future] = (index, stage, cells)+ else:+ ready[index] = (stage, cells, cached, True)+ if next_result in ready:+ yield ready.pop(next_result)+ next_result += 1+ continue+ if pending:+ completed, _ = wait(pending, return_when=FIRST_COMPLETED)+ for future in completed:+ index, stage, cells = pending.pop(future)+ ready[index] = (stage, cells, finish(stage, cells, future), False)+ elif exhausted and ready:+ raise RuntimeError("Tile result ordering is inconsistent")diff --git a/tests/test_real_parallel_retention.py b/tests/test_real_parallel_retention.pyindex d7c8021..78bf72f 100644--- a/tests/test_real_parallel_retention.py+++ b/tests/test_real_parallel_retention.py@@ -52,7 +52,7 @@ def test_real_tiles_match_sequential_scoring_and_preserve_ledgers(tmp_path):from dopplergeist.data_access import FlightIndex, content_hashfrom dopplergeist.matching import prepare_events- from dopplergeist.parallel_tiles import TileFailure, initialize_tile_worker, ordered_tile_futures, score_tile+ from dopplergeist.parallel_tiles import TileFailure, initialize_tile_worker, ordered_tile_results, score_tilefrom dopplergeist.tiled_search import SearchBudgetsource = os.environ.get("DOPPLERGEIST_REAL_JOB_ROOT")@@ -89,12 +89,13 @@ def test_real_tiles_match_sequential_scoring_and_preserve_ledgers(tmp_path):parallel = []with ProcessPoolExecutor(max_workers=2, mp_context=multiprocessing.get_context("spawn"),initializer=initialize_tile_worker, initargs=args) as pool:- for stage, cells, cached, future in ordered_tile_futures(pool, jobs, 2):- if cached is not None:- assert stage == stages[0][0] and cached == serial[0][0] and future is None+ for stage, cells, result, was_cached in ordered_tile_results(+ pool, jobs, 2, lambda stage, cells, future: future.result()):+ if was_cached:+ assert stage == stages[0][0] and result == serial[0][0]else:assert stage == stages[len(parallel)][0] and cells == stages[len(parallel)][1]- parallel.append(future.result())+ parallel.append(result)parent = RetentionLedger(tmp_path / "parallel-parent.sqlite", run_hash, score_kind)assert sum(p[0]["counts"].get("candidates", 0) for p in serial) > 0for (stage, _), (expected, expected_db), (actual, worker_db) in zip(stages, serial, parallel):