Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Clean unconsumed fine-stage worker ledgers
Cinna Davis200289c5fdb9
diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.pyindex d0f1ae0..0320a84 100644--- a/src/dopplergeist/parallel_tiles.py+++ b/src/dopplergeist/parallel_tiles.py@@ -204,6 +204,20 @@ def score_tile(stage, cells, score_floor=None):raise+def discard_pending_fine_ledgers(inflight):+ """Remove successful worker ledgers that were not consumed after an abort."""+ for _, _, future in inflight:+ try:+ _, path = future.result()+ except BaseException:+ continue+ if path is not None:+ try:+ Path(path).unlink()+ except OSError:+ pass++def score_fine_tile(stage, cells, spacing, exclusions, score_floor=None):"""Score one fine-grid piece in a worker with its own retention ledger."""if _worker is None:diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.pyindex 6fd6a7e..738925b 100644--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@ -1107,7 +1107,7 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outif parallel:from collections import dequefrom .data_access import _index_pool- from .parallel_tiles import score_fine_tile+ from .parallel_tiles import discard_pending_fine_ledgers, score_fine_tilewith _index_pool(workers, initialize_tile_worker, initargs) as fine_pool:for seed_index, row in enumerate(chosen):@@ -1117,41 +1117,44 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outfor y0 in range(cy - width, cy + width + 1, budget.tile_cells)for x0 in range(cx - width, cx + width + 1, budget.tile_cells))inflight = deque()- while pending or inflight:- while pending and len(inflight) < workers:- cells = pending.popleft()- stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"- # The currently retained floor is conservative for every- # submitted piece, even if later merges raise it.- floor = _retained_score_floor(ranked, budget.top_k)- future = fine_pool.submit(score_fine_tile, stage, cells, fine,- tuple(completed_rectangles), floor)- inflight.append((cells, stage, future))- cells, stage, future = inflight.popleft()- try:- payload, worker_path = future.result()- except (DataUnavailable, ValueError, OSError) as error:- if isinstance(error, TileFailure):- restore_terrain(error.provenance)- parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []- if not parts:- raise- pending.extendleft(reversed(parts))- continue- restore_terrain(payload)- refinement_points_skipped += payload["skipped"]- if worker_path is None:- continue- import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])- try:- Path(worker_path).unlink()- except OSError:- pass- _record_support(output, stage, payload["counts"], support)- fine_stages += 1- refinement_points_searched += payload["grid_points"]- ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,- ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")+ try:+ while pending or inflight:+ while pending and len(inflight) < workers:+ cells = pending.popleft()+ stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"+ # The currently retained floor is conservative for every+ # submitted piece, even if later merges raise it.+ floor = _retained_score_floor(ranked, budget.top_k)+ future = fine_pool.submit(score_fine_tile, stage, cells, fine,+ tuple(completed_rectangles), floor)+ inflight.append((cells, stage, future))+ cells, stage, future = inflight.popleft()+ try:+ payload, worker_path = future.result()+ except (DataUnavailable, ValueError, OSError) as error:+ if isinstance(error, TileFailure):+ restore_terrain(error.provenance)+ parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []+ if not parts:+ raise+ pending.extendleft(reversed(parts))+ continue+ restore_terrain(payload)+ refinement_points_skipped += payload["skipped"]+ if worker_path is None:+ continue+ import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])+ try:+ Path(worker_path).unlink()+ except OSError:+ pass+ _record_support(output, stage, payload["counts"], support)+ fine_stages += 1+ refinement_points_searched += payload["grid_points"]+ ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,+ ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")+ finally:+ discard_pending_fine_ledgers(inflight)completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))report_search_progress(0.8 + 0.2 * (seed_index + 1) / len(chosen))else:diff --git a/tests/test_real_explicit_terrain_workers.py b/tests/test_real_explicit_terrain_workers.pyindex f9333bc..0fa63f4 100644--- a/tests/test_real_explicit_terrain_workers.py+++ b/tests/test_real_explicit_terrain_workers.py@@ -1,6 +1,6 @@"""Worker initialization against operator-supplied real terrain, without a search."""-from concurrent.futures import ProcessPoolExecutor+from concurrent.futures import Future, ProcessPoolExecutorfrom multiprocessing import get_contextimport jsonimport os@@ -11,7 +11,8 @@ import pytestfrom dopplergeist.data_access import FlightIndexfrom dopplergeist.finite_search import RetentionLedgerfrom dopplergeist.matching import prepare_events-from dopplergeist.parallel_tiles import initialize_tile_worker, score_fine_tile, score_tile+from dopplergeist.parallel_tiles import (discard_pending_fine_ledgers, initialize_tile_worker,+ score_fine_tile, score_tile)from dopplergeist.terrain import Terrainfrom dopplergeist.tiled_search import SearchBudget, _expanded_bounds, _score, _tracks_for_points, tile_points@@ -20,6 +21,22 @@ _POINTS = ((30.386361, -98.860931), (30.38874803589124, -98.87993276822816))_barrier = None+def test_aborted_fine_search_cleans_only_unconsumed_ledgers(tmp_path):+ orphan = tmp_path / "unconsumed.sqlite"+ orphan.touch()+ failed_import = tmp_path / "failed-import.sqlite"+ failed_import.touch()+ completed = Future()+ completed.set_result(({}, str(orphan)))+ failed = Future()+ failed.set_exception(ValueError("piece failed"))+ empty = Future()+ empty.set_result(({}, None))+ discard_pending_fine_ledgers([((), "failed", failed), ((), "done", completed), ((), "empty", empty)])+ assert not orphan.exists()+ assert failed_import.exists()++def _initialize(args, barrier):global _barrierinitialize_tile_worker(*args)