Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Clean unconsumed fine-stage worker ledgers

Cinna Davis200289c5fdb9
diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.py
index d0f1ae0..0320a84 100644
--- a/src/dopplergeist/parallel_tiles.py
+++ b/src/dopplergeist/parallel_tiles.py
@@ -204,6 +204,20 @@ def score_tile(stage, cells, score_floor=None):
raise
+def discard_pending_fine_ledgers(inflight):
+ """Remove successful worker ledgers that were not consumed after an abort."""
+ for _, _, future in inflight:
+ try:
+ _, path = future.result()
+ except BaseException:
+ continue
+ if path is not None:
+ try:
+ Path(path).unlink()
+ except OSError:
+ pass
+
+
def score_fine_tile(stage, cells, spacing, exclusions, score_floor=None):
"""Score one fine-grid piece in a worker with its own retention ledger."""
if _worker is None:
diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.py
index 6fd6a7e..738925b 100644
--- a/src/dopplergeist/tiled_search.py
+++ b/src/dopplergeist/tiled_search.py
@@ -1107,7 +1107,7 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
if parallel:
from collections import deque
from .data_access import _index_pool
- from .parallel_tiles import score_fine_tile
+ from .parallel_tiles import discard_pending_fine_ledgers, score_fine_tile
with _index_pool(workers, initialize_tile_worker, initargs) as fine_pool:
for seed_index, row in enumerate(chosen):
@@ -1117,41 +1117,44 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
for y0 in range(cy - width, cy + width + 1, budget.tile_cells)
for x0 in range(cx - width, cx + width + 1, budget.tile_cells))
inflight = deque()
- while pending or inflight:
- while pending and len(inflight) < workers:
- cells = pending.popleft()
- stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"
- # The currently retained floor is conservative for every
- # submitted piece, even if later merges raise it.
- floor = _retained_score_floor(ranked, budget.top_k)
- future = fine_pool.submit(score_fine_tile, stage, cells, fine,
- tuple(completed_rectangles), floor)
- inflight.append((cells, stage, future))
- cells, stage, future = inflight.popleft()
- try:
- payload, worker_path = future.result()
- except (DataUnavailable, ValueError, OSError) as error:
- if isinstance(error, TileFailure):
- restore_terrain(error.provenance)
- parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []
- if not parts:
- raise
- pending.extendleft(reversed(parts))
- continue
- restore_terrain(payload)
- refinement_points_skipped += payload["skipped"]
- if worker_path is None:
- continue
- import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])
- try:
- Path(worker_path).unlink()
- except OSError:
- pass
- _record_support(output, stage, payload["counts"], support)
- fine_stages += 1
- refinement_points_searched += payload["grid_points"]
- ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,
- ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")
+ try:
+ while pending or inflight:
+ while pending and len(inflight) < workers:
+ cells = pending.popleft()
+ stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"
+ # The currently retained floor is conservative for every
+ # submitted piece, even if later merges raise it.
+ floor = _retained_score_floor(ranked, budget.top_k)
+ future = fine_pool.submit(score_fine_tile, stage, cells, fine,
+ tuple(completed_rectangles), floor)
+ inflight.append((cells, stage, future))
+ cells, stage, future = inflight.popleft()
+ try:
+ payload, worker_path = future.result()
+ except (DataUnavailable, ValueError, OSError) as error:
+ if isinstance(error, TileFailure):
+ restore_terrain(error.provenance)
+ parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []
+ if not parts:
+ raise
+ pending.extendleft(reversed(parts))
+ continue
+ restore_terrain(payload)
+ refinement_points_skipped += payload["skipped"]
+ if worker_path is None:
+ continue
+ import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])
+ try:
+ Path(worker_path).unlink()
+ except OSError:
+ pass
+ _record_support(output, stage, payload["counts"], support)
+ fine_stages += 1
+ refinement_points_searched += payload["grid_points"]
+ ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,
+ ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")
+ finally:
+ discard_pending_fine_ledgers(inflight)
completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))
report_search_progress(0.8 + 0.2 * (seed_index + 1) / len(chosen))
else:
diff --git a/tests/test_real_explicit_terrain_workers.py b/tests/test_real_explicit_terrain_workers.py
index f9333bc..0fa63f4 100644
--- a/tests/test_real_explicit_terrain_workers.py
+++ b/tests/test_real_explicit_terrain_workers.py
@@ -1,6 +1,6 @@
"""Worker initialization against operator-supplied real terrain, without a search."""
-from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures import Future, ProcessPoolExecutor
from multiprocessing import get_context
import json
import os
@@ -11,7 +11,8 @@ import pytest
from dopplergeist.data_access import FlightIndex
from dopplergeist.finite_search import RetentionLedger
from dopplergeist.matching import prepare_events
-from dopplergeist.parallel_tiles import initialize_tile_worker, score_fine_tile, score_tile
+from dopplergeist.parallel_tiles import (discard_pending_fine_ledgers, initialize_tile_worker,
+ score_fine_tile, score_tile)
from dopplergeist.terrain import Terrain
from dopplergeist.tiled_search import SearchBudget, _expanded_bounds, _score, _tracks_for_points, tile_points
@@ -20,6 +21,22 @@ _POINTS = ((30.386361, -98.860931), (30.38874803589124, -98.87993276822816))
_barrier = None
+def test_aborted_fine_search_cleans_only_unconsumed_ledgers(tmp_path):
+ orphan = tmp_path / "unconsumed.sqlite"
+ orphan.touch()
+ failed_import = tmp_path / "failed-import.sqlite"
+ failed_import.touch()
+ completed = Future()
+ completed.set_result(({}, str(orphan)))
+ failed = Future()
+ failed.set_exception(ValueError("piece failed"))
+ empty = Future()
+ empty.set_result(({}, None))
+ discard_pending_fine_ledgers([((), "failed", failed), ((), "done", completed), ((), "empty", empty)])
+ assert not orphan.exists()
+ assert failed_import.exists()
+
+
def _initialize(args, barrier):
global _barrier
initialize_tile_worker(*args)