Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Parallelize fine-grid scoring across tile workers

Cinna Davis76d64e636e36
diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.py
index 954c9d8..d0f1ae0 100644
--- a/src/dopplergeist/parallel_tiles.py
+++ b/src/dopplergeist/parallel_tiles.py
@@ -4,12 +4,13 @@ from pathlib import Path
import tempfile
from threading import Lock
+import numpy as np
import pandas as pd
from .data_access import CachedTerrain, DataUnavailable, FlightIndex
from .finite_search import RetentionLedger
from .tiled_search import (_expanded_bounds, _partition_query_time_range, _sample_tile_terrain, _score, _tile_anchor_window,
- _tile_coordinates, _tracks_for_points)
+ _tile_coordinates, _tracks_for_points, tile_points)
class TileFailure(DataUnavailable):
@@ -203,6 +204,57 @@ def score_tile(stage, cells, score_floor=None):
raise
+def score_fine_tile(stage, cells, spacing, exclusions, score_floor=None):
+ """Score one fine-grid piece in a worker with its own retention ledger."""
+ if _worker is None:
+ raise RuntimeError("Tile worker was not initialized")
+ state = _worker
+ terrain = state["terrain"]
+ terrain.last_sample_files = set()
+ terrain.last_sample_ocean_tiles = set()
+ failures_before = len(getattr(terrain, "failures", ()))
+ path = None
+
+ def observed_terrain():
+ files = getattr(terrain, "files", {})
+ return {"terrain_files": [files[name] for name in sorted(terrain.last_sample_files) if name in files],
+ "terrain_ocean_tiles": sorted(terrain.last_sample_ocean_tiles),
+ "terrain_failures": getattr(terrain, "failures", [])[failures_before:]}
+
+ try:
+ points = tile_points(cells, spacing, state["bounds"], terrain, state["crs"])
+ keep = np.ones(len(points), dtype=bool)
+ for left, bottom, right, top in exclusions:
+ keep &= ~((points.x >= left * spacing) & (points.x < right * spacing)
+ & (points.y >= bottom * spacing) & (points.y < top * spacing))
+ skipped = int((~keep).sum())
+ points = points[keep]
+ if not len(points):
+ return {"skipped": skipped, "grid_points": 0, **observed_terrain()}, None
+ radius = max(event["slant_range_m"][1] for event in state["events"])
+ tracks, points = _tracks_for_points(
+ state["indexes"], _expanded_bounds(cells, spacing, radius, state["crs"]),
+ state["time_range"], state["budget"], points, terrain, radius,
+ state["parameters"].get("c", 343.0))
+ with tempfile.NamedTemporaryFile(prefix="fine-retention-", suffix=".sqlite",
+ dir=state["ledger_directory"], delete=False) as handle:
+ path = Path(handle.name)
+ ledger = RetentionLedger(path, state["run_hash"], state["score_kind"])
+ rows, counts = _score(points, tracks, state["events"], state["measurements"], state["budget"],
+ state["parameters"], state["filters"], state["start_window"],
+ ledger=ledger, stage=stage, score_floor=score_floor)
+ return {"ranked": rows, "counts": counts, "grid_points": len(points), "skipped": skipped,
+ "retention_checksum": ledger.checksum(stage), **observed_terrain()}, str(path)
+ except (DataUnavailable, ValueError, OSError) as error:
+ if path is not None:
+ path.unlink(missing_ok=True)
+ raise TileFailure(str(error), observed_terrain()) from error
+ except BaseException:
+ if path is not None:
+ path.unlink(missing_ok=True)
+ raise
+
+
def score_tile_with_cache_state(stage, cells, score_floor=None):
"""Return the worker's resident DEM keys for scheduling; never persist them."""
payload, worker_path = score_tile(stage, cells, score_floor)
diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.py
index 1ac735e..1d16dea 100644
--- a/src/dopplergeist/tiled_search.py
+++ b/src/dopplergeist/tiled_search.py
@@ -1104,48 +1104,98 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
fine = budget.refine_spacing_m
width = math.ceil(budget.spacing_m / fine)
completed_rectangles = []
- for seed_index, row in enumerate(chosen):
- # Subdivide the refinement square as well, retaining hard grid limits.
- cx, cy = round(row["x"] / fine), round(row["y"] / fine)
- for y0 in range(cy - width, cy + width + 1, budget.tile_cells):
- for x0 in range(cx - width, cx + width + 1, budget.tile_cells):
- root = (x0, y0, min(cx + width + 1, x0 + budget.tile_cells),
- min(cy + width + 1, y0 + budget.tile_cells))
- pending = [root]
- while pending:
- cells = pending.pop()
- points = tile_points(cells, fine, bounds, terrain, crs)
- keep = np.ones(len(points), dtype=bool)
- for left, bottom, right, top in completed_rectangles:
- keep &= ~((points.x >= left * fine) & (points.x < right * fine)
- & (points.y >= bottom * fine) & (points.y < top * fine))
- skipped = int((~keep).sum())
- points = points[keep]
- if not len(points):
- refinement_points_skipped += skipped
- continue
- try:
- tracks, points = _tracks_for_points(
- indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s,
- budget, points, terrain, radius, parameters.get("c", 343.0))
+ if parallel:
+ from collections import deque
+ from .data_access import _index_pool
+ from .parallel_tiles import score_fine_tile
+
+ with _index_pool(workers, initialize_tile_worker, initargs) as fine_pool:
+ for seed_index, row in enumerate(chosen):
+ cx, cy = round(row["x"] / fine), round(row["y"] / fine)
+ pending = deque((x0, y0, min(cx + width + 1, x0 + budget.tile_cells),
+ min(cy + width + 1, y0 + budget.tile_cells))
+ for y0 in range(cy - width, cy + width + 1, budget.tile_cells)
+ for x0 in range(cx - width, cx + width + 1, budget.tile_cells))
+ inflight = deque()
+ while pending or inflight:
+ while pending and len(inflight) < workers:
+ cells = pending.popleft()
stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"
- rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
- ledger=ledger, stage=stage,
- score_floor=_retained_score_floor(ranked, budget.top_k))
+ # The currently retained floor is conservative for every
+ # submitted piece, even if later merges raise it.
+ floor = _retained_score_floor(ranked, budget.top_k)
+ future = fine_pool.submit(score_fine_tile, stage, cells, fine,
+ tuple(completed_rectangles), floor)
+ inflight.append((cells, stage, future))
+ cells, stage, future = inflight.popleft()
+ try:
+ payload, worker_path = future.result()
except (DataUnavailable, ValueError, OSError) as error:
+ if isinstance(error, TileFailure):
+ restore_terrain(error.provenance)
parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []
if not parts:
raise
- pending.extend(reversed(parts))
+ pending.extendleft(reversed(parts))
continue
- refinement_points_skipped += skipped
- _record_support(output, stage, counts, support)
+ restore_terrain(payload)
+ refinement_points_skipped += payload["skipped"]
+ if worker_path is None:
+ continue
+ try:
+ import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])
+ finally:
+ Path(worker_path).unlink(missing_ok=True)
+ _record_support(output, stage, payload["counts"], support)
fine_stages += 1
- refinement_points_searched += len(points)
- ranked = merge_candidates(itertools.chain(ranked, rows), budget.top_k,
+ refinement_points_searched += payload["grid_points"]
+ ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,
ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")
- completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))
- report_search_progress(0.8 + 0.2 * (seed_index + 1) / len(chosen))
+ completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))
+ report_search_progress(0.8 + 0.2 * (seed_index + 1) / len(chosen))
+ else:
+ for seed_index, row in enumerate(chosen):
+ # Subdivide the refinement square as well, retaining hard grid limits.
+ cx, cy = round(row["x"] / fine), round(row["y"] / fine)
+ for y0 in range(cy - width, cy + width + 1, budget.tile_cells):
+ for x0 in range(cx - width, cx + width + 1, budget.tile_cells):
+ root = (x0, y0, min(cx + width + 1, x0 + budget.tile_cells),
+ min(cy + width + 1, y0 + budget.tile_cells))
+ pending = [root]
+ while pending:
+ cells = pending.pop()
+ points = tile_points(cells, fine, bounds, terrain, crs)
+ keep = np.ones(len(points), dtype=bool)
+ for left, bottom, right, top in completed_rectangles:
+ keep &= ~((points.x >= left * fine) & (points.x < right * fine)
+ & (points.y >= bottom * fine) & (points.y < top * fine))
+ skipped = int((~keep).sum())
+ points = points[keep]
+ if not len(points):
+ refinement_points_skipped += skipped
+ continue
+ try:
+ tracks, points = _tracks_for_points(
+ indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s,
+ budget, points, terrain, radius, parameters.get("c", 343.0))
+ stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"
+ rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
+ ledger=ledger, stage=stage,
+ score_floor=_retained_score_floor(ranked, budget.top_k))
+ except (DataUnavailable, ValueError, OSError) as error:
+ parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []
+ if not parts:
+ raise
+ pending.extend(reversed(parts))
+ continue
+ refinement_points_skipped += skipped
+ _record_support(output, stage, counts, support)
+ fine_stages += 1
+ refinement_points_searched += len(points)
+ ranked = merge_candidates(itertools.chain(ranked, rows), budget.top_k,
+ ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")
+ completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))
+ report_search_progress(0.8 + 0.2 * (seed_index + 1) / len(chosen))
except (DataUnavailable, ValueError, OSError) as error:
refinement_failure = str(error)
coverage = [i.coverage_for(bounds, time_range_s) for i in indexes]
diff --git a/tests/test_real_explicit_terrain_workers.py b/tests/test_real_explicit_terrain_workers.py
index eda7aa5..f9333bc 100644
--- a/tests/test_real_explicit_terrain_workers.py
+++ b/tests/test_real_explicit_terrain_workers.py
@@ -8,10 +8,12 @@ from pathlib import Path
import pytest
+from dopplergeist.data_access import FlightIndex
+from dopplergeist.finite_search import RetentionLedger
from dopplergeist.matching import prepare_events
-from dopplergeist.parallel_tiles import initialize_tile_worker, score_tile
+from dopplergeist.parallel_tiles import initialize_tile_worker, score_fine_tile, score_tile
from dopplergeist.terrain import Terrain
-from dopplergeist.tiled_search import SearchBudget
+from dopplergeist.tiled_search import SearchBudget, _expanded_bounds, _score, _tracks_for_points, tile_points
# Published ESID 762 site and a candidate from its saved real-audio search.
_POINTS = ((30.386361, -98.860931), (30.38874803589124, -98.87993276822816))
@@ -37,6 +39,10 @@ def _score_one(task):
return os.getpid(), payload
+def _score_fine_one(task):
+ return score_fine_tile(*task)[0]
+
+
def test_real_explicit_dem_reopened_in_two_workers(tmp_path, capfd):
names = ("DOPPLERGEIST_REAL_TERRAIN_DEM", "DOPPLERGEIST_REAL_TERRAIN_GEOID",
"DOPPLERGEIST_REAL_TERRAIN_FLIGHT_INDEX")
@@ -100,3 +106,66 @@ def test_real_explicit_tiles_match_direct_scoring(tmp_path):
assert len({pid for pid, _ in workers}) == 2
assert all(payload["grid_points"] > 0 and payload["ranked"] for payload in direct)
assert direct == [payload for _, payload in workers]
+
+
+def test_real_fine_piece_matches_spawned_worker(tmp_path):
+ names = ("DOPPLERGEIST_REAL_TERRAIN_DEM", "DOPPLERGEIST_REAL_TERRAIN_GEOID",
+ "DOPPLERGEIST_REAL_TERRAIN_FLIGHT_INDEX", "DOPPLERGEIST_REAL_TILE_JOB")
+ values = [os.environ.get(name) for name in names]
+ if not all(values):
+ pytest.skip("set real terrain, index and ESID 762 job paths to replay a recorded piece")
+ dem, geoid, index, saved_job = values
+ job = Path(saved_job)
+ config = json.loads((job / "search/manifest.json").read_text())["configuration"]
+ raw = json.loads((job / "discovery/discovery.json").read_text())["events"]
+ events, measurements = prepare_events(raw, c=343.0, max_distance_m=20000.0)
+ budget = SearchBudget(**config["budget"])
+ parameters = dict(config["scoring"])
+ window = parameters.pop("recording_start_window_s")
+ options = {"paths": [dem], "constant_m": None, "geoid_path": geoid}
+
+ def args(directory):
+ return ([index], None, options, events, measurements, config["bounds"], config["time_range_s"],
+ budget, parameters, config["filters"], window, config["crs"], tmp_path / directory,
+ "real-fine-replay", "matched_group_support_minus_normalized_residual", "explicit")
+
+ report = json.loads((job / "search/search-report.json").read_text())
+ row = report["coarse_ranked"][0]
+ spacing = budget.refine_spacing_m
+ x, y = round(row["x"] / spacing), round(row["y"] / spacing)
+ task = ("fine:0:real", (x, y, x + 2, y + 2), spacing, [], None)
+ terrain = Terrain([dem], geoid_path=geoid)
+ try:
+ points = tile_points(task[1], spacing, config["bounds"], terrain, config["crs"])
+ radius = max(event["slant_range_m"][1] for event in events)
+ tracks, points = _tracks_for_points(
+ [FlightIndex(index)], _expanded_bounds(task[1], spacing, radius, config["crs"]),
+ config["time_range_s"], budget, points, terrain, radius, parameters.get("c", 343.0))
+ ledger = RetentionLedger(tmp_path / "serial.sqlite", "real-fine-replay",
+ "matched_group_support_minus_normalized_residual")
+ rows, counts = _score(points, tracks, events, measurements, budget, parameters, config["filters"], window,
+ ledger=ledger, stage=task[0], score_floor=None)
+ finally:
+ for raster in terrain.rasters + terrain.geoid.rasters:
+ raster.close()
+ initialize_tile_worker(*args("direct-fine"))
+ direct, _ = score_fine_tile(*task)
+ partial_task = (task[0] + ":partial", task[1], spacing, [(x, y, x + 1, y + 2)], None)
+ partial, _ = score_fine_tile(*partial_task)
+ fully_excluded, empty_path = score_fine_tile(task[0] + ":excluded", task[1], spacing,
+ [(x, y, x + 2, y + 2)], None)
+ with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn"),
+ initializer=initialize_tile_worker, initargs=args("worker-fine")) as pool:
+ spawned = pool.submit(_score_fine_one, task).result(timeout=120)
+ spawned_partial = pool.submit(_score_fine_one, partial_task).result(timeout=120)
+ assert direct["grid_points"] > 0
+ assert direct["grid_points"] == len(points)
+ assert direct["ranked"] == rows
+ assert direct["counts"] == counts
+ assert direct["retention_checksum"] == ledger.checksum(task[0])
+ assert direct == spawned
+ assert partial["grid_points"] == len(points[points.x >= (x + 1) * spacing])
+ assert partial["skipped"] + partial["grid_points"] == direct["grid_points"]
+ assert partial == spawned_partial
+ assert fully_excluded["skipped"] == direct["grid_points"]
+ assert fully_excluded["grid_points"] == 0 and empty_path is None