Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Parallelize fine-grid scoring across tile workers
Cinna Davis76d64e636e36
diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.pyindex 954c9d8..d0f1ae0 100644--- a/src/dopplergeist/parallel_tiles.py+++ b/src/dopplergeist/parallel_tiles.py@@ -4,12 +4,13 @@ from pathlib import Pathimport tempfilefrom threading import Lock+import numpy as npimport pandas as pdfrom .data_access import CachedTerrain, DataUnavailable, FlightIndexfrom .finite_search import RetentionLedgerfrom .tiled_search import (_expanded_bounds, _partition_query_time_range, _sample_tile_terrain, _score, _tile_anchor_window,- _tile_coordinates, _tracks_for_points)+ _tile_coordinates, _tracks_for_points, tile_points)class TileFailure(DataUnavailable):@@ -203,6 +204,57 @@ def score_tile(stage, cells, score_floor=None):raise+def score_fine_tile(stage, cells, spacing, exclusions, score_floor=None):+ """Score one fine-grid piece in a worker with its own retention ledger."""+ if _worker is None:+ raise RuntimeError("Tile worker was not initialized")+ state = _worker+ terrain = state["terrain"]+ terrain.last_sample_files = set()+ terrain.last_sample_ocean_tiles = set()+ failures_before = len(getattr(terrain, "failures", ()))+ path = None++ def observed_terrain():+ files = getattr(terrain, "files", {})+ return {"terrain_files": [files[name] for name in sorted(terrain.last_sample_files) if name in files],+ "terrain_ocean_tiles": sorted(terrain.last_sample_ocean_tiles),+ "terrain_failures": getattr(terrain, "failures", [])[failures_before:]}++ try:+ points = tile_points(cells, spacing, state["bounds"], terrain, state["crs"])+ keep = np.ones(len(points), dtype=bool)+ for left, bottom, right, top in exclusions:+ keep &= ~((points.x >= left * spacing) & (points.x < right * spacing)+ & (points.y >= bottom * spacing) & (points.y < top * spacing))+ skipped = int((~keep).sum())+ points = points[keep]+ if not len(points):+ return {"skipped": skipped, "grid_points": 0, **observed_terrain()}, None+ radius = max(event["slant_range_m"][1] for event in state["events"])+ tracks, points = _tracks_for_points(+ state["indexes"], _expanded_bounds(cells, spacing, radius, state["crs"]),+ state["time_range"], state["budget"], points, terrain, radius,+ state["parameters"].get("c", 343.0))+ with tempfile.NamedTemporaryFile(prefix="fine-retention-", suffix=".sqlite",+ dir=state["ledger_directory"], delete=False) as handle:+ path = Path(handle.name)+ ledger = RetentionLedger(path, state["run_hash"], state["score_kind"])+ rows, counts = _score(points, tracks, state["events"], state["measurements"], state["budget"],+ state["parameters"], state["filters"], state["start_window"],+ ledger=ledger, stage=stage, score_floor=score_floor)+ return {"ranked": rows, "counts": counts, "grid_points": len(points), "skipped": skipped,+ "retention_checksum": ledger.checksum(stage), **observed_terrain()}, str(path)+ except (DataUnavailable, ValueError, OSError) as error:+ if path is not None:+ path.unlink(missing_ok=True)+ raise TileFailure(str(error), observed_terrain()) from error+ except BaseException:+ if path is not None:+ path.unlink(missing_ok=True)+ raise++def score_tile_with_cache_state(stage, cells, score_floor=None):"""Return the worker's resident DEM keys for scheduling; never persist them."""payload, worker_path = score_tile(stage, cells, score_floor)diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.pyindex 1ac735e..1d16dea 100644--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@ -1104,48 +1104,98 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outfine = budget.refine_spacing_mwidth = math.ceil(budget.spacing_m / fine)completed_rectangles = []- for seed_index, row in enumerate(chosen):- # Subdivide the refinement square as well, retaining hard grid limits.- cx, cy = round(row["x"] / fine), round(row["y"] / fine)- for y0 in range(cy - width, cy + width + 1, budget.tile_cells):- for x0 in range(cx - width, cx + width + 1, budget.tile_cells):- root = (x0, y0, min(cx + width + 1, x0 + budget.tile_cells),- min(cy + width + 1, y0 + budget.tile_cells))- pending = [root]- while pending:- cells = pending.pop()- points = tile_points(cells, fine, bounds, terrain, crs)- keep = np.ones(len(points), dtype=bool)- for left, bottom, right, top in completed_rectangles:- keep &= ~((points.x >= left * fine) & (points.x < right * fine)- & (points.y >= bottom * fine) & (points.y < top * fine))- skipped = int((~keep).sum())- points = points[keep]- if not len(points):- refinement_points_skipped += skipped- continue- try:- tracks, points = _tracks_for_points(- indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s,- budget, points, terrain, radius, parameters.get("c", 343.0))+ if parallel:+ from collections import deque+ from .data_access import _index_pool+ from .parallel_tiles import score_fine_tile++ with _index_pool(workers, initialize_tile_worker, initargs) as fine_pool:+ for seed_index, row in enumerate(chosen):+ cx, cy = round(row["x"] / fine), round(row["y"] / fine)+ pending = deque((x0, y0, min(cx + width + 1, x0 + budget.tile_cells),+ min(cy + width + 1, y0 + budget.tile_cells))+ for y0 in range(cy - width, cy + width + 1, budget.tile_cells)+ for x0 in range(cx - width, cx + width + 1, budget.tile_cells))+ inflight = deque()+ while pending or inflight:+ while pending and len(inflight) < workers:+ cells = pending.popleft()stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"- rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,- ledger=ledger, stage=stage,- score_floor=_retained_score_floor(ranked, budget.top_k))+ # The currently retained floor is conservative for every+ # submitted piece, even if later merges raise it.+ floor = _retained_score_floor(ranked, budget.top_k)+ future = fine_pool.submit(score_fine_tile, stage, cells, fine,+ tuple(completed_rectangles), floor)+ inflight.append((cells, stage, future))+ cells, stage, future = inflight.popleft()+ try:+ payload, worker_path = future.result()except (DataUnavailable, ValueError, OSError) as error:+ if isinstance(error, TileFailure):+ restore_terrain(error.provenance)parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []if not parts:raise- pending.extend(reversed(parts))+ pending.extendleft(reversed(parts))continue- refinement_points_skipped += skipped- _record_support(output, stage, counts, support)+ restore_terrain(payload)+ refinement_points_skipped += payload["skipped"]+ if worker_path is None:+ continue+ try:+ import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])+ finally:+ Path(worker_path).unlink(missing_ok=True)+ _record_support(output, stage, payload["counts"], support)fine_stages += 1- refinement_points_searched += len(points)- ranked = merge_candidates(itertools.chain(ranked, rows), budget.top_k,+ refinement_points_searched += payload["grid_points"]+ ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")- completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))- report_search_progress(0.8 + 0.2 * (seed_index + 1) / len(chosen))+ completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))+ report_search_progress(0.8 + 0.2 * (seed_index + 1) / len(chosen))+ else:+ for seed_index, row in enumerate(chosen):+ # Subdivide the refinement square as well, retaining hard grid limits.+ cx, cy = round(row["x"] / fine), round(row["y"] / fine)+ for y0 in range(cy - width, cy + width + 1, budget.tile_cells):+ for x0 in range(cx - width, cx + width + 1, budget.tile_cells):+ root = (x0, y0, min(cx + width + 1, x0 + budget.tile_cells),+ min(cy + width + 1, y0 + budget.tile_cells))+ pending = [root]+ while pending:+ cells = pending.pop()+ points = tile_points(cells, fine, bounds, terrain, crs)+ keep = np.ones(len(points), dtype=bool)+ for left, bottom, right, top in completed_rectangles:+ keep &= ~((points.x >= left * fine) & (points.x < right * fine)+ & (points.y >= bottom * fine) & (points.y < top * fine))+ skipped = int((~keep).sum())+ points = points[keep]+ if not len(points):+ refinement_points_skipped += skipped+ continue+ try:+ tracks, points = _tracks_for_points(+ indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s,+ budget, points, terrain, radius, parameters.get("c", 343.0))+ stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"+ rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,+ ledger=ledger, stage=stage,+ score_floor=_retained_score_floor(ranked, budget.top_k))+ except (DataUnavailable, ValueError, OSError) as error:+ parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []+ if not parts:+ raise+ pending.extend(reversed(parts))+ continue+ refinement_points_skipped += skipped+ _record_support(output, stage, counts, support)+ fine_stages += 1+ refinement_points_searched += len(points)+ ranked = merge_candidates(itertools.chain(ranked, rows), budget.top_k,+ ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")+ completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))+ report_search_progress(0.8 + 0.2 * (seed_index + 1) / len(chosen))except (DataUnavailable, ValueError, OSError) as error:refinement_failure = str(error)coverage = [i.coverage_for(bounds, time_range_s) for i in indexes]diff --git a/tests/test_real_explicit_terrain_workers.py b/tests/test_real_explicit_terrain_workers.pyindex eda7aa5..f9333bc 100644--- a/tests/test_real_explicit_terrain_workers.py+++ b/tests/test_real_explicit_terrain_workers.py@@ -8,10 +8,12 @@ from pathlib import Pathimport pytest+from dopplergeist.data_access import FlightIndex+from dopplergeist.finite_search import RetentionLedgerfrom dopplergeist.matching import prepare_events-from dopplergeist.parallel_tiles import initialize_tile_worker, score_tile+from dopplergeist.parallel_tiles import initialize_tile_worker, score_fine_tile, score_tilefrom dopplergeist.terrain import Terrain-from dopplergeist.tiled_search import SearchBudget+from dopplergeist.tiled_search import SearchBudget, _expanded_bounds, _score, _tracks_for_points, tile_points# Published ESID 762 site and a candidate from its saved real-audio search._POINTS = ((30.386361, -98.860931), (30.38874803589124, -98.87993276822816))@@ -37,6 +39,10 @@ def _score_one(task):return os.getpid(), payload+def _score_fine_one(task):+ return score_fine_tile(*task)[0]++def test_real_explicit_dem_reopened_in_two_workers(tmp_path, capfd):names = ("DOPPLERGEIST_REAL_TERRAIN_DEM", "DOPPLERGEIST_REAL_TERRAIN_GEOID","DOPPLERGEIST_REAL_TERRAIN_FLIGHT_INDEX")@@ -100,3 +106,66 @@ def test_real_explicit_tiles_match_direct_scoring(tmp_path):assert len({pid for pid, _ in workers}) == 2assert all(payload["grid_points"] > 0 and payload["ranked"] for payload in direct)assert direct == [payload for _, payload in workers]+++def test_real_fine_piece_matches_spawned_worker(tmp_path):+ names = ("DOPPLERGEIST_REAL_TERRAIN_DEM", "DOPPLERGEIST_REAL_TERRAIN_GEOID",+ "DOPPLERGEIST_REAL_TERRAIN_FLIGHT_INDEX", "DOPPLERGEIST_REAL_TILE_JOB")+ values = [os.environ.get(name) for name in names]+ if not all(values):+ pytest.skip("set real terrain, index and ESID 762 job paths to replay a recorded piece")+ dem, geoid, index, saved_job = values+ job = Path(saved_job)+ config = json.loads((job / "search/manifest.json").read_text())["configuration"]+ raw = json.loads((job / "discovery/discovery.json").read_text())["events"]+ events, measurements = prepare_events(raw, c=343.0, max_distance_m=20000.0)+ budget = SearchBudget(**config["budget"])+ parameters = dict(config["scoring"])+ window = parameters.pop("recording_start_window_s")+ options = {"paths": [dem], "constant_m": None, "geoid_path": geoid}++ def args(directory):+ return ([index], None, options, events, measurements, config["bounds"], config["time_range_s"],+ budget, parameters, config["filters"], window, config["crs"], tmp_path / directory,+ "real-fine-replay", "matched_group_support_minus_normalized_residual", "explicit")++ report = json.loads((job / "search/search-report.json").read_text())+ row = report["coarse_ranked"][0]+ spacing = budget.refine_spacing_m+ x, y = round(row["x"] / spacing), round(row["y"] / spacing)+ task = ("fine:0:real", (x, y, x + 2, y + 2), spacing, [], None)+ terrain = Terrain([dem], geoid_path=geoid)+ try:+ points = tile_points(task[1], spacing, config["bounds"], terrain, config["crs"])+ radius = max(event["slant_range_m"][1] for event in events)+ tracks, points = _tracks_for_points(+ [FlightIndex(index)], _expanded_bounds(task[1], spacing, radius, config["crs"]),+ config["time_range_s"], budget, points, terrain, radius, parameters.get("c", 343.0))+ ledger = RetentionLedger(tmp_path / "serial.sqlite", "real-fine-replay",+ "matched_group_support_minus_normalized_residual")+ rows, counts = _score(points, tracks, events, measurements, budget, parameters, config["filters"], window,+ ledger=ledger, stage=task[0], score_floor=None)+ finally:+ for raster in terrain.rasters + terrain.geoid.rasters:+ raster.close()+ initialize_tile_worker(*args("direct-fine"))+ direct, _ = score_fine_tile(*task)+ partial_task = (task[0] + ":partial", task[1], spacing, [(x, y, x + 1, y + 2)], None)+ partial, _ = score_fine_tile(*partial_task)+ fully_excluded, empty_path = score_fine_tile(task[0] + ":excluded", task[1], spacing,+ [(x, y, x + 2, y + 2)], None)+ with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn"),+ initializer=initialize_tile_worker, initargs=args("worker-fine")) as pool:+ spawned = pool.submit(_score_fine_one, task).result(timeout=120)+ spawned_partial = pool.submit(_score_fine_one, partial_task).result(timeout=120)+ assert direct["grid_points"] > 0+ assert direct["grid_points"] == len(points)+ assert direct["ranked"] == rows+ assert direct["counts"] == counts+ assert direct["retention_checksum"] == ledger.checksum(task[0])+ assert direct == spawned+ assert partial["grid_points"] == len(points[points.x >= (x + 1) * spacing])+ assert partial["skipped"] + partial["grid_points"] == direct["grid_points"]+ assert partial == spawned_partial+ assert fully_excluded["skipped"] == direct["grid_points"]+ assert fully_excluded["grid_points"] == 0 and empty_path is None