Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Merge remote-tracking branch 'origin/main' into perf/adaptive-search-tiles-20261103

Cinna Davisb700ade2ed13
diff --cc src/dopplergeist/tiled_search.py
index f573457,70f3863..bccbd92
--- a/src/dopplergeist/tiled_search.py
+++ b/src/dopplergeist/tiled_search.py
@@@ -525,6 -503,18 +525,23 @@@ def search_national(indexes, terrain, e
from .parallel_tiles import TileFailure, initialize_tile_worker, ordered_tile_results
xmin, ymin, xmax, ymax = _grid_extent(bounds, budget.spacing_m, crs)
total = (xmax // budget.tile_cells - xmin // budget.tile_cells + 1) * (ymax // budget.tile_cells - ymin // budget.tile_cells + 1)
- coarse_total = min(total, budget.max_tiles or total)
++ if budget.max_tiles is None:
++ coarse_cells = (xmax-xmin+1) * (ymax-ymin+1)
++ else:
++ coarse_cells = sum((cells[2]-cells[0]) * (cells[3]-cells[1])
++ for _, cells in itertools.islice(
++ grid_tiles(bounds, budget.spacing_m, budget.tile_cells, crs), budget.max_tiles))
+ last_progress = 0.0
+
+ def report_search_progress(fraction, *, force=False):
+ nonlocal last_progress
+ if search_progress is None:
+ return
+ fraction = min(1.0, max(0.0, float(fraction)))
+ if fraction > last_progress and (force or fraction - last_progress >= 0.025):
+ search_progress(fraction)
+ last_progress = fraction
+
available = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else (os.cpu_count() or 1)
if workers is None:
workers = max(1, available - 1)
@@@ -572,7 -562,7 +589,7 @@@
else {"robust_ridge": "negative_robust_ridge_loss_hz", "ridge": "negative_ridge_rmse_hz",
"brightness": "summed_spectral_contrast"}[mode])
ledger = RetentionLedger(output / "retention.sqlite", run_hash, score_kind)
- began, ranked, failures, done, resumed, points_searched = time.monotonic(), [], [], 0, 0, 0
- began, ranked, failures, done, processed, resumed, points_searched = time.monotonic(), [], [], 0, 0, 0, 0
++ began, ranked, failures, done, processed_cells, resumed, points_searched = time.monotonic(), [], [], 0, 0, 0, 0
support = dict(unsupported_cpa_track_evaluations=0, unclassified_score_candidates=0,
bounded_score_candidates=0, unbounded_score_candidates=0,
evaluated_trials=0, rejected_trials=0, unknown_trials=0, unreported_trial_envelopes=0)
@@@ -657,62 -626,53 +674,69 @@@
return error
with context as pool:
- requests = tile_requests()
- results = (ordered_tile_results(pool, requests, workers, finish_tile) if pool is not None else
- ((stage, cells, cached, cached is not None) for stage, cells, cached, _ in requests))
- for stage, cells, prepared, was_cached in results:
- tile_id = stage[len("coarse:"):]
- checkpoint = output / f"tile-{tile_id}.json"
- try:
- if isinstance(prepared, BaseException):
- raise prepared
- if prepared is not None:
- payload = json.loads(prepared.read_text())["payload"]
- if was_cached:
- resumed += 1
- restore_terrain(payload)
- else:
- points = _tile_coordinates(cells, budget.spacing_m, bounds, crs)
- tracks = (_tracks(indexes, _expanded_bounds(cells, budget.spacing_m, radius, crs), time_range_s, budget)
- if len(points) else [])
- points = _sample_tile_terrain(points, terrain) if tracks else pd.DataFrame()
- if len(points):
- rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
- ledger=ledger, stage=stage,
- score_floor=(_retained_score_floor(ranked, budget.top_k)
- if floor_policy == "global_and_per_tile" else None))
+ roots = grid_tiles(bounds, budget.spacing_m, budget.tile_cells, crs)
+ wave = itertools.islice(roots, budget.max_tiles) if budget.max_tiles is not None else roots
+ while wave is not None:
+ next_wave = []
+ requests = requests_for_wave(wave, next_wave)
+ results = (ordered_tile_results(pool, requests, workers, finish_tile) if pool is not None else
+ ((stage, cells, cached, cached is not None) for stage, cells, cached, _ in requests))
+ for stage, cells, prepared, was_cached in results:
+ tile_id = stage[len("coarse:"):]
+ checkpoint = output / f"tile-{tile_id}.json"
++ terminal = True
+ try:
+ if isinstance(prepared, BaseException):
+ raise prepared
+ if prepared is not None:
+ payload = json.loads(prepared.read_text())["payload"]
+ if was_cached:
+ resumed += 1
+ restore_terrain(payload)
else:
- rows, counts = [], {}
- ledger.record(stage, [], [], 0)
- used = getattr(terrain, "last_sample_files", set()) if len(points) else set()
- terrain_files = [terrain.files[name] for name in sorted(used)] if used else []
- oceans = sorted(getattr(terrain, "last_sample_ocean_tiles", set())) if len(points) else []
- payload = {"ranked": rows, "counts": counts, "grid_points": len(points), "tracks": len(tracks),
- "cells": list(cells), "terrain_files": terrain_files, "terrain_ocean_tiles": oceans,
- "terrain_failures": [], "retention_checksum": ledger.checksum(stage)}
- write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload})
- done += 1
- points_searched += payload["grid_points"]
- _record_support(output, stage, payload["counts"], support)
- ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,
- ledger=ledger, stage=f"global:{tile_id}")
- except (DataUnavailable, ValueError, OSError) as error:
- if isinstance(error, TileFailure):
- restore_terrain(error.provenance)
- failures.append({"tile": tile_id, "reason": str(error)})
- processed += 1
- if coarse_total:
- report_search_progress(0.8 * processed / coarse_total)
+ points = _tile_coordinates(cells, budget.spacing_m, bounds, crs)
+ tracks = (_tracks(indexes, _expanded_bounds(cells, budget.spacing_m, radius, crs), time_range_s, budget)
+ if len(points) else [])
+ points = _sample_tile_terrain(points, terrain) if tracks else pd.DataFrame()
+ if len(points):
+ rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
+ ledger=ledger, stage=stage,
+ score_floor=(_retained_score_floor(ranked, budget.top_k)
+ if floor_policy == "global_and_per_tile" else None))
+ else:
+ rows, counts = [], {}
+ ledger.record(stage, [], [], 0)
+ used = getattr(terrain, "last_sample_files", set()) if len(points) else set()
+ terrain_files = [terrain.files[name] for name in sorted(used)] if used else []
+ oceans = sorted(getattr(terrain, "last_sample_ocean_tiles", set())) if len(points) else []
+ payload = {"ranked": rows, "counts": counts, "grid_points": len(points), "tracks": len(tracks),
+ "cells": list(cells), "terrain_files": terrain_files, "terrain_ocean_tiles": oceans,
+ "terrain_failures": [], "retention_checksum": ledger.checksum(stage)}
+ write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload})
+ done += 1
+ points_searched += payload["grid_points"]
+ _record_support(output, stage, payload["counts"], support)
+ ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,
+ ledger=ledger, stage=f"global:{tile_id}")
+ except (DataUnavailable, ValueError, OSError) as error:
+ if isinstance(error, TileFailure):
+ restore_terrain(error.provenance)
+ parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []
+ if parts:
++ terminal = False
+ children = [[f"{tile_id}~{i}", list(part)] for i, part in enumerate(parts)]
+ write_json(output / f"split-{tile_id}.json",
+ {"run_hash": run_hash, "cells": list(cells), "children": children})
+ total += len(children)-1
+ next_wave.extend((child_id, tuple(child_cells)) for child_id, child_cells in children)
+ else:
+ failures.append({"tile": tile_id, "reason": str(error)})
++ if terminal:
++ processed_cells += (cells[2]-cells[0]) * (cells[3]-cells[1])
++ if coarse_cells:
++ report_search_progress(0.8 * processed_cells / coarse_cells)
+ wave = sorted(next_wave, key=tile_order) if next_wave else None
+ report_search_progress(0.8, force=True)
coarse = list(ranked)
# Refine only global finalists; tile selection never controls refinement rank.
refinement_failure = None
@@@ -736,40 -696,28 +760,41 @@@
cx, cy = round(row["x"] / fine), round(row["y"] / fine)
for y0 in range(cy - width, cy + width + 1, budget.tile_cells):
for x0 in range(cx - width, cx + width + 1, budget.tile_cells):
- cells = (x0, y0, min(cx + width + 1, x0 + budget.tile_cells), min(cy + width + 1, y0 + budget.tile_cells))
- points = tile_points(cells, fine, bounds, terrain, crs)
- keep = np.ones(len(points), dtype=bool)
- for left, bottom, right, top in completed_rectangles:
- keep &= ~((points.x >= left * fine) & (points.x < right * fine)
- & (points.y >= bottom * fine) & (points.y < top * fine))
- refinement_points_skipped += int((~keep).sum())
- points = points[keep]
- if not len(points):
- continue
- tracks = _tracks(indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s, budget)
- stage = f"fine:{seed_index}:{x0}:{y0}"
- rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
- ledger=ledger, stage=stage,
- score_floor=_retained_score_floor(ranked, budget.top_k))
- _record_support(output, stage, counts, support)
- fine_stages += 1
- refinement_points_searched += len(points)
- ranked = merge_candidates(itertools.chain(ranked, rows), budget.top_k,
- ledger=ledger, stage=f"fine-global:{seed_index}:{x0}:{y0}")
+ root = (x0, y0, min(cx + width + 1, x0 + budget.tile_cells),
+ min(cy + width + 1, y0 + budget.tile_cells))
+ pending = [root]
+ while pending:
+ cells = pending.pop()
+ points = tile_points(cells, fine, bounds, terrain, crs)
+ keep = np.ones(len(points), dtype=bool)
+ for left, bottom, right, top in completed_rectangles:
+ keep &= ~((points.x >= left * fine) & (points.x < right * fine)
+ & (points.y >= bottom * fine) & (points.y < top * fine))
+ skipped = int((~keep).sum())
+ points = points[keep]
+ if not len(points):
+ refinement_points_skipped += skipped
+ continue
+ try:
+ tracks = _tracks(indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s, budget)
+ stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"
+ rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
+ ledger=ledger, stage=stage,
+ score_floor=_retained_score_floor(ranked, budget.top_k))
+ except (DataUnavailable, ValueError, OSError) as error:
+ parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []
+ if not parts:
+ raise
+ pending.extend(reversed(parts))
+ continue
+ refinement_points_skipped += skipped
+ _record_support(output, stage, counts, support)
+ fine_stages += 1
+ refinement_points_searched += len(points)
+ ranked = merge_candidates(itertools.chain(ranked, rows), budget.top_k,
+ ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")
completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))
+ report_search_progress(0.8 + 0.2 * (seed_index + 1) / len(chosen))
except (DataUnavailable, ValueError, OSError) as error:
refinement_failure = str(error)
coverage = [i.coverage_for(bounds, time_range_s) for i in indexes]
diff --cc tests/test_real_adaptive_tiles.py
index 800bd8f,0000000..93b8348
mode 100644,000000..100644
--- a/tests/test_real_adaptive_tiles.py
+++ b/tests/test_real_adaptive_tiles.py
@@@ -1,134 -1,0 +1,138 @@@
+"""Optional subdivision/restart checks on recorded audio and flight tracks."""
+import json
+import os
+from pathlib import Path
+import shutil
+
+import pytest
+
+from dopplergeist.data_access import CachedTerrain, FlightIndex
+from dopplergeist.matching import prepare_events
+from dopplergeist.tiled_search import SearchBudget, search_national
+
+
+def _run_interrupted_search(index_path, terrain_root, geoid, events, measurements, clock, output):
+ os.setsid()
+ index = FlightIndex(index_path)
+ terrain = CachedTerrain(terrain_root, offline=True, geoid_path=geoid, adopt_existing=True)
+ search_national([index], terrain, events, measurements,
+ time_range_s=index.report["coverage"]["time_range_s"], output_dir=output,
+ bounds=(-73.56, 40.85, -73.50, 40.90),
+ budget=SearchBudget(spacing_m=2000, tile_cells=4, top_k=5, refine_spacing_m=2000,
+ max_candidates_per_tile=50),
+ scoring={"mode": "robust_ridge", "time_origin": clock, "time_sweep_s": 0.,
+ "recording_start_window_s": [clock, clock]}, workers=2)
+
+
+def test_real_overloaded_tiles_split_and_resume(tmp_path):
+ import multiprocessing
+ import signal
+ import time
+ source, cache, reference, index_path = (os.environ.get(name) for name in (
+ "DOPPLERGEIST_REAL_JOB_ROOT", "DOPPLERGEIST_REAL_CACHE_ROOT",
+ "DOPPLERGEIST_REAL_REFERENCE_ROOT", "DOPPLERGEIST_REAL_DEVELOPMENT_INDEX"))
+ if not all((source, cache, reference, index_path)):
+ pytest.skip("set the real job, cache, reference and development index paths")
+ job, cache, reference = Path(source), Path(cache), Path(reference)
+ index = FlightIndex(index_path)
+ events, measurements = prepare_events(json.loads((job / "discovery/discovery.json").read_text())["events"])
+ clock = json.loads((reference / "results/reference-local/ranked.json").read_text())[0]["recording_start_time_s"]
+ terrain_root = tmp_path / "terrain"
+ terrain_root.mkdir()
+ for name in ("tileList.txt", "tileList.txt.sha256.json"):
+ shutil.copy2(cache / "terrain" / name, terrain_root / name)
+ raster = "Copernicus_DSM_COG_30_N40_00_W074_00_DEM.tif"
+ shutil.copy2(reference / "data/terrain/copernicus90" / raster, terrain_root / raster)
+ geoid = cache / "terrain/us_nga_egm08_25.tif"
+ scoring = {"mode": "robust_ridge", "time_origin": clock, "time_sweep_s": 0.,
+ "recording_start_window_s": [clock, clock]}
+
- def run(name, capacity, workers, *, fine=False):
++ def run(name, capacity, workers, *, fine=False, progress=None):
+ terrain = CachedTerrain(terrain_root, offline=True, geoid_path=geoid, adopt_existing=True)
+ return search_national([index], terrain, events, measurements,
+ time_range_s=index.report["coverage"]["time_range_s"], output_dir=tmp_path / name,
+ bounds=(-73.56, 40.85, -73.50, 40.90),
+ budget=SearchBudget(spacing_m=2000, tile_cells=4, top_k=5, refine_spacing_m=1000 if fine else 2000,
+ refine_finalists=1 if fine else 10,
- max_candidates_per_tile=capacity), scoring=scoring, workers=workers)
++ max_candidates_per_tile=capacity), scoring=scoring, workers=workers,
++ search_progress=progress)
+
+ # This fixed clock and bounded region are an engineering replay, not location truth.
+ baseline = run("baseline", 20000, 2)
- split = run("split", 50, 2)
++ progress = []
++ split = run("split", 50, 2, progress=progress.append)
+ keys = ("score", "lat", "lon", "elev_m", "flight_ids", "recording_start_time_s", "event_ids")
+ projection = lambda report: [{key: row.get(key) for key in keys} for row in report["ranked"]]
+ assert projection(baseline) == projection(split)
+ assert split["grid_points_searched"] == baseline["grid_points_searched"] > 0
+ assert not split["tile_failures"]
+ assert list((tmp_path / "split").glob("split-*.json"))
+ assert split["finite_search_evidence"]["pruning_accounted_for"]
++ assert progress and progress == sorted(progress) and progress[-1] == 1.0
++ assert any(0 < value < .8 for value in progress) and .8 in progress
+ resumed = run("split", 50, 1)
+ assert resumed["tiles_resumed"] == split["tiles_completed"]
+ assert resumed["ranked"] == split["ranked"]
+ assert resumed["finite_search_evidence"] == split["finite_search_evidence"]
+
+ shutil.copytree(tmp_path / "split", tmp_path / "corrupt")
+ marker = next((tmp_path / "corrupt").glob("split-*.json"))
+ value = json.loads(marker.read_text())
+ value["children"].pop()
+ marker.write_text(json.dumps(value))
+ corrupt = run("corrupt", 50, 1)
+ assert any("Corrupt tile subdivision checkpoint" in item["reason"] for item in corrupt["tile_failures"])
+ assert corrupt["status"] == "partial"
+
+ fine = run("fine", 100, 2, fine=True)
+ assert fine["refinement_failure"] is None and fine["refinement_points_searched"] > 0
+ assert fine["finite_search_evidence"]["stages"]["fine"]["stages"] > 4
+ assert fine["finite_search_evidence"]["pruning_accounted_for"]
+
+ if not hasattr(os, "setsid") or not Path("/proc").is_dir():
+ return
+ output = tmp_path / "interrupted"
+ child = multiprocessing.get_context("spawn").Process(
+ target=_run_interrupted_search,
+ args=(index_path, terrain_root, geoid, events, measurements, clock, output))
+ child.start()
+ checkpoints = []
+ try:
+ deadline = time.monotonic() + 120
+ while time.monotonic() < deadline and child.is_alive():
+ checkpoints = list(output.glob("tile-*~*.json")) if output.exists() else []
+ if checkpoints and os.getpgid(child.pid) == child.pid:
+ break
+ time.sleep(.05)
+ else:
+ pytest.fail("adaptive search did not checkpoint a child tile before interruption")
+ os.killpg(child.pid, signal.SIGTERM)
+ child.join(15)
+ assert not child.is_alive()
+ live = []
+ for _ in range(50):
+ live = []
+ for proc in Path("/proc").iterdir():
+ if not proc.name.isdigit():
+ continue
+ try:
+ fields = (proc / "stat").read_text().split(") ", 1)[1].split()
+ if int(fields[2]) == child.pid and fields[0] != "Z":
+ live.append(int(proc.name))
+ except (OSError, ValueError, IndexError):
+ pass
+ if not live:
+ break
+ time.sleep(.1)
+ assert not live
+ finally:
+ if child.is_alive():
+ if os.getpgid(child.pid) == child.pid:
+ os.killpg(child.pid, signal.SIGTERM)
+ else:
+ child.terminate()
+ child.join(15)
+ interrupted = run("interrupted", 50, 2)
+ assert interrupted["tiles_resumed"] >= len(checkpoints)
+ assert interrupted["ranked"] == split["ranked"]
+ assert interrupted["finite_search_evidence"] == split["finite_search_evidence"]