Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Subdivide tiles that exceed search work budgets

Cinna Davise49c87d80069
diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.py
index 57b405b..f573457 100644
--- a/src/dopplergeist/tiled_search.py
+++ b/src/dopplergeist/tiled_search.py
@@ -135,6 +135,28 @@ def grid_tiles(bounds, spacing_m, tile_cells, crs="EPSG:5070"):
min(xmax + 1, (ix + 1) * tile_cells), min(ymax + 1, (iy + 1) * tile_cells))
+def _split_tile_cells(cells):
+ """Partition a half-open grid rectangle without dropping or repeating cells."""
+ x0, y0, x1, y1 = cells
+ xs = [x0, (x0+x1)//2, x1] if x1-x0 > 1 else [x0, x1]
+ ys = [y0, (y0+y1)//2, y1] if y1-y0 > 1 else [y0, y1]
+ if len(xs) == len(ys) == 2:
+ return []
+ return [(xs[i], ys[j], xs[i+1], ys[j+1])
+ for j in range(len(ys)-1) for i in range(len(xs)-1)]
+
+
+def _tile_capacity_error(error):
+ """Only known per-tile work limits are candidates for subdivision."""
+ return str(error) in {
+ "Tile track/sample memory budget exceeded; reduce tile_cells",
+ "Tile CPA-row memory budget exceeded",
+ "Association work/storage budget exceeded; search is incomplete",
+ "Association candidate budget exceeded; search is incomplete",
+ "Tile candidate memory budget exceeded; reduce tile_cells",
+ }
+
+
@lru_cache(maxsize=8)
def _cached_to_geographic(crs):
return Transformer.from_crs(crs, "EPSG:4326", always_xy=True)
@@ -563,27 +585,48 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
terrain.ocean_tiles.update(provenance.get("terrain_ocean_tiles", []))
terrain.failures.extend(provenance.get("terrain_failures", []))
- def tile_requests():
- for number, (tile_id, cells) in enumerate(grid_tiles(bounds, budget.spacing_m, budget.tile_cells, crs)):
- if budget.max_tiles is not None and number >= budget.max_tiles:
- break
- stage = f"coarse:{tile_id}"
- checkpoint = output / f"tile-{tile_id}.json"
- cached = None
+ def tile_request(tile_id, cells):
+ stage = f"coarse:{tile_id}"
+ checkpoint = output / f"tile-{tile_id}.json"
+ cached = None
+ try:
+ if checkpoint.exists():
+ record = json.loads(checkpoint.read_text())
+ if record["run_hash"] != run_hash or record["payload_hash"] != content_hash(record["payload"]):
+ raise DataUnavailable("Corrupt tile checkpoint")
+ prior = record["payload"]
+ for source in prior.get("terrain_files", []):
+ if sha256(source["path"]) != source["sha256"]:
+ raise DataUnavailable("Terrain changed since tile checkpoint")
+ if ledger.stage_valid(stage, prior.get("retention_checksum")):
+ cached = checkpoint
+ except (DataUnavailable, ValueError, OSError) as error:
+ cached = error
+ return stage, cells, cached, None
+
+ def requests_for_wave(wave, next_wave):
+ nonlocal total
+ for tile_id, cells in wave:
+ marker = output / f"split-{tile_id}.json"
+ if not marker.exists():
+ yield tile_request(tile_id, cells)
+ continue
+ parts = _split_tile_cells(cells)
+ children = [[f"{tile_id}~{i}", list(part)] for i, part in enumerate(parts)]
try:
- if checkpoint.exists():
- record = json.loads(checkpoint.read_text())
- if record["run_hash"] != run_hash or record["payload_hash"] != content_hash(record["payload"]):
- raise DataUnavailable("Corrupt tile checkpoint")
- prior = record["payload"]
- for source in prior.get("terrain_files", []):
- if sha256(source["path"]) != source["sha256"]:
- raise DataUnavailable("Terrain changed since tile checkpoint")
- if ledger.stage_valid(stage, prior.get("retention_checksum")):
- cached = checkpoint
+ record = json.loads(marker.read_text())
+ if not parts or record != {"run_hash": run_hash, "cells": list(cells), "children": children}:
+ raise DataUnavailable("Corrupt tile subdivision checkpoint")
except (DataUnavailable, ValueError, OSError) as error:
- cached = error
- yield stage, cells, cached, None
+ yield f"coarse:{tile_id}", cells, error, None
+ continue
+ total += len(children)-1
+ next_wave.extend((child_id, tuple(child_cells)) for child_id, child_cells in children)
+
+ def tile_order(item):
+ root, *path = item[0].split("~")
+ ix, iy = map(int, root.split("_"))
+ return iy, ix, tuple(map(int, path))
if parallel:
terrain_options = {"offline": terrain.offline, "geoid_path": terrain.geoid_path,
@@ -614,49 +657,62 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
return error
with context as pool:
- requests = tile_requests()
- results = (ordered_tile_results(pool, requests, workers, finish_tile) if pool is not None else
- ((stage, cells, cached, cached is not None) for stage, cells, cached, _ in requests))
- for stage, cells, prepared, was_cached in results:
- tile_id = stage[len("coarse:"):]
- checkpoint = output / f"tile-{tile_id}.json"
- try:
- if isinstance(prepared, BaseException):
- raise prepared
- if prepared is not None:
- payload = json.loads(prepared.read_text())["payload"]
- if was_cached:
- resumed += 1
- restore_terrain(payload)
- else:
- points = _tile_coordinates(cells, budget.spacing_m, bounds, crs)
- tracks = (_tracks(indexes, _expanded_bounds(cells, budget.spacing_m, radius, crs), time_range_s, budget)
- if len(points) else [])
- points = _sample_tile_terrain(points, terrain) if tracks else pd.DataFrame()
- if len(points):
- rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
- ledger=ledger, stage=stage,
- score_floor=(_retained_score_floor(ranked, budget.top_k)
- if floor_policy == "global_and_per_tile" else None))
+ roots = grid_tiles(bounds, budget.spacing_m, budget.tile_cells, crs)
+ wave = itertools.islice(roots, budget.max_tiles) if budget.max_tiles is not None else roots
+ while wave is not None:
+ next_wave = []
+ requests = requests_for_wave(wave, next_wave)
+ results = (ordered_tile_results(pool, requests, workers, finish_tile) if pool is not None else
+ ((stage, cells, cached, cached is not None) for stage, cells, cached, _ in requests))
+ for stage, cells, prepared, was_cached in results:
+ tile_id = stage[len("coarse:"):]
+ checkpoint = output / f"tile-{tile_id}.json"
+ try:
+ if isinstance(prepared, BaseException):
+ raise prepared
+ if prepared is not None:
+ payload = json.loads(prepared.read_text())["payload"]
+ if was_cached:
+ resumed += 1
+ restore_terrain(payload)
else:
- rows, counts = [], {}
- ledger.record(stage, [], [], 0)
- used = getattr(terrain, "last_sample_files", set()) if len(points) else set()
- terrain_files = [terrain.files[name] for name in sorted(used)] if used else []
- oceans = sorted(getattr(terrain, "last_sample_ocean_tiles", set())) if len(points) else []
- payload = {"ranked": rows, "counts": counts, "grid_points": len(points), "tracks": len(tracks),
- "cells": list(cells), "terrain_files": terrain_files, "terrain_ocean_tiles": oceans,
- "terrain_failures": [], "retention_checksum": ledger.checksum(stage)}
- write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload})
- done += 1
- points_searched += payload["grid_points"]
- _record_support(output, stage, payload["counts"], support)
- ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,
- ledger=ledger, stage=f"global:{tile_id}")
- except (DataUnavailable, ValueError, OSError) as error:
- if isinstance(error, TileFailure):
- restore_terrain(error.provenance)
- failures.append({"tile": tile_id, "reason": str(error)})
+ points = _tile_coordinates(cells, budget.spacing_m, bounds, crs)
+ tracks = (_tracks(indexes, _expanded_bounds(cells, budget.spacing_m, radius, crs), time_range_s, budget)
+ if len(points) else [])
+ points = _sample_tile_terrain(points, terrain) if tracks else pd.DataFrame()
+ if len(points):
+ rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
+ ledger=ledger, stage=stage,
+ score_floor=(_retained_score_floor(ranked, budget.top_k)
+ if floor_policy == "global_and_per_tile" else None))
+ else:
+ rows, counts = [], {}
+ ledger.record(stage, [], [], 0)
+ used = getattr(terrain, "last_sample_files", set()) if len(points) else set()
+ terrain_files = [terrain.files[name] for name in sorted(used)] if used else []
+ oceans = sorted(getattr(terrain, "last_sample_ocean_tiles", set())) if len(points) else []
+ payload = {"ranked": rows, "counts": counts, "grid_points": len(points), "tracks": len(tracks),
+ "cells": list(cells), "terrain_files": terrain_files, "terrain_ocean_tiles": oceans,
+ "terrain_failures": [], "retention_checksum": ledger.checksum(stage)}
+ write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload})
+ done += 1
+ points_searched += payload["grid_points"]
+ _record_support(output, stage, payload["counts"], support)
+ ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,
+ ledger=ledger, stage=f"global:{tile_id}")
+ except (DataUnavailable, ValueError, OSError) as error:
+ if isinstance(error, TileFailure):
+ restore_terrain(error.provenance)
+ parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []
+ if parts:
+ children = [[f"{tile_id}~{i}", list(part)] for i, part in enumerate(parts)]
+ write_json(output / f"split-{tile_id}.json",
+ {"run_hash": run_hash, "cells": list(cells), "children": children})
+ total += len(children)-1
+ next_wave.extend((child_id, tuple(child_cells)) for child_id, child_cells in children)
+ else:
+ failures.append({"tile": tile_id, "reason": str(error)})
+ wave = sorted(next_wave, key=tile_order) if next_wave else None
coarse = list(ranked)
# Refine only global finalists; tile selection never controls refinement rank.
refinement_failure = None
@@ -680,26 +736,39 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
cx, cy = round(row["x"] / fine), round(row["y"] / fine)
for y0 in range(cy - width, cy + width + 1, budget.tile_cells):
for x0 in range(cx - width, cx + width + 1, budget.tile_cells):
- cells = (x0, y0, min(cx + width + 1, x0 + budget.tile_cells), min(cy + width + 1, y0 + budget.tile_cells))
- points = tile_points(cells, fine, bounds, terrain, crs)
- keep = np.ones(len(points), dtype=bool)
- for left, bottom, right, top in completed_rectangles:
- keep &= ~((points.x >= left * fine) & (points.x < right * fine)
- & (points.y >= bottom * fine) & (points.y < top * fine))
- refinement_points_skipped += int((~keep).sum())
- points = points[keep]
- if not len(points):
- continue
- tracks = _tracks(indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s, budget)
- stage = f"fine:{seed_index}:{x0}:{y0}"
- rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
- ledger=ledger, stage=stage,
- score_floor=_retained_score_floor(ranked, budget.top_k))
- _record_support(output, stage, counts, support)
- fine_stages += 1
- refinement_points_searched += len(points)
- ranked = merge_candidates(itertools.chain(ranked, rows), budget.top_k,
- ledger=ledger, stage=f"fine-global:{seed_index}:{x0}:{y0}")
+ root = (x0, y0, min(cx + width + 1, x0 + budget.tile_cells),
+ min(cy + width + 1, y0 + budget.tile_cells))
+ pending = [root]
+ while pending:
+ cells = pending.pop()
+ points = tile_points(cells, fine, bounds, terrain, crs)
+ keep = np.ones(len(points), dtype=bool)
+ for left, bottom, right, top in completed_rectangles:
+ keep &= ~((points.x >= left * fine) & (points.x < right * fine)
+ & (points.y >= bottom * fine) & (points.y < top * fine))
+ skipped = int((~keep).sum())
+ points = points[keep]
+ if not len(points):
+ refinement_points_skipped += skipped
+ continue
+ try:
+ tracks = _tracks(indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s, budget)
+ stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"
+ rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
+ ledger=ledger, stage=stage,
+ score_floor=_retained_score_floor(ranked, budget.top_k))
+ except (DataUnavailable, ValueError, OSError) as error:
+ parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []
+ if not parts:
+ raise
+ pending.extend(reversed(parts))
+ continue
+ refinement_points_skipped += skipped
+ _record_support(output, stage, counts, support)
+ fine_stages += 1
+ refinement_points_searched += len(points)
+ ranked = merge_candidates(itertools.chain(ranked, rows), budget.top_k,
+ ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")
completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))
except (DataUnavailable, ValueError, OSError) as error:
refinement_failure = str(error)
diff --git a/tests/test_real_adaptive_tiles.py b/tests/test_real_adaptive_tiles.py
new file mode 100644
index 0000000..800bd8f
--- /dev/null
+++ b/tests/test_real_adaptive_tiles.py
@@ -0,0 +1,134 @@
+"""Optional subdivision/restart checks on recorded audio and flight tracks."""
+import json
+import os
+from pathlib import Path
+import shutil
+
+import pytest
+
+from dopplergeist.data_access import CachedTerrain, FlightIndex
+from dopplergeist.matching import prepare_events
+from dopplergeist.tiled_search import SearchBudget, search_national
+
+
+def _run_interrupted_search(index_path, terrain_root, geoid, events, measurements, clock, output):
+ os.setsid()
+ index = FlightIndex(index_path)
+ terrain = CachedTerrain(terrain_root, offline=True, geoid_path=geoid, adopt_existing=True)
+ search_national([index], terrain, events, measurements,
+ time_range_s=index.report["coverage"]["time_range_s"], output_dir=output,
+ bounds=(-73.56, 40.85, -73.50, 40.90),
+ budget=SearchBudget(spacing_m=2000, tile_cells=4, top_k=5, refine_spacing_m=2000,
+ max_candidates_per_tile=50),
+ scoring={"mode": "robust_ridge", "time_origin": clock, "time_sweep_s": 0.,
+ "recording_start_window_s": [clock, clock]}, workers=2)
+
+
+def test_real_overloaded_tiles_split_and_resume(tmp_path):
+ import multiprocessing
+ import signal
+ import time
+ source, cache, reference, index_path = (os.environ.get(name) for name in (
+ "DOPPLERGEIST_REAL_JOB_ROOT", "DOPPLERGEIST_REAL_CACHE_ROOT",
+ "DOPPLERGEIST_REAL_REFERENCE_ROOT", "DOPPLERGEIST_REAL_DEVELOPMENT_INDEX"))
+ if not all((source, cache, reference, index_path)):
+ pytest.skip("set the real job, cache, reference and development index paths")
+ job, cache, reference = Path(source), Path(cache), Path(reference)
+ index = FlightIndex(index_path)
+ events, measurements = prepare_events(json.loads((job / "discovery/discovery.json").read_text())["events"])
+ clock = json.loads((reference / "results/reference-local/ranked.json").read_text())[0]["recording_start_time_s"]
+ terrain_root = tmp_path / "terrain"
+ terrain_root.mkdir()
+ for name in ("tileList.txt", "tileList.txt.sha256.json"):
+ shutil.copy2(cache / "terrain" / name, terrain_root / name)
+ raster = "Copernicus_DSM_COG_30_N40_00_W074_00_DEM.tif"
+ shutil.copy2(reference / "data/terrain/copernicus90" / raster, terrain_root / raster)
+ geoid = cache / "terrain/us_nga_egm08_25.tif"
+ scoring = {"mode": "robust_ridge", "time_origin": clock, "time_sweep_s": 0.,
+ "recording_start_window_s": [clock, clock]}
+
+ def run(name, capacity, workers, *, fine=False):
+ terrain = CachedTerrain(terrain_root, offline=True, geoid_path=geoid, adopt_existing=True)
+ return search_national([index], terrain, events, measurements,
+ time_range_s=index.report["coverage"]["time_range_s"], output_dir=tmp_path / name,
+ bounds=(-73.56, 40.85, -73.50, 40.90),
+ budget=SearchBudget(spacing_m=2000, tile_cells=4, top_k=5, refine_spacing_m=1000 if fine else 2000,
+ refine_finalists=1 if fine else 10,
+ max_candidates_per_tile=capacity), scoring=scoring, workers=workers)
+
+ # This fixed clock and bounded region are an engineering replay, not location truth.
+ baseline = run("baseline", 20000, 2)
+ split = run("split", 50, 2)
+ keys = ("score", "lat", "lon", "elev_m", "flight_ids", "recording_start_time_s", "event_ids")
+ projection = lambda report: [{key: row.get(key) for key in keys} for row in report["ranked"]]
+ assert projection(baseline) == projection(split)
+ assert split["grid_points_searched"] == baseline["grid_points_searched"] > 0
+ assert not split["tile_failures"]
+ assert list((tmp_path / "split").glob("split-*.json"))
+ assert split["finite_search_evidence"]["pruning_accounted_for"]
+ resumed = run("split", 50, 1)
+ assert resumed["tiles_resumed"] == split["tiles_completed"]
+ assert resumed["ranked"] == split["ranked"]
+ assert resumed["finite_search_evidence"] == split["finite_search_evidence"]
+
+ shutil.copytree(tmp_path / "split", tmp_path / "corrupt")
+ marker = next((tmp_path / "corrupt").glob("split-*.json"))
+ value = json.loads(marker.read_text())
+ value["children"].pop()
+ marker.write_text(json.dumps(value))
+ corrupt = run("corrupt", 50, 1)
+ assert any("Corrupt tile subdivision checkpoint" in item["reason"] for item in corrupt["tile_failures"])
+ assert corrupt["status"] == "partial"
+
+ fine = run("fine", 100, 2, fine=True)
+ assert fine["refinement_failure"] is None and fine["refinement_points_searched"] > 0
+ assert fine["finite_search_evidence"]["stages"]["fine"]["stages"] > 4
+ assert fine["finite_search_evidence"]["pruning_accounted_for"]
+
+ if not hasattr(os, "setsid") or not Path("/proc").is_dir():
+ return
+ output = tmp_path / "interrupted"
+ child = multiprocessing.get_context("spawn").Process(
+ target=_run_interrupted_search,
+ args=(index_path, terrain_root, geoid, events, measurements, clock, output))
+ child.start()
+ checkpoints = []
+ try:
+ deadline = time.monotonic() + 120
+ while time.monotonic() < deadline and child.is_alive():
+ checkpoints = list(output.glob("tile-*~*.json")) if output.exists() else []
+ if checkpoints and os.getpgid(child.pid) == child.pid:
+ break
+ time.sleep(.05)
+ else:
+ pytest.fail("adaptive search did not checkpoint a child tile before interruption")
+ os.killpg(child.pid, signal.SIGTERM)
+ child.join(15)
+ assert not child.is_alive()
+ live = []
+ for _ in range(50):
+ live = []
+ for proc in Path("/proc").iterdir():
+ if not proc.name.isdigit():
+ continue
+ try:
+ fields = (proc / "stat").read_text().split(") ", 1)[1].split()
+ if int(fields[2]) == child.pid and fields[0] != "Z":
+ live.append(int(proc.name))
+ except (OSError, ValueError, IndexError):
+ pass
+ if not live:
+ break
+ time.sleep(.1)
+ assert not live
+ finally:
+ if child.is_alive():
+ if os.getpgid(child.pid) == child.pid:
+ os.killpg(child.pid, signal.SIGTERM)
+ else:
+ child.terminate()
+ child.join(15)
+ interrupted = run("interrupted", 50, 2)
+ assert interrupted["tiles_resumed"] >= len(checkpoints)
+ assert interrupted["ranked"] == split["ranked"]
+ assert interrupted["finite_search_evidence"] == split["finite_search_evidence"]