Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Preserve real terrain provenance across tile workers
Cinna Davis146eb6cd597c
diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.pyindex a9772bb..47cb195 100644--- a/src/dopplergeist/data_access.py+++ b/src/dopplergeist/data_access.py@@ -697,6 +697,7 @@ class CachedTerrain:# Their outer edges are shifted north/west by half a GLO90 pixel.degrees = np.column_stack((np.floor(lat - 1 / 2400).astype(int), np.floor(lon + 1 / 2400).astype(int)))self.last_sample_files = {"tileList.txt", self.geoid_path.name}+ self.last_sample_ocean_tiles = set()tiles, inverse = np.unique(degrees, axis=0, return_inverse=True)height = np.zeros(len(lat))try:@@ -704,6 +705,7 @@ class CachedTerrain:tag = f"Copernicus_DSM_COG_30_{'N' if north >= 0 else 'S'}{abs(north):02d}_00_{'E' if east >= 0 else 'W'}{abs(east):03d}_00_DEM"if tag not in self.catalog:self.ocean_tiles.add(tag)+ self.last_sample_ocean_tiles.add(tag)continuepath = self._file(f"{self.base}/{tag}/{tag}.tif", self.root / f"{tag}.tif", raster=True)self.last_sample_files.add(path.name)diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.pyindex 0fd7e95..b86f6e7 100644--- a/src/dopplergeist/parallel_tiles.py+++ b/src/dopplergeist/parallel_tiles.py@@ -5,11 +5,22 @@ import tempfileimport pandas as pd-from .data_access import CachedTerrain, FlightIndex+from .data_access import CachedTerrain, DataUnavailable, FlightIndexfrom .finite_search import RetentionLedgerfrom .tiled_search import _expanded_bounds, _score, _tracks, tile_points+class TileFailure(DataUnavailable):+ """Recoverable tile failure carrying the terrain provenance it observed."""++ def __init__(self, message, provenance):+ self.provenance = provenance+ super().__init__(message, provenance)++ def __str__(self):+ return str(self.args[0])++_worker = None@@ -36,15 +47,27 @@ def score_tile(stage, cells, score_floor=None):raise RuntimeError("Tile worker was not initialized")state = _workerbudget = state["budget"]- radius = max(event["slant_range_m"][1] for event in state["events"])- tracks = _tracks(state["indexes"], _expanded_bounds(cells, budget.spacing_m, radius, state["crs"]),- state["time_range"], budget)- points = (tile_points(cells, budget.spacing_m, state["bounds"], state["terrain"], state["crs"])- if tracks else pd.DataFrame())- with tempfile.NamedTemporaryFile(prefix="retention-", suffix=".sqlite",- dir=state["ledger_directory"], delete=False) as handle:- path = Path(handle.name)+ terrain = state["terrain"]+ terrain.last_sample_files = set()+ terrain.last_sample_ocean_tiles = set()+ failures_before = len(terrain.failures)+ path = None++ def observed_terrain():+ names = getattr(terrain, "last_sample_files", set())+ return {"terrain_files": [terrain.files[name] for name in sorted(names) if name in terrain.files],+ "terrain_ocean_tiles": sorted(getattr(terrain, "last_sample_ocean_tiles", set())),+ "terrain_failures": terrain.failures[failures_before:]}+try:+ radius = max(event["slant_range_m"][1] for event in state["events"])+ tracks = _tracks(state["indexes"], _expanded_bounds(cells, budget.spacing_m, radius, state["crs"]),+ state["time_range"], budget)+ points = (tile_points(cells, budget.spacing_m, state["bounds"], terrain, state["crs"])+ if tracks else pd.DataFrame())+ with tempfile.NamedTemporaryFile(prefix="retention-", suffix=".sqlite",+ dir=state["ledger_directory"], delete=False) as handle:+ path = Path(handle.name)ledger = RetentionLedger(path, state["run_hash"], state["score_kind"])if len(points):kwargs = {"ledger": ledger, "stage": stage}@@ -55,14 +78,18 @@ def score_tile(stage, cells, score_floor=None):else:rows, counts = [], {}ledger.record(stage, [], [], 0)- used = getattr(state["terrain"], "last_sample_files", set()) if len(points) else set()- terrain_files = [state["terrain"].files[name] for name in sorted(used)] if used else []+ provenance = observed_terrain() if len(points) else {+ "terrain_files": [], "terrain_ocean_tiles": [], "terrain_failures": []}payload = {"ranked": rows, "counts": counts, "grid_points": len(points), "tracks": len(tracks),- "cells": list(cells), "terrain_files": terrain_files,- "retention_checksum": ledger.checksum(stage)}+ "cells": list(cells), **provenance, "retention_checksum": ledger.checksum(stage)}return payload, str(path)+ except (DataUnavailable, ValueError, OSError) as error:+ if path is not None:+ path.unlink(missing_ok=True)+ raise TileFailure(str(error), observed_terrain()) from errorexcept BaseException:- path.unlink(missing_ok=True)+ if path is not None:+ path.unlink(missing_ok=True)raisediff --git a/tests/test_real_parallel_retention.py b/tests/test_real_parallel_retention.pyindex ace7785..d7c8021 100644--- a/tests/test_real_parallel_retention.py+++ b/tests/test_real_parallel_retention.py@@ -47,11 +47,12 @@ def test_real_retention_stages_survive_worker_import(tmp_path):def test_real_tiles_match_sequential_scoring_and_preserve_ledgers(tmp_path):import jsonimport multiprocessing+ import shutilfrom concurrent.futures import ProcessPoolExecutorfrom dopplergeist.data_access import FlightIndex, content_hashfrom dopplergeist.matching import prepare_events- from dopplergeist.parallel_tiles import initialize_tile_worker, ordered_tile_futures, score_tile+ from dopplergeist.parallel_tiles import TileFailure, initialize_tile_worker, ordered_tile_futures, score_tilefrom dopplergeist.tiled_search import SearchBudgetsource = os.environ.get("DOPPLERGEIST_REAL_JOB_ROOT")@@ -102,3 +103,50 @@ def test_real_tiles_match_sequential_scoring_and_preserve_ledgers(tmp_path):assert RetentionLedger(expected_db, run_hash, score_kind).stage_valid(stage, checksum)import_retention_stage(parent, worker_db, stage, checksum)assert parent.stage_valid(stage, checksum)++ # A real tile with its terrain file deliberately unavailable must carry+ # the provider failure back from the worker instead of losing provenance.+ missing = tmp_path / "missing-terrain"+ missing.mkdir()+ for name in ("tileList.txt", "tileList.txt.sha256.json"):+ shutil.copy2(cache / "terrain" / name, missing / name)+ failure_args = list(args)+ failure_args[1] = missing+ failure_args[2] = {"offline": True, "geoid_path": cache / "terrain/us_nga_egm08_25.tif"}+ with ProcessPoolExecutor(max_workers=1, mp_context=multiprocessing.get_context("spawn"),+ initializer=initialize_tile_worker, initargs=tuple(failure_args)) as pool:+ future = pool.submit(score_tile, *stages[0])+ with pytest.raises(TileFailure) as caught:+ future.result()+ assert "Terrain coverage unavailable" in str(caught.value)+ assert caught.value.provenance["terrain_failures"]+ assert any(Path(item["path"]).name == "tileList.txt" for item in caught.value.provenance["terrain_files"])+++def test_real_terrain_tracks_ocean_tiles_per_sample():+ import json+ import math++ from dopplergeist.data_access import CachedTerrain++ source = os.environ.get("DOPPLERGEIST_REAL_JOB_ROOT")+ cache = os.environ.get("DOPPLERGEIST_REAL_CACHE_ROOT")+ if not source or not cache:+ pytest.skip("set the real job and cache roots")+ job, cache = Path(source), Path(cache)+ terrain = CachedTerrain(cache / "terrain", offline=True)+ ocean_rows = json.loads((job / "search/tile-0_2.json").read_text())["payload"]["ranked"]+ for row in ocean_rows:+ north = math.floor(row["lat"]-1/2400)+ east = math.floor(row["lon"]+1/2400)+ tag = (f"Copernicus_DSM_COG_30_{'N' if north >= 0 else 'S'}{abs(north):02d}_00_"+ f"{'E' if east >= 0 else 'W'}{abs(east):03d}_00_DEM")+ if tag not in terrain.catalog:+ break+ else:+ pytest.fail("the saved real tile no longer contains the expected ocean point")+ land = json.loads((job / "search/tile--10_1.json").read_text())["payload"]["ranked"][0]+ terrain.sample([row["lat"]], [row["lon"]])+ assert terrain.last_sample_ocean_tiles == {tag}+ terrain.sample([land["lat"]], [land["lon"]])+ assert not terrain.last_sample_ocean_tiles and tag in terrain.ocean_tiles