Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
perf: scope optional flight preparation to regional uploads (#288)
Cinna Davis4cbdec8ab2a7
diff --git a/docs/DATA_AND_SCALE.md b/docs/DATA_AND_SCALE.mdindex 19ba5ef..ae24ebf 100644--- a/docs/DATA_AND_SCALE.md+++ b/docs/DATA_AND_SCALE.md@@ -20,6 +20,13 @@ separate processes, using the available CPU cores minus one for the SQLite writesource order. SQLite writes and finalization still run in the writer process; astopped partial index must be rebuilt.+For a fresh automatic regional upload, only segments touching its receiver bounds+and emission-time interval get the optional numeric decoding cache. Every original+global segment remains in the index, and queries outside that warm scope use the+verified original payload. This reduces first-use cache construction without+changing source coverage or filtering aircraft. `FlightIndex.prepare` also accepts+`prepare_scope=(bounds_wsen, time_range_s)` for an explicitly scoped index copy.+The on-disk SQLite index has spatial and time indexes, source hashes, explicitsource scope, per-payload checksums and altitude/classification labels. Disk staging supportsarbitrary input order. Query predicates select whole gap-split segments; theydiff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.pyindex 7b48954..85b20c0 100644--- a/src/dopplergeist/data_access.py+++ b/src/dopplergeist/data_access.py@@ -618,8 +618,8 @@ class FlightIndex:return track@staticmethod- def _prepare_tracks(db):- """Store constructor inputs beside the original observations, once."""+ def _prepare_tracks(db, scope=None):+ """Warm optional constructor inputs without removing observations."""from .flight_payload import encode_trackdb.execute("DROP TABLE IF EXISTS prepared_tracks")db.execute("CREATE TABLE prepared_tracks(id INTEGER PRIMARY KEY, source_hash TEXT, payload BLOB, payload_hash TEXT)")@@ -629,8 +629,16 @@ class FlightIndex:or _payload_frame is not _CANONICAL_PAYLOAD_FRAMEor Track is not _CANONICAL_TRACK or np.geterr() != _PREPARED_ERROR_POLICY):return {"version": 1, "builder": None, "tracks": count, "bytes": size}- for identifier, name, payload, expected in db.execute(- "SELECT id,name,payload,payload_hash FROM segments ORDER BY id"):+ query = "SELECT s.id,s.name,s.payload,s.payload_hash FROM segments s"+ parameters = ()+ if scope is not None:+ bounds, times = scope+ west, south, east, north = bounds+ start, end = times+ query += (" JOIN spatial r ON s.id=r.id WHERE r.west<=? AND r.east>=?"+ " AND r.south<=? AND r.north>=? AND s.start<=? AND s.end>=?")+ parameters = (east, west, north, south, end, start)+ for identifier, name, payload, expected in db.execute(query + " ORDER BY s.id", parameters):if hashlib.sha256(payload).hexdigest() != expected:raise DataUnavailable(f"Corrupt flight payload: {name}")# Use the precise values the old reader gets AFTER the index's@@ -644,14 +652,19 @@ class FlightIndex:(identifier, expected, packed, hashlib.sha256(packed).hexdigest()))count += 1size += len(packed)- return {"version": 1, "builder": _prepared_track_identity(), "tracks": count, "bytes": size}+ report = {"version": 1, "builder": _prepared_track_identity(), "tracks": count, "bytes": size}+ if scope is not None:+ report["scope"] = {"bounds": list(bounds), "time_range_s": list(times)}+ return report- def prepare(self, destination):+ def prepare(self, destination, *, prepare_scope=None):"""Create a new index copy with faster verified track decoding.The source stays read-only and the destination must not exist. Coverage,original payloads, spatial bounds and scientific input identity stayunchanged. Old readers ignore the optional numeric representation.+ Optional ``prepare_scope=(bounds_wsen, time_range_s)`` limits warming;+ queries outside it still decode the complete original observations."""from uuid import uuid4destination = Path(destination)@@ -672,7 +685,7 @@ class FlightIndex:raise DataUnavailable("Flight index version mismatch")except (sqlite3.DatabaseError, TypeError, KeyError, json.JSONDecodeError) as error:raise DataUnavailable("Invalid source flight index metadata") from error- report["prepared_tracks"] = self._prepare_tracks(db)+ report["prepared_tracks"] = self._prepare_tracks(db, prepare_scope)db.execute("UPDATE metadata SET value=?", (json.dumps(report),))db.commit()db.close()@@ -685,7 +698,7 @@ class FlightIndex:@classmethoddef build(cls, path, frames, *, sources, coverage, max_gap_s=30.0, max_flight_samples=250000, class_map=None,- prepared=False):+ prepared=False, prepare_scope=None):"""Disk staging permits arbitrary source order without whole-table RAM.Explicit source coverage is required: cropped or prop-only historical@@ -750,7 +763,7 @@ class FlightIndex:report = dict(identity, input_hash=content_hash(identity), rows=rows, invalid_rows=invalid,segments=segments, altitude_basis_counts=dict(altitude), jets_excluded=False,unknown_types_retained=True, geometry_clipped=False, continuous_geometry_bounds=True)- report["prepared_tracks"] = cls._prepare_tracks(db)+ report["prepared_tracks"] = cls._prepare_tracks(db, prepare_scope)db.execute("INSERT INTO metadata VALUES(?)", (json.dumps(report),))db.commit()db.execute("VACUUM")@@ -901,7 +914,7 @@ class FlightCache:self.root, self.offline = Path(root), offlineself.session = session or requests.Session()- def acquire_day(self, day, *, provider="staging-0"):+ def acquire_day(self, day, *, provider="staging-0", prepare_scope=None):from . import tracks as normalizerday = date.fromisoformat(day)tag = f"v{day:%Y.%m.%d}-planes-readsb-{provider}"@@ -970,7 +983,8 @@ class FlightCache:# ICAO T also covers helicopter turboshafts; keep the family label broad.class_map[code] = {"J": "jet", "T": "turbine", "P": "piston"}.get(description[-1:], "unclassified")return FlightIndex.build(index, archive_staging([s["path"] for s in sources], class_map), sources=sources + [type_source],- coverage=coverage, class_map=class_map, prepared=True)+ coverage=coverage, class_map=class_map, prepared=True,+ prepare_scope=prepare_scope)class CachedTerrain:@@ -1156,6 +1170,7 @@ def acquire_inputs(request, recording, *, cache_root, index_paths=None, offline=raise ValueError("Required emission interval must be ordered")times = (first, last)root = Path(cache_root)+ bounds = tuple(request["search"].get("bounds_wsen", CONUS))if index_paths:indexes = [FlightIndex(path) for path in index_paths]else:@@ -1164,9 +1179,8 @@ def acquire_inputs(request, recording, *, cache_root, index_paths=None, offline=indexes = []day = firstwhile day <= last:- indexes.append(cache.acquire_day(day.isoformat()))+ indexes.append(cache.acquire_day(day.isoformat(), prepare_scope=(bounds, times)))day += timedelta(days=1)- bounds = tuple(request["search"].get("bounds_wsen", CONUS))options = dict(terrain_options or {})terrain = CachedTerrain(root / "terrain", offline=offline, **options)coverage = {"requested_area": request["search"], "sources": [i.coverage_for(bounds, times) for i in indexes],diff --git a/tests/test_real_prepared_index.py b/tests/test_real_prepared_index.pyindex 420fcbb..385fbeb 100644--- a/tests/test_real_prepared_index.py+++ b/tests/test_real_prepared_index.py@@ -85,6 +85,60 @@ def test_prepared_index_copy_preserves_source_and_input_identity(observed_index,index.prepare(destination)[email protected]("bounds,times,nonempty", [+ ([-79., 35., -78., 36.], [1788899404., 1788900299.], True),+ ([-180., -90., 180., 90.], [1788899404.72, 1788899405.], True),+ ([-180., -90., 180., 90.], [0., 1.], False),+])+def test_scoped_preparation_preserves_all_query_and_lookup_tracks(+ observed_index, tmp_path, monkeypatch, bounds, times, nonempty):+ index, coverage = observed_index+ prepared = index.prepare(tmp_path / "scoped.sqlite", prepare_scope=(bounds, times))+ with index.connect() as db:+ complete_rows = dict(db.execute("SELECT id,payload FROM prepared_tracks"))+ with prepared.connect() as db:+ scoped_rows = dict(db.execute("SELECT id,payload FROM prepared_tracks"))+ assert bool(scoped_rows) is nonempty+ assert len(scoped_rows) < len(complete_rows)+ assert all(payload == complete_rows[identifier] for identifier, payload in scoped_rows.items())+ assert prepared.report["prepared_tracks"]["scope"] == {"bounds": bounds, "time_range_s": times}+ assert prepared.report["input_hash"] == index.report["input_hash"]+ assert prepared.coverage_for(coverage["bounds"], coverage["time_range_s"]) == index.coverage_for(+ coverage["bounds"], coverage["time_range_s"])+ legacy = FlightIndex(index.path, track_cache_bytes=0)+ legacy._prepared_tracks = False+ expected = queried(legacy, coverage)+ reader = FlightIndex(prepared.path, track_cache_bytes=0)+ from dopplergeist import flight_payload+ decode, calls = flight_payload.decode_track, []++ def counted(*args, **kwargs):+ calls.append(args[0])+ return decode(*args, **kwargs)++ monkeypatch.setattr(flight_payload, "decode_track", counted)+ assert [signature(track) for track in queried(reader, coverage)] == [signature(track) for track in expected]+ assert [signature(track) for track in reader.lookup([track.id for track in expected])] == [+ signature(track) for track in expected]+ assert len(calls) == 2 * len(scoped_rows)+++def test_scoped_build_retains_every_original_segment(observed_index, tmp_path):+ index, coverage = observed_index+ source = Path(__file__).resolve().parents[1] / "research/trajectory-motion/frozen/canonical-observations.json"+ frame = pd.DataFrame(json.loads(source.read_text()))+ scoped = FlightIndex.build(tmp_path / "scoped-build.sqlite", [frame], sources=[], coverage=coverage,+ prepare_scope=([-79., 35., -78., 36.], coverage["time_range_s"]))+ for table in ("segments", "spatial"):+ with index.connect() as db:+ original = db.execute(f"SELECT * FROM {table} ORDER BY id").fetchall()+ with scoped.connect() as db:+ assert db.execute(f"SELECT * FROM {table} ORDER BY id").fetchall() == original+ assert scoped.report["input_hash"] == index.report["input_hash"]+ assert scoped.report["rows"] == index.report["rows"]+ assert 0 < scoped.report["prepared_tracks"]["tracks"] < index.report["prepared_tracks"]["tracks"]++def test_prepare_and_build_coordinate_the_destination(observed_index, tmp_path, monkeypatch):index, coverage = observed_indexoriginal_lock, locks = data_access.cache_lock, []