Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Reuse prepared indexes across CPA-only geometry change (#337)
Cinna Davis31ddb88226f3
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex c0d724d..7e2f351 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -31,7 +31,7 @@ jobs:run: |sudo apt-get updatesudo apt-get install -y ffmpeg- python -m pip install .+ python -m pip install '.[test]'python -m pip check# The owner requires real recordings for further validation. The historical# pytest suite generates synthetic inputs, so this workflow does not run it.@@ -39,6 +39,8 @@ jobs:run: python -m compileall -q src scripts- name: Check benchmark instrumentation detectionrun: python scripts/check_benchmark_instrumentation.py+ - name: Check prepared-index compatibility with recorded observations+ run: python -m pytest -q tests/test_real_prepared_geometry_compatibility.py::test_pinned_legacy_geometry_and_data_source_preserve_real_tracks- name: CLI smoke checkrun: dopplergeist --help- name: Check installed native validator and packaged browser assetsdiff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.pyindex 6847566..47c3a12 100644--- a/src/dopplergeist/data_access.py+++ b/src/dopplergeist/data_access.py@@ -161,9 +161,9 @@ _CANONICAL_TRACK = Track_PREPARED_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}-@lru_cache(maxsize=16)-def _prepared_track_identity(data_access_digest=None):- """Use prepared velocities only with identical normalization and NumPy."""+@lru_cache(maxsize=1)+def _prepared_track_components():+ """Hash immutable runtime inputs once when checking compatible builders."""from . import flight_payloadtry:import numpy._core._multiarray_umath as numpy_core@@ -173,12 +173,21 @@ def _prepared_track_identity(data_access_digest=None):Path(__file__).with_name("geometry.py"), Path(flight_payload.__file__),Path(numpy_core.__file__))sources = [sha256(path) for path in files]+ return {"sources": sources, "numpy": np.__version__, "pandas": pd.__version__,+ "python": sys.version, "cpu": getattr(numpy_core, "__cpu_features__", {})}+++@lru_cache(maxsize=16)+def _prepared_track_identity(data_access_digest=None, geometry_digest=None):+ """Use prepared velocities only with identical normalization and NumPy."""+ components = dict(_prepared_track_components())+ sources = list(components["sources"])if data_access_digest is not None:sources[0] = data_access_digest- return content_hash({"sources": sources,- "numpy": np.__version__,- "pandas": pd.__version__, "python": sys.version,- "cpu": getattr(numpy_core, "__cpu_features__", {})})+ if geometry_digest is not None:+ sources[2] = geometry_digest+ components["sources"] = sources+ return content_hash(components)@lru_cache(maxsize=1)@@ -583,11 +592,14 @@ class FlightIndex:builder = prepared.get("builder")identity = _prepared_track_identity()if builder != identity:- from .index_compatibility import PREPARED_DATA_ACCESS_EQUIVALENCE+ from .index_compatibility import PREPARED_DATA_ACCESS_EQUIVALENCE, PREPARED_GEOMETRY_EQUIVALENCEcurrent = sha256(Path(__file__))- previous = PREPARED_DATA_ACCESS_EQUIVALENCE.get(current, ())- if (_prepared_track_identity(current) != identity- or all(_prepared_track_identity(source) != builder for source in previous)):+ geometry = sha256(Path(__file__).with_name("geometry.py"))+ access_sources = (current, *PREPARED_DATA_ACCESS_EQUIVALENCE.get(current, ()))+ geometry_sources = (geometry, *PREPARED_GEOMETRY_EQUIVALENCE.get(geometry, ()))+ if (_prepared_track_identity(current, geometry) != identity+ or all(_prepared_track_identity(access, geom) != builder+ for access in access_sources for geom in geometry_sources)):return Falsecolumns = [(row[1], row[2].upper(), row[5]) for row indb.execute("PRAGMA table_info(prepared_tracks)")]diff --git a/src/dopplergeist/index_compatibility.py b/src/dopplergeist/index_compatibility.pyindex 391280e..1ad24ba 100644--- a/src/dopplergeist/index_compatibility.py+++ b/src/dopplergeist/index_compatibility.py@@ -3,6 +3,16 @@# The cache-size-only change and this exact compatibility check do not change# decoded observations or normalization. Any source edit disables this mapping.PREPARED_DATA_ACCESS_EQUIVALENCE = {+ # Component hashing and exact geometry-source lookup preserve normalization.+ "d88c4be6668b92c9a1a31b5cd0b4dbdcb3d7cf73c5891ae414f44deca6088644": (+ "77da0d39e1e95476fdf02e29e9a345914545af7e62227a93b4cb6786edb37a71",+ "ddd77e9734fff6a70cc9195bd1305002f9f78e39ea601f27e0f596dcb336033a",+ "275e38a37f82993b609b6b78c113d35c4da269083ceb3afcb1d94ada48bdc0d4",+ "dad02d09d89ad9d267cd8d0424b8e018d0399a28d6b43903b5a55867290b3957",+ "d664caf4b496599d29a81360c4a9d9736a3248c157e71d885463b349aad6dd36",+ # The earlier revision differs only in release-asset discovery.+ "ce78ad66f2d478e1f7ea158ea04c0d3b25021339e34f090652740fe55e46e5ec",+ ),# Read-only digest fallback changes no observation decoding or normalization."77da0d39e1e95476fdf02e29e9a345914545af7e62227a93b4cb6786edb37a71": ("ddd77e9734fff6a70cc9195bd1305002f9f78e39ea601f27e0f596dcb336033a",@@ -21,3 +31,11 @@ PREPARED_DATA_ACCESS_EQUIVALENCE = {"d664caf4b496599d29a81360c4a9d9736a3248c157e71d885463b349aad6dd36",),}++# #329 adds a CPA-only support method without changing Track initialization,+# decoded observations, velocities, or prepared payload construction.+PREPARED_GEOMETRY_EQUIVALENCE = {+ "aff5eb70f00766a1a3ab890d1ba33f1a4989af9cf1c4b628770a8d2a7767fd13": (+ "85336bd0e3739f890386e9f93965cefb78c4477ae2e8a4fe489353501c5e26a2",+ ),+}diff --git a/tests/test_real_prepared_geometry_compatibility.py b/tests/test_real_prepared_geometry_compatibility.pynew file mode 100644index 0000000..95b7585--- /dev/null+++ b/tests/test_real_prepared_geometry_compatibility.py@@ -0,0 +1,63 @@+"""Prepared-index compatibility across exact recorded-source revisions."""+import json+import os+from pathlib import Path+import sqlite3++import pytest++from dopplergeist import data_access, index_compatibility+from test_real_prepared_index import observed_index, queried, signature+++def test_pinned_legacy_geometry_and_data_source_preserve_real_tracks(observed_index, monkeypatch):+ index, coverage = observed_index+ expected = [signature(track) for track in queried(index, coverage)]+ access = data_access.sha256(Path(data_access.__file__))+ geometry = data_access.sha256(Path(data_access.__file__).with_name("geometry.py"))+ previous_access = index_compatibility.PREPARED_DATA_ACCESS_EQUIVALENCE[access][0]+ previous_geometry = index_compatibility.PREPARED_GEOMETRY_EQUIVALENCE[geometry][0]+ # Relabel only the builder metadata in this disposable index. The aircraft+ # observations and prepared bytes are unchanged recorded observations.+ with sqlite3.connect(index.path) as db:+ report = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])+ report["prepared_tracks"]["builder"] = data_access._prepared_track_identity(previous_access, previous_geometry)+ db.execute("UPDATE metadata SET value=?", (json.dumps(report),))+ reused = data_access.FlightIndex(index.path, track_cache_bytes=0)+ assert reused._prepared_tracks+ assert [signature(track) for track in queried(reused, coverage)] == expected+ with monkeypatch.context() as patch:+ patch.setattr(index_compatibility, "PREPARED_GEOMETRY_EQUIVALENCE", {})+ assert not data_access.FlightIndex(index.path)._prepared_tracks+ with monkeypatch.context() as patch:+ patch.setattr(index_compatibility, "PREPARED_DATA_ACCESS_EQUIVALENCE", {})+ assert not data_access.FlightIndex(index.path)._prepared_tracks+ with monkeypatch.context() as patch:+ patch.setattr(data_access, "_prepared_track_identity", lambda *args: "different-runtime")+ assert not data_access.FlightIndex(index.path)._prepared_tracks+ original_sha256 = data_access.sha256+ for modified_name in ("data_access.py", "geometry.py"):+ with monkeypatch.context() as patch:+ patch.setattr(data_access, "sha256", lambda path: "0" * 64 if Path(path).name == modified_name+ else original_sha256(path))+ assert not data_access.FlightIndex(index.path)._prepared_tracks++[email protected]("version", ["7f52ff0-parent", "54bc027", "470b708", "ee23896", "4cbdec8", "0071db4"])+def test_actual_prior_prepared_indexes_match_verified_raw_tracks(version, monkeypatch):+ root = os.environ.get("DOPPLERGEIST_REAL_GEOMETRY_PREPARED_DIR")+ if not root:+ pytest.skip("set the directory containing prepared copies from the pinned real source versions")+ index = data_access.FlightIndex(Path(root) / f"{version}.sqlite", track_cache_bytes=0)+ assert index._prepared_tracks+ scope = index.report["prepared_tracks"]["scope"]+ raw = data_access.FlightIndex(index.path, track_cache_bytes=0)+ raw._prepared_tracks = False+ expected = list(raw.query(scope["bounds"], scope["time_range_s"]))+ actual = list(index.query(scope["bounds"], scope["time_range_s"]))+ assert expected+ assert [signature(track) for track in actual] == [signature(track) for track in expected]+ if version != "0071db4":+ with monkeypatch.context() as patch:+ patch.setattr(index_compatibility, "PREPARED_GEOMETRY_EQUIVALENCE", {})+ assert not data_access.FlightIndex(index.path)._prepared_tracks