Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Reuse prepared indexes across CPA-only geometry change (#337)

Cinna Davis31ddb88226f3
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index c0d724d..7e2f351 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -31,7 +31,7 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y ffmpeg
- python -m pip install .
+ python -m pip install '.[test]'
python -m pip check
# The owner requires real recordings for further validation. The historical
# pytest suite generates synthetic inputs, so this workflow does not run it.
@@ -39,6 +39,8 @@ jobs:
run: python -m compileall -q src scripts
- name: Check benchmark instrumentation detection
run: python scripts/check_benchmark_instrumentation.py
+ - name: Check prepared-index compatibility with recorded observations
+ run: python -m pytest -q tests/test_real_prepared_geometry_compatibility.py::test_pinned_legacy_geometry_and_data_source_preserve_real_tracks
- name: CLI smoke check
run: dopplergeist --help
- name: Check installed native validator and packaged browser assets
diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.py
index 6847566..47c3a12 100644
--- a/src/dopplergeist/data_access.py
+++ b/src/dopplergeist/data_access.py
@@ -161,9 +161,9 @@ _CANONICAL_TRACK = Track
_PREPARED_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}
-@lru_cache(maxsize=16)
-def _prepared_track_identity(data_access_digest=None):
- """Use prepared velocities only with identical normalization and NumPy."""
+@lru_cache(maxsize=1)
+def _prepared_track_components():
+ """Hash immutable runtime inputs once when checking compatible builders."""
from . import flight_payload
try:
import numpy._core._multiarray_umath as numpy_core
@@ -173,12 +173,21 @@ def _prepared_track_identity(data_access_digest=None):
Path(__file__).with_name("geometry.py"), Path(flight_payload.__file__),
Path(numpy_core.__file__))
sources = [sha256(path) for path in files]
+ return {"sources": sources, "numpy": np.__version__, "pandas": pd.__version__,
+ "python": sys.version, "cpu": getattr(numpy_core, "__cpu_features__", {})}
+
+
+@lru_cache(maxsize=16)
+def _prepared_track_identity(data_access_digest=None, geometry_digest=None):
+ """Use prepared velocities only with identical normalization and NumPy."""
+ components = dict(_prepared_track_components())
+ sources = list(components["sources"])
if data_access_digest is not None:
sources[0] = data_access_digest
- return content_hash({"sources": sources,
- "numpy": np.__version__,
- "pandas": pd.__version__, "python": sys.version,
- "cpu": getattr(numpy_core, "__cpu_features__", {})})
+ if geometry_digest is not None:
+ sources[2] = geometry_digest
+ components["sources"] = sources
+ return content_hash(components)
@lru_cache(maxsize=1)
@@ -583,11 +592,14 @@ class FlightIndex:
builder = prepared.get("builder")
identity = _prepared_track_identity()
if builder != identity:
- from .index_compatibility import PREPARED_DATA_ACCESS_EQUIVALENCE
+ from .index_compatibility import PREPARED_DATA_ACCESS_EQUIVALENCE, PREPARED_GEOMETRY_EQUIVALENCE
current = sha256(Path(__file__))
- previous = PREPARED_DATA_ACCESS_EQUIVALENCE.get(current, ())
- if (_prepared_track_identity(current) != identity
- or all(_prepared_track_identity(source) != builder for source in previous)):
+ geometry = sha256(Path(__file__).with_name("geometry.py"))
+ access_sources = (current, *PREPARED_DATA_ACCESS_EQUIVALENCE.get(current, ()))
+ geometry_sources = (geometry, *PREPARED_GEOMETRY_EQUIVALENCE.get(geometry, ()))
+ if (_prepared_track_identity(current, geometry) != identity
+ or all(_prepared_track_identity(access, geom) != builder
+ for access in access_sources for geom in geometry_sources)):
return False
columns = [(row[1], row[2].upper(), row[5]) for row in
db.execute("PRAGMA table_info(prepared_tracks)")]
diff --git a/src/dopplergeist/index_compatibility.py b/src/dopplergeist/index_compatibility.py
index 391280e..1ad24ba 100644
--- a/src/dopplergeist/index_compatibility.py
+++ b/src/dopplergeist/index_compatibility.py
@@ -3,6 +3,16 @@
# The cache-size-only change and this exact compatibility check do not change
# decoded observations or normalization. Any source edit disables this mapping.
PREPARED_DATA_ACCESS_EQUIVALENCE = {
+ # Component hashing and exact geometry-source lookup preserve normalization.
+ "d88c4be6668b92c9a1a31b5cd0b4dbdcb3d7cf73c5891ae414f44deca6088644": (
+ "77da0d39e1e95476fdf02e29e9a345914545af7e62227a93b4cb6786edb37a71",
+ "ddd77e9734fff6a70cc9195bd1305002f9f78e39ea601f27e0f596dcb336033a",
+ "275e38a37f82993b609b6b78c113d35c4da269083ceb3afcb1d94ada48bdc0d4",
+ "dad02d09d89ad9d267cd8d0424b8e018d0399a28d6b43903b5a55867290b3957",
+ "d664caf4b496599d29a81360c4a9d9736a3248c157e71d885463b349aad6dd36",
+ # The earlier revision differs only in release-asset discovery.
+ "ce78ad66f2d478e1f7ea158ea04c0d3b25021339e34f090652740fe55e46e5ec",
+ ),
# Read-only digest fallback changes no observation decoding or normalization.
"77da0d39e1e95476fdf02e29e9a345914545af7e62227a93b4cb6786edb37a71": (
"ddd77e9734fff6a70cc9195bd1305002f9f78e39ea601f27e0f596dcb336033a",
@@ -21,3 +31,11 @@ PREPARED_DATA_ACCESS_EQUIVALENCE = {
"d664caf4b496599d29a81360c4a9d9736a3248c157e71d885463b349aad6dd36",
),
}
+
+# #329 adds a CPA-only support method without changing Track initialization,
+# decoded observations, velocities, or prepared payload construction.
+PREPARED_GEOMETRY_EQUIVALENCE = {
+ "aff5eb70f00766a1a3ab890d1ba33f1a4989af9cf1c4b628770a8d2a7767fd13": (
+ "85336bd0e3739f890386e9f93965cefb78c4477ae2e8a4fe489353501c5e26a2",
+ ),
+}
diff --git a/tests/test_real_prepared_geometry_compatibility.py b/tests/test_real_prepared_geometry_compatibility.py
new file mode 100644
index 0000000..95b7585
--- /dev/null
+++ b/tests/test_real_prepared_geometry_compatibility.py
@@ -0,0 +1,63 @@
+"""Prepared-index compatibility across exact recorded-source revisions."""
+import json
+import os
+from pathlib import Path
+import sqlite3
+
+import pytest
+
+from dopplergeist import data_access, index_compatibility
+from test_real_prepared_index import observed_index, queried, signature
+
+
+def test_pinned_legacy_geometry_and_data_source_preserve_real_tracks(observed_index, monkeypatch):
+ index, coverage = observed_index
+ expected = [signature(track) for track in queried(index, coverage)]
+ access = data_access.sha256(Path(data_access.__file__))
+ geometry = data_access.sha256(Path(data_access.__file__).with_name("geometry.py"))
+ previous_access = index_compatibility.PREPARED_DATA_ACCESS_EQUIVALENCE[access][0]
+ previous_geometry = index_compatibility.PREPARED_GEOMETRY_EQUIVALENCE[geometry][0]
+ # Relabel only the builder metadata in this disposable index. The aircraft
+ # observations and prepared bytes are unchanged recorded observations.
+ with sqlite3.connect(index.path) as db:
+ report = json.loads(db.execute("SELECT value FROM metadata").fetchone()[0])
+ report["prepared_tracks"]["builder"] = data_access._prepared_track_identity(previous_access, previous_geometry)
+ db.execute("UPDATE metadata SET value=?", (json.dumps(report),))
+ reused = data_access.FlightIndex(index.path, track_cache_bytes=0)
+ assert reused._prepared_tracks
+ assert [signature(track) for track in queried(reused, coverage)] == expected
+ with monkeypatch.context() as patch:
+ patch.setattr(index_compatibility, "PREPARED_GEOMETRY_EQUIVALENCE", {})
+ assert not data_access.FlightIndex(index.path)._prepared_tracks
+ with monkeypatch.context() as patch:
+ patch.setattr(index_compatibility, "PREPARED_DATA_ACCESS_EQUIVALENCE", {})
+ assert not data_access.FlightIndex(index.path)._prepared_tracks
+ with monkeypatch.context() as patch:
+ patch.setattr(data_access, "_prepared_track_identity", lambda *args: "different-runtime")
+ assert not data_access.FlightIndex(index.path)._prepared_tracks
+ original_sha256 = data_access.sha256
+ for modified_name in ("data_access.py", "geometry.py"):
+ with monkeypatch.context() as patch:
+ patch.setattr(data_access, "sha256", lambda path: "0" * 64 if Path(path).name == modified_name
+ else original_sha256(path))
+ assert not data_access.FlightIndex(index.path)._prepared_tracks
+
+
[email protected]("version", ["7f52ff0-parent", "54bc027", "470b708", "ee23896", "4cbdec8", "0071db4"])
+def test_actual_prior_prepared_indexes_match_verified_raw_tracks(version, monkeypatch):
+ root = os.environ.get("DOPPLERGEIST_REAL_GEOMETRY_PREPARED_DIR")
+ if not root:
+ pytest.skip("set the directory containing prepared copies from the pinned real source versions")
+ index = data_access.FlightIndex(Path(root) / f"{version}.sqlite", track_cache_bytes=0)
+ assert index._prepared_tracks
+ scope = index.report["prepared_tracks"]["scope"]
+ raw = data_access.FlightIndex(index.path, track_cache_bytes=0)
+ raw._prepared_tracks = False
+ expected = list(raw.query(scope["bounds"], scope["time_range_s"]))
+ actual = list(index.query(scope["bounds"], scope["time_range_s"]))
+ assert expected
+ assert [signature(track) for track in actual] == [signature(track) for track in expected]
+ if version != "0071db4":
+ with monkeypatch.context() as patch:
+ patch.setattr(index_compatibility, "PREPARED_GEOMETRY_EQUIVALENCE", {})
+ assert not data_access.FlightIndex(index.path)._prepared_tracks