Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Accelerate canonical flight-index column decoding (#309)
Cinna Davis54bc0274e072
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex f482815..994ba3d 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -46,12 +46,15 @@ jobs:import jsonfrom pathlib import Pathimport dopplergeist- from dopplergeist import _native_cpa, _native_ridge, automatic, jobs, service+ from dopplergeist import _native_cpa, _native_index, _native_ridge, automatic, jobs, servicepackage = Path(dopplergeist.__file__).resolve().parentassert "site-packages" in package.parts, packageassert Path(_native_ridge.__file__).resolve().parent == packageassert Path(_native_cpa.__file__).resolve().parent == package+ assert Path(_native_index.__file__).resolve().parent == package+ observed = json.loads(Path("research/trajectory-motion/frozen/fresh-a6cc10.json").read_text())+ assert _native_index.payload_columns(observed) is not None, "Native real-column extraction failed"assert callable(_native_ridge.fit_irls_batch), "Native batch fitter did not build"assert callable(_native_ridge.fit_statistics), "Native fit counters did not build"assert callable(_native_cpa.select), "Native CPA reducer did not build"diff --git a/docs/DATA_AND_SCALE.md b/docs/DATA_AND_SCALE.mdindex aae4086..2d4e2bf 100644--- a/docs/DATA_AND_SCALE.md+++ b/docs/DATA_AND_SCALE.md@@ -33,6 +33,13 @@ cache-size change remain usable when the exact source and all other builderidentities match. Unknown source versions continue to use the verified originalpayload instead of trusting incompatible prepared data.+An optional native collector speeds canonical raw-index column extraction;+unsupported representations and installations without the extension use the+existing Python path. The exact compatible prepared-index versions remain usable.+On one real full-day query, 1,060 trajectories decoded with identical fields in+median 6.31 seconds of CPU before and 5.79 seconds after. This measures the query+only, not a complete upload or national search.+Decoded tracks also have a bounded 128 MiB serialized cache per index per worker,with at most 2,048 entries. `FlightIndex(track_cache_bytes=...)` accepts a smallerlimit or zero; cached queries still verify source payloads and return independentdiff --git a/setup.py b/setup.pyindex d5b8184..10c7f52 100644--- a/setup.py+++ b/setup.py@@ -30,4 +30,11 @@ cpa_extension = Extension(libraries=[] if os.name == "nt" else ["m"],)-setup(ext_modules=[extension, acoustic_extension, cpa_extension])+index_extension = Extension(+ "dopplergeist._native_index",+ ["src/dopplergeist/_native_index.c"],+ optional=True,+ extra_compile_args=["/O2"] if os.name == "nt" else ["-O3"],+)++setup(ext_modules=[extension, acoustic_extension, cpa_extension, index_extension])diff --git a/src/dopplergeist/_native_index.c b/src/dopplergeist/_native_index.cnew file mode 100644index 0000000..8066735--- /dev/null+++ b/src/dopplergeist/_native_index.c@@ -0,0 +1,105 @@+/* Exact canonical JSON-column extraction, without NumPy's C API.+ * The Python path owns JSON decoding and all dtype/trajectory conversion.+ * This optional helper only validates built-in types and collects references.+ */+#define PY_SSIZE_T_CLEAN+#include <Python.h>++#define COLUMN_COUNT 10++static const char *column_names[COLUMN_COUNT] = {+ "time_s", "lat", "lon", "alt_m", "east_mps", "north_mps", "up_mps",+ "typecode", "altitude_basis", "aircraft_class"+};++typedef struct {+ unsigned long long successful_calls;+} module_state;++static PyObject *payload_columns(PyObject *self, PyObject *records) {+ PyObject *keys[COLUMN_COUNT] = {NULL};+ PyObject *columns[COLUMN_COUNT] = {NULL};+ PyObject *result = NULL;+ int present[COLUMN_COUNT] = {0};+ if (!PyList_CheckExact(records) || PyList_GET_SIZE(records) == 0)+ Py_RETURN_NONE;+ Py_ssize_t count = PyList_GET_SIZE(records);+ for (int j = 0; j < COLUMN_COUNT; ++j) {+ keys[j] = PyUnicode_InternFromString(column_names[j]);+ if (!keys[j]) goto done;+ if (j < 4) {+ columns[j] = PyList_New(count);+ if (!columns[j]) goto done;+ present[j] = 1;+ }+ }+ for (Py_ssize_t i = 0; i < count; ++i) {+ PyObject *row = PyList_GET_ITEM(records, i);+ if (!PyDict_CheckExact(row)) goto unsupported;+ /* JSON dictionaries have plain string keys. Other mappings keep the+ * original Python path, including custom equality/hash behavior. */+ Py_ssize_t position = 0;+ PyObject *key, *value;+ while (PyDict_Next(row, &position, &key, &value))+ if (!PyUnicode_CheckExact(key)) goto unsupported;+ for (int j = 0; j < COLUMN_COUNT; ++j) {+ value = PyDict_GetItemWithError(row, keys[j]);+ if (!value && PyErr_Occurred()) goto done;+ if (j >= 4 && i == 0) {+ present[j] = value != NULL;+ if (present[j]) {+ columns[j] = PyList_New(count);+ if (!columns[j]) goto done;+ }+ }+ if (!present[j]) {+ if (value) goto unsupported;+ continue;+ }+ if (!value) goto unsupported;+ if (j < 4) {+ if (!PyFloat_CheckExact(value)) goto unsupported;+ } else if (j < 7) {+ if (value != Py_None && !PyFloat_CheckExact(value)) goto unsupported;+ } else if (!PyUnicode_CheckExact(value)) {+ goto unsupported;+ }+ /* Each list slot is initially NULL and filled once. */+ PyList_SET_ITEM(columns[j], i, Py_NewRef(value));+ }+ }+ result = PyTuple_New(COLUMN_COUNT);+ if (!result) goto done;+ for (int j = 0; j < COLUMN_COUNT; ++j) {+ PyTuple_SET_ITEM(result, j, columns[j] ? columns[j] : Py_NewRef(Py_None));+ columns[j] = NULL; /* Tuple owns these references now. */+ }+ ((module_state *)PyModule_GetState(self))->successful_calls++;+ goto done;+unsupported:+ result = Py_NewRef(Py_None);+done:+ for (int j = 0; j < COLUMN_COUNT; ++j) {+ Py_XDECREF(columns[j]);+ Py_XDECREF(keys[j]);+ }+ return result;+}++static PyObject *successful_calls(PyObject *self, PyObject *unused) {+ return PyLong_FromUnsignedLongLong(((module_state *)PyModule_GetState(self))->successful_calls);+}++static PyMethodDef methods[] = {+ {"payload_columns", payload_columns, METH_O,+ "Collect exact canonical recorded columns, or return None for Python fallback."},+ {"successful_calls", successful_calls, METH_NOARGS,+ "Count successful canonical column extractions."},+ {NULL, NULL, 0, NULL}+};++static struct PyModuleDef module = {+ PyModuleDef_HEAD_INIT, "_native_index", NULL, sizeof(module_state), methods+};++PyMODINIT_FUNC PyInit__native_index(void) { return PyModule_Create(&module); }diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.pyindex b3c0e6f..749e387 100644--- a/src/dopplergeist/data_access.py+++ b/src/dopplergeist/data_access.py@@ -29,6 +29,14 @@ import requestsfrom .geometry import Trackfrom .tracks import _segment_velocity, adsblol_frame+try:+ from ._native_index import payload_columns as _native_payload_columns+except ImportError:+ _native_payload_columns = None++_PAYLOAD_COLUMNS = ("time_s", "lat", "lon", "alt_m", "east_mps", "north_mps", "up_mps",+ "typecode", "altitude_basis", "aircraft_class")+CONUS = (-125.0, 24.0, -66.0, 50.0)INDEX_VERSION = 2@@ -115,6 +123,13 @@ def _payload_frame(records):mixed schemas, integer position fields and unusual labels keep pandas'original dtype inference and failure behavior."""+ if _native_payload_columns is not None:+ values = _native_payload_columns(records)+ if values is not None:+ columns = {name: _IndexColumn(np.asarray(column, dtype=float if i < 7 else object))+ for i, (name, column) in enumerate(zip(_PAYLOAD_COLUMNS, values))+ if column is not None}+ return _IndexColumns(columns, len(records))if type(records) is not list or not records or any(type(row) is not dict for row in records):return pd.DataFrame(records)absent = object()@@ -146,7 +161,7 @@ _CANONICAL_TRACK = Track_PREPARED_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}-@lru_cache(maxsize=4)+@lru_cache(maxsize=16)def _prepared_track_identity(data_access_digest=None):"""Use prepared velocities only with identical normalization and NumPy."""from . import flight_payloaddiff --git a/src/dopplergeist/index_compatibility.py b/src/dopplergeist/index_compatibility.pyindex 9f9aa41..f752971 100644--- a/src/dopplergeist/index_compatibility.py+++ b/src/dopplergeist/index_compatibility.py@@ -3,6 +3,12 @@# The cache-size-only change and this exact compatibility check do not change# decoded observations or normalization. Any source edit disables this mapping.PREPARED_DATA_ACCESS_EQUIVALENCE = {+ # The native collector only replaces canonical Python column extraction.+ "ddd77e9734fff6a70cc9195bd1305002f9f78e39ea601f27e0f596dcb336033a": (+ "275e38a37f82993b609b6b78c113d35c4da269083ceb3afcb1d94ada48bdc0d4",+ "dad02d09d89ad9d267cd8d0424b8e018d0399a28d6b43903b5a55867290b3957",+ "d664caf4b496599d29a81360c4a9d9736a3248c157e71d885463b349aad6dd36",+ ),"275e38a37f82993b609b6b78c113d35c4da269083ceb3afcb1d94ada48bdc0d4": ("dad02d09d89ad9d267cd8d0424b8e018d0399a28d6b43903b5a55867290b3957","d664caf4b496599d29a81360c4a9d9736a3248c157e71d885463b349aad6dd36",diff --git a/tests/test_real_native_index.py b/tests/test_real_native_index.pynew file mode 100644index 0000000..0bb7900--- /dev/null+++ b/tests/test_real_native_index.py@@ -0,0 +1,165 @@+"""Native column parity on real ADS-B rows and unchanged representations."""+from copy import deepcopy+import json+from pathlib import Path+import sys++import numpy as np+import pandas as pd+import pytest++from dopplergeist import data_access++ROOT = Path(__file__).resolve().parents[1]+RECORDED = sorted((ROOT / "research/trajectory-motion/frozen").glob("fresh-*.json"))++[email protected](scope="module")+def native_index():+ return pytest.importorskip("dopplergeist._native_index")+++def recorded_rows():+ assert RECORDED, "The archived real observations are required"+ return json.loads(RECORDED[0].read_text())+++def assert_same_columns(actual, expected):+ assert type(actual) is type(expected)+ if isinstance(expected, pd.DataFrame):+ pd.testing.assert_frame_equal(actual, expected, check_exact=True)+ return+ assert list(actual.columns) == list(expected.columns)+ assert len(actual) == len(expected)+ for name, column in expected.columns.items():+ wanted, got = column.values, actual.columns[name].values+ assert got.dtype == wanted.dtype and got.shape == wanted.shape, name+ if got.dtype.hasobject:+ assert got.tolist() == wanted.tolist(), name+ else:+ assert got.tobytes() == wanted.tobytes(), name+++def compare_python_and_native(records, native_index, monkeypatch):+ with monkeypatch.context() as patch:+ patch.setattr(data_access, "_native_payload_columns", None)+ expected = data_access._payload_frame(records)+ with monkeypatch.context() as patch:+ patch.setattr(data_access, "_native_payload_columns", native_index.payload_columns)+ actual = data_access._payload_frame(records)+ assert_same_columns(actual, expected)+ return actual++[email protected]("source", RECORDED, ids=lambda source: source.stem)+def test_native_recorded_columns_match_python_bytes(source, native_index, monkeypatch):+ records = json.loads(source.read_text())+ before = native_index.successful_calls()+ frame = compare_python_and_native(records, native_index, monkeypatch)+ assert isinstance(frame, data_access._IndexColumns)+ assert native_index.successful_calls() == before + 1+++def test_recorded_null_and_absent_optional_columns(native_index, monkeypatch):+ records = recorded_rows()+ assert any(row["east_mps"] is None for row in records)+ compare_python_and_native(records, native_index, monkeypatch)+ for name in data_access._PAYLOAD_COLUMNS[4:]:+ without = [{key: value for key, value in row.items() if key != name} for row in records]+ columns = native_index.payload_columns(without)+ assert columns is not None+ assert columns[data_access._PAYLOAD_COLUMNS.index(name)] is None+ frame = compare_python_and_native(without, native_index, monkeypatch)+ assert name not in frame+++def test_real_noncanonical_representations_keep_python_fallback(native_index, monkeypatch):+ records = recorded_rows()+ variants = []+ for row_index in (0, -1):+ missing_optional = deepcopy(records)+ del missing_optional[row_index]["typecode"]+ variants.append(missing_optional)+ numeric_scalar = deepcopy(records)+ numeric_scalar[-1]["time_s"] = np.float64(numeric_scalar[-1]["time_s"])+ variants.append(numeric_scalar)++ class RecordedRow(dict):+ pass++ mapping_subclass = deepcopy(records)+ mapping_subclass[-1] = RecordedRow(mapping_subclass[-1])+ variants.extend((mapping_subclass, tuple(records)))+ for changed in variants:+ assert native_index.payload_columns(changed) is None+ compare_python_and_native(changed, native_index, monkeypatch)+++def test_native_output_containers_do_not_mutate_recorded_input(native_index, monkeypatch):+ records = recorded_rows()+ original = deepcopy(records)+ first = native_index.payload_columns(records)+ second = native_index.payload_columns(records)+ assert first is not None and second is not None+ for left, right in zip(first, second):+ if left is not None:+ assert left is not right+ left.clear()+ assert records == original+ assert second[0] == [row["time_s"] for row in records]+ compare_python_and_native(records, native_index, monkeypatch)+++def test_native_success_and_partial_fallback_release_references(native_index):+ records = recorded_rows()+ watched = records[0]["time_s"]+ before = sys.getrefcount(watched)+ columns = native_index.payload_columns(records)+ assert columns[0][0] is watched+ assert sys.getrefcount(watched) == before + 1+ del columns+ assert sys.getrefcount(watched) == before++ # A late mismatch leaves earlier list slots populated. Representation+ # changes preserve all recorded positions, timestamps and velocities.+ for name in ("lat", "typecode"):+ changed = deepcopy(records)+ del changed[-1][name]+ watched_values = (changed[0]["time_s"], changed[-1]["time_s"], changed[0]["typecode"])+ before = tuple(sys.getrefcount(value) for value in watched_values)+ for _ in range(32):+ assert native_index.payload_columns(changed) is None+ after = tuple(sys.getrefcount(value) for value in watched_values)+ assert after == before+++def test_previous_real_prepared_index_stays_usable(monkeypatch):+ import os+ path = os.environ.get("DOPPLERGEIST_REAL_PREVIOUS_PREPARED_INDEX")+ if not path:+ pytest.skip("set a completed prepared index built by the previous source")+ from dopplergeist import index_compatibility+ from test_real_prepared_index import signature++ prepared = data_access.FlightIndex(path, track_cache_bytes=0)+ assert prepared._prepared_tracks+ # Reopening an older compatible index must not continually rehash the+ # numerical runtime as the bounded identity cache cycles through versions.+ data_access._prepared_track_identity.cache_clear()+ assert data_access.FlightIndex(path, track_cache_bytes=0)._prepared_tracks+ misses = data_access._prepared_track_identity.cache_info().misses+ assert data_access.FlightIndex(path, track_cache_bytes=0)._prepared_tracks+ assert data_access._prepared_track_identity.cache_info().misses == misses+ scope = prepared.report["prepared_tracks"]["scope"]+ expected = data_access.FlightIndex(path, track_cache_bytes=0)+ expected._prepared_tracks = False+ expected_tracks = list(expected.query(scope["bounds"], scope["time_range_s"]))+ actual_tracks = list(prepared.query(scope["bounds"], scope["time_range_s"]))+ assert expected_tracks+ assert [signature(track) for track in actual_tracks] == [signature(track) for track in expected_tracks]+ with monkeypatch.context() as patch:+ patch.setattr(data_access, "_native_payload_columns", None)+ assert data_access.FlightIndex(path, track_cache_bytes=0)._prepared_tracks+ with monkeypatch.context() as patch:+ patch.setattr(index_compatibility, "PREPARED_DATA_ACCESS_EQUIVALENCE", {})+ assert not data_access.FlightIndex(path, track_cache_bytes=0)._prepared_tracksdiff --git a/tests/test_real_prepared_compatibility.py b/tests/test_real_prepared_compatibility.pyindex 940bb34..b5b07fd 100644--- a/tests/test_real_prepared_compatibility.py+++ b/tests/test_real_prepared_compatibility.py@@ -8,7 +8,8 @@ from dopplergeist import data_access, index_compatibilityfrom test_real_prepared_index import signature[email protected]("variable", ["DOPPLERGEIST_REAL_PREPARED_BEFORE_CACHE", "DOPPLERGEIST_REAL_PREPARED_AFTER_CACHE"])[email protected]("variable", ["DOPPLERGEIST_REAL_PREPARED_BEFORE_CACHE", "DOPPLERGEIST_REAL_PREPARED_AFTER_CACHE",+ "DOPPLERGEIST_REAL_PREPARED_AFTER_COMPAT"])def test_prior_prepared_index_is_reused_only_for_pinned_sources(variable, monkeypatch):value = os.environ.get(variable)if not value: