Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Accelerate canonical flight-index column decoding (#309)

Cinna Davis54bc0274e072
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index f482815..994ba3d 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -46,12 +46,15 @@ jobs:
import json
from pathlib import Path
import dopplergeist
- from dopplergeist import _native_cpa, _native_ridge, automatic, jobs, service
+ from dopplergeist import _native_cpa, _native_index, _native_ridge, automatic, jobs, service
package = Path(dopplergeist.__file__).resolve().parent
assert "site-packages" in package.parts, package
assert Path(_native_ridge.__file__).resolve().parent == package
assert Path(_native_cpa.__file__).resolve().parent == package
+ assert Path(_native_index.__file__).resolve().parent == package
+ observed = json.loads(Path("research/trajectory-motion/frozen/fresh-a6cc10.json").read_text())
+ assert _native_index.payload_columns(observed) is not None, "Native real-column extraction failed"
assert callable(_native_ridge.fit_irls_batch), "Native batch fitter did not build"
assert callable(_native_ridge.fit_statistics), "Native fit counters did not build"
assert callable(_native_cpa.select), "Native CPA reducer did not build"
diff --git a/docs/DATA_AND_SCALE.md b/docs/DATA_AND_SCALE.md
index aae4086..2d4e2bf 100644
--- a/docs/DATA_AND_SCALE.md
+++ b/docs/DATA_AND_SCALE.md
@@ -33,6 +33,13 @@ cache-size change remain usable when the exact source and all other builder
identities match. Unknown source versions continue to use the verified original
payload instead of trusting incompatible prepared data.
+An optional native collector speeds canonical raw-index column extraction;
+unsupported representations and installations without the extension use the
+existing Python path. The exact compatible prepared-index versions remain usable.
+On one real full-day query, 1,060 trajectories decoded with identical fields in
+median 6.31 seconds of CPU before and 5.79 seconds after. This measures the query
+only, not a complete upload or national search.
+
Decoded tracks also have a bounded 128 MiB serialized cache per index per worker,
with at most 2,048 entries. `FlightIndex(track_cache_bytes=...)` accepts a smaller
limit or zero; cached queries still verify source payloads and return independent
diff --git a/setup.py b/setup.py
index d5b8184..10c7f52 100644
--- a/setup.py
+++ b/setup.py
@@ -30,4 +30,11 @@ cpa_extension = Extension(
libraries=[] if os.name == "nt" else ["m"],
)
-setup(ext_modules=[extension, acoustic_extension, cpa_extension])
+index_extension = Extension(
+ "dopplergeist._native_index",
+ ["src/dopplergeist/_native_index.c"],
+ optional=True,
+ extra_compile_args=["/O2"] if os.name == "nt" else ["-O3"],
+)
+
+setup(ext_modules=[extension, acoustic_extension, cpa_extension, index_extension])
diff --git a/src/dopplergeist/_native_index.c b/src/dopplergeist/_native_index.c
new file mode 100644
index 0000000..8066735
--- /dev/null
+++ b/src/dopplergeist/_native_index.c
@@ -0,0 +1,105 @@
+/* Exact canonical JSON-column extraction, without NumPy's C API.
+ * The Python path owns JSON decoding and all dtype/trajectory conversion.
+ * This optional helper only validates built-in types and collects references.
+ */
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#define COLUMN_COUNT 10
+
+static const char *column_names[COLUMN_COUNT] = {
+ "time_s", "lat", "lon", "alt_m", "east_mps", "north_mps", "up_mps",
+ "typecode", "altitude_basis", "aircraft_class"
+};
+
+typedef struct {
+ unsigned long long successful_calls;
+} module_state;
+
+static PyObject *payload_columns(PyObject *self, PyObject *records) {
+ PyObject *keys[COLUMN_COUNT] = {NULL};
+ PyObject *columns[COLUMN_COUNT] = {NULL};
+ PyObject *result = NULL;
+ int present[COLUMN_COUNT] = {0};
+ if (!PyList_CheckExact(records) || PyList_GET_SIZE(records) == 0)
+ Py_RETURN_NONE;
+ Py_ssize_t count = PyList_GET_SIZE(records);
+ for (int j = 0; j < COLUMN_COUNT; ++j) {
+ keys[j] = PyUnicode_InternFromString(column_names[j]);
+ if (!keys[j]) goto done;
+ if (j < 4) {
+ columns[j] = PyList_New(count);
+ if (!columns[j]) goto done;
+ present[j] = 1;
+ }
+ }
+ for (Py_ssize_t i = 0; i < count; ++i) {
+ PyObject *row = PyList_GET_ITEM(records, i);
+ if (!PyDict_CheckExact(row)) goto unsupported;
+ /* JSON dictionaries have plain string keys. Other mappings keep the
+ * original Python path, including custom equality/hash behavior. */
+ Py_ssize_t position = 0;
+ PyObject *key, *value;
+ while (PyDict_Next(row, &position, &key, &value))
+ if (!PyUnicode_CheckExact(key)) goto unsupported;
+ for (int j = 0; j < COLUMN_COUNT; ++j) {
+ value = PyDict_GetItemWithError(row, keys[j]);
+ if (!value && PyErr_Occurred()) goto done;
+ if (j >= 4 && i == 0) {
+ present[j] = value != NULL;
+ if (present[j]) {
+ columns[j] = PyList_New(count);
+ if (!columns[j]) goto done;
+ }
+ }
+ if (!present[j]) {
+ if (value) goto unsupported;
+ continue;
+ }
+ if (!value) goto unsupported;
+ if (j < 4) {
+ if (!PyFloat_CheckExact(value)) goto unsupported;
+ } else if (j < 7) {
+ if (value != Py_None && !PyFloat_CheckExact(value)) goto unsupported;
+ } else if (!PyUnicode_CheckExact(value)) {
+ goto unsupported;
+ }
+ /* Each list slot is initially NULL and filled once. */
+ PyList_SET_ITEM(columns[j], i, Py_NewRef(value));
+ }
+ }
+ result = PyTuple_New(COLUMN_COUNT);
+ if (!result) goto done;
+ for (int j = 0; j < COLUMN_COUNT; ++j) {
+ PyTuple_SET_ITEM(result, j, columns[j] ? columns[j] : Py_NewRef(Py_None));
+ columns[j] = NULL; /* Tuple owns these references now. */
+ }
+ ((module_state *)PyModule_GetState(self))->successful_calls++;
+ goto done;
+unsupported:
+ result = Py_NewRef(Py_None);
+done:
+ for (int j = 0; j < COLUMN_COUNT; ++j) {
+ Py_XDECREF(columns[j]);
+ Py_XDECREF(keys[j]);
+ }
+ return result;
+}
+
+static PyObject *successful_calls(PyObject *self, PyObject *unused) {
+ return PyLong_FromUnsignedLongLong(((module_state *)PyModule_GetState(self))->successful_calls);
+}
+
+static PyMethodDef methods[] = {
+ {"payload_columns", payload_columns, METH_O,
+ "Collect exact canonical recorded columns, or return None for Python fallback."},
+ {"successful_calls", successful_calls, METH_NOARGS,
+ "Count successful canonical column extractions."},
+ {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef module = {
+ PyModuleDef_HEAD_INIT, "_native_index", NULL, sizeof(module_state), methods
+};
+
+PyMODINIT_FUNC PyInit__native_index(void) { return PyModule_Create(&module); }
diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.py
index b3c0e6f..749e387 100644
--- a/src/dopplergeist/data_access.py
+++ b/src/dopplergeist/data_access.py
@@ -29,6 +29,14 @@ import requests
from .geometry import Track
from .tracks import _segment_velocity, adsblol_frame
+try:
+ from ._native_index import payload_columns as _native_payload_columns
+except ImportError:
+ _native_payload_columns = None
+
+_PAYLOAD_COLUMNS = ("time_s", "lat", "lon", "alt_m", "east_mps", "north_mps", "up_mps",
+ "typecode", "altitude_basis", "aircraft_class")
+
CONUS = (-125.0, 24.0, -66.0, 50.0)
INDEX_VERSION = 2
@@ -115,6 +123,13 @@ def _payload_frame(records):
mixed schemas, integer position fields and unusual labels keep pandas'
original dtype inference and failure behavior.
"""
+ if _native_payload_columns is not None:
+ values = _native_payload_columns(records)
+ if values is not None:
+ columns = {name: _IndexColumn(np.asarray(column, dtype=float if i < 7 else object))
+ for i, (name, column) in enumerate(zip(_PAYLOAD_COLUMNS, values))
+ if column is not None}
+ return _IndexColumns(columns, len(records))
if type(records) is not list or not records or any(type(row) is not dict for row in records):
return pd.DataFrame(records)
absent = object()
@@ -146,7 +161,7 @@ _CANONICAL_TRACK = Track
_PREPARED_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}
-@lru_cache(maxsize=4)
+@lru_cache(maxsize=16)
def _prepared_track_identity(data_access_digest=None):
"""Use prepared velocities only with identical normalization and NumPy."""
from . import flight_payload
diff --git a/src/dopplergeist/index_compatibility.py b/src/dopplergeist/index_compatibility.py
index 9f9aa41..f752971 100644
--- a/src/dopplergeist/index_compatibility.py
+++ b/src/dopplergeist/index_compatibility.py
@@ -3,6 +3,12 @@
# The cache-size-only change and this exact compatibility check do not change
# decoded observations or normalization. Any source edit disables this mapping.
PREPARED_DATA_ACCESS_EQUIVALENCE = {
+ # The native collector only replaces canonical Python column extraction.
+ "ddd77e9734fff6a70cc9195bd1305002f9f78e39ea601f27e0f596dcb336033a": (
+ "275e38a37f82993b609b6b78c113d35c4da269083ceb3afcb1d94ada48bdc0d4",
+ "dad02d09d89ad9d267cd8d0424b8e018d0399a28d6b43903b5a55867290b3957",
+ "d664caf4b496599d29a81360c4a9d9736a3248c157e71d885463b349aad6dd36",
+ ),
"275e38a37f82993b609b6b78c113d35c4da269083ceb3afcb1d94ada48bdc0d4": (
"dad02d09d89ad9d267cd8d0424b8e018d0399a28d6b43903b5a55867290b3957",
"d664caf4b496599d29a81360c4a9d9736a3248c157e71d885463b349aad6dd36",
diff --git a/tests/test_real_native_index.py b/tests/test_real_native_index.py
new file mode 100644
index 0000000..0bb7900
--- /dev/null
+++ b/tests/test_real_native_index.py
@@ -0,0 +1,165 @@
+"""Native column parity on real ADS-B rows and unchanged representations."""
+from copy import deepcopy
+import json
+from pathlib import Path
+import sys
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from dopplergeist import data_access
+
+ROOT = Path(__file__).resolve().parents[1]
+RECORDED = sorted((ROOT / "research/trajectory-motion/frozen").glob("fresh-*.json"))
+
+
[email protected](scope="module")
+def native_index():
+ return pytest.importorskip("dopplergeist._native_index")
+
+
+def recorded_rows():
+ assert RECORDED, "The archived real observations are required"
+ return json.loads(RECORDED[0].read_text())
+
+
+def assert_same_columns(actual, expected):
+ assert type(actual) is type(expected)
+ if isinstance(expected, pd.DataFrame):
+ pd.testing.assert_frame_equal(actual, expected, check_exact=True)
+ return
+ assert list(actual.columns) == list(expected.columns)
+ assert len(actual) == len(expected)
+ for name, column in expected.columns.items():
+ wanted, got = column.values, actual.columns[name].values
+ assert got.dtype == wanted.dtype and got.shape == wanted.shape, name
+ if got.dtype.hasobject:
+ assert got.tolist() == wanted.tolist(), name
+ else:
+ assert got.tobytes() == wanted.tobytes(), name
+
+
+def compare_python_and_native(records, native_index, monkeypatch):
+ with monkeypatch.context() as patch:
+ patch.setattr(data_access, "_native_payload_columns", None)
+ expected = data_access._payload_frame(records)
+ with monkeypatch.context() as patch:
+ patch.setattr(data_access, "_native_payload_columns", native_index.payload_columns)
+ actual = data_access._payload_frame(records)
+ assert_same_columns(actual, expected)
+ return actual
+
+
[email protected]("source", RECORDED, ids=lambda source: source.stem)
+def test_native_recorded_columns_match_python_bytes(source, native_index, monkeypatch):
+ records = json.loads(source.read_text())
+ before = native_index.successful_calls()
+ frame = compare_python_and_native(records, native_index, monkeypatch)
+ assert isinstance(frame, data_access._IndexColumns)
+ assert native_index.successful_calls() == before + 1
+
+
+def test_recorded_null_and_absent_optional_columns(native_index, monkeypatch):
+ records = recorded_rows()
+ assert any(row["east_mps"] is None for row in records)
+ compare_python_and_native(records, native_index, monkeypatch)
+ for name in data_access._PAYLOAD_COLUMNS[4:]:
+ without = [{key: value for key, value in row.items() if key != name} for row in records]
+ columns = native_index.payload_columns(without)
+ assert columns is not None
+ assert columns[data_access._PAYLOAD_COLUMNS.index(name)] is None
+ frame = compare_python_and_native(without, native_index, monkeypatch)
+ assert name not in frame
+
+
+def test_real_noncanonical_representations_keep_python_fallback(native_index, monkeypatch):
+ records = recorded_rows()
+ variants = []
+ for row_index in (0, -1):
+ missing_optional = deepcopy(records)
+ del missing_optional[row_index]["typecode"]
+ variants.append(missing_optional)
+ numeric_scalar = deepcopy(records)
+ numeric_scalar[-1]["time_s"] = np.float64(numeric_scalar[-1]["time_s"])
+ variants.append(numeric_scalar)
+
+ class RecordedRow(dict):
+ pass
+
+ mapping_subclass = deepcopy(records)
+ mapping_subclass[-1] = RecordedRow(mapping_subclass[-1])
+ variants.extend((mapping_subclass, tuple(records)))
+ for changed in variants:
+ assert native_index.payload_columns(changed) is None
+ compare_python_and_native(changed, native_index, monkeypatch)
+
+
+def test_native_output_containers_do_not_mutate_recorded_input(native_index, monkeypatch):
+ records = recorded_rows()
+ original = deepcopy(records)
+ first = native_index.payload_columns(records)
+ second = native_index.payload_columns(records)
+ assert first is not None and second is not None
+ for left, right in zip(first, second):
+ if left is not None:
+ assert left is not right
+ left.clear()
+ assert records == original
+ assert second[0] == [row["time_s"] for row in records]
+ compare_python_and_native(records, native_index, monkeypatch)
+
+
+def test_native_success_and_partial_fallback_release_references(native_index):
+ records = recorded_rows()
+ watched = records[0]["time_s"]
+ before = sys.getrefcount(watched)
+ columns = native_index.payload_columns(records)
+ assert columns[0][0] is watched
+ assert sys.getrefcount(watched) == before + 1
+ del columns
+ assert sys.getrefcount(watched) == before
+
+ # A late mismatch leaves earlier list slots populated. Representation
+ # changes preserve all recorded positions, timestamps and velocities.
+ for name in ("lat", "typecode"):
+ changed = deepcopy(records)
+ del changed[-1][name]
+ watched_values = (changed[0]["time_s"], changed[-1]["time_s"], changed[0]["typecode"])
+ before = tuple(sys.getrefcount(value) for value in watched_values)
+ for _ in range(32):
+ assert native_index.payload_columns(changed) is None
+ after = tuple(sys.getrefcount(value) for value in watched_values)
+ assert after == before
+
+
+def test_previous_real_prepared_index_stays_usable(monkeypatch):
+ import os
+ path = os.environ.get("DOPPLERGEIST_REAL_PREVIOUS_PREPARED_INDEX")
+ if not path:
+ pytest.skip("set a completed prepared index built by the previous source")
+ from dopplergeist import index_compatibility
+ from test_real_prepared_index import signature
+
+ prepared = data_access.FlightIndex(path, track_cache_bytes=0)
+ assert prepared._prepared_tracks
+ # Reopening an older compatible index must not continually rehash the
+ # numerical runtime as the bounded identity cache cycles through versions.
+ data_access._prepared_track_identity.cache_clear()
+ assert data_access.FlightIndex(path, track_cache_bytes=0)._prepared_tracks
+ misses = data_access._prepared_track_identity.cache_info().misses
+ assert data_access.FlightIndex(path, track_cache_bytes=0)._prepared_tracks
+ assert data_access._prepared_track_identity.cache_info().misses == misses
+ scope = prepared.report["prepared_tracks"]["scope"]
+ expected = data_access.FlightIndex(path, track_cache_bytes=0)
+ expected._prepared_tracks = False
+ expected_tracks = list(expected.query(scope["bounds"], scope["time_range_s"]))
+ actual_tracks = list(prepared.query(scope["bounds"], scope["time_range_s"]))
+ assert expected_tracks
+ assert [signature(track) for track in actual_tracks] == [signature(track) for track in expected_tracks]
+ with monkeypatch.context() as patch:
+ patch.setattr(data_access, "_native_payload_columns", None)
+ assert data_access.FlightIndex(path, track_cache_bytes=0)._prepared_tracks
+ with monkeypatch.context() as patch:
+ patch.setattr(index_compatibility, "PREPARED_DATA_ACCESS_EQUIVALENCE", {})
+ assert not data_access.FlightIndex(path, track_cache_bytes=0)._prepared_tracks
diff --git a/tests/test_real_prepared_compatibility.py b/tests/test_real_prepared_compatibility.py
index 940bb34..b5b07fd 100644
--- a/tests/test_real_prepared_compatibility.py
+++ b/tests/test_real_prepared_compatibility.py
@@ -8,7 +8,8 @@ from dopplergeist import data_access, index_compatibility
from test_real_prepared_index import signature
[email protected]("variable", ["DOPPLERGEIST_REAL_PREPARED_BEFORE_CACHE", "DOPPLERGEIST_REAL_PREPARED_AFTER_CACHE"])
[email protected]("variable", ["DOPPLERGEIST_REAL_PREPARED_BEFORE_CACHE", "DOPPLERGEIST_REAL_PREPARED_AFTER_CACHE",
+ "DOPPLERGEIST_REAL_PREPARED_AFTER_COMPAT"])
def test_prior_prepared_index_is_reused_only_for_pinned_sources(variable, monkeypatch):
value = os.environ.get(variable)
if not value: