Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Batch first-event fits across shared clocks

Cinna Davisfff5c9388ff3
diff --git a/src/dopplergeist/batch_ridge.py b/src/dopplergeist/batch_ridge.py
new file mode 100644
index 0000000..3b9d579
--- /dev/null
+++ b/src/dopplergeist/batch_ridge.py
@@ -0,0 +1,92 @@
+"""Batched versions of the existing robust first-event ridge fits."""
+import numpy as np
+
+from .matching import _fit_ridge_uncached, _frequency_scale, _ridge_fit_inputs
+
+
+def first_event_fits(measurement, event, factors, emissions):
+ """Fit supported clocks together, or return None for the ordinary fitter.
+
+ This uses the same six iterations, box constraints, and diagnostics as the
+ scalar fitter. Unsupported, degenerate or numerically nonfinite batches
+ fall back rather than changing their classification.
+ """
+ sigma = _frequency_scale(event)
+ fraction = event.get("quality", {}).get("source_drift_fraction", 0.02)
+ times, ridge, weights = _ridge_fit_inputs(measurement, sigma, fraction)
+ measured = np.isfinite(times) & np.isfinite(ridge) & (weights > 0)
+ factors, emissions = np.asarray(factors, dtype=float), np.asarray(emissions, dtype=float)
+ if (measured.sum() < 10 or factors.ndim != 2 or emissions.shape != factors.shape
+ or factors.shape[1] != measured.sum() or not len(factors)
+ or not np.all(np.isfinite(factors)) or not np.all(factors > 0)
+ or not np.all(np.isfinite(emissions)) or fraction <= 0):
+ return None
+ low, high = event["f0_range_hz"]
+ y, w = ridge[measured], weights[measured]
+ independent = len(np.unique(times[measured]))
+ with np.errstate(over="ignore", invalid="ignore", divide="ignore", under="ignore"):
+ w = w * independent / w.sum()
+ span = np.maximum(np.ptp(emissions, axis=1), 1e-9)
+ reference = (emissions.min(axis=1) + emissions.max(axis=1)) / 2
+ u = (emissions - reference[:, None]) / span[:, None]
+ cap = np.clip(np.median(y[None, :] / factors, axis=1), low, high) * fraction
+ if not np.all(np.isfinite(cap) & (cap > 0)) or not np.all(np.isfinite(u)):
+ return None
+ factor2, u2 = factors**2, u**2
+ effective = np.broadcast_to(w, factors.shape).copy()
+ for _ in range(6):
+ weighted_factor2, weighted_factor = effective * factor2, effective * factors
+ a = np.sum(weighted_factor2, axis=1)
+ r = np.sum(weighted_factor * y, axis=1)
+ b = np.sum(weighted_factor2 * u, axis=1)
+ prior = (2 * sigma / cap)**2
+ d = np.sum(weighted_factor2 * u2, axis=1) + prior
+ s = np.sum(weighted_factor * u * y, axis=1)
+ determinant = a*d - b*b
+ if (not all(np.all(np.isfinite(value)) for value in (a, r, b, prior, d, s, determinant))
+ or np.any(a <= 0) or np.any(d <= 0) or np.any(determinant <= 0)):
+ return None
+ alpha, beta = (r*d - s*b)/determinant, (s*a - r*b)/determinant
+ if not np.all(np.isfinite(alpha) & np.isfinite(beta)):
+ return None
+ outside = np.flatnonzero(~((low <= alpha) & (alpha <= high) & (np.abs(beta) <= cap)))
+ if len(outside):
+ ao, ro, bo, do, so, co = (value[outside] for value in (a, r, b, d, s, cap))
+ candidates = np.stack([
+ np.stack([np.full(len(outside), low), np.clip((so-bo*low)/do, -co, co)], axis=1),
+ np.stack([np.full(len(outside), high), np.clip((so-bo*high)/do, -co, co)], axis=1),
+ np.stack([np.clip((ro+bo*co)/ao, low, high), -co], axis=1),
+ np.stack([np.clip((ro-bo*co)/ao, low, high), co], axis=1),
+ ], axis=1)
+ errors = factors[outside, None, :] * (candidates[:, :, 0, None] +
+ candidates[:, :, 1, None] * u[outside, None, :]) - y
+ costs = np.sum(effective[outside, None, :] * errors**2, axis=2) + prior[outside, None] * candidates[:, :, 1]**2
+ if not np.all(np.isfinite(costs)):
+ return None
+ chosen = candidates[np.arange(len(outside)), np.argmin(costs, axis=1)]
+ alpha[outside], beta[outside] = chosen[:, 0], chosen[:, 1]
+ residual = factors * (alpha[:, None] + beta[:, None] * u) - y
+ effective = w / np.sqrt(1 + (residual / sigma)**2)
+ loss = np.average(2 * sigma**2 * (np.sqrt(1 + (residual / sigma)**2) - 1), weights=w, axis=1)
+ penalty = (2 * sigma * beta / cap)**2 / independent
+ score = -np.sqrt(loss + penalty)
+ rmse = np.sqrt(np.average(residual**2, weights=w, axis=1))
+ outlier = np.average(np.abs(residual) > 3 * sigma, weights=w, axis=1)
+ if not all(np.all(np.isfinite(value)) for value in (span, reference, score, rmse, outlier)):
+ return None
+ results = [{"score": float(score[i]), "f0_hz": float(alpha[i]),
+ "source_drift_hz": float(beta[i]), "source_drift_hz_s": float(beta[i]/span[i]),
+ "source_reference_emission_time_s": float(reference[i]), "source_span_s": float(span[i]),
+ "drift_bound_hz": float(cap[i]), "drift_at_bound": bool(cap[i] and abs(beta[i]) > 0.99*cap[i]),
+ "rmse_hz": float(rmse[i]), "support_fraction": 1.0, "sample_count": int(len(y)),
+ "independent_sample_count": independent, "outlier_fraction": float(outlier[i])}
+ for i in range(len(alpha))]
+ # Check representative clocks with the existing fitter. This catches a
+ # platform or arithmetic discrepancy and sends the whole batch down the
+ # established path instead of using inconsistent fit diagnostics.
+ for i in {0, len(results)//2, len(results)-1}:
+ scalar = _fit_ridge_uncached(None, None, measurement, 0., event["f0_range_hz"],
+ 343., sigma, fraction, acoustic_result=(factors[i], emissions[i]))
+ if results[i] != scalar:
+ return None
+ return results
diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.py
index 7b871c6..d50578b 100644
--- a/src/dopplergeist/search.py
+++ b/src/dopplergeist/search.py
@@ -401,7 +401,7 @@ def _validate_score_ceiling_inputs(events, measurements):
raise ValueError("Source frequency bounds must be finite, positive and ordered")
-def _screen_association_trial(tracks, events, measurements, observer, start, c, acoustic_at, floor):
+def _screen_association_trial(tracks, events, measurements, observer, start, c, acoustic_at, floor, first_fit=None):
"""Fit groups until a conservative residual ceiling cannot reach the floor.
Unperformed groups each contribute at most one. The existing validated-fit
@@ -412,7 +412,10 @@ def _screen_association_trial(tracks, events, measurements, observer, start, c,
missing = {"score": -np.inf, "f0_hz": np.nan, "support_fraction": 0.0}
for index, (track, event, measurement) in enumerate(zip(tracks, events, measurements)):
acoustic = None if acoustic_at is None else [acoustic_at(index)]
- single = score_ridges([track], [event], [measurement], observer, start, c, acoustic_results=acoustic)
+ if index == 0 and first_fit is not None:
+ single = {"event_diagnostics": [first_fit]}
+ else:
+ single = score_ridges([track], [event], [measurement], observer, start, c, acoustic_results=acoustic)
diagnostics.extend(single["event_diagnostics"])
remaining = len(events) - len(diagnostics)
if remaining:
@@ -513,6 +516,7 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="bright
batches = []
else:
batches = batch_acoustic_trials(selected_tracks, measurements, observer, anchor + shifts, c)
+ first_fits, first_batch_attempted = None, False
for trial, shift in enumerate(() if floor_pruned else shifts):
if mode == "robust_ridge":
from .matching import score_ridges
@@ -526,9 +530,15 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="bright
if batches is not None and lazy_batches is None else None)
try:
if association and score_floor is not None:
+ if not first_batch_attempted and batches is not None and lazy_batches is not None and time_origin is None:
+ first_batch_attempted = True
+ acoustic_at(0)
+ if batches[0] is not None:
+ from .batch_ridge import first_event_fits
+ first_fits = first_event_fits(measurements[0], events[0], *batches[0])
result, upper = _screen_association_trial(selected_tracks, events, measurements, observer,
anchor+shift, c, acoustic_at if batches is not None else None,
- score_floor)
+ score_floor, first_fits[trial] if first_fits is not None else None)
if result is None:
partial_prunes += 1
assessment["unknown_trials"] += 1
diff --git a/tests/test_real_batch_ridge.py b/tests/test_real_batch_ridge.py
new file mode 100644
index 0000000..5d9b403
--- /dev/null
+++ b/tests/test_real_batch_ridge.py
@@ -0,0 +1,99 @@
+"""Optional batched-fit parity checks using observed audio and aircraft tracks."""
+import json
+import os
+from pathlib import Path
+import random
+from types import SimpleNamespace
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from dopplergeist.batch_ridge import first_event_fits
+from dopplergeist.data_access import FlightIndex
+from dopplergeist.geometry import ecef
+from dopplergeist.matching import (_cache_ridge_fits, _fit_ridge_uncached, _frequency_scale,
+ association_candidates, batch_acoustic_trials, prepare_events)
+from dopplergeist.search import filter_association_event, score_candidate
+
+
+def recorded():
+ source = os.environ.get("DOPPLERGEIST_REAL_CPA_FIXTURE")
+ if not source:
+ pytest.skip("set DOPPLERGEIST_REAL_CPA_FIXTURE to the real reference diagnostic")
+ fixture = json.loads(Path(source).read_text())
+ events, measurements = prepare_events(fixture["events"])
+ table = pd.DataFrame(fixture["cpa_rows"])
+ tracks = {track.id: track for track in FlightIndex(fixture["real_index"]).query(
+ fixture["query_bounds"], fixture["query_time_s"])}
+ return fixture, events, measurements, table, tracks
+
+
+def test_real_batched_event_fits_equal_scalar_diagnostics(recorded):
+ fixture, events, measurements, table, tracks = recorded
+ point = fixture["point"]
+ observer = ecef(point["lat"], point["lon"], point["elev_m"])
+ checked = unsupported = 0
+ for event, measurement in zip(events, measurements):
+ for row in list(filter_association_event(table, event, 5., None, 5.).itertuples())[:5]:
+ if row.flight_id not in tracks:
+ continue
+ starts = row.receive_time - event["cpa_receive_s"] + np.linspace(-5, 5, 21)
+ batch = batch_acoustic_trials([tracks[row.flight_id]], [measurement], observer, starts)[0]
+ if batch is None:
+ unsupported += len(starts)
+ continue
+ factors, emissions = batch
+ complete = np.all(np.isfinite(factors) & np.isfinite(emissions) & (factors > 0), axis=1)
+ if not np.all(complete):
+ assert first_event_fits(measurement, event, factors, emissions) is None
+ unsupported += int((~complete).sum())
+ if not np.any(complete):
+ continue
+ factors, emissions = factors[complete], emissions[complete]
+ results = first_event_fits(measurement, event, factors, emissions)
+ if results is None:
+ continue
+ fraction = event.get("quality", {}).get("source_drift_fraction", .02)
+ sigma = _frequency_scale(event)
+ for result, factor, emitted in zip(results, factors, emissions):
+ scalar = _fit_ridge_uncached(None, None, measurement, 0., event["f0_range_hz"],
+ 343., sigma, fraction, acoustic_result=(factor, emitted))
+ assert json.dumps(result, sort_keys=True) == json.dumps(scalar, sort_keys=True)
+ checked += 1
+ zero_drift = dict(event, quality=dict(event.get("quality", {}), source_drift_fraction=0.))
+ assert first_event_fits(measurement, zero_drift, factors, emissions) is None
+ assert checked > 100
+
+
+def test_real_association_results_match_scalar_fits(recorded, monkeypatch):
+ import dopplergeist.batch_ridge as batch_ridge
+
+ fixture, events, measurements, table, tracks = recorded
+ candidates, _ = association_candidates(table, events, max_candidates=20000)
+ selected = sorted(random.Random(20261103).sample(range(len(candidates)), min(100, len(candidates))))
+ point = SimpleNamespace(**fixture["point"], x=0., y=0.)
+
+ @_cache_ridge_fits
+ def run():
+ return [score_candidate(candidates[i], point, tracks, events, measurements,
+ mode="robust_ridge", score_floor=0.) for i in selected]
+
+ with monkeypatch.context() as patch:
+ patch.setattr(batch_ridge, "first_event_fits", lambda *args: None)
+ scalar = run()
+ calls = 0
+ original = batch_ridge.first_event_fits
+
+ def counted(*args):
+ nonlocal calls
+ result = original(*args)
+ calls += result is not None
+ return result
+
+ with monkeypatch.context() as patch:
+ patch.setattr(batch_ridge, "first_event_fits", counted)
+ batched = run()
+ assert calls > 0
+ assert json.dumps(scalar, sort_keys=True, allow_nan=True) == json.dumps(batched, sort_keys=True, allow_nan=True)