Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Batch first-event fits across shared clocks
Cinna Davisfff5c9388ff3
diff --git a/src/dopplergeist/batch_ridge.py b/src/dopplergeist/batch_ridge.pynew file mode 100644index 0000000..3b9d579--- /dev/null+++ b/src/dopplergeist/batch_ridge.py@@ -0,0 +1,92 @@+"""Batched versions of the existing robust first-event ridge fits."""+import numpy as np++from .matching import _fit_ridge_uncached, _frequency_scale, _ridge_fit_inputs+++def first_event_fits(measurement, event, factors, emissions):+ """Fit supported clocks together, or return None for the ordinary fitter.++ This uses the same six iterations, box constraints, and diagnostics as the+ scalar fitter. Unsupported, degenerate or numerically nonfinite batches+ fall back rather than changing their classification.+ """+ sigma = _frequency_scale(event)+ fraction = event.get("quality", {}).get("source_drift_fraction", 0.02)+ times, ridge, weights = _ridge_fit_inputs(measurement, sigma, fraction)+ measured = np.isfinite(times) & np.isfinite(ridge) & (weights > 0)+ factors, emissions = np.asarray(factors, dtype=float), np.asarray(emissions, dtype=float)+ if (measured.sum() < 10 or factors.ndim != 2 or emissions.shape != factors.shape+ or factors.shape[1] != measured.sum() or not len(factors)+ or not np.all(np.isfinite(factors)) or not np.all(factors > 0)+ or not np.all(np.isfinite(emissions)) or fraction <= 0):+ return None+ low, high = event["f0_range_hz"]+ y, w = ridge[measured], weights[measured]+ independent = len(np.unique(times[measured]))+ with np.errstate(over="ignore", invalid="ignore", divide="ignore", under="ignore"):+ w = w * independent / w.sum()+ span = np.maximum(np.ptp(emissions, axis=1), 1e-9)+ reference = (emissions.min(axis=1) + emissions.max(axis=1)) / 2+ u = (emissions - reference[:, None]) / span[:, None]+ cap = np.clip(np.median(y[None, :] / factors, axis=1), low, high) * fraction+ if not np.all(np.isfinite(cap) & (cap > 0)) or not np.all(np.isfinite(u)):+ return None+ factor2, u2 = factors**2, u**2+ effective = np.broadcast_to(w, factors.shape).copy()+ for _ in range(6):+ weighted_factor2, weighted_factor = effective * factor2, effective * factors+ a = np.sum(weighted_factor2, axis=1)+ r = np.sum(weighted_factor * y, axis=1)+ b = np.sum(weighted_factor2 * u, axis=1)+ prior = (2 * sigma / cap)**2+ d = np.sum(weighted_factor2 * u2, axis=1) + prior+ s = np.sum(weighted_factor * u * y, axis=1)+ determinant = a*d - b*b+ if (not all(np.all(np.isfinite(value)) for value in (a, r, b, prior, d, s, determinant))+ or np.any(a <= 0) or np.any(d <= 0) or np.any(determinant <= 0)):+ return None+ alpha, beta = (r*d - s*b)/determinant, (s*a - r*b)/determinant+ if not np.all(np.isfinite(alpha) & np.isfinite(beta)):+ return None+ outside = np.flatnonzero(~((low <= alpha) & (alpha <= high) & (np.abs(beta) <= cap)))+ if len(outside):+ ao, ro, bo, do, so, co = (value[outside] for value in (a, r, b, d, s, cap))+ candidates = np.stack([+ np.stack([np.full(len(outside), low), np.clip((so-bo*low)/do, -co, co)], axis=1),+ np.stack([np.full(len(outside), high), np.clip((so-bo*high)/do, -co, co)], axis=1),+ np.stack([np.clip((ro+bo*co)/ao, low, high), -co], axis=1),+ np.stack([np.clip((ro-bo*co)/ao, low, high), co], axis=1),+ ], axis=1)+ errors = factors[outside, None, :] * (candidates[:, :, 0, None] ++ candidates[:, :, 1, None] * u[outside, None, :]) - y+ costs = np.sum(effective[outside, None, :] * errors**2, axis=2) + prior[outside, None] * candidates[:, :, 1]**2+ if not np.all(np.isfinite(costs)):+ return None+ chosen = candidates[np.arange(len(outside)), np.argmin(costs, axis=1)]+ alpha[outside], beta[outside] = chosen[:, 0], chosen[:, 1]+ residual = factors * (alpha[:, None] + beta[:, None] * u) - y+ effective = w / np.sqrt(1 + (residual / sigma)**2)+ loss = np.average(2 * sigma**2 * (np.sqrt(1 + (residual / sigma)**2) - 1), weights=w, axis=1)+ penalty = (2 * sigma * beta / cap)**2 / independent+ score = -np.sqrt(loss + penalty)+ rmse = np.sqrt(np.average(residual**2, weights=w, axis=1))+ outlier = np.average(np.abs(residual) > 3 * sigma, weights=w, axis=1)+ if not all(np.all(np.isfinite(value)) for value in (span, reference, score, rmse, outlier)):+ return None+ results = [{"score": float(score[i]), "f0_hz": float(alpha[i]),+ "source_drift_hz": float(beta[i]), "source_drift_hz_s": float(beta[i]/span[i]),+ "source_reference_emission_time_s": float(reference[i]), "source_span_s": float(span[i]),+ "drift_bound_hz": float(cap[i]), "drift_at_bound": bool(cap[i] and abs(beta[i]) > 0.99*cap[i]),+ "rmse_hz": float(rmse[i]), "support_fraction": 1.0, "sample_count": int(len(y)),+ "independent_sample_count": independent, "outlier_fraction": float(outlier[i])}+ for i in range(len(alpha))]+ # Check representative clocks with the existing fitter. This catches a+ # platform or arithmetic discrepancy and sends the whole batch down the+ # established path instead of using inconsistent fit diagnostics.+ for i in {0, len(results)//2, len(results)-1}:+ scalar = _fit_ridge_uncached(None, None, measurement, 0., event["f0_range_hz"],+ 343., sigma, fraction, acoustic_result=(factors[i], emissions[i]))+ if results[i] != scalar:+ return None+ return resultsdiff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.pyindex 7b871c6..d50578b 100644--- a/src/dopplergeist/search.py+++ b/src/dopplergeist/search.py@@ -401,7 +401,7 @@ def _validate_score_ceiling_inputs(events, measurements):raise ValueError("Source frequency bounds must be finite, positive and ordered")-def _screen_association_trial(tracks, events, measurements, observer, start, c, acoustic_at, floor):+def _screen_association_trial(tracks, events, measurements, observer, start, c, acoustic_at, floor, first_fit=None):"""Fit groups until a conservative residual ceiling cannot reach the floor.Unperformed groups each contribute at most one. The existing validated-fit@@ -412,7 +412,10 @@ def _screen_association_trial(tracks, events, measurements, observer, start, c,missing = {"score": -np.inf, "f0_hz": np.nan, "support_fraction": 0.0}for index, (track, event, measurement) in enumerate(zip(tracks, events, measurements)):acoustic = None if acoustic_at is None else [acoustic_at(index)]- single = score_ridges([track], [event], [measurement], observer, start, c, acoustic_results=acoustic)+ if index == 0 and first_fit is not None:+ single = {"event_diagnostics": [first_fit]}+ else:+ single = score_ridges([track], [event], [measurement], observer, start, c, acoustic_results=acoustic)diagnostics.extend(single["event_diagnostics"])remaining = len(events) - len(diagnostics)if remaining:@@ -513,6 +516,7 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="brightbatches = []else:batches = batch_acoustic_trials(selected_tracks, measurements, observer, anchor + shifts, c)+ first_fits, first_batch_attempted = None, Falsefor trial, shift in enumerate(() if floor_pruned else shifts):if mode == "robust_ridge":from .matching import score_ridges@@ -526,9 +530,15 @@ def score_candidate(candidate, point, tracks, events, measurements, mode="brightif batches is not None and lazy_batches is None else None)try:if association and score_floor is not None:+ if not first_batch_attempted and batches is not None and lazy_batches is not None and time_origin is None:+ first_batch_attempted = True+ acoustic_at(0)+ if batches[0] is not None:+ from .batch_ridge import first_event_fits+ first_fits = first_event_fits(measurements[0], events[0], *batches[0])result, upper = _screen_association_trial(selected_tracks, events, measurements, observer,anchor+shift, c, acoustic_at if batches is not None else None,- score_floor)+ score_floor, first_fits[trial] if first_fits is not None else None)if result is None:partial_prunes += 1assessment["unknown_trials"] += 1diff --git a/tests/test_real_batch_ridge.py b/tests/test_real_batch_ridge.pynew file mode 100644index 0000000..5d9b403--- /dev/null+++ b/tests/test_real_batch_ridge.py@@ -0,0 +1,99 @@+"""Optional batched-fit parity checks using observed audio and aircraft tracks."""+import json+import os+from pathlib import Path+import random+from types import SimpleNamespace++import numpy as np+import pandas as pd+import pytest++from dopplergeist.batch_ridge import first_event_fits+from dopplergeist.data_access import FlightIndex+from dopplergeist.geometry import ecef+from dopplergeist.matching import (_cache_ridge_fits, _fit_ridge_uncached, _frequency_scale,+ association_candidates, batch_acoustic_trials, prepare_events)+from dopplergeist.search import filter_association_event, score_candidate+++def recorded():+ source = os.environ.get("DOPPLERGEIST_REAL_CPA_FIXTURE")+ if not source:+ pytest.skip("set DOPPLERGEIST_REAL_CPA_FIXTURE to the real reference diagnostic")+ fixture = json.loads(Path(source).read_text())+ events, measurements = prepare_events(fixture["events"])+ table = pd.DataFrame(fixture["cpa_rows"])+ tracks = {track.id: track for track in FlightIndex(fixture["real_index"]).query(+ fixture["query_bounds"], fixture["query_time_s"])}+ return fixture, events, measurements, table, tracks+++def test_real_batched_event_fits_equal_scalar_diagnostics(recorded):+ fixture, events, measurements, table, tracks = recorded+ point = fixture["point"]+ observer = ecef(point["lat"], point["lon"], point["elev_m"])+ checked = unsupported = 0+ for event, measurement in zip(events, measurements):+ for row in list(filter_association_event(table, event, 5., None, 5.).itertuples())[:5]:+ if row.flight_id not in tracks:+ continue+ starts = row.receive_time - event["cpa_receive_s"] + np.linspace(-5, 5, 21)+ batch = batch_acoustic_trials([tracks[row.flight_id]], [measurement], observer, starts)[0]+ if batch is None:+ unsupported += len(starts)+ continue+ factors, emissions = batch+ complete = np.all(np.isfinite(factors) & np.isfinite(emissions) & (factors > 0), axis=1)+ if not np.all(complete):+ assert first_event_fits(measurement, event, factors, emissions) is None+ unsupported += int((~complete).sum())+ if not np.any(complete):+ continue+ factors, emissions = factors[complete], emissions[complete]+ results = first_event_fits(measurement, event, factors, emissions)+ if results is None:+ continue+ fraction = event.get("quality", {}).get("source_drift_fraction", .02)+ sigma = _frequency_scale(event)+ for result, factor, emitted in zip(results, factors, emissions):+ scalar = _fit_ridge_uncached(None, None, measurement, 0., event["f0_range_hz"],+ 343., sigma, fraction, acoustic_result=(factor, emitted))+ assert json.dumps(result, sort_keys=True) == json.dumps(scalar, sort_keys=True)+ checked += 1+ zero_drift = dict(event, quality=dict(event.get("quality", {}), source_drift_fraction=0.))+ assert first_event_fits(measurement, zero_drift, factors, emissions) is None+ assert checked > 100+++def test_real_association_results_match_scalar_fits(recorded, monkeypatch):+ import dopplergeist.batch_ridge as batch_ridge++ fixture, events, measurements, table, tracks = recorded+ candidates, _ = association_candidates(table, events, max_candidates=20000)+ selected = sorted(random.Random(20261103).sample(range(len(candidates)), min(100, len(candidates))))+ point = SimpleNamespace(**fixture["point"], x=0., y=0.)++ @_cache_ridge_fits+ def run():+ return [score_candidate(candidates[i], point, tracks, events, measurements,+ mode="robust_ridge", score_floor=0.) for i in selected]++ with monkeypatch.context() as patch:+ patch.setattr(batch_ridge, "first_event_fits", lambda *args: None)+ scalar = run()+ calls = 0+ original = batch_ridge.first_event_fits++ def counted(*args):+ nonlocal calls+ result = original(*args)+ calls += result is not None+ return result++ with monkeypatch.context() as patch:+ patch.setattr(batch_ridge, "first_event_fits", counted)+ batched = run()+ assert calls > 0+ assert json.dumps(scalar, sort_keys=True, allow_nan=True) == json.dumps(batched, sort_keys=True, allow_nan=True)