Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
from types import SimpleNamespace
import numpy as np
import pandas as pd
import pytest
from pyproj import Transformer
from dopplergeist.geometry import Track, ecef, straight_doppler
from dopplergeist.matching import RidgeTimes, fit_ridge, prepare_events, refine_candidate
from dopplergeist.search import event_score, filter_event, pair_candidates, score_candidate
class Flyover:
def __init__(self, center=30, support=None):
self.center, self.support = center, support
def acoustic(self, t, observer, c):
t = np.asarray(t)
factor = 1.1 - 0.25 * np.tanh((t - self.center) / 8)
if self.support is not None:
factor = np.where(t <= self.support, factor, np.nan)
return factor, t - 5, np.ones_like(t) * 1700
def measurement(center=30, drift=0.0, noisy=False):
t = np.arange(center - 25, center + 25, 0.5)
factor = Flyover(center).acoustic(t, None, 343)[0]
ridge = factor * (120 + drift * (t - (t[0] + t[-1]) / 2) / np.ptp(t))
if noisy:
ridge += np.random.default_rng(17).normal(0, 0.15, len(t))
ridge[::11] += 18
return RidgeTimes(t), ridge, np.ones(len(t)), {}
def test_robust_source_drift_and_outliers_preserve_wrong_track_penalty():
measured = measurement(drift=1.8, noisy=True)
correct = fit_ridge(Flyover(), None, measured, 0, [118, 122])
wrong = fit_ridge(Flyover(38), None, measured, 0, [118, 122])
legacy = event_score(Flyover(), None, *measured[:3], 0, [118, 122], "ridge")
assert abs(correct["f0_hz"] - 120) < 0.2
assert abs(correct["source_drift_hz"] - 1.8) < 0.5
assert correct["score"] > legacy[0]
assert correct["score"] > wrong["score"] + 1
assert correct["outlier_fraction"] > 0.05
assert correct["support_fraction"] == 1
noise = list(measured)
noise[1] = np.random.default_rng(3).uniform(90, 160, len(measured[1]))
assert fit_ridge(Flyover(), None, noise, 0, [118, 122])["score"] < correct["score"] - 1
def test_drift_is_bounded_and_incomplete_support_cannot_win():
measured = measurement(drift=18)
fit = fit_ridge(Flyover(), None, measured, 0, [118, 122])
assert fit["drift_at_bound"]
assert abs(fit["source_drift_hz"]) <= fit["drift_bound_hz"]
assert not np.isfinite(fit_ridge(Flyover(support=54), None, measured, 0, [118, 122])["score"])
zero = list(measured)
zero[2] = np.zeros(len(measured[1]))
assert not np.isfinite(fit_ridge(Flyover(), None, zero, 0, [118, 122])["score"])
def test_robust_mode_locks_all_event_clocks_and_rejects_missing_passes():
point = SimpleNamespace(point_id=0, lat=40, lon=-100, elev_m=0, x=0, y=0)
events = [{"cpa_receive_s": t, "f0_range_hz": [119.9, 120.1]} for t in (30, 950)]
candidate = {"passes": [{"flight_id": "a", "receive_time": 30}, {"flight_id": "b", "receive_time": 950}]}
kwargs = dict(mode="robust_ridge", time_origin=0, time_sweep_s=10, time_step_s=1)
good = score_candidate(candidate, point, {"a": Flyover(30), "b": Flyover(950)},
events, [measurement(30), measurement(950)], **kwargs)
bad = score_candidate(candidate, point, {"a": Flyover(30), "b": Flyover(958)},
events, [measurement(30), measurement(950)], **kwargs)
assert abs(good["score"]) < 1e-8
assert bad["score"] < -2
assert len(bad["event_diagnostics"]) == 2
with pytest.raises(ValueError, match="exactly one"):
score_candidate({"passes": candidate["passes"][:1]}, point, {}, events, [], **kwargs)
def detected(identity="a", harmonic=1):
m = measurement()
return {"event_id": identity, "start_s": 4, "end_s": 56, "band_hz": [80 * harmonic, 170 * harmonic],
"harmonic_group_id": "group", "cpa_receive_s": 30,
"speed_range_mps": [70, 90], "slant_range_m": [800, 1400],
"ridge": [{"time_s": float(t), "frequency_hz": float(f * harmonic), "weight": 2}
for t, f in zip(m[0].time, m[1])]}
def test_adapter_retains_jet_with_misleading_straight_geometry_and_merges_harmonics():
primary = dict(detected(), quality={"frequency_uncertainty_hz": 1.5})
events, measurements = prepare_events([primary, detected("harmonic", 2)])
table = pd.DataFrame([dict(point_id=0, flight_id="prop", receive_time=42, emission_time=28,
speed_mps=155, distance_m=4900)])
assert len(events) == 1
assert len(measurements[0][1]) == 200
assert len(filter_event(table, events[0], time_origin=0)) == 1
assert events[0]["event_ids"] == ["a", "harmonic"]
assert events[0]["quality"]["frequency_loss_scale_hz"] == 1.5
assert events[0]["quality"]["frequency_uncertainty_hz"] == 1.5
legacy_scale, _ = prepare_events([dict(detected(), quality={"frequency_sigma_hz": 0.2})])
assert legacy_scale[0]["quality"]["frequency_loss_scale_hz"] == 1.0
# Duplicating harmonic evidence changes neither the fit nor its normalization.
single, sm = prepare_events([detected()])
one = fit_ridge(Flyover(), None, sm[0], 0, single[0]["f0_range_hz"])
two = fit_ridge(Flyover(), None, measurements[0], 0, events[0]["f0_range_hz"])
assert abs(one["score"] - two["score"]) < 1e-8
inconsistent = detected("bad", 2)
for p in inconsistent["ridge"]:
p["frequency_hz"] *= 0.73
with pytest.raises(ValueError, match="supported frequency ratio"):
prepare_events([detected(), inconsistent])
def test_uncertainty_requires_one_common_recording_start_interval():
events = [dict(cpa_receive_s=t, speed_range_mps=[70, 90], slant_range_m=[800, 1400],
quality={"cpa_tolerance_s": 1}) for t in (30, 130, 230)]
rows = [dict(point_id=0, flight_id=str(i), receive_time=t + delta, emission_time=t + delta - 5,
speed_mps=80, distance_m=1000) for i, (t, delta) in enumerate(zip((30, 130, 230), (100, 103, 97)))]
assert pair_candidates(pd.DataFrame(rows), events, offset_tolerance_s=2)[0] == []
rows[-1]["receive_time"] += 6
candidates, _ = pair_candidates(pd.DataFrame(rows), events, offset_tolerance_s=2)
assert len(candidates) == 1
lower, upper = candidates[0]["recording_start_bounds_s"]
assert lower <= upper
def physical_case():
observer = ecef(40, -100, 100)
epoch = 1_790_000_000.0
inverse = Transformer.from_crs("EPSG:4978", "EPSG:4979", always_xy=True)
tracks, events, measurements = {}, [], []
for identity, center, f0, velocity, cross in [
("a", 70, 120, [80, 0, 0], [0, 1600, 800]),
("b", 250, 160, [0, 80, 0], [1700, 0, 1000])]:
t = np.arange(center - 120, center + 121, 1.0)
xyz = observer + (t - center)[:, None] * np.array(velocity) + cross
lon, lat, altitude = inverse.transform(*xyz.T)
tracks[identity] = Track(identity, t + epoch, lat, lon, altitude,
velocity=np.tile(velocity, (len(t), 1)))
tracks[identity].altitude_basis = ["ellipsoidal"]*len(t)
receive = np.arange(center - 35, center + 35, 1.0)
slant = np.linalg.norm(cross)
ridge = straight_doppler(receive, f0, 80, slant, center + slant / 343)
measurements.append((RidgeTimes(receive), ridge, np.ones(len(receive)), {}))
events.append(dict(f0_range_hz=[f0 - 1, f0 + 1], quality={"source_drift_fraction": 0}))
seed = dict(lat=40.0015, lon=-99.998, elev_m=100, recording_start_time_s=epoch + 0.5,
flight_ids=["a", "b"], point_id=0)
terrain = SimpleNamespace(sample=lambda lat, lon: np.array(100.0), vertical_datum="ellipsoidal")
return observer, epoch, seed, tracks, events, measurements, terrain
def test_joint_refinement_uses_nonparallel_passes_and_profiles_altitude():
observer, epoch, seed, tracks, events, measurements, terrain = physical_case()
result = refine_candidate(seed, tracks, events, measurements, terrain, radius_m=500, clock_radius_s=3)
error = np.linalg.norm(ecef(result["lat"], result["lon"], 100) - observer)
assert error < 10
assert abs(result["recording_start_time_s"] - epoch) < 0.1
assert len(result["sensitivity"]["profiles"]) == 3
assert result["sensitivity"]["radius_m"] > 0
assert result["sensitivity"]["kind"] == "sensitivity"
assert result["sensitivity"]["complete"]
@pytest.mark.parametrize("window", [(0.0, 0.0), (0.0, 0.25)])
def test_refinement_cannot_move_a_trusted_clock_outside_its_window(window):
seed = dict(lat=40, lon=-100, recording_start_time_s=0.0, flight_ids=["a"])
terrain = SimpleNamespace(sample=lambda lat, lon: np.array(100.0))
event = dict(f0_range_hz=[119, 121], quality={"source_drift_fraction": 0})
# A freely fitted clock would move +2 seconds; the request forbids that.
result = refine_candidate(seed, {"a": Flyover(32)}, [event], [measurement()], terrain,
altitude_offsets_m=(0.0,), recording_start_window_s=window)
assert window[0] <= result["recording_start_time_s"] <= window[1]
assert result["refinement_clock_bounds_s"] == list(window)
if window[0] == window[1]:
assert result["recording_start_time_s"] == 0
assert not result["clock_boundary_limited"]
with pytest.raises(ValueError, match="do not intersect"):
refine_candidate(seed, {"a": Flyover()}, [event], [measurement()], terrain,
recording_start_window_s=(100, 101))
def test_source_fit_matches_independent_bounded_solver_on_interior_and_edges():
from scipy.optimize import lsq_linear
from dopplergeist.matching import _source_fit
rng = np.random.default_rng(440)
factor, u = rng.uniform(0.7, 1.4, 80), np.linspace(-0.5, 0.5, 80)
for _ in range(30):
y = factor*(rng.uniform(110, 130)+rng.uniform(-15, 15)*u)
weight = rng.uniform(0.1, 2, len(y))
cap, scale = 2.4, 1.0
matrix = np.vstack((np.column_stack((factor, factor*u))*np.sqrt(weight[:, None]), [0, 2*scale/cap]))
target = np.r_[y*np.sqrt(weight), 0]
expected = lsq_linear(matrix, target, bounds=([119, -cap], [121, cap]), tol=1e-12).x
actual = _source_fit(factor, u, y, weight, [119, 121], cap, scale)
np.testing.assert_allclose(actual, expected, atol=1e-7, rtol=0)
def test_full_sensitivity_reoptimizes_all_sources_without_moving_exact_utc():
from dopplergeist.matching import profile_candidate
observer, epoch, seed, tracks, events, measurements, terrain = physical_case()
for event in events:
event["quality"]["frequency_loss_scale_hz"] = 0.05
result = profile_candidate(seed, tracks, events, measurements, terrain,
recording_start_window_s=(epoch, epoch), altitude_uncertainty_m=25,
sound_speed_uncertainty_mps=1, radius_m=500, clock_radius_s=3)
sensitivity = result["sensitivity"]
assert set(sensitivity["sources"]) >= {"frequency", "clock", "altitude", "sound_speed", "source_frequency"}
assert sensitivity["complete"] and not sensitivity["boundary_limited"]
assert sensitivity["terrain_vertical_datum"] == "ellipsoidal"
assert sensitivity["radius_m"] > 0
assert all(p["recording_start_time_s"] == epoch for p in sensitivity["profiles"])
assert len(sensitivity["profiles"]) >= 15
assert len([p for p in sensitivity["profiles"] if p["flight_altitude_bias"] is not None]) == 4
assert np.linalg.norm(ecef(result["lat"], result["lon"], 100)-observer) < 10
def test_missing_sensitivity_scenario_is_retained_as_incomplete_evidence():
from dopplergeist.matching import profile_candidate
class LimitedTrack(Flyover):
def acoustic(self, t, observer, c):
factor, emitted, distance = super().acoustic(t, observer, c)
return (factor if c >= 343 else np.full_like(factor, np.nan)), emitted, distance
seed = dict(lat=40, lon=-100, recording_start_time_s=0, flight_ids=["a"])
terrain = SimpleNamespace(sample=lambda lat, lon: np.array(100.0))
event = dict(f0_range_hz=[119, 121])
result = profile_candidate(seed, {"a": LimitedTrack()}, [event], [measurement()], terrain,
recording_start_window_s=(0, 0), altitude_uncertainty_m=0)
assert not result["sensitivity"]["complete"]
assert any(p["source"] == "sound_speed" for p in result["sensitivity"]["failures"])
def test_exhausted_real_optimizer_cannot_report_complete_sensitivity(monkeypatch):
import dopplergeist.matching as matching
real_minimize = matching.minimize
def exhausted(*args, **kwargs):
kwargs["options"] = dict(kwargs.get("options", {}), maxiter=0)
return real_minimize(*args, **kwargs)
monkeypatch.setattr(matching, "minimize", exhausted)
seed = dict(lat=40, lon=-100, recording_start_time_s=0, flight_ids=["a"])
terrain = SimpleNamespace(sample=lambda lat, lon: np.array(100.0))
result = refine_candidate(seed, {"a": Flyover()}, [dict(f0_range_hz=[119, 121])], [measurement()], terrain)
assert not result["sensitivity"]["complete"]
assert not any(p["optimizer_success"] for p in result["sensitivity"]["profiles"])
def test_flight_altitude_bias_preserves_derivatives_and_original_track():
from dopplergeist.matching import _altitude_bias
observer, epoch, seed, tracks, events, measurements, terrain = physical_case()
original = tracks["a"]
shifted = _altitude_bias(original, 150)
np.testing.assert_allclose(shifted.altitude-original.altitude, 150)
assert np.max(np.linalg.norm(shifted.position(original.time)-original.position(original.time), axis=1)) > 149.9
delta = shifted.xyz-original.xyz
expected = original.speed_vector(original.time)+np.gradient(delta, original.relative_time, axis=0)
np.testing.assert_allclose(shifted.speed_vector(original.time), expected, atol=1e-9)
assert tracks["b"].altitude_basis == ["ellipsoidal"]*len(tracks["b"].time)
def test_unestablished_flight_datum_prevents_complete_sensitivity():
from dopplergeist.matching import profile_candidate
track = Flyover()
track.altitude_basis = ["barometric", "unspecified"]
seed = dict(lat=40, lon=-100, recording_start_time_s=0, flight_ids=["a"])
terrain = SimpleNamespace(sample=lambda lat, lon: np.array(100.0))
result = profile_candidate(seed, {"a": track}, [dict(f0_range_hz=[119, 121])], [measurement()], terrain,
recording_start_window_s=(0, 0), altitude_uncertainty_m=0,
sound_speed_uncertainty_mps=0)
assert not result["sensitivity"]["complete"]
assert result["sensitivity"]["altitude_basis"]["a"] == ["barometric", "unspecified"]
assert any("datum" in p["reason"] for p in result["sensitivity"]["failures"])
def test_overlay_prediction_uses_fitted_emission_drift_and_preserves_support():
from dopplergeist.matching import predict_fitted_ridge
epoch = 1_790_000_000.0
measured = measurement(drift=1.8)
track = Flyover(epoch+30)
fit = fit_ridge(track, None, measured, epoch, [118, 122])
predicted = predict_fitted_ridge(track, None, epoch+measured[0].time, fit)
rms = np.sqrt(np.mean((predicted-measured[1])**2))
assert abs(rms-fit["rmse_hz"]) < 1e-10
assert rms < 0.1
constant = fit["f0_hz"]*track.acoustic(epoch+measured[0].time, None, 343)[0]
assert np.sqrt(np.mean((constant-measured[1])**2)) > 0.4
assert epoch < fit["source_reference_emission_time_s"] < epoch+60
limited = Flyover(epoch+30, support=epoch+54)
assert np.isnan(predict_fitted_ridge(limited, None, [epoch+55], fit)[0])
incomplete = {k:v for k,v in fit.items() if k != "source_reference_emission_time_s"}
with pytest.raises(ValueError, match="reference"):
predict_fitted_ridge(track, None, epoch+measured[0].time, incomplete)