Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

feat(#341): adapt coherent real-audio traces without type gates

Cinna Davis99bdf5eebbb4
diff --git a/docs/COHERENT_DISCOVERY.md b/docs/COHERENT_DISCOVERY.md
new file mode 100644
index 0000000..1ea3867
--- /dev/null
+++ b/docs/COHERENT_DISCOVERY.md
@@ -0,0 +1,38 @@
+# Experimental coherent audio proposals
+
+The ordinary per-frame peak linker can miss a continuous weak curve. Issue
+[#341](https://github.com/1337isnot1337/DOPPLERGEIST/issues/341) evaluates an
+additional whole-file detector using
+[digitalis-p/doppler-locate](https://github.com/digitalis-p/doppler-locate).
+The tested reference revision is `ed84e0f`, under its MIT license. This optional
+research script requires that package and its dependencies in the executing
+Python environment; the default DOPPLERGEIST installation is unchanged. Python
+3.12 was used for the reference detector.
+
+From this checkout, with an installed reference package:
+
+```bash
+PYTHONPATH=src OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 \
+ python scripts/discover_coherent_audio.py original.mp3 \
+ --output coherent-events.json --workers 2
+```
+
+Alternatively, add the reference checkout's `src` to `PYTHONPATH`. The script
+processes the entire original recording with the reference template bank. It
+captures actual measured ridge samples, skips unused straight-pass profile-range
+calculations, and exports the existing detected-event schema. It reuses the
+reference's 40-point and 1% relative-fit-residual requirements without its hard
+aircraft/close type gate. Slow and nearby proposals remain eligible. Duplicate
+and harmonic ridges are grouped; their count is not an independent aircraft count.
+
+These are acoustic proposals. They can include ground traffic, and short curves
+can fit several unrelated archived flights. Neither a low residual nor ADS-B
+presence establishes audibility. Use the historical physical matcher and shared
+timing; never present these outputs alone as locations. This script does not
+change `analyze`, its discovery defaults, or existing job checkpoints.
+
+The regression fixture contains only measured frequencies/times derived from
+[XC1092456](https://xeno-canto.org/1092456) by Valerie Heemstra, with the public recording link retained
+for attribution. It contains no audio, coordinates or flight IDs. The original
+recording is [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/);
+see the source page for the recordist attribution.
diff --git a/scripts/discover_coherent_audio.py b/scripts/discover_coherent_audio.py
new file mode 100644
index 0000000..7fb769e
--- /dev/null
+++ b/scripts/discover_coherent_audio.py
@@ -0,0 +1,60 @@
+"""Experimental whole-file coherent discovery using an installed dopploc source.
+
+Run in an environment with digitalis-p/doppler-locate and this checkout's src
+on PYTHONPATH. No aircraft identity, location, clock or selected band is needed.
+"""
+from concurrent.futures import ProcessPoolExecutor
+from dataclasses import asdict
+from pathlib import Path
+import argparse
+import json
+import multiprocessing as mp
+import time
+
+from dopplergeist.coherent_discovery import events_from_coherent_traces
+
+
+def capture_traces(source, workers=2):
+ from dopploc import detect
+
+ traces = []
+ original_refine, original_pool = detect._refine_all, detect._pool
+
+ def capture(found, spectrum, settings, executor):
+ original_refine(found, spectrum, settings, executor)
+ for proposal in found:
+ if not hasattr(proposal, "_trace"):
+ continue
+ t, f, _ = proposal._trace
+ traces.append({"proposal": asdict(proposal), "ridge_points": [
+ {"time_s": float(tt), "frequency_hz": float(ff)} for tt, ff in zip(t, f)]})
+ # Automatic matching uses broad measured-ridge envelopes, not
+ # straight-pass profile ranges. Avoid that unused nonlinear work.
+ del proposal._trace
+
+ detect._refine_all = capture
+ detect._pool = lambda: ProcessPoolExecutor(max_workers=workers, mp_context=mp.get_context("spawn"))
+ try:
+ detect.detect(source)
+ finally:
+ detect._refine_all, detect._pool = original_refine, original_pool
+ return traces
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("audio", type=Path)
+ parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument("--workers", type=int, default=2)
+ args = parser.parse_args()
+ if args.workers < 1:
+ parser.error("workers must be positive")
+ if args.output.exists():
+ parser.error("output exists; preserve it and choose a new path")
+ started = time.monotonic()
+ traces = capture_traces(args.audio, args.workers)
+ events = events_from_coherent_traces(traces)
+ args.output.write_text(json.dumps({"events": events, "raw_refined_traces": len(traces),
+ "elapsed_s": time.monotonic() - started,
+ "scope": "Experimental acoustic proposals; not aircraft identifications or geolocation"}, indent=2))
+ print(f"{len(events)} measured-ridge proposals saved to {args.output}")
diff --git a/src/dopplergeist/coherent_discovery.py b/src/dopplergeist/coherent_discovery.py
new file mode 100644
index 0000000..9862bbd
--- /dev/null
+++ b/src/dopplergeist/coherent_discovery.py
@@ -0,0 +1,66 @@
+"""Adapt measured coherent traces without aircraft-type or geometry rejection.
+
+Experimental audio proposals only. Straight-pass fit quality does not identify
+an aircraft; the historical-track matcher and decision stage remain necessary.
+"""
+import numpy as np
+
+from .contracts import validate_event
+from .discovery import _HARMONIC_RATIOS, _merge_duplicates, _overlap
+
+
+def events_from_coherent_traces(traces):
+ """Use the reference detector's 40-point/1% quality gate, excluding kind.
+
+ Rows contain ``proposal`` and its actual inlier ``ridge_points``. No fitted
+ curve samples, speed/range priors, coordinates or flight IDs are consumed.
+ """
+ paths = []
+ for row in traces:
+ proposal, ridge = row["proposal"], row["ridge_points"]
+ if not proposal["refined"] or len(ridge) < 40:
+ continue
+ rms, frequency = float(proposal["rms"]), float(proposal["f_main"])
+ if not np.isfinite([rms, frequency]).all() or rms < 0 or frequency <= 0:
+ raise ValueError("Coherent fit residual/frequency must be finite and physical")
+ if rms / frequency > 0.010:
+ continue
+ points = np.array([[p["time_s"], p["frequency_hz"], p.get("weight", 1.)] for p in ridge])
+ if (not np.isfinite(points).all() or np.any(np.diff(points[:, 0]) <= 0)
+ or np.any(points[:, 1] <= 0) or np.any(points[:, 2] < 0)):
+ raise ValueError("Coherent traces require ordered finite measured ridge samples")
+ paths.append(points)
+
+ # Reference traces have a 0.25 s hop. Use elapsed overlap rather than a
+ # frame count tied to the ordinary discovery backend's 0.5 s hop.
+ groups = []
+ for path in sorted(_merge_duplicates(paths), key=lambda p: np.median(p[:, 1])):
+ for group in groups:
+ shared = _overlap(group[0], path)
+ if shared is None:
+ continue
+ t, base, frequency = shared
+ if np.ptp(t) < .6 * min(np.ptp(group[0][:, 0]), np.ptp(path[:, 0])):
+ continue
+ ratio = frequency / base
+ harmonic = min(_HARMONIC_RATIOS, key=lambda h: abs(h - np.median(ratio)))
+ if np.quantile(np.abs(ratio - harmonic) / harmonic, .9) < .008:
+ group.append(path)
+ break
+ else:
+ groups.append([path])
+
+ events = []
+ for i, group in enumerate(sorted(groups, key=lambda g: g[0][0, 0]), 1):
+ for j, points in enumerate(group, 1):
+ t, f, weight = points.T
+ events.append(validate_event({
+ "event_id": f"coherent-{i:04d}-{j:02d}",
+ "harmonic_group_id": f"coherent-group-{i:04d}",
+ "start_s": float(t[0]), "end_s": float(t[-1]),
+ "band_hz": [float(max(f.min() - 1., 1e-6)), float(f.max() + 1.)],
+ "ridge": [{"time_s": float(a), "frequency_hz": float(b), "weight": float(c)}
+ for a, b, c in points],
+ "quality": {"frequency_resolution_hz": 1., "frequency_uncertainty_hz": 1.},
+ }))
+ return events
diff --git a/tests/fixtures/real_coherent_michigan.json b/tests/fixtures/real_coherent_michigan.json
new file mode 100644
index 0000000..6bc9f1f
--- /dev/null
+++ b/tests/fixtures/real_coherent_michigan.json
@@ -0,0 +1 @@
+{"source_url":"https://xeno-canto.org/1092456","measurement":"Actual inlier ridge samples from unchanged whole-file dopploc ed84e0f default replay; no generated frequencies. Aircraft identities unconfirmed.","traces":[{"proposal":{"t_cpa":418.04817008930166,"f0":65.4011046829452,"v":25.024149441469007,"d":197.80329144045598,"z":29.5952091217041,"contrast":5.632354259490967,"harmonics":[1],"freqs":[65.4011046829452],"refined":true,"rms":0.2627589378645541,"points":66,"ranges":null,"confidence":"low","f_main":66.03582393481692,"kind":"aircraft"},"fit":{"f0":65.4011046829452,"v":25.024149441469007,"d":197.80329144045598,"t_cpa":418.04817008930166,"rms":0.2627589378645541,"harmonic":1},"ridge_points":[{"time_s":407.25,"frequency_hz":69.56145262070893},{"time_s":407.75,"frequency_hz":68.85612638528497},{"time_s":408.0,"frequency_hz":69.32649812702088},{"time_s":408.25,"frequency_hz":69.3366441775898},{"time_s":408.5,"frequency_hz":69.28139295253555},{"time_s":408.75,"frequency_hz":69.41428114744734},{"time_s":409.0,"frequency_hz":69.29475802072675},{"time_s":409.25,"frequency_hz":69.17741623532353},{"time_s":409.5,"frequency_hz":69.21751065567159},{"time_s":409.75,"frequency_hz":69.12996381108427},{"time_s":410.0,"frequency_hz":69.08914157436912},{"time_s":410.25,"frequency_hz":69.05517237758845},{"time_s":410.5,"frequency_hz":69.003547214289},{"time_s":410.75,"frequency_hz":69.091796875},{"time_s":411.0,"frequency_hz":69.0802513745449},{"time_s":411.25,"frequency_hz":69.08469977744126},{"time_s":411.5,"frequency_hz":68.91952039862895},{"time_s":411.75,"frequency_hz":68.75781891814346},{"time_s":412.0,"frequency_hz":68.38639680823928},{"time_s":412.25,"frequency_hz":68.55668371387911},{"time_s":412.5,"frequency_hz":68.41466904259285},{"time_s":412.75,"frequency_hz":68.16131384543822},{"time_s":413.0,"frequency_hz":68.08878635441324},{"time_s":413.25,"frequency_hz":67.962466685468},{"time_s":413.5,"frequency_hz":67.57493984890415},{"time_s":413.75,"frequency_hz":67.53435739245312},{"time_s":414.0,"frequency_hz":67.40508685869834},{"time_s":414.25,"frequency_hz":67.39457623421038},{"time_s":414.5,"frequency_hz":67.39725102290737},{"time_s":414.75,"frequency_hz":67.25100275656179},{"time_s":415.0,"frequency_hz":67.0503543023396},{"time_s":415.25,"frequency_hz":66.83267616865065},{"time_s":415.5,"frequency_hz":66.80639771925942},{"time_s":415.75,"frequency_hz":66.55583323735159},{"time_s":416.0,"frequency_hz":66.59099949123404},{"time_s":416.25,"frequency_hz":66.27527702094544},{"time_s":416.5,"frequency_hz":66.162109375},{"time_s":416.75,"frequency_hz":66.21754222030738},{"time_s":417.0,"frequency_hz":66.02448291015519},{"time_s":417.25,"frequency_hz":66.26197507927404},{"time_s":417.5,"frequency_hz":66.34132182544587},{"time_s":417.75,"frequency_hz":66.27336915766008},{"time_s":418.5,"frequency_hz":65.85705891366446},{"time_s":418.75,"frequency_hz":65.57741516418709},{"time_s":419.0,"frequency_hz":64.96512153610894},{"time_s":419.25,"frequency_hz":64.72563795599645},{"time_s":419.5,"frequency_hz":64.46899713822336},{"time_s":419.75,"frequency_hz":64.37626578533485},{"time_s":420.0,"frequency_hz":64.42471338223645},{"time_s":420.25,"frequency_hz":64.29258954463162},{"time_s":420.5,"frequency_hz":64.01446442748765},{"time_s":420.75,"frequency_hz":63.706103876999116},{"time_s":421.0,"frequency_hz":63.59753108972695},{"time_s":421.25,"frequency_hz":63.41305834057959},{"time_s":421.5,"frequency_hz":63.170498430014646},{"time_s":421.75,"frequency_hz":63.10635794674039},{"time_s":422.0,"frequency_hz":63.180655566609616},{"time_s":422.25,"frequency_hz":63.19346733931474},{"time_s":422.5,"frequency_hz":62.90204020733921},{"time_s":422.75,"frequency_hz":62.744140625},{"time_s":423.0,"frequency_hz":62.91090481108957},{"time_s":423.25,"frequency_hz":62.8784150946634},{"time_s":423.5,"frequency_hz":62.94188733477378},{"time_s":423.75,"frequency_hz":63.201847015267674},{"time_s":424.0,"frequency_hz":62.98752619856925},{"time_s":424.25,"frequency_hz":63.173457248261876}],"observed_span_s":17.0},{"proposal":{"t_cpa":418.6156834518105,"f0":103.72091209818434,"v":22.51765966679437,"d":124.55207218403818,"z":18.042387008666992,"contrast":3.9637610912323,"harmonics":[1],"freqs":[103.72091209818434],"refined":true,"rms":0.678120899805757,"points":58,"ranges":null,"confidence":"low","f_main":103.5647873558515,"kind":"aircraft"},"fit":{"f0":103.72091209818434,"v":22.51765966679437,"d":124.55207218403818,"t_cpa":418.6156834518105,"rms":0.678120899805757,"harmonic":1},"ridge_points":[{"time_s":411.0,"frequency_hz":107.45142827002135},{"time_s":411.25,"frequency_hz":109.39775554008435},{"time_s":411.5,"frequency_hz":109.74947949732828},{"time_s":411.75,"frequency_hz":109.8527422687063},{"time_s":412.0,"frequency_hz":109.59933230991317},{"time_s":412.25,"frequency_hz":109.42787384130133},{"time_s":412.5,"frequency_hz":109.28672736262932},{"time_s":412.75,"frequency_hz":109.52878996658664},{"time_s":413.5,"frequency_hz":108.70025759845659},{"time_s":413.75,"frequency_hz":108.80257466464866},{"time_s":414.0,"frequency_hz":108.49131654427578},{"time_s":414.25,"frequency_hz":107.99046329955246},{"time_s":414.5,"frequency_hz":107.49052338680978},{"time_s":414.75,"frequency_hz":107.666015625},{"time_s":415.0,"frequency_hz":107.43786159891806},{"time_s":415.25,"frequency_hz":107.03678522026644},{"time_s":415.5,"frequency_hz":106.60595065659163},{"time_s":415.75,"frequency_hz":106.59749099769402},{"time_s":416.0,"frequency_hz":106.18461307146237},{"time_s":416.25,"frequency_hz":106.1673977184193},{"time_s":416.5,"frequency_hz":106.02853280764906},{"time_s":416.75,"frequency_hz":105.90920425783462},{"time_s":417.0,"frequency_hz":105.87137134371658},{"time_s":417.25,"frequency_hz":105.9832179117113},{"time_s":417.5,"frequency_hz":105.2971082618221},{"time_s":417.75,"frequency_hz":104.75303514105764},{"time_s":418.0,"frequency_hz":105.224609375},{"time_s":418.5,"frequency_hz":103.271484375},{"time_s":418.75,"frequency_hz":103.60736076971818},{"time_s":419.0,"frequency_hz":103.66800300249338},{"time_s":419.25,"frequency_hz":103.39813748473583},{"time_s":419.5,"frequency_hz":103.17729951901758},{"time_s":419.75,"frequency_hz":103.271484375},{"time_s":420.0,"frequency_hz":102.60709199919964},{"time_s":420.25,"frequency_hz":102.64243091291944},{"time_s":420.5,"frequency_hz":102.59700866085346},{"time_s":420.75,"frequency_hz":102.294921875},{"time_s":421.0,"frequency_hz":99.3710011766077},{"time_s":421.25,"frequency_hz":99.39467083527543},{"time_s":421.5,"frequency_hz":99.2017033834089},{"time_s":421.75,"frequency_hz":99.06084186002846},{"time_s":422.0,"frequency_hz":99.27985176798174},{"time_s":422.25,"frequency_hz":99.23821500623711},{"time_s":422.5,"frequency_hz":99.22083549795262},{"time_s":422.75,"frequency_hz":99.25550620278801},{"time_s":423.0,"frequency_hz":99.19736616490174},{"time_s":423.25,"frequency_hz":99.25310880419269},{"time_s":423.5,"frequency_hz":99.16701879472942},{"time_s":423.75,"frequency_hz":99.27839462267472},{"time_s":424.0,"frequency_hz":99.3205299989156},{"time_s":424.25,"frequency_hz":99.22956413570161},{"time_s":424.5,"frequency_hz":99.2909502489028},{"time_s":424.75,"frequency_hz":99.29074294546457},{"time_s":425.0,"frequency_hz":99.2199006389337},{"time_s":425.25,"frequency_hz":99.0113146582264},{"time_s":425.5,"frequency_hz":98.90033836726198},{"time_s":425.75,"frequency_hz":99.06016911252256},{"time_s":426.0,"frequency_hz":99.2156093033119}],"observed_span_s":15.0},{"proposal":{"t_cpa":95.83120127482073,"f0":57.7222775485059,"v":58.27055484516291,"d":125.25759407045565,"z":11.266520500183105,"contrast":4.085392355918884,"harmonics":[1],"freqs":[57.7222775485059],"refined":true,"rms":0.6491233096289355,"points":40,"ranges":null,"confidence":"low","f_main":54.17379918324255,"kind":"aircraft"},"fit":{"f0":57.7222775485059,"v":58.27055484516291,"d":125.25759407045565,"t_cpa":95.83120127482073,"rms":0.6491233096289355,"harmonic":1},"ridge_points":[{"time_s":89.0,"frequency_hz":67.626953125},{"time_s":89.25,"frequency_hz":68.56061331569813},{"time_s":89.5,"frequency_hz":69.428094112509},{"time_s":89.75,"frequency_hz":69.99859064039681},{"time_s":90.0,"frequency_hz":69.091796875},{"time_s":90.25,"frequency_hz":68.73838582429346},{"time_s":90.5,"frequency_hz":70.068359375},{"time_s":90.75,"frequency_hz":68.8981294882543},{"time_s":91.0,"frequency_hz":68.98053901171733},{"time_s":91.25,"frequency_hz":67.626953125},{"time_s":91.5,"frequency_hz":68.115234375},{"time_s":91.75,"frequency_hz":68.04813858597474},{"time_s":92.0,"frequency_hz":66.65342770146967},{"time_s":92.25,"frequency_hz":67.93711362203781},{"time_s":92.5,"frequency_hz":67.81106840843114},{"time_s":92.75,"frequency_hz":66.73310953141288},{"time_s":93.0,"frequency_hz":67.12404505997878},{"time_s":93.25,"frequency_hz":66.90431536168225},{"time_s":93.75,"frequency_hz":65.6691484751383},{"time_s":94.0,"frequency_hz":65.67175500800279},{"time_s":94.25,"frequency_hz":64.64278750518832},{"time_s":94.5,"frequency_hz":63.232421875},{"time_s":94.75,"frequency_hz":62.514546150770286},{"time_s":95.0,"frequency_hz":61.345627030584055},{"time_s":95.25,"frequency_hz":60.791015625},{"time_s":95.5,"frequency_hz":59.40188792107092},{"time_s":95.75,"frequency_hz":58.12964337346426},{"time_s":96.0,"frequency_hz":57.09679174755355},{"time_s":96.25,"frequency_hz":56.07694425716856},{"time_s":96.5,"frequency_hz":55.186758295258514},{"time_s":96.75,"frequency_hz":53.99528521737051},{"time_s":97.0,"frequency_hz":53.14350001311681},{"time_s":97.25,"frequency_hz":52.367461403647155},{"time_s":97.5,"frequency_hz":52.001953125},{"time_s":98.0,"frequency_hz":53.373730816051534},{"time_s":98.25,"frequency_hz":52.490234375},{"time_s":98.5,"frequency_hz":51.228671186734715},{"time_s":98.75,"frequency_hz":50.048828125},{"time_s":99.0,"frequency_hz":50.381546724283},{"time_s":99.25,"frequency_hz":51.855745308176466}],"observed_span_s":10.25}],"recordist":"Valerie Heemstra","source_audio_license":"CC BY-NC-SA 4.0"}
diff --git a/tests/test_real_coherent_discovery.py b/tests/test_real_coherent_discovery.py
new file mode 100644
index 0000000..d0e6e9c
--- /dev/null
+++ b/tests/test_real_coherent_discovery.py
@@ -0,0 +1,33 @@
+"""Software regression on measured public Michigan recording traces only."""
+import json
+from pathlib import Path
+
+from dopplergeist.coherent_discovery import events_from_coherent_traces
+from dopplergeist.contracts import validate_event
+from dopplergeist.matching import prepare_events
+
+
+def test_real_slow_traces_survive_without_geometry_or_kind_gates():
+ fixture = Path(__file__).parent / "fixtures/real_coherent_michigan.json"
+ rows = json.loads(fixture.read_text())["traces"]
+ events = events_from_coherent_traces(rows)
+ assert len(events) == 2
+ assert sorted(len(event["ridge"]) for event in events) == [58, 66]
+ assert all(validate_event(event) == event for event in events)
+ assert all(row["proposal"]["v"] < 30 for row in rows[:2])
+ observed = {(p["time_s"], p["frequency_hz"]) for row in rows for p in row["ridge_points"]}
+ assert all((p["time_s"], p["frequency_hz"]) in observed for e in events for p in e["ridge"])
+ settings, measurements = prepare_events(events, c=333.3)
+ assert sum(len(measurement[1]) for measurement in measurements) == 124
+ assert all(event["speed_range_mps"] == [0., 333.3] for event in settings)
+ assert all(event["slant_range_m"] == [0., 20000.] for event in settings)
+
+
+def test_real_noisier_trace_fails_reference_quality_gate():
+ fixture = Path(__file__).parent / "fixtures/real_coherent_michigan.json"
+ rows = json.loads(fixture.read_text())["traces"]
+ # This actual 40-point trace fails the existing 1% fit criterion; the two
+ # slower traces above pass. No aircraft-type conclusion follows either way.
+ assert len(rows[2]["ridge_points"]) == 40
+ assert rows[2]["proposal"]["rms"] / rows[2]["proposal"]["f_main"] > .01
+ assert events_from_coherent_traces(rows[2:]) == []