Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
"""Optional integration regression against a local, real ADS-B archive part."""
import gzip
import io
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tarfile
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest
from dopplergeist.data_access import DataUnavailable, FlightIndex
from dopplergeist.flight_selection import intersects_index_query
from dopplergeist.tracks import adsblol_frame
def test_real_boundary_segments_survive_selection(tmp_path):
source = os.environ.get("DOPPLERGEIST_REAL_ARCHIVE_PART")
if not source:
pytest.skip("set DOPPLERGEIST_REAL_ARCHIVE_PART to a local readsb tar part")
type_path = os.environ.get("DOPPLERGEIST_TYPE_DATABASE")
if not type_path:
pytest.skip("set DOPPLERGEIST_TYPE_DATABASE to the ICAO type database")
from types import SimpleNamespace
from dopplergeist.aircraft_types import load_classes, track_class
classes, _ = load_classes(type_path)
frames = []
first_member = None
with Path(source).open("rb") as stream, tarfile.open(fileobj=stream, mode="r|") as archive:
for member in archive:
if not member.isfile() or "trace_full_" not in member.name:
continue
raw = archive.extractfile(member).read()
data = json.loads(gzip.decompress(raw) if raw[:2] == b"\x1f\x8b" else raw)
codes = {str(data.get("t", "")).upper()} if data.get("t") else set()
codes.update(str(row[8]["t"]).upper() for row in data.get("trace", [])
if len(row) > 8 and isinstance(row[8], dict) and row[8].get("t"))
labels = {track_class(SimpleNamespace(typecode=code, aircraft_class="unclassified"), classes)
for code in codes}
if len(labels) != 1 or not labels.issubset({"piston", "turbine"}):
continue
frame = adsblol_frame(data)
if len(frame) > 50:
frames.append(frame)
if first_member is None:
first_member = member.name, raw
if len(frames) == 10:
break
assert len(frames) == 10
bounds = (-180, -90, 180, 90)
coverage = {"bounds": list(bounds), "time_range_s": [0, 4102444800],
"all_aircraft": False, "release_tag": "real-regression"}
full = FlightIndex.build(tmp_path / "full.sqlite", frames, sources=[], coverage=coverage)
checked = old_filter_misses = 0
for frame in frames:
rows = frame.dropna(subset=["time_s", "lat", "lon", "alt_m"]).sort_values("time_s").drop_duplicates("time_s")
gaps = np.diff(rows.time_s.to_numpy())
choices = np.flatnonzero((gaps > 0.02) & (gaps <= 30))
if not len(choices):
continue
i = int(choices[len(choices) // 2])
a, b = float(rows.time_s.iloc[i]), float(rows.time_s.iloc[i + 1])
interval = (a + (b - a) * 0.4, a + (b - a) * 0.6)
selected_ids = {str(f.flight_id.iloc[0]) for f in frames
if intersects_index_query(f, bounds, interval)}
matches = list(full.query(bounds, interval))
assert all(track.id.split("@")[0] in selected_ids for track in matches)
own_id = str(frame.flight_id.iloc[0])
if any(track.id.split("@")[0] == own_id for track in matches):
old_filter_misses += not frame.time_s.between(*interval).any()
checked += 1
if checked == 5:
break
assert checked == 5
assert old_filter_misses > 0
# Real rows with embedded source limits exercise index-side scope checks.
day_window = [1788825600.0, 1788912000.0]
selection = {"bounds": list(bounds), "time_range_s": day_window, "archive_day_utc": "2026-09-08",
"all_aircraft": False, "max_gap_s": 30.0}
table = pa.Table.from_pandas(frames[0], preserve_index=False)
table = table.replace_schema_metadata({b"dopplergeist.selection.v2": json.dumps(selection).encode()})
source_parquet = tmp_path / "real-selected.parquet"
pq.write_table(table, source_parquet)
restricted = dict(coverage, all_aircraft=False, time_range_s=day_window)
FlightIndex.from_parquet(tmp_path / "valid.sqlite", source_parquet, coverage=restricted)
with pytest.raises(DataUnavailable, match="excluded aircraft"):
FlightIndex.from_parquet(tmp_path / "types.sqlite", source_parquet,
coverage=dict(restricted, all_aircraft=True))
with pytest.raises(DataUnavailable, match="gap limit"):
FlightIndex.from_parquet(tmp_path / "gap.sqlite", source_parquet, coverage=restricted, max_gap_s=60)
with pytest.raises(DataUnavailable, match="selection bounds"):
FlightIndex.from_parquet(tmp_path / "bounds.sqlite", source_parquet,
coverage=dict(restricted, bounds=[-180, -90, 181, 90]))
with pytest.raises(DataUnavailable, match="selection time"):
FlightIndex.from_parquet(tmp_path / "time.sqlite", source_parquet,
coverage=dict(restricted, time_range_s=[day_window[0] - 1, day_window[1]]))
for i, (tag, selected, message) in enumerate((
(b"dopplergeist.selection.v1", dict(selection, time_range_s=None), "Legacy selected Parquet"),
(b"dopplergeist.selection.v1", dict(selection, time_range_s=[0, 4102444800]), "Legacy selected Parquet"),
(b"dopplergeist.selection.v2", dict(selection, time_range_s=None), "selection time"),
(b"dopplergeist.selection.v2", dict(selection, time_range_s=[0, 4102444800]), "selection time"),
)):
malformed = tmp_path / f"malformed-{i}.parquet"
pq.write_table(table.replace_schema_metadata({tag: json.dumps(selected).encode()}), malformed)
with pytest.raises(DataUnavailable, match=message):
FlightIndex.from_parquet(tmp_path / "malformed.sqlite", malformed, coverage=restricted)
untagged = tmp_path / "real-untagged.parquet"
pq.write_table(table.replace_schema_metadata(None), untagged)
FlightIndex.from_parquet(tmp_path / "untagged.sqlite", untagged, coverage=coverage)
# Exercise the actual ingest CLI with one unmodified September 8 trace.
if "2026.09.08" in Path(source).name:
project = tmp_path / "real-ingest"
(project / "scripts").mkdir(parents=True)
shutil.copyfile(Path(__file__).resolve().parents[1] / "scripts/ingest_adsblol_day.py",
project / "scripts/ingest_adsblol_day.py")
(project / "data/source").mkdir(parents=True)
import gzip as gz
type_data = Path(type_path).read_bytes()
if type_data[:2] == b"\x1f\x8b":
type_data = gz.decompress(type_data)
text = type_data.decode()
(project / "data/source/icao_aircraft_types.json").write_text(text[text.index("{"):text.rindex("}") + 1])
day = project / "data/adsb/2026-09-08"
day.mkdir(parents=True)
name, raw = first_member
archive_bytes = io.BytesIO()
with tarfile.open(fileobj=archive_bytes, mode="w") as archive:
member = tarfile.TarInfo(name)
member.size = len(raw)
archive.addfile(member, io.BytesIO(raw))
blob = archive_bytes.getvalue()
(day / "real.tar.aa").write_bytes(blob[:len(blob) // 2])
(day / "real.tar.ab").write_bytes(blob[len(blob) // 2:])
for output, extra in (("whole", []), ("wide", ["--time-range", "2026-09-07T23:00:00Z", "2026-09-09T01:00:00Z"])):
command = [sys.executable, str(project / "scripts/ingest_adsblol_day.py"),
"2026-09-08", "--bounds", "-180", "-90", "180", "90",
"--output", output + ".parquet", *extra]
env = dict(os.environ, PYTHONPATH=str(Path(__file__).resolve().parents[1] / "src"))
completed = subprocess.run(command, env=env, capture_output=True, text=True)
assert completed.returncode == 0, completed.stderr
meta = pq.read_schema(day / (output + ".parquet")).metadata[b"dopplergeist.selection.v2"]
assert json.loads(meta)["time_range_s"] == day_window
assert json.loads(meta)["archive_day_utc"] == "2026-09-08"