Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
import gzip
import hashlib
import io
import json
import tarfile
import numpy as np
import pandas as pd
import pytest
from dopplergeist.data_access import CachedTerrain, DataUnavailable, FlightCache, FlightIndex, archive_frames, download
def samples():
return pd.DataFrame({"flight_id": ["prop"] * 6 + ["unknown"] * 3,
"time_s": [0, 5, 10, 100, 105, 110, 0, 5, 10],
"lat": [40.] * 9, "lon": np.linspace(-100.2, -99.8, 9),
"alt_m": [1000.] * 9, "typecode": ["C172"] * 6 + [""] * 3,
"altitude_basis": ["ellipsoidal"] * 3 + ["barometric"] * 6})
def test_whole_segments_nonjet_unknown_altitude_and_repeat(tmp_path):
parquet = tmp_path / "input.parquet"
frame = samples()
frame.to_parquet(parquet)
coverage = {"bounds": [-125, 24, -66, 50], "time_range_s": [0, 200], "all_aircraft": True}
index = FlightIndex.from_parquet(tmp_path / "index.sqlite", parquet, coverage=coverage, class_map={"C172": "piston"})
tracks = list(index.query([-101, 39, -99, 41], [4, 6]))
assert [t.id for t in tracks] == ["prop:0", "unknown"]
assert tracks[0].time.tolist() == [0, 5, 10] # query does not clip
assert tracks[0].typecode == "C172"
assert tracks[0].aircraft_class == "piston" and tracks[1].aircraft_class == "unclassified"
assert tracks[0].altitude_basis == ["ellipsoidal"] * 3
assert np.isnan(tracks[0].position(11)).all() # no gap extrapolation
later = list(index.query([-101, 39, -99, 41], [104, 106]))
assert later[0].id == "prop:1"
assert later[0].altitude_basis == ["barometric"] * 3
before = index.path.stat().st_mtime_ns
replay = FlightIndex.from_parquet(index.path, parquet, coverage=coverage, class_map={"C172": "piston"})
assert replay.path.stat().st_mtime_ns == before
assert replay.coverage_for([-101, 39, -99, 41], [0, 200])["scope_complete"]
assert not replay.coverage_for([-130, 39, -99, 41], [0, 200])["scope_complete"]
with pytest.raises(DataUnavailable, match="different inputs"):
FlightIndex.from_parquet(index.path, parquet, coverage=dict(coverage, all_aircraft=False), class_map={"C172": "piston"})
def test_truncated_parquet_and_payload_corruption(tmp_path):
path = tmp_path / "bad.parquet"
path.write_bytes(b"PAR1truncated")
with pytest.raises(DataUnavailable, match="Parquet"):
FlightIndex.from_parquet(tmp_path / "bad.sqlite", path, coverage={})
index = FlightIndex.build(tmp_path / "index.sqlite", [samples()], sources=[], coverage={})
import sqlite3
with sqlite3.connect(index.path) as db:
db.execute("UPDATE segments SET payload=?", (b"corrupt",))
with pytest.raises(DataUnavailable, match="Corrupt flight payload"):
list(index.query([-180, -90, 180, 90], [0, 200]))
class Response:
def __init__(self, data, status=200, headers=None):
self.data, self.status_code, self.headers = data, status, headers or {}
def __enter__(self): return self
def __exit__(self, *args): pass
def raise_for_status(self): pass
def json(self): return json.loads(self.data)
def iter_content(self, size): yield self.data
class Session:
def __init__(self, response): self.response, self.calls = response, []
def get(self, url, **kwargs):
self.calls.append(kwargs)
return self.response
def test_resumable_download_validates_ranges_and_offline_cache(tmp_path):
path = tmp_path / "asset"
path.with_name("asset.part").write_bytes(b"abc")
session = Session(Response(b"def", 206, {"Content-Range": "bytes 3-5/6"}))
digest = hashlib.sha256(b"abcdef").hexdigest()
result = download("url", path, size=6, digest=digest, session=session)
assert path.read_bytes() == b"abcdef" and result["sha256"] == digest
assert session.calls[0]["headers"]["Range"] == "bytes=3-"
download("url", path, size=6, offline=True)
path.write_bytes(b"broken")
with pytest.raises(DataUnavailable, match="corrupt"):
download("url", path, size=6, offline=True)
with pytest.raises(DataUnavailable, match="Offline"):
download("url", tmp_path / "absent", offline=True)
other = tmp_path / "other"
other.with_name("other.part").write_bytes(b"abc")
with pytest.raises(DataUnavailable, match="byte range"):
download("url", other, size=6, session=Session(Response(b"abcdef")))
assert not other.exists()
with pytest.raises(DataUnavailable, match="Truncated download"):
download("url", tmp_path / "truncated", size=6, session=Session(Response(b"abc")))
def test_streaming_split_readsb_archive(tmp_path):
trace = {"icao": "a12345", "timestamp": 1000, "t": "C172", "trace": [
[i, 40, -100 + i * .001, 3000, 100, 90, 0, 0, {}, None, 3200, 0] for i in [0, 5, 10]]}
data = gzip.compress(json.dumps(trace).encode())
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode="w") as archive:
member = tarfile.TarInfo("traces/trace_full_a12345.json")
member.size = len(data)
archive.addfile(member, io.BytesIO(data))
raw = stream.getvalue()
paths = [tmp_path / "archive.aa", tmp_path / "archive.ab"]
paths[0].write_bytes(raw[:600]); paths[1].write_bytes(raw[600:])
frames = list(archive_frames(paths))
assert frames[0].typecode.tolist() == ["C172"] * 3
assert frames[0].altitude_basis.tolist() == ["ellipsoidal"] * 3
assert np.allclose(frames[0].alt_m, 3200 * .3048)
paths[1].write_bytes(b"broken")
with pytest.raises(DataUnavailable, match="archive"):
list(archive_frames(paths))
def test_archive_ignores_readsb_partial_members(tmp_path):
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode="w") as archive:
member = tarfile.TarInfo("traces/trace_full_a4414a.json.readsb_tmp")
member.size = 0
archive.addfile(member, io.BytesIO())
path = tmp_path / "archive.tar"
path.write_bytes(stream.getvalue())
assert list(archive_frames([path])) == []
def test_terrain_offline_datum_missing_and_corruption(tmp_path):
import rasterio
from rasterio.transform import from_origin
tag = "Copernicus_DSM_COG_30_N40_00_W100_00_DEM"
(tmp_path / "tileList.txt").write_text(tag + "/\n")
for name, value in [(tag + ".tif", 100.), ("geoid.tif", -30.)]:
with rasterio.open(tmp_path / name, "w", driver="GTiff", height=2, width=2, count=1,
dtype="float32", crs="EPSG:4326", transform=from_origin(-100, 41, .5, .5)) as raster:
raster.write(np.full((2, 2), value, dtype="float32"), 1)
terrain = CachedTerrain(tmp_path, offline=True, geoid_path=tmp_path / "geoid.tif", adopt_existing=True)
assert terrain.sample([40.5], [-99.5]).tolist() == [70.]
assert terrain.provenance()["vertical_datum"] == "EGM2008 converted to ellipsoidal"
(tmp_path / (tag + ".tif")).unlink()
with pytest.raises(DataUnavailable, match="Offline"):
CachedTerrain(tmp_path, offline=True, geoid_path=tmp_path / "geoid.tif").sample([40.5], [-99.5])
(tmp_path / "tileList.txt").write_text("broken")
with pytest.raises(DataUnavailable, match="corrupt"):
CachedTerrain(tmp_path, offline=True, geoid_path=tmp_path / "geoid.tif")
def test_terrain_degree_boundary_uses_published_cog_extent(tmp_path):
import rasterio
from rasterio.transform import from_origin
tag = "Copernicus_DSM_COG_30_N39_00_W100_00_DEM"
(tmp_path / "tileList.txt").write_text(tag + "/\n")
for name, value in [(tag + ".tif", 100.), ("geoid.tif", -30.)]:
with rasterio.open(tmp_path / name, "w", driver="GTiff", height=2, width=2, count=1,
dtype="float32", crs="EPSG:4326", transform=from_origin(-100, 40.000416666667, .5, .5)) as raster:
raster.write(np.full((2, 2), value, dtype="float32"), 1)
terrain = CachedTerrain(tmp_path, offline=True, geoid_path=tmp_path / "geoid.tif", adopt_existing=True)
assert terrain.sample([40.], [-99.5]).tolist() == [70.]
assert tag in str(terrain.provenance()["files"])
def test_index_bounds_include_hermite_excursion_between_samples(tmp_path):
from pyproj import Transformer
from dopplergeist.geometry import Track, velocity_ecef
frame = pd.DataFrame({"flight_id": ["turn"] * 3, "time_s": [0., 10., 20.],
"lat": [40.] * 3, "lon": [-100.] * 3, "alt_m": [1000.] * 3,
"east_mps": [300., -300., 300.], "north_mps": [0.] * 3, "up_mps": [0.] * 3})
velocity = velocity_ecef(frame.lat.to_numpy(), frame.lon.to_numpy(), frame.east_mps.to_numpy(), np.zeros(3), np.zeros(3))
oracle = Track("turn", frame.time_s, frame.lat, frame.lon, frame.alt_m, velocity=velocity)
lon, lat, _ = Transformer.from_crs("EPSG:4978", "EPSG:4979", always_xy=True).transform(*oracle.position(5))
assert lon > -99.995 # the whole measured-position box is far outside this query
index = FlightIndex.build(tmp_path / "curve.sqlite", [frame], sources=[], coverage={})
found = list(index.query([lon-.00001, lat-.00001, lon+.00001, lat+.00001], [4., 6.]))
assert len(found) == 1
assert np.allclose(found[0].position(5), oracle.position(5))
def test_index_partial_velocity_inference_preserves_measured_zero(tmp_path):
frame = pd.DataFrame({"flight_id": ["partial"] * 3, "time_s": [0., 5., 10.],
"lat": [40.] * 3, "lon": [-100., -99.999, -99.998], "alt_m": [1000., 1050., 1100.],
"east_mps": [np.nan] * 3, "north_mps": [np.nan] * 3, "up_mps": [0.] * 3})
index = FlightIndex.build(tmp_path / "partial.sqlite", [frame], sources=[], coverage={})
track = list(index.query([-101, 39, -99, 41], [0, 10]))[0]
lat, lon = np.deg2rad([40., -99.999])
vertical = np.array([np.cos(lat)*np.cos(lon), np.cos(lat)*np.sin(lon), np.sin(lat)])
assert abs(np.dot(track.speed_vector(5), vertical)) < 1e-9
assert np.linalg.norm(track.speed_vector(5)) > 10
assert track.velocity_inferred_components["up_mps"] == 0
def test_date_acquisition_ingests_all_types_and_reuses_validated_offline_index(tmp_path):
start = pd.Timestamp("2026-09-08T00:00:00Z").timestamp()
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode="w") as archive:
for identity, code in [("a12345", "C172"), ("b12345", "H500"), ("c12345", "ZZZZ")]:
trace = {"icao": identity, "timestamp": start, "t": code, "trace": [
[i, 40, -100+i*.001, 3000, 100, 90, 0, 0, {}, None, 3200, 0] for i in [0, 5, 10]]}
raw = gzip.compress(json.dumps(trace).encode())
member = tarfile.TarInfo(f"trace_full_{identity}.json"); member.size = len(raw)
archive.addfile(member, io.BytesIO(raw))
payload = stream.getvalue()
asset = {"name": "day.tar.aa", "size": len(payload), "digest": "sha256:"+hashlib.sha256(payload).hexdigest(),
"browser_download_url": "https://fixture.invalid/day.tar.aa"}
class Provider:
def __init__(self): self.calls = []
def get(self, url, **kwargs):
self.calls.append(url)
if url.startswith("https://api.github.com/"):
return Response(json.dumps({"tag_name": "v2026.09.08-planes-readsb-staging-0", "assets": [asset]}).encode())
if url == asset["browser_download_url"]:
return Response(payload)
if url.endswith("icao_aircraft_types.js"):
return Response(b'let types={"C172":{"desc":"L1P"},"H500":{"desc":"H1T"}};')
raise AssertionError(f"Unexpected network request: {url}")
provider = Provider()
index = FlightCache(tmp_path, session=provider).acquire_day("2026-09-08")
tracks = list(index.query([-101, 39, -99, 41], [start, start+10]))
assert {t.typecode: t.aircraft_class for t in tracks} == {"C172": "piston", "H500": "turbine"}
assert all(t.altitude_basis == ["ellipsoidal"]*3 for t in tracks)
assert len(provider.calls) == 3 and index.report["rows"] == 6
repeat = FlightCache(tmp_path, offline=True, session=provider).acquire_day("2026-09-08")
assert repeat.report == index.report and len(provider.calls) == 3
assert index.report["coverage"]["release_tag"] == "v2026.09.08-planes-readsb-staging-0"
index.path.unlink()
(tmp_path / "2026-09-08/staging-0/day.tar.aa").write_bytes(payload[:100])
with pytest.raises(DataUnavailable, match="Truncated cache"):
FlightCache(tmp_path, offline=True, session=provider).acquire_day("2026-09-08")
release_path = tmp_path / "2026-09-08/staging-0/release.json"
release_path.write_text(json.dumps({"tag_name": "wrong-day", "assets": [asset]}))
with pytest.raises(DataUnavailable, match="requested tag"):
FlightCache(tmp_path, offline=True, session=provider).acquire_day("2026-09-08")
with pytest.raises(DataUnavailable, match="Offline release"):
FlightCache(tmp_path, offline=True, session=provider).acquire_day("2026-09-09")