Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
ingest_adsblol_day: decode trace members in a worker pool (--workers), output unchanged
Henrik Lied586697797d96
diff --git a/scripts/ingest_adsblol_day.py b/scripts/ingest_adsblol_day.pyindex aa18d2a..79f3d81 100644--- a/scripts/ingest_adsblol_day.py+++ b/scripts/ingest_adsblol_day.py@@ -1,5 +1,6 @@"""Stream a split daily tar archive into a CONUS prop/unknown parquet table."""import argparse+from collections import dequefrom datetime import date, datetime, time, timezoneimport gzipimport io@@ -16,48 +17,13 @@ import requestsfrom dopplergeist.tracks import adsblol_frame,unix_timefrom dopplergeist.flight_selection import intersects_index_query+from dopplergeist.data_access import _index_pool,_index_workers-parser=argparse.ArgumentParser()-parser.add_argument("date")-parser.add_argument("--bounds",type=float,nargs=4,default=[-125,24,-66,50],metavar=("WEST","SOUTH","EAST","NORTH"))-parser.add_argument("--output",default="conus-prop-unknown.parquet")-parser.add_argument("--keep-jets",action="store_true",help="Include jet types for the single-event ablation")-parser.add_argument("--time-range",nargs=2,metavar=("START_UTC","END_UTC"))-args=parser.parse_args()-root=Path(__file__).resolve().parents[1]-directory=root/"data/adsb"/args.date-west,south,east,north=args.bounds-time_range=[unix_time(t) for t in args.time_range] if args.time_range else None-day_start=datetime.combine(date.fromisoformat(args.date),time.min,tzinfo=timezone.utc).timestamp()-day_end=day_start+86400-selected_start=max(day_start,time_range[0]) if time_range else day_start-selected_end=min(day_end,time_range[1]) if time_range else day_end-selected_time=[selected_start,selected_end] if selected_start<=selected_end else []-output_path=directory/args.output-report_path=directory/(Path(args.output).stem+"-ingestion-report.json")-if output_path.exists() or report_path.exists():- raise FileExistsError("Ingestion output exists; choose a new --output name to preserve historical evidence")-types_path=root/"data/source/icao_aircraft_types.json"-if not types_path.exists():- r=requests.get("https://raw.githubusercontent.com/wiedehopf/tar1090-db/master/db/icao_aircraft_types.js",timeout=60)- r.raise_for_status()- raw=r.content- if raw[:2]==b"\x1f\x8b":raw=gzip.decompress(raw)- text=raw.decode()- # The public database is a JS assignment containing one JSON dictionary.- start,end=text.find("{"),text.rfind("}")+1- json.loads(text[start:end])- # Publish atomically: a concurrent ingest that sees the path must never read a partial file.- temporary=types_path.with_name(f"{types_path.name}.{os.getpid()}.tmp")- temporary.write_text(text[start:end])- os.replace(temporary,types_path)-types=json.loads(types_path.read_text())-print("Type examples",list(types.items())[:3],flush=True)-engines={}-for code,value in types.items():- classification=value.get("desc","") if isinstance(value,dict) else (value[2] if len(value)>2 else "")- engines[code]="Jet" if classification.endswith("J") else "Other/unknown"-pd.DataFrame({"designator":list(engines),"engine_type":list(engines.values())}).to_csv(directory/"type-database.csv",index=False)+# Members per worker task, and tasks in flight per worker. The reader stays at most+# BATCH*INFLIGHT*workers members ahead of the writer, so memory stays bounded.+BATCH=64+INFLIGHT=4+_SELECT={}class SplitReader(io.RawIOBase):@@ -76,49 +42,146 @@ class SplitReader(io.RawIOBase):self.current.close();self.current=None-paths=sorted(directory.glob("*.tar.a?"))-if len(paths)<2:raise ValueError("Daily archive download is incomplete")-stats={"trace_files":0,"known_jets_removed":0,"no_conus_points":0,"retained_aircraft":0,"rows":0,"barometric_rows":0}-stats["normalization_scope"]="full_native_trace_before_segment_selection"-stats["selection_max_gap_s"]=30.0-selection={"bounds":[west,south,east,north],"time_range_s":selected_time,"archive_day_utc":args.date,- "all_aircraft":args.keep_jets,"max_gap_s":30.0}-writer=None-with io.BufferedReader(SplitReader(paths),buffer_size=1024*1024) as stream:- with tarfile.open(fileobj=stream,mode="r|") as archive:- for member in archive:- if not member.isfile() or "trace_full_" not in member.name:continue- raw=archive.extractfile(member).read()- d=json.loads(gzip.decompress(raw) if raw[:2]==b"\x1f\x8b" else raw)- stats["trace_files"]+=1- if not args.keep_jets and engines.get(d.get("t",""))=="Jet":- stats["known_jets_removed"]+=1- continue- # A segment can cross a query without a sample inside it. Temporal- # extents can reject only traces entirely before or after the query.- if time_range is not None:- offsets=[p[0] for p in d.get("trace",[]) if isinstance(p[0],(int,float)) and np.isfinite(p[0])]- if not offsets or d["timestamp"]+min(offsets)>time_range[1] or d["timestamp"]+max(offsets)<time_range[0]:+def _configure(engines,keep_jets,bounds,time_range):+ _SELECT.update(engines=engines,keep_jets=keep_jets,bounds=bounds,time_range=time_range)+++def select_trace(raw):+ """Decode one trace member and apply the jet, time and bounds selection.++ Returns ("jet", None), ("outside", None), ("empty", None) or ("kept", frame).+ """+ d=json.loads(gzip.decompress(raw) if raw[:2]==b"\x1f\x8b" else raw)+ if not _SELECT["keep_jets"] and _SELECT["engines"].get(d.get("t",""))=="Jet":return "jet",None+ time_range=_SELECT["time_range"]+ # A segment can cross a query without a sample inside it. Temporal+ # extents can reject only traces entirely before or after the query.+ if time_range is not None:+ offsets=[p[0] for p in d.get("trace",[]) if isinstance(p[0],(int,float)) and np.isfinite(p[0])]+ if not offsets or d["timestamp"]+min(offsets)>time_range[1] or d["timestamp"]+max(offsets)<time_range[0]:+ return "outside",None+ frame=adsblol_frame(d)+ if frame.empty:return "empty",None+ if not intersects_index_query(frame,_SELECT["bounds"],time_range):return "outside",None+ # Keep the native trace intact: FlightIndex performs its own+ # gap splitting and uses neighboring samples to infer velocity.+ frame["heading_deg"]=frame.heading_deg.astype(float)+ return "kept",frame+++def select_batch(raws):+ return [select_trace(raw) for raw in raws]+++def trace_members(archive):+ for member in archive:+ if not member.isfile() or "trace_full_" not in member.name:continue+ yield archive.extractfile(member).read()+++def batches(items,size):+ batch=[]+ for item in items:+ batch.append(item)+ if len(batch)==size:yield batch;batch=[]+ if batch:yield batch+++def selected_traces(raws,workers):+ """Selection results in archive order, decoded serially or by a spawned pool."""+ if workers<=1:+ yield from map(select_trace,raws)+ return+ with _index_pool(workers,initializer=_configure,initargs=(_SELECT["engines"],_SELECT["keep_jets"],+ _SELECT["bounds"],_SELECT["time_range"])) as pool:+ pending=deque()+ for batch in batches(raws,BATCH):+ pending.append(pool.submit(select_batch,batch))+ if len(pending)>=workers*INFLIGHT:yield from pending.popleft().result()+ while pending:yield from pending.popleft().result()+++def main():+ parser=argparse.ArgumentParser()+ parser.add_argument("date")+ parser.add_argument("--bounds",type=float,nargs=4,default=[-125,24,-66,50],metavar=("WEST","SOUTH","EAST","NORTH"))+ parser.add_argument("--output",default="conus-prop-unknown.parquet")+ parser.add_argument("--keep-jets",action="store_true",help="Include jet types for the single-event ablation")+ parser.add_argument("--time-range",nargs=2,metavar=("START_UTC","END_UTC"))+ parser.add_argument("--workers",type=int,default=1,+ help=f"Processes decoding trace members; 0 = all but one core ({_index_workers()} here). Output is identical for any value")+ args=parser.parse_args()+ if args.workers<0:parser.error("--workers must be >= 0")+ workers=args.workers or _index_workers()+ root=Path(__file__).resolve().parents[1]+ directory=root/"data/adsb"/args.date+ west,south,east,north=args.bounds+ time_range=[unix_time(t) for t in args.time_range] if args.time_range else None+ day_start=datetime.combine(date.fromisoformat(args.date),time.min,tzinfo=timezone.utc).timestamp()+ day_end=day_start+86400+ selected_start=max(day_start,time_range[0]) if time_range else day_start+ selected_end=min(day_end,time_range[1]) if time_range else day_end+ selected_time=[selected_start,selected_end] if selected_start<=selected_end else []+ output_path=directory/args.output+ report_path=directory/(Path(args.output).stem+"-ingestion-report.json")+ if output_path.exists() or report_path.exists():+ raise FileExistsError("Ingestion output exists; choose a new --output name to preserve historical evidence")+ types_path=root/"data/source/icao_aircraft_types.json"+ if not types_path.exists():+ r=requests.get("https://raw.githubusercontent.com/wiedehopf/tar1090-db/master/db/icao_aircraft_types.js",timeout=60)+ r.raise_for_status()+ raw=r.content+ if raw[:2]==b"\x1f\x8b":raw=gzip.decompress(raw)+ text=raw.decode()+ # The public database is a JS assignment containing one JSON dictionary.+ start,end=text.find("{"),text.rfind("}")+1+ json.loads(text[start:end])+ # Publish atomically: a concurrent ingest that sees the path must never read a partial file.+ temporary=types_path.with_name(f"{types_path.name}.{os.getpid()}.tmp")+ temporary.write_text(text[start:end])+ os.replace(temporary,types_path)+ types=json.loads(types_path.read_text())+ print("Type examples",list(types.items())[:3],flush=True)+ engines={}+ for code,value in types.items():+ classification=value.get("desc","") if isinstance(value,dict) else (value[2] if len(value)>2 else "")+ engines[code]="Jet" if classification.endswith("J") else "Other/unknown"+ pd.DataFrame({"designator":list(engines),"engine_type":list(engines.values())}).to_csv(directory/"type-database.csv",index=False)+ _configure(engines,args.keep_jets,(west,south,east,north),time_range)++ paths=sorted(directory.glob("*.tar.a?"))+ if len(paths)<2:raise ValueError("Daily archive download is incomplete")+ stats={"trace_files":0,"known_jets_removed":0,"no_conus_points":0,"retained_aircraft":0,"rows":0,"barometric_rows":0}+ stats["normalization_scope"]="full_native_trace_before_segment_selection"+ stats["selection_max_gap_s"]=30.0+ selection={"bounds":[west,south,east,north],"time_range_s":selected_time,"archive_day_utc":args.date,+ "all_aircraft":args.keep_jets,"max_gap_s":30.0}+ writer=None+ with io.BufferedReader(SplitReader(paths),buffer_size=1024*1024) as stream:+ with tarfile.open(fileobj=stream,mode="r|") as archive:+ for outcome,frame in selected_traces(trace_members(archive),workers):+ stats["trace_files"]+=1+ if outcome=="jet":+ stats["known_jets_removed"]+=1+ continue+ if outcome=="outside":stats["no_conus_points"]+=1continue- frame=adsblol_frame(d)- if frame.empty:continue- if not intersects_index_query(frame,(west,south,east,north),time_range):- stats["no_conus_points"]+=1- continue- # Keep the native trace intact: FlightIndex performs its own- # gap splitting and uses neighboring samples to infer velocity.- frame["heading_deg"]=frame.heading_deg.astype(float)- # Bound schema explicitly so an all-null metadata field in one- # aircraft cannot change a later parquet batch's physical type.- schema=pa.schema([(c,pa.string() if c in ["flight_id","typecode","altitude_basis"] else pa.float64()) for c in frame.columns],- metadata={b"dopplergeist.selection.v2":json.dumps(selection,allow_nan=False).encode()})- batch=pa.Table.from_pandas(frame,schema=schema,preserve_index=False)- if writer is None:writer=pq.ParquetWriter(output_path,schema,compression="zstd")- writer.write_table(batch)- stats["retained_aircraft"]+=1;stats["rows"]+=len(frame)- stats["barometric_rows"]+=int((frame.altitude_basis=="barometric").sum())- if stats["retained_aircraft"]%500==0:print(stats,flush=True)-if writer:writer.close()-report_path.write_text(json.dumps(stats,indent=2)+"\n")-print(stats,flush=True)+ if outcome=="empty":continue+ # Bound schema explicitly so an all-null metadata field in one+ # aircraft cannot change a later parquet batch's physical type.+ schema=pa.schema([(c,pa.string() if c in ["flight_id","typecode","altitude_basis"] else pa.float64()) for c in frame.columns],+ metadata={b"dopplergeist.selection.v2":json.dumps(selection,allow_nan=False).encode()})+ batch=pa.Table.from_pandas(frame,schema=schema,preserve_index=False)+ if writer is None:writer=pq.ParquetWriter(output_path,schema,compression="zstd")+ writer.write_table(batch)+ stats["retained_aircraft"]+=1;stats["rows"]+=len(frame)+ stats["barometric_rows"]+=int((frame.altitude_basis=="barometric").sum())+ if stats["retained_aircraft"]%500==0:print(stats,flush=True)+ if writer:writer.close()+ report_path.write_text(json.dumps(stats,indent=2)+"\n")+ print(stats,flush=True)+++if __name__=="__main__":+ main()diff --git a/tests/test_ingest_workers.py b/tests/test_ingest_workers.pynew file mode 100644index 0000000..d564a15--- /dev/null+++ b/tests/test_ingest_workers.py@@ -0,0 +1,58 @@+"""Software check: the parallel ingest writes the same table and report as the serial one."""+import gzip+import io+import json+import os+from pathlib import Path+import shutil+import subprocess+import sys+import tarfile++import pyarrow.parquet as pq+++def _member(index):+ lat = 40.05 if index % 3 else 45.0+ trace = {"icao": f"f{index:04d}", "timestamp": 1790000000 + index, "t": "B738" if index % 7 == 0 else "PC12",+ "trace": [[t, lat + t * 1e-4, -100, 5000, 100, 90, 0, 0, {}, 0, 5450, None] for t in (0, 10, 20)]}+ raw = json.dumps(trace).encode()+ return gzip.compress(raw) if index % 2 else raw+++def test_worker_count_does_not_change_output(tmp_path):+ root = Path(__file__).resolve().parents[1]+ project = tmp_path / "isolated-project"+ (project / "scripts").mkdir(parents=True)+ script = project / "scripts/ingest_adsblol_day.py"+ shutil.copyfile(root / "scripts/ingest_adsblol_day.py", script)+ (project / "data/source").mkdir(parents=True)+ (project / "data/source/icao_aircraft_types.json").write_text('{"B738": {"desc": "L2J"}, "PC12": {"desc": "L1T"}}')+ day = project / "data/adsb/2026-09-22"+ day.mkdir(parents=True)+ archive = io.BytesIO()+ with tarfile.open(fileobj=archive, mode="w") as output:+ output.addfile(tarfile.TarInfo("traces/readme.txt"), io.BytesIO(b""))+ # More members than one task batch x in-flight window, so ordering across batches is exercised.+ for index in range(700):+ member = _member(index)+ info = tarfile.TarInfo(f"traces/trace_full_f{index:04d}.json")+ info.size = len(member)+ output.addfile(info, io.BytesIO(member))+ raw = archive.getvalue()+ (day / "archive.tar.aa").write_bytes(raw[:len(raw) // 2])+ (day / "archive.tar.ab").write_bytes(raw[len(raw) // 2:])+ env = dict(os.environ, PYTHONPATH=str(root / "src"))+ outputs = {}+ for workers in (1, 3):+ name = f"w{workers}.parquet"+ result = subprocess.run([sys.executable, str(script), "2026-09-22", "--output", name, "--workers", str(workers),+ "--bounds", "-101", "40", "-99", "41"], cwd=project, env=env, capture_output=True, text=True)+ assert result.returncode == 0, result.stderr+ outputs[workers] = (pq.read_table(day / name), json.loads((day / f"w{workers}-ingestion-report.json").read_text()))+ (serial, serial_report), (parallel, parallel_report) = outputs[1], outputs[3]+ assert serial_report == parallel_report+ assert serial_report["trace_files"] == 700 and serial_report["known_jets_removed"] == 100+ assert serial_report["retained_aircraft"] > 0 and serial_report["no_conus_points"] > 0+ assert parallel.schema.equals(serial.schema, check_metadata=True)+ assert parallel.equals(serial)