Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

9.0 KB
Raw Blame
"""Stream a split daily tar archive into a confirmed non-jet parquet table."""
import argparse
from collections import deque
from datetime import date, datetime, time, timezone
import gzip
import io
import json
import os
import tarfile
from pathlib import Path

import numpy as np
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
import requests

from dopplergeist.tracks import adsblol_frame,unix_time
from dopplergeist.flight_selection import intersects_index_query
from dopplergeist.data_access import _index_pool,_index_workers

# Members per worker task, and tasks in flight per worker. The reader stays at most
# BATCH*INFLIGHT*workers members ahead of the writer, so memory stays bounded.
BATCH=64
INFLIGHT=4
_SELECT={}


class SplitReader(io.RawIOBase):
    def __init__(self,paths):
        self.paths=iter(paths)
        self.current=None
    def readable(self):return True
    def readinto(self,b):
        while True:
            if self.current is None:
                path=next(self.paths,None)
                if path is None:return 0
                self.current=open(path,"rb")
            n=self.current.readinto(b)
            if n:return n
            self.current.close();self.current=None


def _configure(engines,bounds,time_range):
    _SELECT.update(engines=engines,bounds=bounds,time_range=time_range)


def select_trace(raw):
    """Decode one trace member and apply the jet, time and bounds selection.

    Returns ("jet", None), ("unknown", None), ("outside", None), ("empty", None) or ("kept", frame).
    """
    d=json.loads(gzip.decompress(raw) if raw[:2]==b"\x1f\x8b" else raw)
    codes={str(d.get("t", "")).upper()} if d.get("t") else set()
    for row in d.get("trace", []):
        if len(row)>8 and isinstance(row[8],dict) and row[8].get("t"):
            codes.add(str(row[8]["t"]).upper())
    kinds={_SELECT["engines"].get(code, "Unknown") for code in codes}
    if "Jet" in kinds:return "jet",None
    if len(kinds)!=1 or not kinds.issubset({"Piston", "Turbine"}):return "unknown",None
    time_range=_SELECT["time_range"]
    # A segment can cross a query without a sample inside it. Temporal
    # extents can reject only traces entirely before or after the query.
    if time_range is not None:
        offsets=[p[0] for p in d.get("trace",[]) if isinstance(p[0],(int,float)) and np.isfinite(p[0])]
        if not offsets or d["timestamp"]+min(offsets)>time_range[1] or d["timestamp"]+max(offsets)<time_range[0]:
            return "outside",None
    frame=adsblol_frame(d)
    if frame.empty:return "empty",None
    if not intersects_index_query(frame,_SELECT["bounds"],time_range):return "outside",None
    # Keep the native trace intact: FlightIndex performs its own
    # gap splitting and uses neighboring samples to infer velocity.
    frame["heading_deg"]=frame.heading_deg.astype(float)
    return "kept",frame


def select_batch(raws):
    return [select_trace(raw) for raw in raws]


def trace_members(archive):
    for member in archive:
        if not member.isfile() or "trace_full_" not in member.name:continue
        yield archive.extractfile(member).read()


def batches(items,size):
    batch=[]
    for item in items:
        batch.append(item)
        if len(batch)==size:yield batch;batch=[]
    if batch:yield batch


def selected_traces(raws,workers):
    """Selection results in archive order, decoded serially or by a spawned pool."""
    if workers<=1:
        yield from map(select_trace,raws)
        return
    with _index_pool(workers,initializer=_configure,initargs=(_SELECT["engines"], _SELECT["bounds"], _SELECT["time_range"])) as pool:
        pending=deque()
        for batch in batches(raws,BATCH):
            pending.append(pool.submit(select_batch,batch))
            if len(pending)>=workers*INFLIGHT:yield from pending.popleft().result()
        while pending:yield from pending.popleft().result()


def main():
    parser=argparse.ArgumentParser()
    parser.add_argument("date")
    parser.add_argument("--bounds",type=float,nargs=4,default=[-125,24,-66,50],metavar=("WEST","SOUTH","EAST","NORTH"))
    parser.add_argument("--output",default="conus-confirmed-nonjet.parquet")
    parser.add_argument("--time-range",nargs=2,metavar=("START_UTC","END_UTC"))
    parser.add_argument("--workers",type=int,default=1,
                        help=f"Processes decoding trace members; 0 = all but one core ({_index_workers()} here). Output is identical for any value")
    args=parser.parse_args()
    if args.workers<0:parser.error("--workers must be >= 0")
    workers=args.workers or _index_workers()
    root=Path(__file__).resolve().parents[1]
    directory=root/"data/adsb"/args.date
    west,south,east,north=args.bounds
    time_range=[unix_time(t) for t in args.time_range] if args.time_range else None
    day_start=datetime.combine(date.fromisoformat(args.date),time.min,tzinfo=timezone.utc).timestamp()
    day_end=day_start+86400
    selected_start=max(day_start,time_range[0]) if time_range else day_start
    selected_end=min(day_end,time_range[1]) if time_range else day_end
    selected_time=[selected_start,selected_end] if selected_start<=selected_end else []
    output_path=directory/args.output
    report_path=directory/(Path(args.output).stem+"-ingestion-report.json")
    if output_path.exists() or report_path.exists():
        raise FileExistsError("Ingestion output exists; choose a new --output name to preserve historical evidence")
    types_path=root/"data/source/icao_aircraft_types.json"
    if not types_path.exists():
        r=requests.get("https://raw.githubusercontent.com/wiedehopf/tar1090-db/master/db/icao_aircraft_types.js",timeout=60)
        r.raise_for_status()
        raw=r.content
        if raw[:2]==b"\x1f\x8b":raw=gzip.decompress(raw)
        text=raw.decode()
        # The public database is a JS assignment containing one JSON dictionary.
        start,end=text.find("{"),text.rfind("}")+1
        json.loads(text[start:end])
        # Publish atomically: a concurrent ingest that sees the path must never read a partial file.
        temporary=types_path.with_name(f"{types_path.name}.{os.getpid()}.tmp")
        temporary.write_text(text[start:end])
        os.replace(temporary,types_path)
    types=json.loads(types_path.read_text())
    print("Type examples",list(types.items())[:3],flush=True)
    engines={}
    for code,value in types.items():
        classification=value.get("desc","") if isinstance(value,dict) else (value[2] if len(value)>2 else "")
        engines[code.upper()]={"J":"Jet", "T":"Turbine", "P":"Piston"}.get(classification[-1:], "Unknown")
    pd.DataFrame({"designator":list(engines),"engine_type":list(engines.values())}).to_csv(directory/"type-database.csv",index=False)
    _configure(engines,(west,south,east,north),time_range)

    paths=sorted(directory.glob("*.tar.a?"))
    if len(paths)<2:raise ValueError("Daily archive download is incomplete")
    stats={"trace_files":0,"known_jets_removed":0,"unknown_types_removed":0,"no_conus_points":0,"retained_aircraft":0,"rows":0,"barometric_rows":0}
    stats["normalization_scope"]="full_native_trace_before_segment_selection"
    stats["selection_max_gap_s"]=30.0
    selection={"bounds":[west,south,east,north],"time_range_s":selected_time,"archive_day_utc":args.date,
               "all_aircraft":False,"aircraft_policy":"confirmed_nonjet","max_gap_s":30.0}
    writer=None
    with io.BufferedReader(SplitReader(paths),buffer_size=1024*1024) as stream:
        with tarfile.open(fileobj=stream,mode="r|") as archive:
            for outcome,frame in selected_traces(trace_members(archive),workers):
                stats["trace_files"]+=1
                if outcome=="jet":
                    stats["known_jets_removed"]+=1
                    continue
                if outcome=="unknown":
                    stats["unknown_types_removed"]+=1
                    continue
                if outcome=="outside":
                    stats["no_conus_points"]+=1
                    continue
                if outcome=="empty":continue
                # Bound schema explicitly so an all-null metadata field in one
                # aircraft cannot change a later parquet batch's physical type.
                schema=pa.schema([(c,pa.string() if c in ["flight_id","typecode","altitude_basis"] else pa.float64()) for c in frame.columns],
                                 metadata={b"dopplergeist.selection.v2":json.dumps(selection,allow_nan=False).encode()})
                batch=pa.Table.from_pandas(frame,schema=schema,preserve_index=False)
                if writer is None:writer=pq.ParquetWriter(output_path,schema,compression="zstd")
                writer.write_table(batch)
                stats["retained_aircraft"]+=1;stats["rows"]+=len(frame)
                stats["barometric_rows"]+=int((frame.altitude_basis=="barometric").sum())
                if stats["retained_aircraft"]%500==0:print(stats,flush=True)
    if writer:writer.close()
    report_path.write_text(json.dumps(stats,indent=2)+"\n")
    if writer is None:
        raise ValueError("No confirmed non-jet traces matched the selection; see the ingestion report")
    print(stats,flush=True)


if __name__=="__main__":
    main()