Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

13.0 KB
Raw Blame
#!/usr/bin/env python3
"""Compare discovery normalization on an existing real decoded recording.

The untimed check compares every Spectrum array directly as bytes. Timed workers
run uninstrumented discovery, including its normal JSON, CSV, and summary PNG.
No audio or aircraft observations are generated by this script.
"""
from __future__ import annotations

import argparse
from dataclasses import replace
import importlib.util
import json
import os
from pathlib import Path
import resource
import statistics
import subprocess
import sys
import time


def write_json(path, value):
    with Path(path).open("x") as stream:
        json.dump(value, stream, indent=2, allow_nan=False)
        stream.write("\n")


def canonical(value):
    return json.dumps(value, sort_keys=True, allow_nan=False)


def compare_array(left, right, label):
    signature = lambda value: (
        value.dtype.str, value.shape, value.strides,
        value.flags.c_contiguous, value.flags.f_contiguous, value.flags.writeable,
    )
    if signature(left) != signature(right):
        raise AssertionError(f"Array dtype, shape, or layout differs: {label}")
    if left.tobytes(order="C") != right.tobytes(order="C"):
        raise AssertionError(f"Array bytes differ: {label}")


def compare_spectrum(left, right, label):
    if left is None or right is None:
        if left is not right:
            raise AssertionError(f"Spectrum availability differs: {label}")
        return
    for field in ("time", "frequency", "db", "contrast"):
        compare_array(getattr(left, field), getattr(right, field), f"{label}.{field}")


def compare_artifacts(reference, current):
    from PIL import Image

    reference, current = Path(reference), Path(current)
    names = lambda path: sorted(p.name for p in path.iterdir() if p.is_file())
    if names(reference) != names(current):
        raise AssertionError("Discovery diagnostic file set differs")
    png_bytes_exact, csv_count = True, 0
    for name in names(reference):
        left, right = reference / name, current / name
        if name.endswith(".png"):
            png_bytes_exact &= left.read_bytes() == right.read_bytes()
            with Image.open(left) as a, Image.open(right) as b:
                if a.mode != b.mode or a.size != b.size or a.tobytes() != b.tobytes():
                    raise AssertionError(f"PNG pixels differ: {name}")
        elif left.read_bytes() != right.read_bytes():
            raise AssertionError(f"Diagnostic bytes differ: {name}")
        csv_count += name.endswith(".csv")
    return {"exact_json_and_csv_bytes": True, "csv_files": csv_count,
            "exact_png_pixels": True, "exact_png_file_bytes": bool(png_bytes_exact)}


def load_baseline_spectrum(source):
    # Only discovery.py differs in this experiment. Relative imports deliberately
    # use the same unchanged audio/contracts modules and numerical dependencies.
    name = "dopplergeist._normalization_comparison_baseline"
    path = Path(source).resolve() / "dopplergeist/discovery.py"
    spec = importlib.util.spec_from_file_location(name, path)
    module = importlib.util.module_from_spec(spec)
    sys.modules[name] = module
    spec.loader.exec_module(module)
    return module._spectrum


def worker(args):
    sys.path.insert(0, str(args.source.resolve()))
    import numpy as np
    import scipy
    from dopplergeist import discovery
    from dopplergeist.intake import IntakeRecording

    args.output.mkdir(parents=True, exist_ok=False)
    reference = json.loads((args.job / "discovery/discovery.json").read_text())
    recording = IntakeRecording(
        args.job / "intake/recording.f32",
        json.loads((args.job / "intake/recording.json").read_text()),
        json.loads((args.job / "intake/intake.json").read_text()),
    )
    if recording.samples_path.stat().st_size != 4 * recording.diagnostics["sample_count"]:
        raise AssertionError("Saved decoded recording size differs from intake metadata")
    settings = discovery.DiscoverySettings(**reference["settings"])
    spectra, original = [], discovery._spectrum
    baseline_spectrum = None
    if args.compare_spectrum_source is not None:
        baseline_spectrum = load_baseline_spectrum(args.compare_spectrum_source)

        def checked_spectrum(data, sr, start, settings):
            expected = baseline_spectrum(data, sr, start, settings)
            result = original(data, sr, start, settings)
            compare_spectrum(expected, result, f"chunk-{len(spectra)}")
            spectra.append({
                "start_s": start, "samples": len(data), "available": result is not None,
                "shape": list(result.db.shape) if result is not None else None,
                "finite_float32": bool(result is not None and result.db.dtype == np.float32
                                       and np.isfinite(result.db).all()),
            })
            return result

        discovery._spectrum = checked_spectrum
    wall, cpu = time.perf_counter(), time.process_time()
    try:
        events = discovery.discover_events(
            recording, output_dir=args.output / "discovery", settings=settings,
            plot_summary=True, plot_event_details=False,
        )
    finally:
        elapsed, used = time.perf_counter() - wall, time.process_time() - cpu
        discovery._spectrum = original
    if canonical(events) != canonical(reference["events"]):
        raise AssertionError("Complete discovered event payload differs from the recorded job")
    artifacts = compare_artifacts(args.job / "discovery", args.output / "discovery")
    extra_checks = {}
    if baseline_spectrum is not None:
        if len(spectra) != len(reference["chunks"]):
            raise AssertionError("Not every original chunk received an array comparison")
        first, data = next(recording.iter_chunks(settings.chunk_s, settings.overlap_s))
        sr = recording.metadata["sample_rate_hz"]
        double = data.astype(np.float64)
        compare_spectrum(baseline_spectrum(double, sr, first, settings),
                         original(double, sr, first, settings), "real-float64-fallback")
        short = data[:round(sr * settings.window_s) - 1]
        compare_spectrum(baseline_spectrum(short, sr, first, settings),
                         original(short, sr, first, settings), "real-short-prefix")
        median, calls = np.median, []

        def recording_median(*values, **keywords):
            calls.append(keywords.get("axis"))
            return median(*values, **keywords)

        try:
            np.median = recording_median
            expected = baseline_spectrum(data, sr, first, settings)
            baseline_axes = calls[:]
            calls.clear()
            actual = original(data, sr, first, settings)
            current_axes = calls[:]
        finally:
            np.median = median
        compare_spectrum(expected, actual, "custom-median-fallback")
        if baseline_axes != [1, 0, 0] or current_axes != baseline_axes:
            raise AssertionError("Custom median dispatch no longer uses the original axes")
        upper = expected.frequency[-2 if len(expected.frequency) % 2 else -1]
        even_settings = replace(settings, max_frequency_hz=float(upper))
        even_expected = baseline_spectrum(data, sr, first, even_settings)
        even_actual = original(data, sr, first, even_settings)
        compare_spectrum(even_expected, even_actual, "real-even-frequency-median")
        if len(even_actual.frequency) % 2:
            raise AssertionError("The even-median check did not use an even frequency count")
        extra_checks = {"real_float64_fallback_exact": True, "short_real_prefix_exact": True,
                        "custom_median_fallback_exact": True, "median_axes": current_axes,
                        "real_even_frequency_median_exact": True,
                        "even_median_frequency_bins": len(even_actual.frequency)}
    report = {
        "source": str(args.source.resolve()), "job": str(args.job.resolve()),
        "wall_s": elapsed, "cpu_s": used,
        "peak_rss_mib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024,
        "timed_run": baseline_spectrum is None,
        "event_records": len(events),
        "independent_groups": len({event["harmonic_group_id"] for event in events}),
        "exact_events": True, **artifacts,
        "sample_count": recording.diagnostics["sample_count"],
        "duration_s": recording.metadata["duration_s"],
        "source_sha256": recording.metadata["source_sha256"],
        "settings": reference["settings"],
        "python": sys.version, "numpy": np.__version__, "scipy": scipy.__version__,
        "spectrum_array_checks": len(spectra), "spectra": spectra, **extra_checks,
    }
    write_json(args.output / "summary.json", report)
    print(json.dumps({key: value for key, value in report.items() if key != "spectra"}), flush=True)


def compare(args):
    args.output.mkdir(parents=True, exist_ok=False)
    environment = dict(os.environ)
    environment.update(PYTHONDONTWRITEBYTECODE="1", OPENBLAS_NUM_THREADS="1",
                       OMP_NUM_THREADS="1", MKL_NUM_THREADS="1", NUMEXPR_NUM_THREADS="1")

    def run(name, source, baseline=None):
        output = args.output / name
        command = [sys.executable, str(Path(__file__).resolve()), "worker",
                   "--source", str(source.resolve()), "--job", str(args.job.resolve()),
                   "--output", str(output.resolve())]
        if baseline is not None:
            command += ["--compare-spectrum-source", str(baseline.resolve())]
        with (args.output / f"{name}.log").open("x") as stream:
            result = subprocess.run(command, env=environment, stdout=stream, stderr=subprocess.STDOUT)
        if result.returncode:
            raise RuntimeError(f"Worker {name} failed; see {args.output / (name + '.log')}")
        report = json.loads((output / "summary.json").read_text())
        print(json.dumps({"worker": name, "wall_s": report["wall_s"],
                          "cpu_s": report["cpu_s"], "exact_events": True}), flush=True)
        return report

    check = run("array-check", args.current_source, args.baseline_source)
    pairs, first_pair_passed = [], False
    for index in range(1, args.pairs + 1):
        order = ("baseline", "current") if index % 2 else ("current", "baseline")
        reports = {}
        for arm in order:
            source = args.baseline_source if arm == "baseline" else args.current_source
            reports[arm] = run(f"{index}-{arm}", source)
        artifacts = compare_artifacts(args.output / f"{index}-baseline/discovery",
                                      args.output / f"{index}-current/discovery")
        pairs.append({"pair": index, "order": list(order), **reports, **artifacts})
        if index == 1:
            first_pair_passed = reports["current"]["wall_s"] <= 0.95 * reports["baseline"]["wall_s"]
            if not first_pair_passed:
                break
    medians = {arm: {metric: statistics.median(pair[arm][metric] for pair in pairs)
                     for metric in ("wall_s", "cpu_s", "peak_rss_mib")}
               for arm in ("baseline", "current")}
    report = {
        "scope": "Whole real saved-PCM discovery with normal JSON/CSV/summary PNG; no intake, decoding, or flight search.",
        "method": "Fresh interpreter per arm; pre-call imports excluded and lazy in-call imports included; no numerical instrumentation in timed workers; three alternating pairs only if first exact pair lowers wall time at least 5%.",
        "cache_conditions": "OS caches are not flushed; recording already existed; no CPU affinity; team coordinates an exclusive local timing slot, unrelated host load is uncontrolled.",
        "array_check": check, "pairs": pairs, "medians": medians,
        "first_pair_5_percent_gate_passed": first_pair_passed,
        "wall_speedup": medians["baseline"]["wall_s"] / medians["current"]["wall_s"],
        "cpu_speedup": medians["baseline"]["cpu_s"] / medians["current"]["cpu_s"],
        "wall_time_reduction_fraction": 1 - medians["current"]["wall_s"] / medians["baseline"]["wall_s"],
        "all_event_and_diagnostic_checks_exact": True,
    }
    write_json(args.output / "comparison.json", report)
    print(json.dumps({key: value for key, value in report.items()
                      if key not in ("array_check", "pairs")}, indent=2), flush=True)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    subparsers = parser.add_subparsers(dest="command", required=True)
    run_parser = subparsers.add_parser("worker")
    run_parser.add_argument("--source", type=Path, required=True)
    run_parser.add_argument("--compare-spectrum-source", type=Path)
    compare_parser = subparsers.add_parser("compare")
    compare_parser.add_argument("--baseline-source", type=Path, required=True)
    compare_parser.add_argument("--current-source", type=Path, required=True)
    compare_parser.add_argument("--pairs", type=int, choices=(1, 2, 3), default=3)
    for child in (run_parser, compare_parser):
        child.add_argument("--job", type=Path, required=True)
        child.add_argument("--output", type=Path, required=True)
    args = parser.parse_args()
    (worker if args.command == "worker" else compare)(args)


if __name__ == "__main__":
    main()