Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

perf: skip empty association traversal frames (#7)

Cinna Davisdd394de2a8c3
diff --git a/scripts/benchmark_real_association.py b/scripts/benchmark_real_association.py
new file mode 100644
index 0000000..e0d3f37
--- /dev/null
+++ b/scripts/benchmark_real_association.py
@@ -0,0 +1,117 @@
+"""Compare association traversal with a Git revision on an observed CPA fixture.
+
+Use the real JSON fixture accepted by benchmark_real_clock_batch.py. No audio,
+flight observations or proposal measurements are generated. This measures the
+join, not whole-search runtime or localization accuracy.
+"""
+import argparse
+import json
+from pathlib import Path
+import statistics
+import subprocess
+import sys
+import time
+from types import ModuleType
+
+import numpy as np
+import pandas as pd
+
+from dopplergeist.matching import association_candidates, prepare_events
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("fixture", type=Path)
+ parser.add_argument("output", type=Path, help="New JSON output file")
+ parser.add_argument("--baseline-ref", required=True)
+ parser.add_argument("--repetitions", type=int, default=5)
+ args = parser.parse_args()
+ if args.output.exists():
+ raise FileExistsError("Use a new output file")
+ if args.repetitions < 1:
+ parser.error("--repetitions must be positive")
+ root = Path(__file__).resolve().parents[1]
+ source = subprocess.check_output(
+ ["git", "show", f"{args.baseline_ref}:src/dopplergeist/matching.py"],
+ cwd=root, text=True)
+ baseline = ModuleType("dopplergeist._association_reference")
+ baseline.__package__ = "dopplergeist"
+ sys.modules[baseline.__name__] = baseline
+ exec(compile(source, "<baseline matching.py>", "exec"), baseline.__dict__)
+ fixture = json.loads(args.fixture.read_text())
+ events, _ = prepare_events(fixture["events"])
+ table = pd.DataFrame(fixture["cpa_rows"])
+ reference, counts = baseline.association_candidates(table, events)
+ if not reference:
+ raise ValueError("Fixture must have real association hypotheses")
+ first = reference[0]
+ clock = sum(first["recording_start_bounds_s"])/2
+ anchor = first["passes"][0]["receive_time"] - events[first["event_indices"][0]]["cpa_receive_s"]
+ cases = [
+ ("all_events", table, events, {}),
+ ("minimum_two", table, events, {"min_events": 2}),
+ ("minimum_four", table, events, {"min_events": 4}),
+ ("explicit_cap_one", table, events, {"max_events": 1}),
+ ("explicit_cap_two", table, events, {"max_events": 2}),
+ ("fixed_clock", table, events, {"time_origin": clock, "time_tolerance_s": 0}),
+ ("clock_boundary", table, events, {"time_origin": first["recording_start_bounds_s"][0],
+ "time_tolerance_s": 0}),
+ ("anchor_exact_start", table, events,
+ {"anchor_window": [anchor, float(np.nextafter(anchor, np.inf))]}),
+ ("anchor_excludes_start", table, events,
+ {"anchor_window": [float(np.nextafter(anchor, -np.inf)), anchor]}),
+ ("silence", table, events, {"silence": {"edge_margin_s": 60}}),
+ ("silence_no_margin", table, events, {"silence": {"edge_margin_s": 0}}),
+ ("candidate_budget_one", table, events, {"max_candidates": 1}),
+ ("candidate_budget_hundred", table, events, {"max_candidates": 100}),
+ ("no_observations", table.iloc[:0], events, {}),
+ ("last_proposal", table, events[-1:], {}),
+ ]
+
+ def outcome(function, observed, proposals, settings):
+ try:
+ return {"result": function(observed, proposals, **settings)}
+ except ValueError as error:
+ return {"error": str(error)}
+
+ checks = []
+ for name, observed, proposals, settings in cases:
+ expected = outcome(baseline.association_candidates, observed, proposals, settings)
+ actual = outcome(association_candidates, observed, proposals, settings)
+ if actual != expected:
+ raise AssertionError(f"Candidate order, bounds, counts or error differ: {name}")
+ check = {"case": name, "exact_parity": True}
+ if "error" in actual:
+ check["error"] = actual["error"]
+ else:
+ check["counts"] = actual["result"][1]
+ checks.append(check)
+ print(f"Parity passed: {name}", flush=True)
+ runs = []
+ functions = {"baseline": baseline.association_candidates, "optimized": association_candidates}
+ for repeat in range(args.repetitions):
+ order = ("baseline", "optimized") if repeat % 2 == 0 else ("optimized", "baseline")
+ for name in order:
+ wall, cpu = time.perf_counter(), time.process_time()
+ actual = functions[name](table, events)
+ run = {"implementation": name, "cpu_s": time.process_time()-cpu,
+ "wall_s": time.perf_counter()-wall}
+ if actual != (reference, counts):
+ raise AssertionError(f"Timing replay differed: {name}")
+ runs.append(run)
+ print(json.dumps(run), flush=True)
+ medians = {name: statistics.median(r["cpu_s"] for r in runs if r["implementation"] == name)
+ for name in functions}
+ report = {"fixture": str(args.fixture), "baseline_ref": args.baseline_ref,
+ "events": len(events), "cpa_rows": len(table), "candidates": len(reference),
+ "exact_candidate_order_and_work_accounting": True, "checks": checks, "runs": runs,
+ "median_cpu_s": medians, "cpu_speedup": medians["baseline"]/medians["optimized"]}
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ with args.output.open("x") as output:
+ json.dump(report, output, indent=2, allow_nan=False)
+ output.write("\n")
+ print(json.dumps({"median_cpu_s": medians, "cpu_speedup": report["cpu_speedup"]}))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/dopplergeist/matching.py b/src/dopplergeist/matching.py
index 65f4e31..3a8fb47 100644
--- a/src/dopplergeist/matching.py
+++ b/src/dopplergeist/matching.py
@@ -267,8 +267,12 @@ def association_candidates(table, events, offset_tolerance_s=5.0, time_origin=No
# The returned original-order slice must itself fit the work budget;
# reserve before allocating and sorting it at every branch.
selected = max(0, right - left)
- reserve(selected * max(1, selected.bit_length()))
- return chain((records[i] for i in sorted(indices[left:right])), [None])
+ if selected:
+ reserve(selected * selected.bit_length())
+ return sorted(indices[left:right])
+
+ def row_options(indices):
+ return chain((records[i] for i in indices), [None])
for point_id in sorted(set().union(*(g.keys() for g in grouped))):
available = [i for i, group in enumerate(grouped) if point_id in group]
@@ -277,7 +281,7 @@ def association_candidates(table, events, offset_tolerance_s=5.0, time_origin=No
lower, upper = (-np.inf, np.inf) if time_origin is None else (time_origin-time_tolerance_s, time_origin+time_tolerance_s)
path, used = [], set()
option_lo, option_hi = anchor_window if anchor_window is not None else (lower, upper)
- frames = [(0, iter(indexed_options(available[0], point_id, option_lo, option_hi)), lower, upper, 0)]
+ frames = [(0, row_options(indexed_options(available[0], point_id, option_lo, option_hi)), lower, upper, 0)]
while frames:
cursor, options, lower, upper, depth = frames[-1]
while len(path) > depth:
@@ -307,9 +311,25 @@ def association_candidates(table, events, offset_tolerance_s=5.0, time_origin=No
following = cursor+1
if len(path)+len(available)-following < min_events:
continue
- if following < len(available) and len(path) < max_events:
+ while following < len(available) and len(path) < max_events:
option_lo, option_hi = anchor_window if not path and anchor_window is not None else (lo, hi)
- frames.append((following, iter(indexed_options(available[following], point_id, option_lo, option_hi)), lo, hi, len(path)))
+ indices = indexed_options(available[following], point_id, option_lo, option_hi)
+ if indices:
+ frame = (following, row_options(indices), lo, hi, len(path))
+ if row is None:
+ # None is the last choice, so its exhausted parent does
+ # not need to remain on the stack while skipping ahead.
+ frames[-1] = frame
+ else:
+ frames.append(frame)
+ break
+ # An empty interval has only the unmatched choice. Charge that
+ # visit exactly as before, without a frame/iterator round trip.
+ reserve(1)
+ following += 1
+ if len(path)+len(available)-following < min_events:
+ break
+ if following < len(available) and len(path) < max_events:
continue
if len(path) < min_events:
continue