Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Merge current pipeline improvements and reuse CPA preparation across chunks
Cinna Davis3c8f335e1d31
diff --cc .github/workflows/real-performance.ymlindex f7ce090,e7a813c..2f127b9--- a/.github/workflows/real-performance.yml+++ b/.github/workflows/real-performance.yml@@@ -3,34 -3,38 +3,44 @@@ onworkflow_dispatch:inputs:baseline:- description: Scalar-consistent baseline commit- default: d7cc1dd2e8089768253492b3269dfae97b3ad1a4+ description: Exact baseline commit for both real-input comparisons- default: 833c96c98366e18cc93562499f169d45bc0a9e06++ default: d9649ee5e369b31072d40c4882a5b7e0a7bfa177required: true+ regional:+ description: Also compare the complete regional HTTP workflow+ type: boolean+ default: truepush:- branches: [perf/measured-hotpath-20261103, perf/single-pass-motion-20261103]- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103]++ branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103]paths:- .github/workflows/real-performance.yml- src/dopplergeist/**- scripts/benchmark_prepared_measurements.py+ - scripts/benchmark_real_cpa_rows.py+ - scripts/benchmark_real_regional_upload.py+ - scripts/benchmark_real_bounded_upload.py+ - scripts/prepare_reference_upload.py+ - scripts/restore_artifacts.py+ - scripts/compare_real_upload_performance.py+ - scripts/check_real_upload_arrays.py+ - pyproject.toml- tests/test_real_*.py+ - tests/data/real-ridge-batch-arithmetic.json+ - tests/data/reference-empty-tiles-real.json+ - tests/data/reference-retention-real.jsonpermissions:contents: read+env:+ OPENBLAS_NUM_THREADS: '1'+ OMP_NUM_THREADS: '1'+ MKL_NUM_THREADS: '1'+ PYTHONPATH: src- BASELINE_REF: ${{ inputs.baseline || '833c96c98366e18cc93562499f169d45bc0a9e06' }}++ BASELINE_REF: ${{ inputs.baseline || 'd9649ee5e369b31072d40c4882a5b7e0a7bfa177' }}jobs:compare:+ if: github.event_name != 'push' || github.ref != 'refs/heads/perf/real-pipeline-throughput-20261103'runs-on: ubuntu-latesttimeout-minutes: 20- env:- OPENBLAS_NUM_THREADS: '1'- OMP_NUM_THREADS: '1'- MKL_NUM_THREADS: '1'- PYTHONPATH: src- BASELINE_REF: ${{ inputs.baseline || 'd7cc1dd2e8089768253492b3269dfae97b3ad1a4' }}steps:- uses: actions/checkout@v4with:@@@ -60,26 -61,19 +70,59 @@@- name: Real-input regression testsenv:DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py++ run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py+ - name: Compare real CPA preparation+ run: python scripts/benchmark_real_cpa_rows.py --baseline-source "$RUNNER_TEMP/baseline/src" --output results/cpa-rows.json --repeats 5 --iterations 20- name: Compare complete payloads and timingsrun: |+ native_option=()+ if [ -f src/dopplergeist/_native_ridge.c ]; then+ native_option=(--require-current-native)+ fifor width in 10 120; do- python scripts/benchmark_prepared_measurements.py compare --fixture "benchmark-inputs/point-$width.json" --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src --repeats 2 --timeout-s 600 --output "results/performance-$width"+ python scripts/benchmark_prepared_measurements.py compare --fixture "benchmark-inputs/point-$width.json" --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src --repeats 2 --timeout-s 600 --output "results/performance-$width" "${native_option[@]}"done++ - name: Profile remaining point work outside the timing comparison++ run: |++ python -m cProfile -o results/current-point.prof scripts/benchmark_prepared_measurements.py _worker --source src --fixture benchmark-inputs/point-120.json --output results/current-point-profile++ python - <<'PY'++ import json++ from pathlib import Path++ import pstats++++ statistics = pstats.Stats("results/current-point.prof")++ root = Path.cwd()++ records = []++ for (filename, line, function), (primitive, total, own, cumulative, callers) in statistics.stats.items():++ path = Path(filename)++ relative = path.relative_to(root).as_posix() if path.is_relative_to(root) else filename++ module = (relative[4:-3].replace("/", ".") if relative.startswith("src/")++ and relative.endswith(".py") else path.stem)++ records.append({"module": module, "file": relative, "line": line, "function": function,++ "primitive_calls": primitive, "total_calls": total,++ "self_seconds": own, "cumulative_seconds": cumulative})++ records.sort(key=lambda row: (-row["cumulative_seconds"], -row["self_seconds"],++ row["file"], row["line"], row["function"]))++ summary = {"scope": "Separate profiled width-120 worker, including imports and input loading; excluded from timing medians",++ "timer": "cProfile default elapsed timer; diagnostic attribution, not an uninstrumented CPU measurement",++ "total_calls": statistics.total_calls, "primitive_calls": statistics.prim_calls,++ "total_profiled_seconds": statistics.total_tt, "top_50": records[:50],++ "dopplergeist_functions": [row for row in records if row["module"].startswith("dopplergeist")]}++ output = Path("results/current-point-profile-stats.json")++ output.write_text(json.dumps(summary, indent=2) + "\n")++ print(json.dumps({"profile_summary": str(output), "excluded_from_timing_medians": True,++ "top_10": records[:10]}, indent=2))++ PY- uses: actions/upload-artifact@v4if: always()with:name: real-performance-summarypath: |+ results/cpa-rows.jsonresults/performance-*/comparison.jsonresults/performance-*/*/summary.json++ results/current-point.prof++ results/current-point-profile-stats.jsonif-no-files-found: ignoreretention-days: 14- uses: actions/upload-artifact@v4@@@ -89,70 -83,172 +132,240 @@@path: results/performance-*if-no-files-found: ignoreretention-days: 3+ regional:+ if: github.ref == 'refs/heads/perf/single-pass-motion-20261103' || (github.event_name == 'workflow_dispatch' && inputs.regional)- runs-on: ubuntu-latest++ # Keep the reference FFmpeg family; a new encoder can change detected ridges.++ runs-on: ubuntu-24.04+ timeout-minutes: 20+ steps:+ - uses: actions/checkout@v4+ with:+ fetch-depth: 0+ - uses: actions/setup-python@v5+ with:+ python-version: '3.12'+ - name: Install runtime and source-tree extensions+ run: |+ python -m pip install --editable '.[test]'+ if ! command -v ffmpeg >/dev/null; then+ sudo apt-get update+ sudo apt-get install -y ffmpeg+ fi+ - name: Restore the complete real recording and observed inputs+ env:+ GH_TOKEN: ${{ github.token }}+ run: python scripts/restore_artifacts.py --bundle data-source --bundle data-adsb-derived --bundle data-terrain+ - name: Prepare common inputs and the baseline+ run: |+ python scripts/prepare_reference_upload.py --data-root data --output benchmark-inputs+ baseline_commit=$(git rev-parse --verify --end-of-options "${BASELINE_REF}^{commit}")+ git worktree add --detach "$RUNNER_TEMP/baseline" "$baseline_commit"+ if [ -f "$RUNNER_TEMP/baseline/setup.py" ]; then+ (cd "$RUNNER_TEMP/baseline" && python setup.py build_ext --inplace)+ fi+ python - <<'PY'+ import json+ from pathlib import Path+ path = Path("benchmark-inputs/operator.json")+ operator = json.loads(path.read_text())+ operator["search_workers"] = 1+ path.write_text(json.dumps(operator, indent=2) + "\n")+ PY+ ffmpeg -nostdin -i data/source/reference-recording.wav -ac 1 -ar 48000 -c:a libmp3lame -b:a 96k benchmark-inputs/reference.mp3+ - name: Measure both complete HTTP arms+ run: |+ python scripts/benchmark_real_regional_upload.py --source "$RUNNER_TEMP/baseline/src" --audio benchmark-inputs/reference.mp3 --request benchmarks/reference-regional-upload-request.json --operator benchmark-inputs/operator.json --output results/regional-baseline --timeout-s 600 --threads 1+ python scripts/benchmark_real_regional_upload.py --source src --audio benchmark-inputs/reference.mp3 --request benchmarks/reference-regional-upload-request.json --operator benchmark-inputs/operator.json --output results/regional-current --timeout-s 600 --threads 1+ - name: Require complete scientific parity+ run: python scripts/benchmark_real_regional_upload.py --compare results/regional-baseline results/regional-current --output results/regional-comparison.json+ - uses: actions/upload-artifact@v4+ if: always()+ with:+ name: real-regional-summary+ path: |+ results/regional-*/timing.json+ results/regional-*/result.json+ results/regional-*/http-verification.json+ results/regional-*/pipeline-evidence.json+ results/regional-comparison.json+ if-no-files-found: ignore+ retention-days: 14+ - uses: actions/upload-artifact@v4+ if: failure()+ with:+ name: real-regional-failure-diagnostics+ path: |+ results/regional-*/jobs/*/search/search-report.json+ results/regional-*/jobs/*/provenance.json+ results/regional-*/*.log+ if-no-files-found: ignore+ retention-days: 3++ bounded-http:+ if: github.event_name == 'push' && github.ref == 'refs/heads/perf/real-pipeline-throughput-20261103'+ runs-on: ubuntu-latest+ timeout-minutes: 45+ env:+ OPENBLAS_NUM_THREADS: '1'+ OMP_NUM_THREADS: '1'+ MKL_NUM_THREADS: '1'+ NUMEXPR_NUM_THREADS: '1'+ PYTHONPATH: src+ BASELINE_REF: aa3dff686c989524daa20c081a816b792d9c8890+ steps:+ - uses: actions/checkout@v4+ with:+ fetch-depth: 0+ - uses: actions/setup-python@v5+ with:+ python-version: '3.12'+ - name: Install matched runtime+ run: |+ python -m pip install '.[test]' 'numpy==2.3.5' 'scipy==1.17.0' 'pandas==2.2.3' 'pyarrow==26.0.1' 'pyproj==3.8.0' 'matplotlib==3.10.8'+ sudo apt-get update+ sudo apt-get install -y ffmpeg+ - name: Restore public recording and observed inputs+ env:+ GH_TOKEN: ${{ github.token }}+ run: python scripts/restore_artifacts.py --bundle data-source --bundle data-adsb-derived --bundle data-terrain+ - name: Prepare one shared workload+ run: |+ python scripts/prepare_reference_upload.py --data-root data --output benchmark-http-inputs+ git worktree add --detach "$RUNNER_TEMP/http-baseline-source" "$BASELINE_REF"+ python - <<'PY'+ import json+ from pathlib import Path+ root = Path("benchmark-http-inputs")+ request = json.loads((root / "request.json").read_text())+ request["search"]["bounds_wsen"] = [-73.545, 40.865, -73.505, 40.895]+ request["time_window"]["source_detail"] = (+ "Source-informed integration replay around the historical fitted recording "+ "start and benchmark receiver region; not independently measured recording "+ "time or a location-accuracy evaluation."+ )+ operator = json.loads((root / "operator.json").read_text())+ operator["search_workers"] = 1+ for name, value in (("request.json", request), ("operator.json", operator)):+ (root / name).write_text(json.dumps(value, indent=2) + "\n")+ PY+ ffmpeg -n -nostdin -v error -i data/source/reference-recording.wav -map 0:a:0 -vn -map_metadata -1 -ac 1 -ar 48000 -c:a libmp3lame -b:a 96k benchmark-http-inputs/recording.mp3+ python scripts/benchmark_prepared_measurements.py prepare --discovery evidence/audio/w1-sensitivity-20261102/reference/discovery.json --index benchmark-http-inputs/reference-region.sqlite --clock-half-width-s 120 --output benchmark-http-inputs/point-120.json+ - name: Recorded-input regression checks+ id: real_checks+ env:+ DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-http-inputs/point-120.json+ run: |+ mkdir -p results+ python -m pytest -q \+ tests/test_real_prepared_measurements.py \+ tests/test_real_geometry_kinematics.py \+ tests/test_real_batch_scalar_parity.py \+ tests/test_real_complete_batch_cache.py \+ tests/test_real_legacy_batch_cache.py \+ tests/test_real_refinement_lookup.py \+ tests/test_real_track_cache.py \+ tests/test_real_retention_batch.py \+ tests/test_real_retention_floor.py \+ tests/test_real_performance_helpers.py::test_cached_raster_transform_preserves_actual_candidate_heights \+ --junitxml=results/ci-real-tests.xml+ python - <<'CHECKS'+ import json+ from pathlib import Path+ import xml.etree.ElementTree as ET+ cases = ET.parse(Path("results/ci-real-tests.xml")).getroot().findall(".//testcase")+ skipped = [case.attrib.get("name") for case in cases if case.find("skipped") is not None]+ print(json.dumps({"selected_real_tests": len(cases), "skipped": skipped}))+ if not cases or skipped:+ raise SystemExit("Selected real-input checks must execute without skips")+ CHECKS+ - name: Record this new workload and runtime+ id: context+ run: |+ python - <<'PY'+ import hashlib+ import importlib.metadata+ import json+ import os+ from pathlib import Path+ import platform+ import subprocess+ output = Path("results")+ output.mkdir(exist_ok=True)+ audio = Path("benchmark-http-inputs/recording.mp3")+ with audio.open("rb") as stream:+ digest = hashlib.file_digest(stream, "sha256").hexdigest()+ record = {+ "scope": "One fresh sequential matched HTTP pair on a new MP3 encoding of the public real WAV. Timings and event counts do not inherit the earlier original-MP3 measurements. Source-informed region/time; not independent location accuracy.",+ "baseline": os.environ["BASELINE_REF"],+ "current": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),+ "python": platform.python_version(),+ "packages": {name: importlib.metadata.version(name) for name in ("numpy", "scipy", "pandas", "pyarrow", "pyproj", "rasterio", "matplotlib")},+ "ffmpeg": subprocess.check_output(["ffmpeg", "-version"], text=True).splitlines()[0],+ "encoding": {"codec": "libmp3lame", "bitrate": "96k", "sample_rate_hz": 48000, "channels": 1},+ "source_sha256": digest,+ "source_bytes": audio.stat().st_size,+ "request": json.loads(Path("benchmark-http-inputs/request.json").read_text()),+ "operator": json.loads(Path("benchmark-http-inputs/operator.json").read_text()),+ }+ (output / "http-comparison-context.json").write_text(json.dumps(record, indent=2, allow_nan=False) + "\n")+ PY+ - name: Fresh baseline HTTP arm+ id: baseline+ run: python scripts/benchmark_real_bounded_upload.py --source "$RUNNER_TEMP/http-baseline-source/src" --inputs benchmark-http-inputs --output results/http-baseline --timeout-s 900+ - name: Fresh integrated HTTP arm+ id: current+ if: ${{ !cancelled() && steps.context.outcome == 'success' }}+ run: python scripts/benchmark_real_bounded_upload.py --source "$GITHUB_WORKSPACE/src" --inputs benchmark-http-inputs --output results/http-current --timeout-s 900+ - name: Compare complete scientific JSON and SQLite+ if: always() && steps.baseline.outcome == 'success' && steps.current.outcome == 'success'+ run: python scripts/compare_real_upload_performance.py --baseline results/http-baseline --current results/http-current --output results/http-science-comparison.json+ - name: Compare intake, decoded PCM, arrays, and PNG pixels+ if: always() && steps.baseline.outcome == 'success' && steps.current.outcome == 'success'+ run: python scripts/check_real_upload_arrays.py --baseline results/http-baseline --current results/http-current --output results/http-array-comparison.json+ - name: Print compact comparison results+ if: always()+ run: |+ python - <<'REPORT'+ import json+ from pathlib import Path+ def read(name):+ path = Path("results") / name+ if not path.is_file():+ return {"missing": str(path)}+ try:+ return json.loads(path.read_text())+ except (OSError, ValueError) as error:+ return {"read_error": str(error), "path": str(path)}+ def select(value, fields):+ if "missing" in value or "read_error" in value:+ return value+ return {key: value[key] for key in fields if key in value}+ science = read("http-science-comparison.json")+ arrays = read("http-array-comparison.json")+ passed = science.get("passed") is True and arrays.get("passed") is True+ result = {+ "both_parity_checks_passed": passed,+ "matched_pair_wall_speedup": science.get("verified_full_workflow_speedup") if passed else None,+ "scientific_comparison": select(science, ("passed", "scientific_payloads_exact", "run_kinds", "full_pipeline_verified", "checked_artifacts", "retention_ledgers", "outcomes", "differences")),+ "intake_array_comparison": select(arrays, ("passed", "checked", "manifest_verified_artifacts", "difference_count", "differences")),+ "http_receipts": {name: select(read(f"http-{name}/http-verification.json"), ("verification_passed", "implementation_unchanged", "full_pipeline_observed", "full_pipeline_requirements_met", "elapsed_s", "status", "reason_code", "event_count", "candidate_count", "source_sha256", "source_bytes", "service")) for name in ("baseline", "current")},+ "context": read("http-comparison-context.json"),+ }+ print("BOUNDED_HTTP_COMPARISON_BEGIN")+ print(json.dumps(result, indent=2, allow_nan=False))+ print("BOUNDED_HTTP_COMPARISON_END")+ REPORT+ - uses: actions/upload-artifact@v4+ if: always()+ with:+ name: bounded-http-comparison+ path: |+ results/ci-real-tests.xml+ results/http-comparison-context.json+ results/http-science-comparison.json+ results/http-array-comparison.json+ results/http-*/http-verification.json+ results/http-*/pipeline-evidence.json+ results/http-*/service.log+ if-no-files-found: ignore+ retention-days: 14diff --cc scripts/benchmark_real_cpa_rows.pyindex e2d632b,0000000..03d74d5mode 100644,000000..100644--- a/scripts/benchmark_real_cpa_rows.py+++ b/scripts/benchmark_real_cpa_rows.py@@@ -1,158 -1,0 +1,236 @@@+#!/usr/bin/env python3+"""Compare CPA preparation on archived aircraft observations, without fake tracks."""+from __future__ import annotations++import argparse+import importlib.util+import json++import math++import os+from pathlib import Path+import statistics+import sys+import time++import numpy as np+import pandas as pd++from dopplergeist.geometry import AcousticSupportError, Track, ecef+from dopplergeist import search+from dopplergeist.tracks import _segment_velocity+++ROOT = Path(__file__).resolve().parents[1]+++def recorded_tracks():+ """Use the normal gap split and velocity inference on frozen real traces."""+ tracks = []+ for source in sorted((ROOT / "research/trajectory-motion/frozen").glob("fresh-*.json")):+ frame = pd.DataFrame(json.loads(source.read_text())).sort_values("time_s")+ for part, (_, rows) in enumerate(frame.groupby((frame.time_s.diff() > 30.).cumsum())):+ if len(rows) < 3:+ continue+ velocity, _ = _segment_velocity(rows)+ tracks.append(Track(f"{rows.flight_id.iloc[0]}:{part}", rows.time_s.to_numpy(),+ rows.lat.to_numpy(), rows.lon.to_numpy(), rows.alt_m.to_numpy(),+ str(rows.typecode.iloc[0]), velocity))+ if not tracks:+ raise ValueError("Archived real observation inputs are missing")+ return tracks+++def recorded_points(tracks, count):+ """Candidate coordinates sampled from the real traces; no accuracy claim."""+ locations = np.concatenate([np.column_stack((track.lat, track.lon)) for track in tracks])+ selected = locations[np.linspace(0, len(locations)-1, count, dtype=int)]+ return pd.DataFrame({"point_id": np.arange(count), "lat": selected[:, 0],+ "lon": selected[:, 1], "elev_m": np.zeros(count)})+++def cpa_stage(module, tracks, points, radius=20000., c=343., step=2., chunk_size=512,- row_mode=False):++ row_mode=False, *, reuse_samples=True, max_cpa_rows=100000, counters=None):+ """The bounded track-isolation loop, before acoustic candidate scoring."""+ observers = ecef(points.lat.to_numpy(), points.lon.to_numpy(), points.elev_m.to_numpy())+ point_ids = points.point_id.to_numpy(dtype=int)+ tables, batches, unsupported = [], [], []++ count = 0+ for track in tracks:++ samples = math.ceil((track.time[-1]-track.time[0])/step)+1++ receiver_chunk = max(1, max_cpa_rows//samples)++ track_tables, track_batches, track_rows = [], [], 0++ prepared = None+ try:+ if track.outside_observer_radius(observers, radius, c):+ continue- if row_mode:- result = module._cpa_rows([track], observers, point_ids, radius, c, step, chunk_size)- else:- result = module.cpa_table([track], points, radius, c, step, chunk_size)++ if row_mode and reuse_samples and hasattr(module, "_PreparedCPATrack"):++ prepared = module._PreparedCPATrack(track, c, step)++ if counters is not None:++ counters["sample_preparations"] += 1++ for first in range(0, len(points), receiver_chunk):++ if counters is not None:++ counters["receiver_chunks"] += 1++ if prepared is not None:++ result = module._ordered_cpa_rows(prepared.raw_rows(++ observers[first:first+receiver_chunk], point_ids[first:first+receiver_chunk],++ radius, chunk_size), step)++ else:++ if counters is not None:++ counters["sample_preparations"] += 1++ result = (module._cpa_rows([track], observers[first:first+receiver_chunk],++ point_ids[first:first+receiver_chunk], radius, c, step, chunk_size) if row_mode else++ module.cpa_table([track], points.iloc[first:first+receiver_chunk],++ radius, c, step, chunk_size))++ track_rows += len(result)++ if count+track_rows > max_cpa_rows:++ raise ValueError("Tile CPA-row memory budget exceeded")++ if len(result):++ if row_mode:++ track_batches.append(np.array(result, dtype=search._CPA_DTYPE))++ else:++ track_tables.append(result)++ del result+ except AcousticSupportError as error:+ unsupported.append(vars(error))+ continue++ finally:++ prepared = None++ count += track_rows+ if row_mode:- if result:- batches.append(np.array(result, dtype=search._CPA_DTYPE))- elif len(result):- tables.append(result)++ batches.extend(track_batches)++ else:++ tables.extend(track_tables)+ if row_mode:- table = (pd.DataFrame.from_records(np.concatenate(batches)) if batches else++ table = (pd.DataFrame({name: np.concatenate([batch[name] for batch in batches])++ for name in search._CPA_DTYPE.names}, copy=False) if batches else+ pd.DataFrame(columns=search._CPA_COLUMNS))+ else:+ table = pd.concat(tables, ignore_index=True) if tables else pd.DataFrame(columns=search._CPA_COLUMNS)+ return table, unsupported+++def compare(before, after):+ pd.testing.assert_frame_equal(before[0], after[0], check_exact=True)+ assert before[1] == after[1], "Unsupported-track accounting changed"++++def counted_chunked_stage(tracks, points, reuse):++ """Count actual tree construction while keeping every recorded input fixed."""++ counters = {"sample_preparations": 0, "tree_builds": 0, "receiver_chunks": 0}++ tree = search.cKDTree++++ def counted_tree(*args, **kwargs):++ counters["tree_builds"] += 1++ return tree(*args, **kwargs)++++ search.cKDTree = counted_tree++ try:++ result = cpa_stage(search, tracks, points, row_mode=True, reuse_samples=reuse,++ counters=counters)++ finally:++ search.cKDTree = tree++ return result, counters+++++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--baseline-source", type=Path, required=True,+ help="Baseline src directory from a separate checkout")+ parser.add_argument("--output", type=Path, required=True)+ parser.add_argument("--repeats", type=int, default=5)+ parser.add_argument("--iterations", type=int, default=20)+ args = parser.parse_args()+ if args.output.exists():+ parser.error("Use a new output path")+ if args.repeats < 1 or args.iterations < 1:+ parser.error("repeats and iterations must be positive")+ path = args.baseline_source / "dopplergeist/search.py"+ spec = importlib.util.spec_from_file_location("dopplergeist._cpa_before", path)+ baseline = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = baseline+ spec.loader.exec_module(baseline)+ tracks = recorded_tracks()+ workloads = {str(count): recorded_points(tracks, count) for count in (1, 32, 128)}+ checks = 0+ for points in workloads.values():+ wide_ids = pd.concat([points.iloc[:1], points.iloc[:1]], ignore_index=True)+ wide_ids["point_id"] = [2**53, 2**53+1]+ for radius, c, step, chunk_size in ((1000., 343., 2., 512), (20000., 343., 2., 512),+ (20000., 338., 1., 7), (20000., 348., 8., 1),+ (20000., 343., np.float32(2), 512)):+ for candidate_points in (points, points.iloc[::-1], pd.concat([points, points], ignore_index=True), wide_ids):+ old = cpa_stage(baseline, tracks, candidate_points, radius, c, step, chunk_size)+ new = cpa_stage(search, tracks, candidate_points, radius, c, step, chunk_size, True)+ compare(old, new)+ checks += 1+ supported = []+ for track in tracks:+ try:+ track.validate_motion_support(c)+ supported.append(track)+ except AcousticSupportError:+ pass+ pd.testing.assert_frame_equal(+ baseline.cpa_table(supported, candidate_points, radius, c, step, chunk_size),+ search.cpa_table(supported, candidate_points, radius, c, step, chunk_size),+ check_exact=True)+ checks += 1+ timings = []+ for name, points in workloads.items():+ runs = {"baseline": [], "current": []}+ final = None+ for repeat in range(args.repeats):+ arms = [("baseline", baseline, False), ("current", search, True)]+ if repeat % 2:+ arms.reverse()+ for label, module, row_mode in arms:+ start = time.process_time()+ for _ in range(args.iterations):+ result = cpa_stage(module, tracks, points, row_mode=row_mode)+ runs[label].append(time.process_time()-start)+ final = result+ medians = {name: statistics.median(values) for name, values in runs.items()}+ timings.append({"receiver_points": int(name), "cpa_rows": len(final[0]),+ "unsupported_tracks": len(final[1]), "iterations_per_trial": args.iterations,+ "cpu_s": runs, "median_cpu_s": medians,+ "speedup": medians["baseline"]/medians["current"]})++ chunked_timings = []++ for size in (1024, 4096):++ points = recorded_points(tracks, size)++ before, old_counts = counted_chunked_stage(tracks, points, False)++ after, new_counts = counted_chunked_stage(tracks, points, True)++ compare(before, after)++ checks += 1++ runs = {"repeated_preparation": [], "reused_preparation": []}++ for repeat in range(args.repeats):++ arms = [("repeated_preparation", False), ("reused_preparation", True)]++ if repeat % 2:++ arms.reverse()++ for label, reuse in arms:++ started = time.process_time()++ for _ in range(args.iterations):++ cpa_stage(search, tracks, points, row_mode=True, reuse_samples=reuse)++ runs[label].append(time.process_time()-started)++ medians = {label: statistics.median(values) for label, values in runs.items()}++ chunked_timings.append({"receiver_points": size, "cpa_rows": len(after[0]),++ "unsupported_tracks": len(after[1]), "max_cpa_rows": 100000,++ "iterations_per_trial": args.iterations, "cpu_s": runs, "median_cpu_s": medians,++ "speedup": medians["repeated_preparation"]/medians["reused_preparation"],++ "operations": {"repeated_preparation": old_counts, "reused_preparation": new_counts}})+ result = {"scope": "CPA preparation on archived real ADS-B observations; excludes acoustic fitting, downloads, and localization accuracy.",+ "baseline_source": str(args.baseline_source), "tracks": len(tracks),++ "runtime": {"python": sys.version.split()[0], "numpy": np.__version__, "pandas": pd.__version__,++ "cpu_affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None},+ "observed_samples": sum(len(track.time) for track in tracks),- "exact_comparisons": checks, "timings": timings}++ "exact_comparisons": checks, "timings": timings,++ "chunked_comparison": "Same current code and recorded inputs, trajectory/tree reuse disabled versus enabled; normal 100000-row budget",++ "chunked_timings": chunked_timings}+ args.output.parent.mkdir(parents=True, exist_ok=True)+ with args.output.open("x") as stream:+ json.dump(result, stream, indent=2)+ stream.write("\n")+ print(json.dumps(result, indent=2))+++if __name__ == "__main__":+ main()diff --cc src/dopplergeist/search.pyindex 7c2c2dc,b1e5759..18260aa--- a/src/dopplergeist/search.py+++ b/src/dopplergeist/search.py@@@ -85,31 -81,20 +85,50 @@@ def cpa_table(tracks,points,max_distanc"""observers=ecef(points.lat.to_numpy(),points.lon.to_numpy(),points.elev_m.to_numpy())point_ids=points.point_id.to_numpy(dtype=int)+ return pd.DataFrame(_cpa_rows(tracks, observers, point_ids, max_distance, c, step_s, chunk_size),+ columns=_CPA_COLUMNS)+++def _cpa_rows(tracks, observers, point_ids, max_distance, c=343.0, step_s=2.0, chunk_size=512):+ """Return the same ordered CPA records without allocating per-track tables.++ Tiled scoring already has ECEF observers and needs to isolate unsupported+ tracks. It can reuse those observers and materialize one table after the+ bounded per-track work instead of rebuilding a DataFrame for every query.+ """rows=[]for track in tracks:++ prepared = _PreparedCPATrack(track, c, step_s)++ rows.extend(prepared.raw_rows(observers, point_ids, max_distance, chunk_size))++ return _ordered_cpa_rows(rows, step_s)++++++class _PreparedCPATrack:++ """One validated track/step, reused only across one call's receiver chunks."""++++ def __init__(self, track, c, step_s):track.validate_motion_support(c)-- t=np.unique(np.r_[np.arange(track.time[0],track.time[-1],step_s),track.time[-1]])-- pos=track.position(t)-- max_step=np.max(np.linalg.norm(np.diff(pos,axis=0),axis=1))-- near=np.flatnonzero(np.all((observers>=pos.min(axis=0)-max_distance)&-- (observers<=pos.max(axis=0)+max_distance),axis=1))-- if not len(near):continue-- tree=cKDTree(pos)++ self.track, self.c, self.step_s = track, c, step_s++ self.time = np.unique(np.r_[np.arange(track.time[0], track.time[-1], step_s), track.time[-1]])++ self.position = track.position(self.time)++ self.max_step = np.max(np.linalg.norm(np.diff(self.position, axis=0), axis=1))++ self.lower, self.upper = self.position.min(axis=0), self.position.max(axis=0)++ self.tree = None++++ def raw_rows(self, observers, point_ids, max_distance, chunk_size=512):++ rows = []++ track, t, pos, c = self.track, self.time, self.position, self.c++ near=np.flatnonzero(np.all((observers>=self.lower-max_distance)&++ (observers<=self.upper+max_distance),axis=1))++ if not len(near):++ return rows++ if self.tree is None:++ self.tree = cKDTree(pos)++ tree = self.treefor base in range(0,len(near),chunk_size):chosen=near[base:base+chunk_size]obs=observers[chosen]-- neighbours=tree.query_ball_point(obs,max_distance+2*max_step,return_sorted=True)++ neighbours=tree.query_ball_point(obs,max_distance+2*self.max_step,return_sorted=True)counts=np.fromiter((len(a) for a in neighbours),dtype=int,count=len(neighbours))if not counts.sum():continuesample=np.concatenate(neighbours).astype(int)@@@ -172,27 -158,17 +191,31 @@@when=t[start]+fraction[n,edge]*(t[start+1]-t[start])speed=np.linalg.norm(track.speed_vector(when),axis=1)for j in np.flatnonzero(distance<=max_distance):- rows.append({"point_id":int(point_ids[chosen[oi[j]]]),"flight_id":track.id,- "emission_time":float(when[j]),"receive_time":float(when[j]+distance[j]/c),- "distance_m":float(distance[j]),"speed_mps":float(speed[j])})- result=pd.DataFrame(rows,columns=["point_id","flight_id","emission_time","receive_time","distance_m","speed_mps"])- if not result.empty:+ rows.append((int(point_ids[chosen[oi[j]]]), track.id,+ float(when[j]), float(when[j]+distance[j]/c),+ float(distance[j]), float(speed[j])))++ return rows++++++def _ordered_cpa_rows(rows, step_s):+ if rows:# A CPA exactly at a sample can be proposed twice under roundoff.- result=result.sort_values(["point_id","flight_id","emission_time"])- duplicate=(result.point_id.eq(result.point_id.shift())&result.flight_id.eq(result.flight_id.shift())&- result.emission_time.diff().abs().lt(step_s*1.1))- result=result[~duplicate].reset_index(drop=True)- return result+ rows.sort(key=lambda row: (row[0], row[1], row[2]))+ duplicate_tolerance = step_s*1.1+ previous, retained = None, []+ for row in rows:+ # Match pandas' shifted int64 IDs (which become float64) and the+ # original float64-difference promotion against the caller's step.+ duplicate = (previous is not None and float(row[0]) == float(previous[0])+ and row[1] == previous[1]+ and np.float64(abs(row[2]-previous[2])) < duplicate_tolerance)+ if not duplicate:+ retained.append(row)+ # Match the original simultaneous adjacent-row mask, including a+ # chain of close proposals: compare with the preceding raw row.+ previous = row+ return retained+ return rowsdef filter_event(table, event, time_origin=None, time_tolerance=5.0):diff --cc src/dopplergeist/tiled_search.pyindex 4d00c74,8ad2639..cb22fe4--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@@ -24,7 -24,7 +24,8 @@@ from .finite_search import RetentionLedfrom .resource_usage import process_peak_rss_kibfrom .geometry import AcousticSupportError, eceffrom .matching import _cache_ridge_fits- from .search import _CPA_DTYPE, _cpa_rows, filter_association_event, pair_candidates, score_candidate-from .search import cpa_table, filter_association_event, pair_candidates, score_candidate++from .search import (_CPA_DTYPE, _PreparedCPATrack, _ordered_cpa_rows,++ filter_association_event, pair_candidates, score_candidate)def _support_error(error):@@@ -365,20 -345,17 +366,21 @@@ def _score(points, tracks, events, meas# Even a pathological looping track cannot allocate an unbounded list# of minima before we inspect the returned table.chunk_points = max(1, budget.max_cpa_rows // samples)- track_tables, track_rows = [], 0+ track_batches, track_rows = [], 0++ prepared = Nonetry:if track.outside_observer_radius(observers, radius, parameters.get("c", 343.0)):continue++ prepared = _PreparedCPATrack(track, parameters.get("c", 343.0), budget.cpa_step_s)for first in range(0, len(points), chunk_points):- rows = _cpa_rows([track], observers[first:first + chunk_points],- point_ids[first:first + chunk_points], radius,- parameters.get("c", 343.0), budget.cpa_step_s)- table = cpa_table([track], points.iloc[first:first + chunk_points], radius, parameters.get("c", 343.0), budget.cpa_step_s)- track_rows += len(table)++ rows = _ordered_cpa_rows(prepared.raw_rows(observers[first:first + chunk_points],++ point_ids[first:first + chunk_points], radius), budget.cpa_step_s)+ track_rows += len(rows)if count + track_rows > budget.max_cpa_rows:raise DataUnavailable("Tile CPA-row memory budget exceeded")- if len(table):- track_tables.append(table)+ if rows:+ track_batches.append(np.array(rows, dtype=_CPA_DTYPE))+ del rowsexcept AcousticSupportError as error:# The typed failure does not identify an exact bad interval. Keep# the entire track/stage as explicitly unassessed, and retain rows@@@ -390,14 -367,13 +392,17 @@@float(points.lon.max()), float(points.lat.max())],"receiver_point_count": len(points), "coverage_envelope": "whole_track_whole_stage_conservative"})continue++ finally:++ prepared = Nonecount += track_rows- tables.extend(track_tables)- if not tables:+ cpa_batches.extend(track_batches)+ if not cpa_batches:if ledger is not None:ledger.record(stage, [], [], 0)return [], {"cpa_rows": 0, "candidates": 0, "unsupported_cpa_tracks": unsupported}- table = pd.DataFrame.from_records(np.concatenate(cpa_batches))- table = pd.concat(tables, ignore_index=True)++ table = pd.DataFrame({name: np.concatenate([batch[name] for batch in cpa_batches])++ for name in _CPA_DTYPE.names}, copy=False)+ del cpa_batches, track_batcheskwargs = dict(offset_tolerance_s=filters.get("offset_tolerance_s", 5.0),time_origin=parameters.get("time_origin"), time_tolerance_s=filters.get("time_tolerance_s", 5.0),silence=filters.get("silence"))diff --cc tests/test_real_cpa_preparation.pyindex 0000000,0000000..3827288new file mode 100644--- /dev/null+++ b/tests/test_real_cpa_preparation.py@@@ -1,0 -1,0 +1,85 @@@++"""CPA preparation reuse on unchanged archived aircraft observations."""++import json++from pathlib import Path++import unittest++from unittest.mock import patch++++import numpy as np++import pandas as pd++++from dopplergeist import search++from dopplergeist.geometry import AcousticSupportError, Track, ecef++from dopplergeist.tracks import _segment_velocity++++++class RealCPAPreparationTests(unittest.TestCase):++ @classmethod++ def setUpClass(cls):++ cls.tracks = []++ root = Path(__file__).resolve().parents[1]++ for source in sorted((root / "research/trajectory-motion/frozen").glob("fresh-*.json")):++ frame = pd.DataFrame(json.loads(source.read_text())).sort_values("time_s")++ for number, (_, rows) in enumerate(frame.groupby((frame.time_s.diff() > 30.).cumsum())):++ if len(rows) < 3:++ continue++ velocity, _ = _segment_velocity(rows)++ cls.tracks.append(Track(f"{rows.flight_id.iloc[0]}:{number}", rows.time_s.to_numpy(),++ rows.lat.to_numpy(), rows.lon.to_numpy(), rows.alt_m.to_numpy(),++ str(rows.typecode.iloc[0]), velocity))++ cls.supported, cls.unsupported = [], []++ for track in cls.tracks:++ try:++ track.validate_motion_support()++ cls.supported.append(track)++ except AcousticSupportError:++ cls.unsupported.append(track)++ cls.long_track = next(track for track in cls.supported if track.time[-1]-track.time[0] > 800.)++++ def test_one_sampling_and_tree_for_several_real_receiver_chunks(self):++ track = self.long_track++ indices = np.linspace(1, len(track.time)-2, 24, dtype=int)++ observers = ecef(track.lat[indices], track.lon[indices], np.zeros(len(indices)))++ point_ids = np.arange(len(indices))++ for step in (np.float32(2), np.float64(2), np.longdouble(2)):++ chunks = [(observers[first:first+4], point_ids[first:first+4]) for first in range(0, len(indices), 4)]++ expected = [search._cpa_rows([track], points, ids, 20000., 343., step) for points, ids in chunks]++ with patch.object(track, "position", wraps=track.position) as position, \++ patch.object(search, "cKDTree", wraps=search.cKDTree) as tree:++ prepared = search._PreparedCPATrack(track, 343., step)++ actual = [search._ordered_cpa_rows(prepared.raw_rows(points, ids, 20000.), step)++ for points, ids in chunks]++ self.assertEqual(expected, actual)++ self.assertGreater(sum(map(len, actual)), 0)++ self.assertEqual(tree.call_count, 1)++ sampled = [call for call in position.call_args_list++ if np.asarray(call.args[0]).ndim == 1 and len(np.asarray(call.args[0])) > 3]++ self.assertEqual(len(sampled), 1)++++ def test_tree_stays_lazy_for_distant_recorded_receiver(self):++ track = self.long_track++ distant = next(other for other in self.supported if abs(other.lon[0]-track.lon[0]) > 3.)++ far = ecef(distant.lat[:1], distant.lon[:1], np.zeros(1))++ midpoint = len(track.time)//2++ near = ecef(track.lat[midpoint:midpoint+1], track.lon[midpoint:midpoint+1], np.zeros(1))++ prepared = search._PreparedCPATrack(track, 343., 2.)++ self.assertIsNone(prepared.tree)++ self.assertEqual(prepared.raw_rows(far, np.array([0]), 20000.), [])++ self.assertIsNone(prepared.tree)++ self.assertTrue(prepared.raw_rows(near, np.array([1]), 20000.))++ tree = prepared.tree++ self.assertIsNotNone(tree)++ self.assertEqual(prepared.raw_rows(far, np.array([2]), 20000.), [])++ self.assertIs(prepared.tree, tree)++++ def test_unsupported_motion_keeps_original_error(self):++ self.assertTrue(self.unsupported)++ for track in self.unsupported:++ with self.assertRaises(AcousticSupportError) as direct:++ track.validate_motion_support()++ with self.assertRaises(AcousticSupportError) as prepared:++ search._PreparedCPATrack(track, 343., 2.)++ self.assertEqual(vars(direct.exception), vars(prepared.exception))++++++if __name__ == "__main__":++ unittest.main()