Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Merge current pipeline improvements and reuse CPA preparation across chunks

Cinna Davis3c8f335e1d31
diff --cc .github/workflows/real-performance.yml
index f7ce090,e7a813c..2f127b9
--- a/.github/workflows/real-performance.yml
+++ b/.github/workflows/real-performance.yml
@@@ -3,34 -3,38 +3,44 @@@ on
workflow_dispatch:
inputs:
baseline:
- description: Scalar-consistent baseline commit
- default: d7cc1dd2e8089768253492b3269dfae97b3ad1a4
+ description: Exact baseline commit for both real-input comparisons
- default: 833c96c98366e18cc93562499f169d45bc0a9e06
++ default: d9649ee5e369b31072d40c4882a5b7e0a7bfa177
required: true
+ regional:
+ description: Also compare the complete regional HTTP workflow
+ type: boolean
+ default: true
push:
- branches: [perf/measured-hotpath-20261103, perf/single-pass-motion-20261103]
- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103]
++ branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103]
paths:
- .github/workflows/real-performance.yml
- src/dopplergeist/**
- scripts/benchmark_prepared_measurements.py
+ - scripts/benchmark_real_cpa_rows.py
+ - scripts/benchmark_real_regional_upload.py
+ - scripts/benchmark_real_bounded_upload.py
+ - scripts/prepare_reference_upload.py
+ - scripts/restore_artifacts.py
+ - scripts/compare_real_upload_performance.py
+ - scripts/check_real_upload_arrays.py
+ - pyproject.toml
- tests/test_real_*.py
+ - tests/data/real-ridge-batch-arithmetic.json
+ - tests/data/reference-empty-tiles-real.json
+ - tests/data/reference-retention-real.json
permissions:
contents: read
+env:
+ OPENBLAS_NUM_THREADS: '1'
+ OMP_NUM_THREADS: '1'
+ MKL_NUM_THREADS: '1'
+ PYTHONPATH: src
- BASELINE_REF: ${{ inputs.baseline || '833c96c98366e18cc93562499f169d45bc0a9e06' }}
++ BASELINE_REF: ${{ inputs.baseline || 'd9649ee5e369b31072d40c4882a5b7e0a7bfa177' }}
jobs:
compare:
+ if: github.event_name != 'push' || github.ref != 'refs/heads/perf/real-pipeline-throughput-20261103'
runs-on: ubuntu-latest
timeout-minutes: 20
- env:
- OPENBLAS_NUM_THREADS: '1'
- OMP_NUM_THREADS: '1'
- MKL_NUM_THREADS: '1'
- PYTHONPATH: src
- BASELINE_REF: ${{ inputs.baseline || 'd7cc1dd2e8089768253492b3269dfae97b3ad1a4' }}
steps:
- uses: actions/checkout@v4
with:
@@@ -60,26 -61,19 +70,59 @@@
- name: Real-input regression tests
env:
DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json
- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py
- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py
++ run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py
+ - name: Compare real CPA preparation
+ run: python scripts/benchmark_real_cpa_rows.py --baseline-source "$RUNNER_TEMP/baseline/src" --output results/cpa-rows.json --repeats 5 --iterations 20
- name: Compare complete payloads and timings
run: |
+ native_option=()
+ if [ -f src/dopplergeist/_native_ridge.c ]; then
+ native_option=(--require-current-native)
+ fi
for width in 10 120; do
- python scripts/benchmark_prepared_measurements.py compare --fixture "benchmark-inputs/point-$width.json" --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src --repeats 2 --timeout-s 600 --output "results/performance-$width"
+ python scripts/benchmark_prepared_measurements.py compare --fixture "benchmark-inputs/point-$width.json" --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src --repeats 2 --timeout-s 600 --output "results/performance-$width" "${native_option[@]}"
done
++ - name: Profile remaining point work outside the timing comparison
++ run: |
++ python -m cProfile -o results/current-point.prof scripts/benchmark_prepared_measurements.py _worker --source src --fixture benchmark-inputs/point-120.json --output results/current-point-profile
++ python - <<'PY'
++ import json
++ from pathlib import Path
++ import pstats
++
++ statistics = pstats.Stats("results/current-point.prof")
++ root = Path.cwd()
++ records = []
++ for (filename, line, function), (primitive, total, own, cumulative, callers) in statistics.stats.items():
++ path = Path(filename)
++ relative = path.relative_to(root).as_posix() if path.is_relative_to(root) else filename
++ module = (relative[4:-3].replace("/", ".") if relative.startswith("src/")
++ and relative.endswith(".py") else path.stem)
++ records.append({"module": module, "file": relative, "line": line, "function": function,
++ "primitive_calls": primitive, "total_calls": total,
++ "self_seconds": own, "cumulative_seconds": cumulative})
++ records.sort(key=lambda row: (-row["cumulative_seconds"], -row["self_seconds"],
++ row["file"], row["line"], row["function"]))
++ summary = {"scope": "Separate profiled width-120 worker, including imports and input loading; excluded from timing medians",
++ "timer": "cProfile default elapsed timer; diagnostic attribution, not an uninstrumented CPU measurement",
++ "total_calls": statistics.total_calls, "primitive_calls": statistics.prim_calls,
++ "total_profiled_seconds": statistics.total_tt, "top_50": records[:50],
++ "dopplergeist_functions": [row for row in records if row["module"].startswith("dopplergeist")]}
++ output = Path("results/current-point-profile-stats.json")
++ output.write_text(json.dumps(summary, indent=2) + "\n")
++ print(json.dumps({"profile_summary": str(output), "excluded_from_timing_medians": True,
++ "top_10": records[:10]}, indent=2))
++ PY
- uses: actions/upload-artifact@v4
if: always()
with:
name: real-performance-summary
path: |
+ results/cpa-rows.json
results/performance-*/comparison.json
results/performance-*/*/summary.json
++ results/current-point.prof
++ results/current-point-profile-stats.json
if-no-files-found: ignore
retention-days: 14
- uses: actions/upload-artifact@v4
@@@ -89,70 -83,172 +132,240 @@@
path: results/performance-*
if-no-files-found: ignore
retention-days: 3
+ regional:
+ if: github.ref == 'refs/heads/perf/single-pass-motion-20261103' || (github.event_name == 'workflow_dispatch' && inputs.regional)
- runs-on: ubuntu-latest
++ # Keep the reference FFmpeg family; a new encoder can change detected ridges.
++ runs-on: ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ - name: Install runtime and source-tree extensions
+ run: |
+ python -m pip install --editable '.[test]'
+ if ! command -v ffmpeg >/dev/null; then
+ sudo apt-get update
+ sudo apt-get install -y ffmpeg
+ fi
+ - name: Restore the complete real recording and observed inputs
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: python scripts/restore_artifacts.py --bundle data-source --bundle data-adsb-derived --bundle data-terrain
+ - name: Prepare common inputs and the baseline
+ run: |
+ python scripts/prepare_reference_upload.py --data-root data --output benchmark-inputs
+ baseline_commit=$(git rev-parse --verify --end-of-options "${BASELINE_REF}^{commit}")
+ git worktree add --detach "$RUNNER_TEMP/baseline" "$baseline_commit"
+ if [ -f "$RUNNER_TEMP/baseline/setup.py" ]; then
+ (cd "$RUNNER_TEMP/baseline" && python setup.py build_ext --inplace)
+ fi
+ python - <<'PY'
+ import json
+ from pathlib import Path
+ path = Path("benchmark-inputs/operator.json")
+ operator = json.loads(path.read_text())
+ operator["search_workers"] = 1
+ path.write_text(json.dumps(operator, indent=2) + "\n")
+ PY
+ ffmpeg -nostdin -i data/source/reference-recording.wav -ac 1 -ar 48000 -c:a libmp3lame -b:a 96k benchmark-inputs/reference.mp3
+ - name: Measure both complete HTTP arms
+ run: |
+ python scripts/benchmark_real_regional_upload.py --source "$RUNNER_TEMP/baseline/src" --audio benchmark-inputs/reference.mp3 --request benchmarks/reference-regional-upload-request.json --operator benchmark-inputs/operator.json --output results/regional-baseline --timeout-s 600 --threads 1
+ python scripts/benchmark_real_regional_upload.py --source src --audio benchmark-inputs/reference.mp3 --request benchmarks/reference-regional-upload-request.json --operator benchmark-inputs/operator.json --output results/regional-current --timeout-s 600 --threads 1
+ - name: Require complete scientific parity
+ run: python scripts/benchmark_real_regional_upload.py --compare results/regional-baseline results/regional-current --output results/regional-comparison.json
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: real-regional-summary
+ path: |
+ results/regional-*/timing.json
+ results/regional-*/result.json
+ results/regional-*/http-verification.json
+ results/regional-*/pipeline-evidence.json
+ results/regional-comparison.json
+ if-no-files-found: ignore
+ retention-days: 14
+ - uses: actions/upload-artifact@v4
+ if: failure()
+ with:
+ name: real-regional-failure-diagnostics
+ path: |
+ results/regional-*/jobs/*/search/search-report.json
+ results/regional-*/jobs/*/provenance.json
+ results/regional-*/*.log
+ if-no-files-found: ignore
+ retention-days: 3
+
+ bounded-http:
+ if: github.event_name == 'push' && github.ref == 'refs/heads/perf/real-pipeline-throughput-20261103'
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ env:
+ OPENBLAS_NUM_THREADS: '1'
+ OMP_NUM_THREADS: '1'
+ MKL_NUM_THREADS: '1'
+ NUMEXPR_NUM_THREADS: '1'
+ PYTHONPATH: src
+ BASELINE_REF: aa3dff686c989524daa20c081a816b792d9c8890
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.12'
+ - name: Install matched runtime
+ run: |
+ python -m pip install '.[test]' 'numpy==2.3.5' 'scipy==1.17.0' 'pandas==2.2.3' 'pyarrow==26.0.1' 'pyproj==3.8.0' 'matplotlib==3.10.8'
+ sudo apt-get update
+ sudo apt-get install -y ffmpeg
+ - name: Restore public recording and observed inputs
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: python scripts/restore_artifacts.py --bundle data-source --bundle data-adsb-derived --bundle data-terrain
+ - name: Prepare one shared workload
+ run: |
+ python scripts/prepare_reference_upload.py --data-root data --output benchmark-http-inputs
+ git worktree add --detach "$RUNNER_TEMP/http-baseline-source" "$BASELINE_REF"
+ python - <<'PY'
+ import json
+ from pathlib import Path
+ root = Path("benchmark-http-inputs")
+ request = json.loads((root / "request.json").read_text())
+ request["search"]["bounds_wsen"] = [-73.545, 40.865, -73.505, 40.895]
+ request["time_window"]["source_detail"] = (
+ "Source-informed integration replay around the historical fitted recording "
+ "start and benchmark receiver region; not independently measured recording "
+ "time or a location-accuracy evaluation."
+ )
+ operator = json.loads((root / "operator.json").read_text())
+ operator["search_workers"] = 1
+ for name, value in (("request.json", request), ("operator.json", operator)):
+ (root / name).write_text(json.dumps(value, indent=2) + "\n")
+ PY
+ ffmpeg -n -nostdin -v error -i data/source/reference-recording.wav -map 0:a:0 -vn -map_metadata -1 -ac 1 -ar 48000 -c:a libmp3lame -b:a 96k benchmark-http-inputs/recording.mp3
+ python scripts/benchmark_prepared_measurements.py prepare --discovery evidence/audio/w1-sensitivity-20261102/reference/discovery.json --index benchmark-http-inputs/reference-region.sqlite --clock-half-width-s 120 --output benchmark-http-inputs/point-120.json
+ - name: Recorded-input regression checks
+ id: real_checks
+ env:
+ DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-http-inputs/point-120.json
+ run: |
+ mkdir -p results
+ python -m pytest -q \
+ tests/test_real_prepared_measurements.py \
+ tests/test_real_geometry_kinematics.py \
+ tests/test_real_batch_scalar_parity.py \
+ tests/test_real_complete_batch_cache.py \
+ tests/test_real_legacy_batch_cache.py \
+ tests/test_real_refinement_lookup.py \
+ tests/test_real_track_cache.py \
+ tests/test_real_retention_batch.py \
+ tests/test_real_retention_floor.py \
+ tests/test_real_performance_helpers.py::test_cached_raster_transform_preserves_actual_candidate_heights \
+ --junitxml=results/ci-real-tests.xml
+ python - <<'CHECKS'
+ import json
+ from pathlib import Path
+ import xml.etree.ElementTree as ET
+ cases = ET.parse(Path("results/ci-real-tests.xml")).getroot().findall(".//testcase")
+ skipped = [case.attrib.get("name") for case in cases if case.find("skipped") is not None]
+ print(json.dumps({"selected_real_tests": len(cases), "skipped": skipped}))
+ if not cases or skipped:
+ raise SystemExit("Selected real-input checks must execute without skips")
+ CHECKS
+ - name: Record this new workload and runtime
+ id: context
+ run: |
+ python - <<'PY'
+ import hashlib
+ import importlib.metadata
+ import json
+ import os
+ from pathlib import Path
+ import platform
+ import subprocess
+ output = Path("results")
+ output.mkdir(exist_ok=True)
+ audio = Path("benchmark-http-inputs/recording.mp3")
+ with audio.open("rb") as stream:
+ digest = hashlib.file_digest(stream, "sha256").hexdigest()
+ record = {
+ "scope": "One fresh sequential matched HTTP pair on a new MP3 encoding of the public real WAV. Timings and event counts do not inherit the earlier original-MP3 measurements. Source-informed region/time; not independent location accuracy.",
+ "baseline": os.environ["BASELINE_REF"],
+ "current": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
+ "python": platform.python_version(),
+ "packages": {name: importlib.metadata.version(name) for name in ("numpy", "scipy", "pandas", "pyarrow", "pyproj", "rasterio", "matplotlib")},
+ "ffmpeg": subprocess.check_output(["ffmpeg", "-version"], text=True).splitlines()[0],
+ "encoding": {"codec": "libmp3lame", "bitrate": "96k", "sample_rate_hz": 48000, "channels": 1},
+ "source_sha256": digest,
+ "source_bytes": audio.stat().st_size,
+ "request": json.loads(Path("benchmark-http-inputs/request.json").read_text()),
+ "operator": json.loads(Path("benchmark-http-inputs/operator.json").read_text()),
+ }
+ (output / "http-comparison-context.json").write_text(json.dumps(record, indent=2, allow_nan=False) + "\n")
+ PY
+ - name: Fresh baseline HTTP arm
+ id: baseline
+ run: python scripts/benchmark_real_bounded_upload.py --source "$RUNNER_TEMP/http-baseline-source/src" --inputs benchmark-http-inputs --output results/http-baseline --timeout-s 900
+ - name: Fresh integrated HTTP arm
+ id: current
+ if: ${{ !cancelled() && steps.context.outcome == 'success' }}
+ run: python scripts/benchmark_real_bounded_upload.py --source "$GITHUB_WORKSPACE/src" --inputs benchmark-http-inputs --output results/http-current --timeout-s 900
+ - name: Compare complete scientific JSON and SQLite
+ if: always() && steps.baseline.outcome == 'success' && steps.current.outcome == 'success'
+ run: python scripts/compare_real_upload_performance.py --baseline results/http-baseline --current results/http-current --output results/http-science-comparison.json
+ - name: Compare intake, decoded PCM, arrays, and PNG pixels
+ if: always() && steps.baseline.outcome == 'success' && steps.current.outcome == 'success'
+ run: python scripts/check_real_upload_arrays.py --baseline results/http-baseline --current results/http-current --output results/http-array-comparison.json
+ - name: Print compact comparison results
+ if: always()
+ run: |
+ python - <<'REPORT'
+ import json
+ from pathlib import Path
+ def read(name):
+ path = Path("results") / name
+ if not path.is_file():
+ return {"missing": str(path)}
+ try:
+ return json.loads(path.read_text())
+ except (OSError, ValueError) as error:
+ return {"read_error": str(error), "path": str(path)}
+ def select(value, fields):
+ if "missing" in value or "read_error" in value:
+ return value
+ return {key: value[key] for key in fields if key in value}
+ science = read("http-science-comparison.json")
+ arrays = read("http-array-comparison.json")
+ passed = science.get("passed") is True and arrays.get("passed") is True
+ result = {
+ "both_parity_checks_passed": passed,
+ "matched_pair_wall_speedup": science.get("verified_full_workflow_speedup") if passed else None,
+ "scientific_comparison": select(science, ("passed", "scientific_payloads_exact", "run_kinds", "full_pipeline_verified", "checked_artifacts", "retention_ledgers", "outcomes", "differences")),
+ "intake_array_comparison": select(arrays, ("passed", "checked", "manifest_verified_artifacts", "difference_count", "differences")),
+ "http_receipts": {name: select(read(f"http-{name}/http-verification.json"), ("verification_passed", "implementation_unchanged", "full_pipeline_observed", "full_pipeline_requirements_met", "elapsed_s", "status", "reason_code", "event_count", "candidate_count", "source_sha256", "source_bytes", "service")) for name in ("baseline", "current")},
+ "context": read("http-comparison-context.json"),
+ }
+ print("BOUNDED_HTTP_COMPARISON_BEGIN")
+ print(json.dumps(result, indent=2, allow_nan=False))
+ print("BOUNDED_HTTP_COMPARISON_END")
+ REPORT
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: bounded-http-comparison
+ path: |
+ results/ci-real-tests.xml
+ results/http-comparison-context.json
+ results/http-science-comparison.json
+ results/http-array-comparison.json
+ results/http-*/http-verification.json
+ results/http-*/pipeline-evidence.json
+ results/http-*/service.log
+ if-no-files-found: ignore
+ retention-days: 14
diff --cc scripts/benchmark_real_cpa_rows.py
index e2d632b,0000000..03d74d5
mode 100644,000000..100644
--- a/scripts/benchmark_real_cpa_rows.py
+++ b/scripts/benchmark_real_cpa_rows.py
@@@ -1,158 -1,0 +1,236 @@@
+#!/usr/bin/env python3
+"""Compare CPA preparation on archived aircraft observations, without fake tracks."""
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
++import math
++import os
+from pathlib import Path
+import statistics
+import sys
+import time
+
+import numpy as np
+import pandas as pd
+
+from dopplergeist.geometry import AcousticSupportError, Track, ecef
+from dopplergeist import search
+from dopplergeist.tracks import _segment_velocity
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def recorded_tracks():
+ """Use the normal gap split and velocity inference on frozen real traces."""
+ tracks = []
+ for source in sorted((ROOT / "research/trajectory-motion/frozen").glob("fresh-*.json")):
+ frame = pd.DataFrame(json.loads(source.read_text())).sort_values("time_s")
+ for part, (_, rows) in enumerate(frame.groupby((frame.time_s.diff() > 30.).cumsum())):
+ if len(rows) < 3:
+ continue
+ velocity, _ = _segment_velocity(rows)
+ tracks.append(Track(f"{rows.flight_id.iloc[0]}:{part}", rows.time_s.to_numpy(),
+ rows.lat.to_numpy(), rows.lon.to_numpy(), rows.alt_m.to_numpy(),
+ str(rows.typecode.iloc[0]), velocity))
+ if not tracks:
+ raise ValueError("Archived real observation inputs are missing")
+ return tracks
+
+
+def recorded_points(tracks, count):
+ """Candidate coordinates sampled from the real traces; no accuracy claim."""
+ locations = np.concatenate([np.column_stack((track.lat, track.lon)) for track in tracks])
+ selected = locations[np.linspace(0, len(locations)-1, count, dtype=int)]
+ return pd.DataFrame({"point_id": np.arange(count), "lat": selected[:, 0],
+ "lon": selected[:, 1], "elev_m": np.zeros(count)})
+
+
+def cpa_stage(module, tracks, points, radius=20000., c=343., step=2., chunk_size=512,
- row_mode=False):
++ row_mode=False, *, reuse_samples=True, max_cpa_rows=100000, counters=None):
+ """The bounded track-isolation loop, before acoustic candidate scoring."""
+ observers = ecef(points.lat.to_numpy(), points.lon.to_numpy(), points.elev_m.to_numpy())
+ point_ids = points.point_id.to_numpy(dtype=int)
+ tables, batches, unsupported = [], [], []
++ count = 0
+ for track in tracks:
++ samples = math.ceil((track.time[-1]-track.time[0])/step)+1
++ receiver_chunk = max(1, max_cpa_rows//samples)
++ track_tables, track_batches, track_rows = [], [], 0
++ prepared = None
+ try:
+ if track.outside_observer_radius(observers, radius, c):
+ continue
- if row_mode:
- result = module._cpa_rows([track], observers, point_ids, radius, c, step, chunk_size)
- else:
- result = module.cpa_table([track], points, radius, c, step, chunk_size)
++ if row_mode and reuse_samples and hasattr(module, "_PreparedCPATrack"):
++ prepared = module._PreparedCPATrack(track, c, step)
++ if counters is not None:
++ counters["sample_preparations"] += 1
++ for first in range(0, len(points), receiver_chunk):
++ if counters is not None:
++ counters["receiver_chunks"] += 1
++ if prepared is not None:
++ result = module._ordered_cpa_rows(prepared.raw_rows(
++ observers[first:first+receiver_chunk], point_ids[first:first+receiver_chunk],
++ radius, chunk_size), step)
++ else:
++ if counters is not None:
++ counters["sample_preparations"] += 1
++ result = (module._cpa_rows([track], observers[first:first+receiver_chunk],
++ point_ids[first:first+receiver_chunk], radius, c, step, chunk_size) if row_mode else
++ module.cpa_table([track], points.iloc[first:first+receiver_chunk],
++ radius, c, step, chunk_size))
++ track_rows += len(result)
++ if count+track_rows > max_cpa_rows:
++ raise ValueError("Tile CPA-row memory budget exceeded")
++ if len(result):
++ if row_mode:
++ track_batches.append(np.array(result, dtype=search._CPA_DTYPE))
++ else:
++ track_tables.append(result)
++ del result
+ except AcousticSupportError as error:
+ unsupported.append(vars(error))
+ continue
++ finally:
++ prepared = None
++ count += track_rows
+ if row_mode:
- if result:
- batches.append(np.array(result, dtype=search._CPA_DTYPE))
- elif len(result):
- tables.append(result)
++ batches.extend(track_batches)
++ else:
++ tables.extend(track_tables)
+ if row_mode:
- table = (pd.DataFrame.from_records(np.concatenate(batches)) if batches else
++ table = (pd.DataFrame({name: np.concatenate([batch[name] for batch in batches])
++ for name in search._CPA_DTYPE.names}, copy=False) if batches else
+ pd.DataFrame(columns=search._CPA_COLUMNS))
+ else:
+ table = pd.concat(tables, ignore_index=True) if tables else pd.DataFrame(columns=search._CPA_COLUMNS)
+ return table, unsupported
+
+
+def compare(before, after):
+ pd.testing.assert_frame_equal(before[0], after[0], check_exact=True)
+ assert before[1] == after[1], "Unsupported-track accounting changed"
+
+
++def counted_chunked_stage(tracks, points, reuse):
++ """Count actual tree construction while keeping every recorded input fixed."""
++ counters = {"sample_preparations": 0, "tree_builds": 0, "receiver_chunks": 0}
++ tree = search.cKDTree
++
++ def counted_tree(*args, **kwargs):
++ counters["tree_builds"] += 1
++ return tree(*args, **kwargs)
++
++ search.cKDTree = counted_tree
++ try:
++ result = cpa_stage(search, tracks, points, row_mode=True, reuse_samples=reuse,
++ counters=counters)
++ finally:
++ search.cKDTree = tree
++ return result, counters
++
++
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--baseline-source", type=Path, required=True,
+ help="Baseline src directory from a separate checkout")
+ parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument("--repeats", type=int, default=5)
+ parser.add_argument("--iterations", type=int, default=20)
+ args = parser.parse_args()
+ if args.output.exists():
+ parser.error("Use a new output path")
+ if args.repeats < 1 or args.iterations < 1:
+ parser.error("repeats and iterations must be positive")
+ path = args.baseline_source / "dopplergeist/search.py"
+ spec = importlib.util.spec_from_file_location("dopplergeist._cpa_before", path)
+ baseline = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = baseline
+ spec.loader.exec_module(baseline)
+ tracks = recorded_tracks()
+ workloads = {str(count): recorded_points(tracks, count) for count in (1, 32, 128)}
+ checks = 0
+ for points in workloads.values():
+ wide_ids = pd.concat([points.iloc[:1], points.iloc[:1]], ignore_index=True)
+ wide_ids["point_id"] = [2**53, 2**53+1]
+ for radius, c, step, chunk_size in ((1000., 343., 2., 512), (20000., 343., 2., 512),
+ (20000., 338., 1., 7), (20000., 348., 8., 1),
+ (20000., 343., np.float32(2), 512)):
+ for candidate_points in (points, points.iloc[::-1], pd.concat([points, points], ignore_index=True), wide_ids):
+ old = cpa_stage(baseline, tracks, candidate_points, radius, c, step, chunk_size)
+ new = cpa_stage(search, tracks, candidate_points, radius, c, step, chunk_size, True)
+ compare(old, new)
+ checks += 1
+ supported = []
+ for track in tracks:
+ try:
+ track.validate_motion_support(c)
+ supported.append(track)
+ except AcousticSupportError:
+ pass
+ pd.testing.assert_frame_equal(
+ baseline.cpa_table(supported, candidate_points, radius, c, step, chunk_size),
+ search.cpa_table(supported, candidate_points, radius, c, step, chunk_size),
+ check_exact=True)
+ checks += 1
+ timings = []
+ for name, points in workloads.items():
+ runs = {"baseline": [], "current": []}
+ final = None
+ for repeat in range(args.repeats):
+ arms = [("baseline", baseline, False), ("current", search, True)]
+ if repeat % 2:
+ arms.reverse()
+ for label, module, row_mode in arms:
+ start = time.process_time()
+ for _ in range(args.iterations):
+ result = cpa_stage(module, tracks, points, row_mode=row_mode)
+ runs[label].append(time.process_time()-start)
+ final = result
+ medians = {name: statistics.median(values) for name, values in runs.items()}
+ timings.append({"receiver_points": int(name), "cpa_rows": len(final[0]),
+ "unsupported_tracks": len(final[1]), "iterations_per_trial": args.iterations,
+ "cpu_s": runs, "median_cpu_s": medians,
+ "speedup": medians["baseline"]/medians["current"]})
++ chunked_timings = []
++ for size in (1024, 4096):
++ points = recorded_points(tracks, size)
++ before, old_counts = counted_chunked_stage(tracks, points, False)
++ after, new_counts = counted_chunked_stage(tracks, points, True)
++ compare(before, after)
++ checks += 1
++ runs = {"repeated_preparation": [], "reused_preparation": []}
++ for repeat in range(args.repeats):
++ arms = [("repeated_preparation", False), ("reused_preparation", True)]
++ if repeat % 2:
++ arms.reverse()
++ for label, reuse in arms:
++ started = time.process_time()
++ for _ in range(args.iterations):
++ cpa_stage(search, tracks, points, row_mode=True, reuse_samples=reuse)
++ runs[label].append(time.process_time()-started)
++ medians = {label: statistics.median(values) for label, values in runs.items()}
++ chunked_timings.append({"receiver_points": size, "cpa_rows": len(after[0]),
++ "unsupported_tracks": len(after[1]), "max_cpa_rows": 100000,
++ "iterations_per_trial": args.iterations, "cpu_s": runs, "median_cpu_s": medians,
++ "speedup": medians["repeated_preparation"]/medians["reused_preparation"],
++ "operations": {"repeated_preparation": old_counts, "reused_preparation": new_counts}})
+ result = {"scope": "CPA preparation on archived real ADS-B observations; excludes acoustic fitting, downloads, and localization accuracy.",
+ "baseline_source": str(args.baseline_source), "tracks": len(tracks),
++ "runtime": {"python": sys.version.split()[0], "numpy": np.__version__, "pandas": pd.__version__,
++ "cpu_affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None},
+ "observed_samples": sum(len(track.time) for track in tracks),
- "exact_comparisons": checks, "timings": timings}
++ "exact_comparisons": checks, "timings": timings,
++ "chunked_comparison": "Same current code and recorded inputs, trajectory/tree reuse disabled versus enabled; normal 100000-row budget",
++ "chunked_timings": chunked_timings}
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ with args.output.open("x") as stream:
+ json.dump(result, stream, indent=2)
+ stream.write("\n")
+ print(json.dumps(result, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --cc src/dopplergeist/search.py
index 7c2c2dc,b1e5759..18260aa
--- a/src/dopplergeist/search.py
+++ b/src/dopplergeist/search.py
@@@ -85,31 -81,20 +85,50 @@@ def cpa_table(tracks,points,max_distanc
"""
observers=ecef(points.lat.to_numpy(),points.lon.to_numpy(),points.elev_m.to_numpy())
point_ids=points.point_id.to_numpy(dtype=int)
+ return pd.DataFrame(_cpa_rows(tracks, observers, point_ids, max_distance, c, step_s, chunk_size),
+ columns=_CPA_COLUMNS)
+
+
+def _cpa_rows(tracks, observers, point_ids, max_distance, c=343.0, step_s=2.0, chunk_size=512):
+ """Return the same ordered CPA records without allocating per-track tables.
+
+ Tiled scoring already has ECEF observers and needs to isolate unsupported
+ tracks. It can reuse those observers and materialize one table after the
+ bounded per-track work instead of rebuilding a DataFrame for every query.
+ """
rows=[]
for track in tracks:
++ prepared = _PreparedCPATrack(track, c, step_s)
++ rows.extend(prepared.raw_rows(observers, point_ids, max_distance, chunk_size))
++ return _ordered_cpa_rows(rows, step_s)
++
++
++class _PreparedCPATrack:
++ """One validated track/step, reused only across one call's receiver chunks."""
++
++ def __init__(self, track, c, step_s):
track.validate_motion_support(c)
-- t=np.unique(np.r_[np.arange(track.time[0],track.time[-1],step_s),track.time[-1]])
-- pos=track.position(t)
-- max_step=np.max(np.linalg.norm(np.diff(pos,axis=0),axis=1))
-- near=np.flatnonzero(np.all((observers>=pos.min(axis=0)-max_distance)&
-- (observers<=pos.max(axis=0)+max_distance),axis=1))
-- if not len(near):continue
-- tree=cKDTree(pos)
++ self.track, self.c, self.step_s = track, c, step_s
++ self.time = np.unique(np.r_[np.arange(track.time[0], track.time[-1], step_s), track.time[-1]])
++ self.position = track.position(self.time)
++ self.max_step = np.max(np.linalg.norm(np.diff(self.position, axis=0), axis=1))
++ self.lower, self.upper = self.position.min(axis=0), self.position.max(axis=0)
++ self.tree = None
++
++ def raw_rows(self, observers, point_ids, max_distance, chunk_size=512):
++ rows = []
++ track, t, pos, c = self.track, self.time, self.position, self.c
++ near=np.flatnonzero(np.all((observers>=self.lower-max_distance)&
++ (observers<=self.upper+max_distance),axis=1))
++ if not len(near):
++ return rows
++ if self.tree is None:
++ self.tree = cKDTree(pos)
++ tree = self.tree
for base in range(0,len(near),chunk_size):
chosen=near[base:base+chunk_size]
obs=observers[chosen]
-- neighbours=tree.query_ball_point(obs,max_distance+2*max_step,return_sorted=True)
++ neighbours=tree.query_ball_point(obs,max_distance+2*self.max_step,return_sorted=True)
counts=np.fromiter((len(a) for a in neighbours),dtype=int,count=len(neighbours))
if not counts.sum():continue
sample=np.concatenate(neighbours).astype(int)
@@@ -172,27 -158,17 +191,31 @@@
when=t[start]+fraction[n,edge]*(t[start+1]-t[start])
speed=np.linalg.norm(track.speed_vector(when),axis=1)
for j in np.flatnonzero(distance<=max_distance):
- rows.append({"point_id":int(point_ids[chosen[oi[j]]]),"flight_id":track.id,
- "emission_time":float(when[j]),"receive_time":float(when[j]+distance[j]/c),
- "distance_m":float(distance[j]),"speed_mps":float(speed[j])})
- result=pd.DataFrame(rows,columns=["point_id","flight_id","emission_time","receive_time","distance_m","speed_mps"])
- if not result.empty:
+ rows.append((int(point_ids[chosen[oi[j]]]), track.id,
+ float(when[j]), float(when[j]+distance[j]/c),
+ float(distance[j]), float(speed[j])))
++ return rows
++
++
++def _ordered_cpa_rows(rows, step_s):
+ if rows:
# A CPA exactly at a sample can be proposed twice under roundoff.
- result=result.sort_values(["point_id","flight_id","emission_time"])
- duplicate=(result.point_id.eq(result.point_id.shift())&result.flight_id.eq(result.flight_id.shift())&
- result.emission_time.diff().abs().lt(step_s*1.1))
- result=result[~duplicate].reset_index(drop=True)
- return result
+ rows.sort(key=lambda row: (row[0], row[1], row[2]))
+ duplicate_tolerance = step_s*1.1
+ previous, retained = None, []
+ for row in rows:
+ # Match pandas' shifted int64 IDs (which become float64) and the
+ # original float64-difference promotion against the caller's step.
+ duplicate = (previous is not None and float(row[0]) == float(previous[0])
+ and row[1] == previous[1]
+ and np.float64(abs(row[2]-previous[2])) < duplicate_tolerance)
+ if not duplicate:
+ retained.append(row)
+ # Match the original simultaneous adjacent-row mask, including a
+ # chain of close proposals: compare with the preceding raw row.
+ previous = row
+ return retained
+ return rows
def filter_event(table, event, time_origin=None, time_tolerance=5.0):
diff --cc src/dopplergeist/tiled_search.py
index 4d00c74,8ad2639..cb22fe4
--- a/src/dopplergeist/tiled_search.py
+++ b/src/dopplergeist/tiled_search.py
@@@ -24,7 -24,7 +24,8 @@@ from .finite_search import RetentionLed
from .resource_usage import process_peak_rss_kib
from .geometry import AcousticSupportError, ecef
from .matching import _cache_ridge_fits
- from .search import _CPA_DTYPE, _cpa_rows, filter_association_event, pair_candidates, score_candidate
-from .search import cpa_table, filter_association_event, pair_candidates, score_candidate
++from .search import (_CPA_DTYPE, _PreparedCPATrack, _ordered_cpa_rows,
++ filter_association_event, pair_candidates, score_candidate)
def _support_error(error):
@@@ -365,20 -345,17 +366,21 @@@ def _score(points, tracks, events, meas
# Even a pathological looping track cannot allocate an unbounded list
# of minima before we inspect the returned table.
chunk_points = max(1, budget.max_cpa_rows // samples)
- track_tables, track_rows = [], 0
+ track_batches, track_rows = [], 0
++ prepared = None
try:
if track.outside_observer_radius(observers, radius, parameters.get("c", 343.0)):
continue
++ prepared = _PreparedCPATrack(track, parameters.get("c", 343.0), budget.cpa_step_s)
for first in range(0, len(points), chunk_points):
- rows = _cpa_rows([track], observers[first:first + chunk_points],
- point_ids[first:first + chunk_points], radius,
- parameters.get("c", 343.0), budget.cpa_step_s)
- table = cpa_table([track], points.iloc[first:first + chunk_points], radius, parameters.get("c", 343.0), budget.cpa_step_s)
- track_rows += len(table)
++ rows = _ordered_cpa_rows(prepared.raw_rows(observers[first:first + chunk_points],
++ point_ids[first:first + chunk_points], radius), budget.cpa_step_s)
+ track_rows += len(rows)
if count + track_rows > budget.max_cpa_rows:
raise DataUnavailable("Tile CPA-row memory budget exceeded")
- if len(table):
- track_tables.append(table)
+ if rows:
+ track_batches.append(np.array(rows, dtype=_CPA_DTYPE))
+ del rows
except AcousticSupportError as error:
# The typed failure does not identify an exact bad interval. Keep
# the entire track/stage as explicitly unassessed, and retain rows
@@@ -390,14 -367,13 +392,17 @@@
float(points.lon.max()), float(points.lat.max())],
"receiver_point_count": len(points), "coverage_envelope": "whole_track_whole_stage_conservative"})
continue
++ finally:
++ prepared = None
count += track_rows
- tables.extend(track_tables)
- if not tables:
+ cpa_batches.extend(track_batches)
+ if not cpa_batches:
if ledger is not None:
ledger.record(stage, [], [], 0)
return [], {"cpa_rows": 0, "candidates": 0, "unsupported_cpa_tracks": unsupported}
- table = pd.DataFrame.from_records(np.concatenate(cpa_batches))
- table = pd.concat(tables, ignore_index=True)
++ table = pd.DataFrame({name: np.concatenate([batch[name] for batch in cpa_batches])
++ for name in _CPA_DTYPE.names}, copy=False)
+ del cpa_batches, track_batches
kwargs = dict(offset_tolerance_s=filters.get("offset_tolerance_s", 5.0),
time_origin=parameters.get("time_origin"), time_tolerance_s=filters.get("time_tolerance_s", 5.0),
silence=filters.get("silence"))
diff --cc tests/test_real_cpa_preparation.py
index 0000000,0000000..3827288
new file mode 100644
--- /dev/null
+++ b/tests/test_real_cpa_preparation.py
@@@ -1,0 -1,0 +1,85 @@@
++"""CPA preparation reuse on unchanged archived aircraft observations."""
++import json
++from pathlib import Path
++import unittest
++from unittest.mock import patch
++
++import numpy as np
++import pandas as pd
++
++from dopplergeist import search
++from dopplergeist.geometry import AcousticSupportError, Track, ecef
++from dopplergeist.tracks import _segment_velocity
++
++
++class RealCPAPreparationTests(unittest.TestCase):
++ @classmethod
++ def setUpClass(cls):
++ cls.tracks = []
++ root = Path(__file__).resolve().parents[1]
++ for source in sorted((root / "research/trajectory-motion/frozen").glob("fresh-*.json")):
++ frame = pd.DataFrame(json.loads(source.read_text())).sort_values("time_s")
++ for number, (_, rows) in enumerate(frame.groupby((frame.time_s.diff() > 30.).cumsum())):
++ if len(rows) < 3:
++ continue
++ velocity, _ = _segment_velocity(rows)
++ cls.tracks.append(Track(f"{rows.flight_id.iloc[0]}:{number}", rows.time_s.to_numpy(),
++ rows.lat.to_numpy(), rows.lon.to_numpy(), rows.alt_m.to_numpy(),
++ str(rows.typecode.iloc[0]), velocity))
++ cls.supported, cls.unsupported = [], []
++ for track in cls.tracks:
++ try:
++ track.validate_motion_support()
++ cls.supported.append(track)
++ except AcousticSupportError:
++ cls.unsupported.append(track)
++ cls.long_track = next(track for track in cls.supported if track.time[-1]-track.time[0] > 800.)
++
++ def test_one_sampling_and_tree_for_several_real_receiver_chunks(self):
++ track = self.long_track
++ indices = np.linspace(1, len(track.time)-2, 24, dtype=int)
++ observers = ecef(track.lat[indices], track.lon[indices], np.zeros(len(indices)))
++ point_ids = np.arange(len(indices))
++ for step in (np.float32(2), np.float64(2), np.longdouble(2)):
++ chunks = [(observers[first:first+4], point_ids[first:first+4]) for first in range(0, len(indices), 4)]
++ expected = [search._cpa_rows([track], points, ids, 20000., 343., step) for points, ids in chunks]
++ with patch.object(track, "position", wraps=track.position) as position, \
++ patch.object(search, "cKDTree", wraps=search.cKDTree) as tree:
++ prepared = search._PreparedCPATrack(track, 343., step)
++ actual = [search._ordered_cpa_rows(prepared.raw_rows(points, ids, 20000.), step)
++ for points, ids in chunks]
++ self.assertEqual(expected, actual)
++ self.assertGreater(sum(map(len, actual)), 0)
++ self.assertEqual(tree.call_count, 1)
++ sampled = [call for call in position.call_args_list
++ if np.asarray(call.args[0]).ndim == 1 and len(np.asarray(call.args[0])) > 3]
++ self.assertEqual(len(sampled), 1)
++
++ def test_tree_stays_lazy_for_distant_recorded_receiver(self):
++ track = self.long_track
++ distant = next(other for other in self.supported if abs(other.lon[0]-track.lon[0]) > 3.)
++ far = ecef(distant.lat[:1], distant.lon[:1], np.zeros(1))
++ midpoint = len(track.time)//2
++ near = ecef(track.lat[midpoint:midpoint+1], track.lon[midpoint:midpoint+1], np.zeros(1))
++ prepared = search._PreparedCPATrack(track, 343., 2.)
++ self.assertIsNone(prepared.tree)
++ self.assertEqual(prepared.raw_rows(far, np.array([0]), 20000.), [])
++ self.assertIsNone(prepared.tree)
++ self.assertTrue(prepared.raw_rows(near, np.array([1]), 20000.))
++ tree = prepared.tree
++ self.assertIsNotNone(tree)
++ self.assertEqual(prepared.raw_rows(far, np.array([2]), 20000.), [])
++ self.assertIs(prepared.tree, tree)
++
++ def test_unsupported_motion_keeps_original_error(self):
++ self.assertTrue(self.unsupported)
++ for track in self.unsupported:
++ with self.assertRaises(AcousticSupportError) as direct:
++ track.validate_motion_support()
++ with self.assertRaises(AcousticSupportError) as prepared:
++ search._PreparedCPATrack(track, 343., 2.)
++ self.assertEqual(vars(direct.exception), vars(prepared.exception))
++
++
++if __name__ == "__main__":
++ unittest.main()