Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Accelerate partitioned flight checks with exact native batches

Cinna Davisc5476bd0ff12
diff --git a/.github/workflows/real-performance.yml b/.github/workflows/real-performance.yml
index d727775..afd1d3b 100644
--- a/.github/workflows/real-performance.yml
+++ b/.github/workflows/real-performance.yml
@@ -11,7 +11,7 @@ on:
type: boolean
default: true
push:
- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103]
+ branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103, perf/flight-sieve-94a6]
paths:
- .github/workflows/real-performance.yml
- src/dopplergeist/**
@@ -38,7 +38,7 @@ env:
NUMEXPR_NUM_THREADS: '1'
CC: gcc
PYTHONPATH: src
- BASELINE_REF: ${{ inputs.baseline || '27da08a878ac2ec90cd7e2c07f43765ceeea609b' }}
+ BASELINE_REF: ${{ inputs.baseline || (github.ref == 'refs/heads/perf/flight-sieve-94a6' && 'b5ff48fe0e9d6e6feac7a4faf5268630f0c85d8b') || '27da08a878ac2ec90cd7e2c07f43765ceeea609b' }}
jobs:
compare:
if: github.event_name != 'push' || github.ref != 'refs/heads/perf/real-pipeline-throughput-20261103'
@@ -56,12 +56,13 @@ jobs:
python -m pip install 'setuptools>=68'
python -m pip install --editable '.[test]'
python - <<'PY'
- from dopplergeist import matching, native_acoustic
+ from dopplergeist import matching, native_acoustic, native_cpa
assert callable(matching._native_fit_irls), "Native ridge extension did not build"
assert callable(matching._native_validate_ridge_events), "Native evidence validator did not build"
assert matching.RIDGE_FIT_BACKEND.startswith("native-c-")
assert native_acoustic._native is not None, "Native acoustic extension did not build"
assert native_acoustic.ACOUSTIC_BACKEND.startswith("native-c-")
+ assert native_cpa._native is not None, "Native CPA extension did not build"
print({"ridge_fit_backend": matching.RIDGE_FIT_BACKEND,
"acoustic_backend": native_acoustic.ACOUSTIC_BACKEND})
PY
@@ -100,14 +101,17 @@ jobs:
- name: Real-input regression tests
env:
DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json
+ DOPPLERGEIST_REAL_DENSE_QUERY: benchmark-inputs/dense-query.json
+ DOPPLERGEIST_REAL_DEVELOPMENT_INDEX: benchmark-inputs/reference-region.sqlite
DOPPLERGEIST_REAL_REFERENCE_ROOT: benchmark-inputs/reference-checkout
DOPPLERGEIST_REAL_REFERENCE_DISCOVERY: evidence/audio/w1-sensitivity-20261102/reference/discovery.json
- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py
+ run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_partition_query.py tests/test_real_native_cpa.py tests/test_real_native_ridge.py tests/test_real_native_ridge_batch.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py
- name: Verify the explicit Python fallback on recorded arithmetic
env:
DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json
DOPPLERGEIST_RIDGE_BACKEND: python
DOPPLERGEIST_ACOUSTIC_BACKEND: python
+ DOPPLERGEIST_CPA_BACKEND: python
run: python -m pytest -q tests/test_real_batch_scalar_parity.py tests/test_real_legacy_batch_cache.py
- name: Compare real CPA preparation
run: python scripts/benchmark_real_cpa_rows.py --baseline-source "$RUNNER_TEMP/baseline/src" --output results/cpa-rows.json --repeats 5 --iterations 20
@@ -121,11 +125,14 @@ jobs:
if [ -f src/dopplergeist/_native_acoustic.c ]; then
native_option+=(--require-current-native-acoustic)
fi
+ if [ -f src/dopplergeist/_native_cpa.c ]; then
+ native_option+=(--require-current-native-cpa)
+ fi
for width in 10 120; do
taskset -c "$benchmark_cpu" python scripts/benchmark_prepared_measurements.py compare --fixture "benchmark-inputs/point-$width.json" --baseline-source "$RUNNER_TEMP/baseline/src" --current-source src --repeats 3 --timeout-s 600 --output "results/performance-$width" "${native_option[@]}"
done
taskset -c "$benchmark_cpu" python scripts/benchmark_prepared_measurements.py compare \
- --fixture benchmark-inputs/dense-query.json --include-track-query \
+ --fixture benchmark-inputs/dense-query.json --include-track-query --clock-partition-query \
--baseline-source "$RUNNER_TEMP/baseline/src" --current-source src \
--repeats 3 --timeout-s 600 --output results/performance-dense-query \
"${native_option[@]}"
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 3a0d11e..71210f0 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -39,19 +39,24 @@ jobs:
run: python -m compileall -q src scripts
- name: CLI smoke check
run: dopplergeist --help
- - name: Check installed modules and packaged browser assets
+ - name: Check installed native acceleration and packaged browser assets
run: |
python -I - <<'PY'
from importlib import resources
from pathlib import Path
import dopplergeist
- from dopplergeist import automatic, jobs, service
+ from dopplergeist import _native_cpa, _native_ridge, automatic, jobs, service
package = Path(dopplergeist.__file__).resolve().parent
assert "site-packages" in package.parts, package
+ assert Path(_native_ridge.__file__).resolve().parent == package
+ assert Path(_native_cpa.__file__).resolve().parent == package
+ assert callable(_native_ridge.fit_irls_batch), "Native batch fitter did not build"
+ assert callable(_native_ridge.fit_statistics), "Native fit counters did not build"
+ assert callable(_native_cpa.select), "Native CPA reducer did not build"
web = resources.files("dopplergeist.web")
for name in ("index.html", "app.js", "style.css"):
assert web.joinpath(name).is_file(), name
assert web.joinpath(name).read_bytes(), name
- print("Installed package imports and browser assets passed; no audio or tracks generated.")
+ print("Installed native acceleration and browser assets passed; no audio or tracks generated.")
PY
diff --git a/docs/FLIGHT_SIEVE_PERFORMANCE.md b/docs/FLIGHT_SIEVE_PERFORMANCE.md
new file mode 100644
index 0000000..73c538e
--- /dev/null
+++ b/docs/FLIGHT_SIEVE_PERFORMANCE.md
@@ -0,0 +1,149 @@
+# Flight checking and sieve performance
+
+## Measured result
+
+The clock-partition query bound removes impossible whole flight segments before
+JSON decoding, spline construction, distance checks, and closest-approach work.
+The dense recorded replay queried **197 segments instead of 4,694**. It preserved
+all **16,101 scored candidate payloads**, the **100 retained results**, and the
+complete retention records and trial evidence.
+
+Three alternating baseline/current pairs gave these median CPU times:
+
+| Scope | Baseline | This change | Speedup |
+| --- | ---: | ---: | ---: |
+| Index query and conservative distance sieve | 6.413 s | 0.398 s | 16.11x |
+| CPA, association scoring, and retention | 26.500 s | 24.964 s | 1.06x |
+| Complete measured hot path | 34.073 s | 25.356 s | 1.34x |
+
+Each row has its own median; stage medians need not sum to the total median.
+Total wall-time medians were 34.146 and 25.400 seconds. The baseline is
+`b5ff48fe0e9d6e6feac7a4faf5268630f0c85d8b`, which already includes native ridge and
+acoustic solvers and numeric flight decoding. Earlier gains are not credited here.
+
+This replay uses the original recording's 16 prepared event groups and 724 ridge
+samples, one source-informed receiver, a full-day regional observation index, and one existing
+348-second association ownership partition. It is a computation benchmark, not
+an independent location-accuracy or nationwide-throughput result.
+
+Timing includes index traversal, track decoding, the distance sieve, CPA
+processing, association scoring, and retention. It excludes imports, index
+metadata, event preparation, downloads, terrain sampling, audio discovery, and
+refinement. Both sources used the same Python 3.12 runtime, dependencies,
+single-thread numerical settings, CPU affinity, recorded inputs, and GCC flags.
+The measured numerical functions were not replaced by counting wrappers.
+
+## What changed
+
+### Select possible source times before decoding tracks
+
+Let the first matched passage imply recording start `A` inside the existing
+half-open partition `[L, U)`. Its clock interval has tolerance `t_first`.
+Each effective tolerance is `quality.cpa_tolerance_s + offset_tolerance_s / 2`.
+A later passage with implied start `B` and tolerance `t_event` can participate
+only if its interval intersects the first interval. Thus it lies within the
+partition expanded by both tolerances. Any event may be first, so the bound uses
+the maximum effective tolerance across every event.
+
+Add each event's audio-relative CPA time to bound receive time. Subtract at most
+`maximum_CPA_radius / sound_speed` to bound emission time. Every inverse operation
+rounds outward, including subtraction at Unix timestamp precision.
+
+The query selects **complete segments** touching this envelope. It never clips a
+trajectory, removes an interior sample, or changes its spline. Serial and worker
+paths derive the same anchor once and use it for both querying and scoring.
+Silence filters, noncanonical settings, invalid tolerances, and disjoint query
+windows keep the original broad query.
+
+On the dense replay:
+
+| Work | Baseline | This change |
+| --- | ---: | ---: |
+| Decoded flight segments | 4,694 | 197 |
+| Decoded source samples | 713,100 | 40,690 |
+| CPA rows | 944 | 95 |
+| Association work units | 302,818 | 147,985 |
+| Unsupported CPA track evaluations | 62 | 4 |
+
+Every removed unsupported track's complete emission interval misses the proven
+query envelope. All four retained unsupported reports remain identical. The
+590,900 unknown trials and 15,777 unclassified candidate assessments also remain
+unchanged. The replay remains incomplete where its original evidence was incomplete.
+
+### Fuse closest-approach neighbor calculations
+
+An optional C extension traverses the existing bounded spatial-neighbor graph.
+It combines squared distances, local-minimum selection, and adjacent-segment
+projection without creating expanded observer-index and coordinate arrays.
+It preserves arithmetic order, asymmetric minima, first-edge ties, and row order.
+Python still performs endpoint spline optimization and final row construction.
+
+Five alternating trials of ten iterations on eight archived ADS-B segments gave
+these full CPA-stage results, including preparation:
+
+| Receiver points | Baseline CPU per iteration | Native CPA | Speedup |
+| ---: | ---: | ---: | ---: |
+| 1 | 1.411 ms | 1.091 ms | 1.29x |
+| 32 | 3.598 ms | 2.767 ms | 1.30x |
+| 128 | 4.977 ms | 3.572 ms | 1.39x |
+| 1,024 | 15.017 ms | 10.183 ms | 1.47x |
+| 4,096 | 48.600 ms | 38.321 ms | 1.27x |
+
+All 205 baseline comparisons matched exactly. This small real-track workload
+measures the CPA implementation, not complete search throughput.
+
+The extension is optional. Unsupported layouts, floating-point policies, or
+exceptional arithmetic retain the Python path. Set
+`DOPPLERGEIST_CPA_BACKEND=python` before starting the process to select it explicitly.
+Checkpoints record the active backend. Existing implementation-mismatch checks
+require a new output directory for ordinary checkpoints from older code.
+The separately supported legacy continuation requires Python ridge, acoustic,
+and CPA backends.
+
+### Batch the unchanged native ridge fit
+
+The native ridge extension now accepts a matrix of clock trials. It reuses scratch
+storage and releases the GIL once per batch. Each row keeps the existing six IRLS
+iterations, pairwise reductions, box constraints, and output arithmetic.
+Custom scalar hooks, old extension binaries, unusual inputs, and failed probes
+retain the established fallback. Native output may not overlap its inputs.
+
+The dense replay uses 5,421 native batches for 227,378 trial rows, plus 862 scalar
+fits. Batching alone measured a modest 1.06x dense improvement; its sparse replay
+was approximately neutral. The main sieve gain comes from doing less impossible
+work before decoding.
+
+## Reproduce and validate
+
+Build the optional extensions with `CC=gcc python setup.py build_ext --inplace`.
+Use the recorded-input preparation in `.github/workflows/real-performance.yml`;
+it restores public observations and creates `benchmark-inputs/dense-query.json`.
+Build the same existing native extensions in the pinned baseline checkout.
+
+```bash
+OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 \
+python scripts/benchmark_prepared_measurements.py compare \
+ --baseline-source /path/to/baseline/src --current-source src \
+ --fixture benchmark-inputs/dense-query.json \
+ --index benchmark-inputs/reference-region.sqlite \
+ --include-track-query --clock-partition-query \
+ --require-current-native --require-current-native-acoustic \
+ --require-current-native-cpa --repeats 3 --output results/flight-sieve
+
+python scripts/benchmark_real_cpa_rows.py \
+ --baseline-source /path/to/baseline/src --points 1 32 128 1024 4096 \
+ --repeats 5 --iterations 10 --skip-reuse-comparison \
+ --output results/flight-cpa.json
+```
+
+Use one available CPU affinity for both arms when comparing elapsed time.
+The normal comparison requires exact complete output equality. Partition-query
+mode also requires exact scored outcomes, rankings, trial evidence, and retention.
+Only declared early-work counts may decrease; each removed unsupported interval
+must be outside the query. Repeated outputs must be deterministic within each arm.
+Profiler runs and numerical wrapper instrumentation cannot produce speed claims.
+
+The added recorded-input tests cover native arithmetic, buffer overlap, fallback
+behavior, outward clock boundaries, complete-track preservation, serial/worker
+query integration, and checkpoint backend identity. No generated recordings or
+aircraft tracks are used. The remaining dominant cost is association scoring.
diff --git a/scripts/benchmark_prepared_measurements.py b/scripts/benchmark_prepared_measurements.py
index 038c87a..1f6720e 100644
--- a/scripts/benchmark_prepared_measurements.py
+++ b/scripts/benchmark_prepared_measurements.py
@@ -7,6 +7,7 @@ and identical observed inputs. This script generates no audio or trajectories.
from __future__ import annotations
import argparse
+from collections import Counter
from dataclasses import asdict
import json
import math
@@ -44,9 +45,15 @@ def canonical(value):
allow_nan=True, default=json_scalar)
-def query_tracks(index, points, events, budget, scoring, bounds, time_range):
+def query_tracks(index, points, events, budget, scoring, bounds, time_range, *,
+ anchor_window=None, clock_partition_query=False):
"""Count actual index traversal while exercising the production overload path."""
- from dopplergeist.tiled_search import _tracks_for_points
+ from dopplergeist import tiled_search
+
+ requested_time_range = list(time_range)
+ narrow_query = getattr(tiled_search, "_partition_query_time_range", None)
+ if clock_partition_query and callable(narrow_query):
+ time_range = narrow_query(time_range, events, scoring, {}, anchor_window)
if "elev_m" not in points:
raise ValueError("The recorded receiver hypothesis must include its elevation")
@@ -63,10 +70,13 @@ def query_tracks(index, points, events, budget, scoring, bounds, time_range):
yield track
radius = max(event["slant_range_m"][1] for event in events)
- tracks, points = _tracks_for_points(
+ tracks, points = tiled_search._tracks_for_points(
[CountedIndex()], bounds, time_range, budget, points, None, radius,
scoring.get("c", 343.0))
accounting = {
+ "requested_time_range_s": requested_time_range,
+ "effective_time_range_s": list(time_range),
+ "clock_partition_time_narrowed": list(time_range) != requested_time_range,
"queried_track_ids": sorted(inventory),
"queried_trajectory_samples": sum(inventory.values()),
"selected_track_ids": [track.id for track in tracks],
@@ -75,6 +85,72 @@ def query_tracks(index, points, events, budget, scoring, bounds, time_range):
return tracks, points, operations, accounting
+_PARTITION_WORK_COUNTS = {"cpa_rows", "per_event_survivors", "work_units", "unsupported_cpa_tracks"}
+
+
+def partition_outcomes(payload):
+ """Project only the explicitly allowed early-work diagnostics out of parity.
+
+ Scored candidate payloads, rankings, trial evidence, support during scoring,
+ retention records/bounds/stages, and all other counts remain exact checks.
+ The removed fields receive separate monotonicity and time-exclusion checks.
+ """
+ return {key: ({name: value for name, value in item.items()
+ if name not in _PARTITION_WORK_COUNTS} if key == "counts" else item)
+ for key, item in payload.items() if key != "track_query_accounting"}
+
+
+def verify_partition_work(baseline, current):
+ """Explain every permitted diagnostic change; reject unexplained omissions."""
+ before, after = (payload["track_query_accounting"] for payload in (baseline, current))
+ if before["requested_time_range_s"] != after["requested_time_range_s"]:
+ raise AssertionError("Partition comparison changed the requested source interval")
+ lower, upper = after["effective_time_range_s"]
+ broad_lower, broad_upper = before["effective_time_range_s"]
+ if not broad_lower <= lower <= upper <= broad_upper:
+ raise AssertionError("Partition query did not conservatively narrow the original query")
+ if not set(after["queried_track_ids"]) <= set(before["queried_track_ids"]):
+ raise AssertionError("Partition query introduced unobserved source tracks")
+ if after["queried_trajectory_samples"] > before["queried_trajectory_samples"]:
+ raise AssertionError("Partition query increased source sample traversal")
+ old_counts, new_counts = baseline["counts"], current["counts"]
+ for field in ("cpa_rows", "work_units"):
+ if new_counts.get(field, 0) > old_counts.get(field, 0):
+ raise AssertionError(f"Partition unexpectedly increased {field}")
+ old_survivors = old_counts.get("per_event_survivors", [])
+ new_survivors = new_counts.get("per_event_survivors", [])
+ if len(old_survivors) != len(new_survivors) or any(
+ new > old for old, new in zip(old_survivors, new_survivors)):
+ raise AssertionError("Partition unexpectedly changed event filtering")
+ old_support = {canonical(item): item for item in old_counts.get("unsupported_cpa_tracks", [])}
+ old_multiplicity = Counter(canonical(item) for item in old_counts.get("unsupported_cpa_tracks", []))
+ new_multiplicity = Counter(canonical(item) for item in new_counts.get("unsupported_cpa_tracks", []))
+ if not new_multiplicity <= old_multiplicity:
+ raise AssertionError("Partition introduced or changed an unsupported-track report")
+ removed = old_multiplicity - new_multiplicity
+ for identity in removed:
+ lo, hi = old_support[identity]["emission_time_range_s"]
+ if lo <= upper and hi >= lower:
+ raise AssertionError("Partition dropped an unsupported track touching the feasible emission interval")
+ return {
+ "requested_time_range_s": before["requested_time_range_s"],
+ "effective_time_range_s": after["effective_time_range_s"],
+ "excluded_unsupported_tracks_proved_outside_time_range": removed.total(),
+ "baseline": {"queried_tracks": len(before["queried_track_ids"]),
+ "queried_samples": before["queried_trajectory_samples"],
+ "selected_tracks": len(before["selected_track_ids"]),
+ "cpa_rows": old_counts["cpa_rows"],
+ "unsupported_cpa_tracks": old_multiplicity.total(),
+ "association_work_units": old_counts.get("work_units")},
+ "current": {"queried_tracks": len(after["queried_track_ids"]),
+ "queried_samples": after["queried_trajectory_samples"],
+ "selected_tracks": len(after["selected_track_ids"]),
+ "cpa_rows": new_counts["cpa_rows"],
+ "unsupported_cpa_tracks": new_multiplicity.total(),
+ "association_work_units": new_counts.get("work_units")},
+ }
+
+
def prepare(args):
import pandas as pd
from dopplergeist.data_access import FlightIndex
@@ -182,16 +258,24 @@ def worker(args):
if Path(dopplergeist.__file__).resolve().parent != source / "dopplergeist":
raise RuntimeError("Comparison imported a different source tree")
native_fit = getattr(matching, "_native_fit_irls", None)
+ native_module = getattr(matching, "_native_ridge_module", None)
+ native_statistics = getattr(native_module, "fit_statistics", None)
ridge_backend = getattr(matching, "RIDGE_FIT_BACKEND", "python-irls-v1")
acoustic_backend = getattr(geometry, "ACOUSTIC_BACKEND", "python-acoustic-v1")
try:
from dopplergeist.native_acoustic import _native as native_acoustic
except ImportError:
native_acoustic = None
+ try:
+ from dopplergeist.native_cpa import _native as native_cpa
+ except ImportError:
+ native_cpa = None
if args.require_native and not callable(native_fit):
raise RuntimeError("The selected source's native ridge fitter is unavailable")
if args.require_native_acoustic and native_acoustic is None:
raise RuntimeError("The selected source's native acoustic solver is unavailable")
+ if args.require_native_cpa and native_cpa is None:
+ raise RuntimeError("The selected source's native CPA reducer is unavailable")
fixture = json.loads(args.fixture.read_text())
index = FlightIndex(args.index or fixture["real_index"])
if index.report["input_hash"] != fixture["index_input_hash"]:
@@ -278,11 +362,17 @@ def worker(args):
all_rows.append(result)
return result
- matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic
+ if args.instrument_numerics:
+ matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic
+ else:
+ for name in ("scalar_fit_calls", "acoustic_calls", "native_fit_calls", "native_fit_successes"):
+ operations[name] = None
tiled_search.score_candidate = captured_score
- if callable(native_fit):
+ native_before = native_statistics() if callable(native_statistics) else None
+ if args.instrument_numerics and callable(native_fit) and native_before is None:
matching._native_fit_irls = counted_native
native_acoustic_before = native_acoustic.successful_calls() if native_acoustic is not None else 0
+ native_cpa_before = native_cpa.successful_calls() if native_cpa is not None else 0
query_accounting, query_operations, execution_stages = None, None, {}
began, cpu = time.perf_counter(), time.process_time()
try:
@@ -290,20 +380,23 @@ def worker(args):
query_began, query_cpu = time.perf_counter(), time.process_time()
tracks, points, query_operations, query_accounting = query_tracks(
index, points, events, budget, fixture["scoring"],
- fixture["query_bounds"], fixture["query_time_s"])
+ fixture["query_bounds"], fixture["query_time_s"],
+ anchor_window=fixture.get("anchor_window_s"),
+ clock_partition_query=args.clock_partition_query)
execution_stages["track_query_and_distance_sieve"] = {
"wall_seconds": time.perf_counter() - query_began,
"cpu_seconds": time.process_time() - query_cpu,
}
- if query_accounting["selected_track_ids"] != fixture["track_ids"]:
- raise AssertionError("The real query changed the ordered selected-track inventory")
- if query_accounting["selected_trajectory_samples"] != fixture["counts"]["trajectory_samples"]:
- raise AssertionError("The real query changed the selected trajectory sample count")
- prepared_query = fixture.get("track_query_preparation")
- if prepared_query is not None:
- if (len(query_accounting["queried_track_ids"]) != prepared_query["queried_trajectories"]
- or query_accounting["queried_trajectory_samples"] != prepared_query["queried_trajectory_samples"]):
- raise AssertionError("The real query changed the complete observed-track inventory")
+ if not query_accounting["clock_partition_time_narrowed"]:
+ if query_accounting["selected_track_ids"] != fixture["track_ids"]:
+ raise AssertionError("The real query changed the ordered selected-track inventory")
+ if query_accounting["selected_trajectory_samples"] != fixture["counts"]["trajectory_samples"]:
+ raise AssertionError("The real query changed the selected trajectory sample count")
+ prepared_query = fixture.get("track_query_preparation")
+ if prepared_query is not None:
+ if (len(query_accounting["queried_track_ids"]) != prepared_query["queried_trajectories"]
+ or query_accounting["queried_trajectory_samples"] != prepared_query["queried_trajectory_samples"]):
+ raise AssertionError("The real query changed the complete observed-track inventory")
scoring_began, scoring_cpu = time.perf_counter(), time.process_time()
partition = ({"anchor_window": fixture["anchor_window_s"]}
if "anchor_window_s" in fixture else {})
@@ -321,31 +414,59 @@ def worker(args):
tiled_search.score_candidate = original_score
if callable(native_fit):
matching._native_fit_irls = native_fit
+ if native_before is not None:
+ native_after = native_statistics()
+ delta = {key: native_after[key] - native_before[key] for key in native_before}
+ operations["native_fit_calls"] = delta["scalar_calls"] + delta["batch_rows"]
+ operations["native_fit_successes"] = delta["scalar_successes"] + delta["batch_successful_rows"]
+ operations["native_batch_calls"] = delta["batch_calls"]
+ operations["native_batch_rows"] = delta["batch_rows"]
if native_acoustic is not None:
operations["native_acoustic_successes"] = native_acoustic.successful_calls() - native_acoustic_before
+ if native_cpa is not None:
+ operations["native_cpa_successes"] = native_cpa.successful_calls() - native_cpa_before
for owner, attribute, function in reversed(phase_patches):
setattr(owner, attribute, function)
expected_counts = fixture.get("expected_counts")
if expected_counts is not None:
actual_counts = {"cpa_rows": counts["cpa_rows"],
"associations": counts["candidates"], "retained": len(ranked)}
- if actual_counts != expected_counts:
+ changed_work_only = (args.clock_partition_query
+ and actual_counts["cpa_rows"] <= expected_counts["cpa_rows"]
+ and all(actual_counts[key] == expected_counts[key] for key in ("associations", "retained")))
+ if actual_counts != expected_counts and not changed_work_only:
raise AssertionError(f"Recorded workload counts changed: {actual_counts}")
- # macOS reports bytes and Linux reports KiB. Keep scoring usable where
- # resource or its unit convention is unavailable, without guessing a value.
+ # Linux ru_maxrss can include the comparison parent's high-water mark
+ # inherited across fork/exec. VmHWM belongs to this worker's address space.
+ # Keep unknown platforms usable without guessing a memory unit or scope.
peak_rss_mib = None
- if resource is not None and (sys.platform == "darwin" or sys.platform.startswith("linux")):
- rss_divisor = 1024 * 1024 if sys.platform == "darwin" else 1024
- peak_rss_mib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rss_divisor
+ peak_rss_method = "Unavailable on this platform"
+ if sys.platform.startswith("linux"):
+ try:
+ for line in Path("/proc/self/status").read_text().splitlines():
+ if line.startswith("VmHWM:"):
+ fields = line.split()
+ if len(fields) == 3 and fields[2] == "kB":
+ peak_rss_mib = int(fields[1]) / 1024
+ peak_rss_method = "/proc/self/status VmHWM after exec"
+ break
+ except (OSError, ValueError):
+ pass
+ elif resource is not None and sys.platform == "darwin":
+ peak_rss_mib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (1024 * 1024)
+ peak_rss_method = "resource.getrusage(RUSAGE_SELF).ru_maxrss"
checksum = ledger.checksum(stage)
if not ledger.stage_valid(stage, checksum):
raise AssertionError("The scoring pass produced an invalid retention ledger")
if not operations["scored_candidates"]:
raise AssertionError("The benchmark did not score any real associations")
- if args.require_native and not operations["native_fit_successes"]:
+ if (args.require_native and (args.instrument_numerics or native_before is not None)
+ and not operations["native_fit_successes"]):
raise AssertionError("No successful native ridge fit occurred inside the timed scoring call")
if args.require_native_acoustic and not operations["native_acoustic_successes"]:
raise AssertionError("No successful native acoustic solve occurred inside the timed scoring call")
+ if args.require_native_cpa and not operations.get("native_cpa_successes"):
+ raise AssertionError("No successful native CPA reduction occurred inside the timed scoring call")
with ledger.connect() as database:
retention = {name: database.execute(f"SELECT * FROM {name} ORDER BY 1,2").fetchall()
for name in ("records", "bounds", "stages")}
@@ -354,11 +475,18 @@ def worker(args):
if query_accounting is not None:
results["track_query_accounting"] = query_accounting
write_new(args.output / "results.json", results)
+ diagnostic = args.instrument_numerics or args.stage_timing or sys.getprofile() is not None
summary = {"wall_seconds": elapsed, "cpu_seconds": cpu_seconds,
"load_seconds_excluded": load_seconds, "operations": operations,
+ "numerical_functions_canonical": not (args.instrument_numerics or args.stage_timing),
+ "performance_timing_eligible": not diagnostic,
+ "ridge_execution_check": ("In-module successful-call counters" if native_before is not None else
+ "Wrapped successful-call counters" if args.instrument_numerics else
+ "Backend availability; successful-call count unmeasured"),
"ridge_fit_backend": ridge_backend, "acoustic_backend": acoustic_backend,
"native_extension_available": callable(native_fit),
"peak_rss_mib": peak_rss_mib,
+ "peak_rss_measurement": peak_rss_method,
"peak_rss_scope": ("Worker lifetime through scoring, including imports and input loading"
if peak_rss_mib is not None else "Unavailable on this platform"),
"retained": len(ranked), "independent_groups": len(events), "scope": fixture["scope"]}
@@ -366,6 +494,7 @@ def worker(args):
summary["anchor_window_s"] = fixture["anchor_window_s"]
if args.include_track_query:
summary["track_query_operations"] = query_operations
+ summary["clock_partition_time_narrowed"] = query_accounting["clock_partition_time_narrowed"]
summary["execution_stage_seconds"] = execution_stages
summary["timing_scope"] = (
"Initial local index traversal and conservative distance sieve, followed by complete "
@@ -384,7 +513,7 @@ def compare(args):
environment = dict(os.environ)
for variable in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS"):
environment[variable] = "1"
- runs, expected = [], None
+ runs, expected, source_payloads, source_canonical = [], None, {}, {}
for repetition in range(args.repeats):
order = ("baseline", "current") if repetition % 2 == 0 else ("current", "baseline")
for name in order:
@@ -397,16 +526,28 @@ def compare(args):
command.append("--stage-timing")
if args.include_track_query:
command.append("--include-track-query")
+ if args.clock_partition_query:
+ command.append("--clock-partition-query")
+ if args.instrument_numerics:
+ command.append("--instrument-numerics")
if name == "current" and args.require_current_native:
command.append("--require-native")
if name == "current" and args.require_current_native_acoustic:
command.append("--require-native-acoustic")
+ if name == "current" and args.require_current_native_cpa:
+ command.append("--require-native-cpa")
subprocess.run(command, check=True, env=environment, timeout=args.timeout_s)
- payload = canonical(json.loads((output / "results.json").read_text()))
+ result = json.loads((output / "results.json").read_text())
+ complete = canonical(result)
+ if name in source_canonical and source_canonical[name] != complete:
+ raise AssertionError(f"Repeated source output was not deterministic: {output}")
+ source_canonical[name] = complete
+ source_payloads[name] = result
+ payload = canonical(partition_outcomes(result) if args.clock_partition_query else result)
if expected is None:
expected = payload
elif payload != expected:
- raise AssertionError(f"Exact selected-track, candidate, diagnostic, support, or retention mismatch: {output}")
+ raise AssertionError(f"Exact candidate, diagnostic, support, or retention mismatch: {output}")
summary = json.loads((output / "summary.json").read_text())
runs.append({"source": name, "repetition": repetition + 1, **summary})
medians = {}
@@ -415,12 +556,24 @@ def compare(args):
for metric in ("wall_seconds", "cpu_seconds", "peak_rss_mib"):
values = [row[metric] for row in runs if row["source"] == name and row[metric] is not None]
medians[name][metric] = statistics.median(values) if values else None
- report = {"identical_results": True, "repeats": args.repeats,
+ report = {"identical_results": source_canonical["baseline"] == source_canonical["current"],
+ "identical_scored_candidates_rankings_and_retention": True, "repeats": args.repeats,
"fixture": str(args.fixture.resolve()),
"sources": {name: str(path) for name, path in sources.items()}, "medians": medians,
"speedup": {metric: medians["baseline"][metric] / medians["current"][metric]
for metric in ("wall_seconds", "cpu_seconds")},
"scope": runs[0]["scope"], "runs": runs}
+ if args.clock_partition_query:
+ report["clock_partition_query_verification"] = verify_partition_work(
+ source_payloads["baseline"], source_payloads["current"])
+ report["equality_scope"] = (
+ "Exact scored candidate payloads, rankings, trial/support evidence and retention records. "
+ "Early query/CPA/event/work counts may decrease; every removed unsupported track's "
+ "complete emission interval must miss the proved partition interval.")
+ report["performance_timing_eligible"] = all(row["performance_timing_eligible"] for row in runs)
+ if not report["performance_timing_eligible"]:
+ report["diagnostic_ratio"] = report["speedup"]
+ report["speedup"] = None
if args.include_track_query:
report["timing_scope"] = runs[0]["timing_scope"]
report["track_query_included"] = True
@@ -466,19 +619,28 @@ def main():
help="Collect inclusive phase times in a separate diagnostic run")
comparison.add_argument("--include-track-query", action="store_true",
help="Include initial index traversal and the production overload sieve in each timed worker")
+ comparison.add_argument("--clock-partition-query", action="store_true",
+ help="Exercise the production clock-partition query bound where available; verify exact outcomes and explain reduced work")
+ comparison.add_argument("--instrument-numerics", action="store_true",
+ help="Diagnostic only: wrap numerical calls for counters; changes execution overhead")
comparison.add_argument("--require-current-native", action="store_true",
help="Require successful native ridge fits inside each current scoring run")
comparison.add_argument("--require-current-native-acoustic", action="store_true",
help="Require successful native acoustic solves inside each current scoring run")
+ comparison.add_argument("--require-current-native-cpa", action="store_true",
+ help="Require successful native CPA reductions inside each current scoring run")
internal = commands.add_parser("_worker", help=argparse.SUPPRESS)
internal.add_argument("--source", required=True, type=Path)
internal.add_argument("--fixture", required=True, type=Path)
internal.add_argument("--index", type=Path)
internal.add_argument("--output", required=True, type=Path)
internal.add_argument("--stage-timing", action="store_true", help=argparse.SUPPRESS)
+ internal.add_argument("--instrument-numerics", action="store_true", help=argparse.SUPPRESS)
internal.add_argument("--include-track-query", action="store_true", help=argparse.SUPPRESS)
+ internal.add_argument("--clock-partition-query", action="store_true", help=argparse.SUPPRESS)
internal.add_argument("--require-native", action="store_true", help=argparse.SUPPRESS)
internal.add_argument("--require-native-acoustic", action="store_true", help=argparse.SUPPRESS)
+ internal.add_argument("--require-native-cpa", action="store_true", help=argparse.SUPPRESS)
args = parser.parse_args()
if args.command == "prepare":
if args.clock_half_width_s < 0:
@@ -492,6 +654,10 @@ def main():
elif args.command == "compare":
if args.repeats < 1 or args.timeout_s <= 0:
parser.error("--repeats and --timeout-s must be positive")
+ if args.clock_partition_query:
+ fixture = json.loads(args.fixture.read_text())
+ if not args.include_track_query or "anchor_window_s" not in fixture:
+ parser.error("--clock-partition-query requires --include-track-query and a recorded anchor partition")
compare(args)
else:
worker(args)
diff --git a/scripts/benchmark_real_cpa_rows.py b/scripts/benchmark_real_cpa_rows.py
index 03d74d5..bcbbb88 100644
--- a/scripts/benchmark_real_cpa_rows.py
+++ b/scripts/benchmark_real_cpa_rows.py
@@ -139,18 +139,26 @@ def main():
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--repeats", type=int, default=5)
parser.add_argument("--iterations", type=int, default=20)
+ parser.add_argument("--points", type=int, nargs="+", default=[1, 32, 128])
+ parser.add_argument("--skip-reuse-comparison", action="store_true",
+ help="Skip the separate historical preparation-reuse comparison")
args = parser.parse_args()
if args.output.exists():
parser.error("Use a new output path")
if args.repeats < 1 or args.iterations < 1:
parser.error("repeats and iterations must be positive")
+ if any(count < 1 for count in args.points):
+ parser.error("points must be positive")
path = args.baseline_source / "dopplergeist/search.py"
spec = importlib.util.spec_from_file_location("dopplergeist._cpa_before", path)
baseline = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = baseline
spec.loader.exec_module(baseline)
tracks = recorded_tracks()
- workloads = {str(count): recorded_points(tracks, count) for count in (1, 32, 128)}
+ workloads = {str(count): recorded_points(tracks, count) for count in args.points}
+ # Recent baselines already support the bounded row path. Comparing their
+ # per-track DataFrames against current rows would remeasure an older change.
+ baseline_row_mode = hasattr(baseline, "_cpa_rows")
checks = 0
for points in workloads.values():
wide_ids = pd.concat([points.iloc[:1], points.iloc[:1]], ignore_index=True)
@@ -175,12 +183,12 @@ def main():
search.cpa_table(supported, candidate_points, radius, c, step, chunk_size),
check_exact=True)
checks += 1
- timings = []
+ timings, raw_timings = [], []
for name, points in workloads.items():
runs = {"baseline": [], "current": []}
final = None
for repeat in range(args.repeats):
- arms = [("baseline", baseline, False), ("current", search, True)]
+ arms = [("baseline", baseline, baseline_row_mode), ("current", search, True)]
if repeat % 2:
arms.reverse()
for label, module, row_mode in arms:
@@ -194,8 +202,41 @@ def main():
"unsupported_tracks": len(final[1]), "iterations_per_trial": args.iterations,
"cpu_s": runs, "median_cpu_s": medians,
"speedup": medians["baseline"]/medians["current"]})
+ if hasattr(baseline, "_PreparedCPATrack"):
+ observers = ecef(points.lat.to_numpy(), points.lon.to_numpy(), points.elev_m.to_numpy())
+ point_ids = points.point_id.to_numpy(dtype=int)
+ prepared = {"baseline": [], "current": []}
+ for label, module in (("baseline", baseline), ("current", search)):
+ for track in tracks:
+ try:
+ if not track.outside_observer_radius(observers, 20000., 343.):
+ prepared[label].append(module._PreparedCPATrack(track, 343., 2.))
+ except AcousticSupportError:
+ pass
+
+ def raw_rows(label):
+ return [row for item in prepared[label]
+ for row in item.raw_rows(observers, point_ids, 20000.)]
+
+ # Warm each lazy tree once; sampling/tree preparation is included
+ # in the complete-stage timing above, not this loop measurement.
+ old_rows, new_rows = raw_rows("baseline"), raw_rows("current")
+ assert old_rows == new_rows, "Raw CPA proposals or order changed"
+ checks += 1
+ raw_runs = {"baseline": [], "current": []}
+ for repeat in range(args.repeats):
+ for label in (("current", "baseline") if repeat % 2 else ("baseline", "current")):
+ started = time.process_time()
+ for _ in range(args.iterations):
+ raw_rows(label)
+ raw_runs[label].append(time.process_time()-started)
+ raw_medians = {label: statistics.median(values) for label, values in raw_runs.items()}
+ raw_timings.append({"receiver_points": int(name), "raw_cpa_rows": len(new_rows),
+ "iterations_per_trial": args.iterations, "cpu_s": raw_runs,
+ "median_cpu_s": raw_medians,
+ "speedup": raw_medians["baseline"]/raw_medians["current"]})
chunked_timings = []
- for size in (1024, 4096):
+ for size in (() if args.skip_reuse_comparison else (1024, 4096)):
points = recorded_points(tracks, size)
before, old_counts = counted_chunked_stage(tracks, points, False)
after, new_counts = counted_chunked_stage(tracks, points, True)
@@ -223,6 +264,9 @@ def main():
"cpu_affinity": sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else None},
"observed_samples": sum(len(track.time) for track in tracks),
"exact_comparisons": checks, "timings": timings,
+ "baseline_row_mode": baseline_row_mode,
+ "raw_rows_scope": "Same sorted spatial query, spline endpoint solves, speed evaluation and row assembly; sampled track and lazy tree already prepared; internal chunks of at most 512 observers",
+ "raw_rows_timings": raw_timings,
"chunked_comparison": "Same current code and recorded inputs, trajectory/tree reuse disabled versus enabled; normal 100000-row budget",
"chunked_timings": chunked_timings}
args.output.parent.mkdir(parents=True, exist_ok=True)
diff --git a/setup.py b/setup.py
index d8f2687..d5b8184 100644
--- a/setup.py
+++ b/setup.py
@@ -21,4 +21,13 @@ acoustic_extension = Extension(
libraries=[] if os.name == "nt" else ["m"],
)
-setup(ext_modules=[extension, acoustic_extension])
+cpa_extension = Extension(
+ "dopplergeist._native_cpa",
+ ["src/dopplergeist/_native_cpa.c"],
+ optional=True,
+ extra_compile_args=["/O2", "/fp:strict"] if os.name == "nt" else
+ ["-O3", "-ffp-contract=off", "-fno-fast-math"],
+ libraries=[] if os.name == "nt" else ["m"],
+)
+
+setup(ext_modules=[extension, acoustic_extension, cpa_extension])
diff --git a/src/dopplergeist/_native_cpa.c b/src/dopplergeist/_native_cpa.c
new file mode 100644
index 0000000..f768c7e
--- /dev/null
+++ b/src/dopplergeist/_native_cpa.c
@@ -0,0 +1,184 @@
+/* Optional exact-order CPA neighbour reduction. No NumPy C headers.
+ * Keep the Python spatial query, endpoint spline solver and row construction.
+ * This loop removes the expanded observer indices and temporary N-by-3 arrays.
+ */
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <fenv.h>
+#include <math.h>
+#include <stdint.h>
+#include <string.h>
+
+typedef struct { unsigned long long successful_calls; } module_state;
+
+/* NumPy reduces the three coordinates in order. Do not reassociate or fuse
+ * these additions and multiplications; setup.py disables both optimizations.
+ */
+static double sum3(double a, double b, double c) {
+ return ((0.0 + a) + b) + c;
+}
+
+static double distance2(const double *position, const double *observer) {
+ double x = position[0] - observer[0];
+ double y = position[1] - observer[1];
+ double z = position[2] - observer[2];
+ return sum3(x*x, y*y, z*z);
+}
+
+static int project(const double *first, const double *last,
+ const double *observer, double *fraction, double *squared) {
+ double delta[3], residual[3], projected[3];
+ for (int j = 0; j < 3; ++j) {
+ delta[j] = last[j] - first[j];
+ residual[j] = first[j] - observer[j];
+ }
+ double length2 = sum3(delta[0]*delta[0], delta[1]*delta[1], delta[2]*delta[2]);
+ double dot = sum3(residual[0]*delta[0], residual[1]*delta[1], residual[2]*delta[2]);
+ double denominator = length2 < 1e-12 ? 1e-12 : length2;
+ double f = -dot / denominator;
+ if (!isfinite(length2) || !isfinite(dot) || !isfinite(f)) return 0;
+ if (f < 0.0) f = 0.0;
+ else if (f > 1.0) f = 1.0;
+ for (int j = 0; j < 3; ++j)
+ projected[j] = residual[j] + f*delta[j];
+ *squared = sum3(projected[0]*projected[0], projected[1]*projected[1],
+ projected[2]*projected[2]);
+ *fraction = f;
+ return isfinite(*squared);
+}
+
+static int reduce_graph(const double *position, Py_ssize_t position_count,
+ const double *observers, Py_ssize_t observer_count,
+ const Py_ssize_t *counts, const Py_ssize_t *sample,
+ Py_ssize_t sample_count, Py_ssize_t *endpoints,
+ Py_ssize_t *indices, double *values, Py_ssize_t capacity,
+ Py_ssize_t *sizes) {
+ Py_ssize_t base = 0;
+ for (Py_ssize_t oi = 0; oi < observer_count; ++oi) {
+ Py_ssize_t count = counts[oi];
+ if (count < 0 || count > sample_count-base) return 0;
+ const Py_ssize_t *samples = sample + base;
+ const double *observer = observers + 3*oi;
+ for (int j = 0; j < 3; ++j) if (!isfinite(observer[j])) return 0;
+ for (Py_ssize_t j = 0; j < count; ++j)
+ if (samples[j] < 0 || samples[j] >= position_count) return 0;
+ base += count;
+ if (!count) continue;
+ double previous = distance2(position + 3*samples[0], observer);
+ if (!isfinite(previous)) return 0;
+ if (count == 1) continue;
+ double current = distance2(position + 3*samples[1], observer);
+ if (!isfinite(current)) return 0;
+ if (samples[0] == 0 && samples[1] == 1 && previous <= current)
+ endpoints[sizes[0]++] = oi;
+ for (Py_ssize_t j = 1; j < count-1; ++j) {
+ double following = distance2(position + 3*samples[j+1], observer);
+ if (!isfinite(following)) return 0;
+ Py_ssize_t k = samples[j];
+ if (samples[j-1] == k-1 && samples[j+1] == k+1
+ && current <= previous && current < following) {
+ double fraction[2], squared[2];
+ if (!project(position + 3*(k-1), position + 3*k, observer,
+ fraction, squared)
+ || !project(position + 3*k, position + 3*(k+1), observer,
+ fraction+1, squared+1)) return 0;
+ /* np.argmin selects the first edge on an exact tie. */
+ int edge = squared[1] < squared[0];
+ Py_ssize_t out = sizes[2]++;
+ if (out >= capacity) return 0;
+ indices[out] = oi;
+ indices[capacity+out] = edge ? k : k-1;
+ values[out] = fraction[edge];
+ values[capacity+out] = squared[edge];
+ }
+ previous = current;
+ current = following;
+ }
+ if (samples[count-1] == position_count-1
+ && samples[count-2] == position_count-2 && current <= previous)
+ endpoints[observer_count+sizes[1]++] = oi;
+ }
+ return base == sample_count;
+}
+
+static int view_array(PyObject *object, Py_buffer *view, int integer, int writable) {
+ int flags = PyBUF_FORMAT | PyBUF_ND | PyBUF_STRIDES;
+ if (writable) flags |= PyBUF_WRITABLE;
+ if (PyObject_GetBuffer(object, view, flags) < 0) return 0;
+ size_t size = integer ? sizeof(Py_ssize_t) : sizeof(double);
+ if (view->itemsize != (Py_ssize_t)size || !view->format
+ || !PyBuffer_IsContiguous(view, 'C')
+ || (uintptr_t)view->buf % size != 0) return 0;
+ if (!integer) return strcmp(view->format, "d") == 0;
+ return strlen(view->format) == 1 && strchr("nliq", view->format[0]) != NULL;
+}
+
+static int matrix_shape(const Py_buffer *view, Py_ssize_t rows, Py_ssize_t columns) {
+ return view->ndim == 2 && view->shape[0] == rows && view->shape[1] == columns;
+}
+
+static int disjoint(const Py_buffer *a, const Py_buffer *b) {
+ uintptr_t first = (uintptr_t)a->buf, second = (uintptr_t)b->buf;
+ if (a->len < 0 || b->len < 0
+ || (uintptr_t)a->len > UINTPTR_MAX-first
+ || (uintptr_t)b->len > UINTPTR_MAX-second) return 0;
+ return !a->len || !b->len || first+(uintptr_t)a->len <= second
+ || second+(uintptr_t)b->len <= first;
+}
+
+static PyObject *select_edges(PyObject *self, PyObject *args) {
+ PyObject *objects[7];
+ Py_buffer views[7] = {{0}};
+ Py_ssize_t sizes[3] = {0, 0, 0};
+ int success = 0;
+ if (!PyArg_ParseTuple(args, "OOOOOOO", &objects[0], &objects[1], &objects[2],
+ &objects[3], &objects[4], &objects[5], &objects[6])) return NULL;
+ if (fegetround() != FE_TONEAREST) Py_RETURN_NONE;
+ for (int i = 0; i < 7; ++i)
+ if (!view_array(objects[i], views+i, i >= 2 && i <= 5, i >= 4)) goto done;
+ for (int i = 4; i < 7; ++i)
+ for (int j = 0; j < i; ++j)
+ if (!disjoint(views+i, views+j)) goto done;
+ if (views[0].ndim != 2 || views[0].shape[0] < 2 || views[0].shape[1] != 3
+ || views[1].ndim != 2 || views[1].shape[1] != 3
+ || views[2].ndim != 1 || views[2].shape[0] != views[1].shape[0]
+ || views[3].ndim != 1
+ || !matrix_shape(views+4, 2, views[1].shape[0])
+ || !matrix_shape(views+5, 2, views[3].shape[0]/2)
+ || !matrix_shape(views+6, 2, views[3].shape[0]/2)) goto done;
+ /* Failed probes leave NumPy's warning/exception behavior to the reference
+ * path. Do not leak the probe's floating-point flags into that fallback.
+ */
+ fenv_t previous_environment;
+ if (fegetenv(&previous_environment) != 0) goto done;
+ Py_BEGIN_ALLOW_THREADS
+ if (feclearexcept(FE_ALL_EXCEPT) == 0) {
+ success = reduce_graph(views[0].buf, views[0].shape[0], views[1].buf,
+ views[1].shape[0], views[2].buf, views[3].buf,
+ views[3].shape[0], views[4].buf, views[5].buf,
+ views[6].buf, views[3].shape[0]/2, sizes);
+ if (fetestexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW)) success = 0;
+ }
+ if (fesetenv(&previous_environment) != 0) success = 0;
+ Py_END_ALLOW_THREADS
+done:
+ for (int i = 0; i < 7; ++i) if (views[i].obj) PyBuffer_Release(views+i);
+ if (PyErr_Occurred()) PyErr_Clear();
+ if (!success) Py_RETURN_NONE;
+ ((module_state *)PyModule_GetState(self))->successful_calls++;
+ return Py_BuildValue("nnn", sizes[0], sizes[1], sizes[2]);
+}
+
+static PyObject *successful_calls(PyObject *self, PyObject *unused) {
+ return PyLong_FromUnsignedLongLong(((module_state *)PyModule_GetState(self))->successful_calls);
+}
+
+static PyMethodDef methods[] = {
+ {"select", select_edges, METH_VARARGS, "Reduce a canonical sorted CPA neighbour graph."},
+ {"successful_calls", successful_calls, METH_NOARGS, "Count successful native CPA reductions."},
+ {NULL, NULL, 0, NULL}
+};
+static struct PyModuleDef module = {
+ PyModuleDef_HEAD_INIT, "_native_cpa", NULL, sizeof(module_state), methods
+};
+PyMODINIT_FUNC PyInit__native_cpa(void) { return PyModule_Create(&module); }
diff --git a/src/dopplergeist/_native_ridge.c b/src/dopplergeist/_native_ridge.c
index 6dfc964..b343ab3 100644
--- a/src/dopplergeist/_native_ridge.c
+++ b/src/dopplergeist/_native_ridge.c
@@ -1,8 +1,14 @@
#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <math.h>
+#include <stdint.h>
#include <string.h>
+typedef struct {
+ unsigned long long scalar_calls, scalar_successes;
+ unsigned long long batch_calls, batch_rows, batch_successful_rows;
+} ridge_state;
+
static int
get_vector(PyObject *object, Py_buffer *view)
{
@@ -151,43 +157,13 @@ source_fit(const Py_buffer *factor, const Py_buffer *u, const Py_buffer *y,
}
}
-static PyObject *
-fit_irls(PyObject *self, PyObject *args)
+/* One numerical implementation serves scalar and batch entry points. The caller
+ * owns the six scratch vectors, so a batch does not allocate them per clock. */
+static int
+fit_row(const Py_buffer *views, Py_ssize_t count, double lower, double upper,
+ double cap, double sigma, Py_ssize_t independent_samples,
+ double *effective, double *output)
{
- PyObject *objects[4];
- double lower, upper, cap, sigma;
- Py_ssize_t independent_samples;
- Py_buffer views[4] = {{0}};
- int acquired = 0;
- double *effective = NULL;
- PyObject *result = NULL;
-
- if (!PyArg_ParseTuple(args, "OOOOddddn", &objects[0], &objects[1],
- &objects[2], &objects[3], &lower, &upper, &cap,
- &sigma, &independent_samples)) {
- return NULL;
- }
- for (int i = 0; i < 4; ++i) {
- if (get_vector(objects[i], &views[i]) < 0) goto done;
- ++acquired;
- }
- Py_ssize_t count = views[0].shape[0];
- if (count == 0 || views[1].shape[0] != count || views[2].shape[0] != count ||
- views[3].shape[0] != count || !isfinite(lower) || !isfinite(upper) ||
- lower > upper || !isfinite(cap) || cap < 0.0 || !isfinite(sigma) ||
- sigma <= 0.0 || independent_samples < 1) {
- PyErr_SetString(PyExc_ValueError, "invalid robust ridge-fit inputs");
- goto done;
- }
- if ((size_t)count > SIZE_MAX / (6 * sizeof(double))) {
- PyErr_NoMemory();
- goto done;
- }
- effective = PyMem_Malloc((size_t)count * 6 * sizeof(double));
- if (effective == NULL) {
- PyErr_NoMemory();
- goto done;
- }
int valid = 1;
for (Py_ssize_t i = 0; i < count; ++i) {
double fi = at(&views[0], i), ui = at(&views[1], i);
@@ -199,18 +175,13 @@ fit_irls(PyObject *self, PyObject *args)
}
effective[i] = wi;
}
- if (!valid) {
- Py_INCREF(Py_None);
- result = Py_None;
- goto done;
- }
+ if (!valid) return 0;
double f0 = NAN, drift = NAN, loss = NAN, rmse = NAN, outlier_fraction = NAN;
double *scratch = effective + count;
double prior = cap == 0.0 ? 0.0 : scalar_square(2.0 * sigma / cap);
double loss_scale = 2.0 * scalar_square(sigma);
valid = 1;
- Py_BEGIN_ALLOW_THREADS
for (int iteration = 0; iteration < 6; ++iteration) {
source_fit(&views[0], &views[1], &views[2], effective, scratch, count,
lower, upper, cap, prior, &f0, &drift);
@@ -253,9 +224,60 @@ fit_irls(PyObject *self, PyObject *args)
loss = -sqrt(loss + penalty);
valid = isfinite(loss) && isfinite(rmse) && isfinite(outlier_fraction);
}
- Py_END_ALLOW_THREADS
+ output[0] = loss;
+ output[1] = f0;
+ output[2] = drift;
+ output[3] = rmse;
+ output[4] = outlier_fraction;
+ return valid;
+}
- if (valid) result = Py_BuildValue("(ddddd)", loss, f0, drift, rmse, outlier_fraction);
+static PyObject *
+fit_irls(PyObject *self, PyObject *args)
+{
+ ridge_state *state = PyModule_GetState(self);
+ ++state->scalar_calls;
+ PyObject *objects[4];
+ double lower, upper, cap, sigma, output[5];
+ Py_ssize_t independent_samples;
+ Py_buffer views[4] = {{0}};
+ int acquired = 0;
+ double *effective = NULL;
+ PyObject *result = NULL;
+
+ if (!PyArg_ParseTuple(args, "OOOOddddn", &objects[0], &objects[1],
+ &objects[2], &objects[3], &lower, &upper, &cap,
+ &sigma, &independent_samples)) return NULL;
+ for (int i = 0; i < 4; ++i) {
+ if (get_vector(objects[i], &views[i]) < 0) goto done;
+ ++acquired;
+ }
+ Py_ssize_t count = views[0].shape[0];
+ if (count == 0 || views[1].shape[0] != count || views[2].shape[0] != count ||
+ views[3].shape[0] != count || !isfinite(lower) || !isfinite(upper) ||
+ lower > upper || !isfinite(cap) || cap < 0.0 || !isfinite(sigma) ||
+ sigma <= 0.0 || independent_samples < 1) {
+ PyErr_SetString(PyExc_ValueError, "invalid robust ridge-fit inputs");
+ goto done;
+ }
+ if ((size_t)count > SIZE_MAX / (6 * sizeof(double))) {
+ PyErr_NoMemory();
+ goto done;
+ }
+ effective = PyMem_Malloc((size_t)count * 6 * sizeof(double));
+ if (effective == NULL) {
+ PyErr_NoMemory();
+ goto done;
+ }
+ int valid;
+ Py_BEGIN_ALLOW_THREADS
+ valid = fit_row(views, count, lower, upper, cap, sigma, independent_samples,
+ effective, output);
+ Py_END_ALLOW_THREADS
+ if (valid) {
+ ++state->scalar_successes;
+ result = Py_BuildValue("(ddddd)", output[0], output[1], output[2], output[3], output[4]);
+ }
else {
Py_INCREF(Py_None);
result = Py_None;
@@ -267,6 +289,146 @@ done:
return result;
}
+static int
+get_matrix(PyObject *object, Py_buffer *view, int writable)
+{
+ int flags = PyBUF_FORMAT | PyBUF_ND | PyBUF_STRIDES;
+ if (writable) flags |= PyBUF_WRITABLE;
+ if (PyObject_GetBuffer(object, view, flags) < 0) return -1;
+ if (view->ndim != 2 || view->itemsize != (Py_ssize_t)sizeof(double) ||
+ view->shape == NULL || view->strides == NULL || view->format == NULL ||
+ strcmp(view->format, "d") != 0 || !PyBuffer_IsContiguous(view, 'C')) {
+ PyBuffer_Release(view);
+ view->obj = NULL;
+ PyErr_SetString(PyExc_TypeError, "expected a contiguous native float64 matrix");
+ return -1;
+ }
+ return 0;
+}
+
+/* Include the holes between strided elements. A conservative span is enough to
+ * prove that writing a result cannot change a later row's input. */
+static int
+buffer_span(const Py_buffer *view, uintptr_t *begin, uintptr_t *end)
+{
+ uintptr_t low = (uintptr_t)view->buf, high = low;
+ if (view->buf == NULL || view->itemsize <= 0) return 0;
+ for (int axis = 0; axis < view->ndim; ++axis) {
+ if (view->shape[axis] <= 0 ||
+ (view->suboffsets != NULL && view->suboffsets[axis] >= 0)) return 0;
+ Py_ssize_t stride = view->strides[axis];
+ uintptr_t magnitude = stride >= 0 ? (uintptr_t)stride :
+ (uintptr_t)(-(stride + 1)) + 1;
+ uintptr_t steps = (uintptr_t)(view->shape[axis] - 1);
+ if (magnitude != 0 && steps > UINTPTR_MAX / magnitude) return 0;
+ uintptr_t offset = steps * magnitude;
+ if (stride < 0) {
+ if (low < offset) return 0;
+ low -= offset;
+ } else {
+ if (high > UINTPTR_MAX - offset) return 0;
+ high += offset;
+ }
+ }
+ if (high > UINTPTR_MAX - (uintptr_t)view->itemsize) return 0;
+ *begin = low;
+ *end = high + (uintptr_t)view->itemsize;
+ return 1;
+}
+
+static PyObject *
+fit_irls_batch(PyObject *self, PyObject *args)
+{
+ ridge_state *state = PyModule_GetState(self);
+ ++state->batch_calls;
+ PyObject *objects[6];
+ Py_buffer views[6] = {{0}};
+ double lower, upper, sigma;
+ Py_ssize_t independent_samples;
+ int acquired = 0, valid = 1;
+ double *scratch = NULL;
+ PyObject *result = NULL;
+ if (!PyArg_ParseTuple(args, "OOOOOdddnO", &objects[0], &objects[1], &objects[2],
+ &objects[3], &objects[4], &lower, &upper, &sigma,
+ &independent_samples, &objects[5])) return NULL;
+ for (int i = 0; i < 6; ++i) {
+ int status = (i < 2 || i == 5) ? get_matrix(objects[i], &views[i], i == 5) :
+ get_vector(objects[i], &views[i]);
+ if (status < 0) goto done;
+ ++acquired;
+ }
+ Py_ssize_t rows = views[0].shape[0], count = views[0].shape[1];
+ if (rows == 0 || count == 0 || views[1].shape[0] != rows || views[1].shape[1] != count ||
+ views[2].shape[0] != count || views[3].shape[0] != count || views[4].shape[0] != rows ||
+ views[5].shape[0] != rows || views[5].shape[1] != 5 ||
+ !isfinite(lower) || !isfinite(upper) || lower > upper ||
+ !isfinite(sigma) || sigma <= 0.0 || independent_samples < 1) {
+ PyErr_SetString(PyExc_ValueError, "invalid robust ridge-fit batch inputs");
+ goto done;
+ }
+ uintptr_t output_begin, output_end;
+ if (!buffer_span(&views[5], &output_begin, &output_end)) {
+ PyErr_SetString(PyExc_ValueError, "unsupported robust ridge-fit output buffer");
+ goto done;
+ }
+ for (int i = 0; i < 5; ++i) {
+ uintptr_t input_begin, input_end;
+ if (!buffer_span(&views[i], &input_begin, &input_end) ||
+ (output_begin < input_end && input_begin < output_end)) {
+ PyErr_SetString(PyExc_ValueError, "robust ridge-fit output must not overlap inputs");
+ goto done;
+ }
+ }
+ if ((size_t)count > SIZE_MAX / (6 * sizeof(double))) {
+ PyErr_NoMemory();
+ goto done;
+ }
+ scratch = PyMem_Malloc((size_t)count * 6 * sizeof(double));
+ if (scratch == NULL) {
+ PyErr_NoMemory();
+ goto done;
+ }
+ Py_buffer row_views[4] = {views[0], views[1], views[2], views[3]};
+ row_views[0].strides = views[0].strides + 1;
+ row_views[1].strides = views[1].strides + 1;
+ state->batch_rows += (unsigned long long)rows;
+ Py_BEGIN_ALLOW_THREADS
+ for (Py_ssize_t row = 0; row < rows; ++row) {
+ double cap = at(&views[4], row), output[5];
+ if (!isfinite(cap) || cap < 0.0) {
+ valid = 0;
+ break;
+ }
+ row_views[0].buf = (char *)views[0].buf + row * views[0].strides[0];
+ row_views[1].buf = (char *)views[1].buf + row * views[1].strides[0];
+ if (!fit_row(row_views, count, lower, upper, cap, sigma,
+ independent_samples, scratch, output)) {
+ valid = 0;
+ break;
+ }
+ memcpy((char *)views[5].buf + row * views[5].strides[0], output, sizeof(output));
+ }
+ Py_END_ALLOW_THREADS
+ if (valid) state->batch_successful_rows += (unsigned long long)rows;
+ result = PyBool_FromLong(valid);
+done:
+ PyMem_Free(scratch);
+ for (int i = 0; i < acquired; ++i) PyBuffer_Release(&views[i]);
+ return result;
+}
+
+static PyObject *
+fit_statistics(PyObject *self, PyObject *unused)
+{
+ ridge_state *state = PyModule_GetState(self);
+ return Py_BuildValue("{sK,sK,sK,sK,sK}",
+ "scalar_calls", state->scalar_calls,
+ "scalar_successes", state->scalar_successes,
+ "batch_calls", state->batch_calls,
+ "batch_rows", state->batch_rows,
+ "batch_successful_rows", state->batch_successful_rows);
+}
+
/* This is a positive-only shortcut for the Python evidence validator. Any
* noncanonical or invalid value returns False, so Python retains its original
* error reason, ordering, subclass behavior and unbounded-integer semantics.
@@ -367,13 +529,16 @@ valid_ridge_events(PyObject *self, PyObject *args)
static PyMethodDef methods[] = {
{"fit_irls", fit_irls, METH_VARARGS, "Run six robust ridge-fit iterations in C."},
+ {"fit_irls_batch", fit_irls_batch, METH_VARARGS,
+ "Run the same robust fit for contiguous clock rows with shared scratch storage."},
+ {"fit_statistics", fit_statistics, METH_NOARGS, "Read native fit operation counters."},
{"valid_ridge_events", valid_ridge_events, METH_VARARGS,
"Recognize valid canonical ridge evidence; other inputs retain Python validation."},
{NULL, NULL, 0, NULL}
};
static struct PyModuleDef module = {
- PyModuleDef_HEAD_INIT, "_native_ridge", NULL, -1, methods
+ PyModuleDef_HEAD_INIT, "_native_ridge", NULL, sizeof(ridge_state), methods
};
PyMODINIT_FUNC
diff --git a/src/dopplergeist/batch_ridge.py b/src/dopplergeist/batch_ridge.py
index d9bc57a..f1caf0f 100644
--- a/src/dopplergeist/batch_ridge.py
+++ b/src/dopplergeist/batch_ridge.py
@@ -185,12 +185,19 @@ def _first_event_fits_uncached(measurement, event, factors, emissions, *, legacy
and factors.flags.aligned and emissions.flags.aligned):
native_rows = []
try:
- for factor, normalized_time, bound in zip(factors, u, cap):
- fit = native(factor, normalized_time, y, w, low, high,
- float(bound), sigma, independent)
- if fit is None:
- break
- native_rows.append(fit)
+ batch = matching._native_fit_irls_batch
+ if batch is not None and native is matching._CANONICAL_NATIVE_FIT_IRLS:
+ output = np.empty((len(factors), 5), dtype=float)
+ if batch(factors, u, y, w, cap, low, high, sigma, independent, output):
+ native_rows = output
+ else:
+ # Preserve custom scalar hooks and older installed binaries.
+ for factor, normalized_time, bound in zip(factors, u, cap):
+ fit = native(factor, normalized_time, y, w, low, high,
+ float(bound), sigma, independent)
+ if fit is None:
+ break
+ native_rows.append(fit)
if len(native_rows) == len(factors):
score, alpha, beta, rmse, outlier = np.asarray(native_rows, dtype=float).T
if all(np.all(np.isfinite(value)) for value in
diff --git a/src/dopplergeist/checkpoint_resume.py b/src/dopplergeist/checkpoint_resume.py
index 5ff63b4..0e58d94 100644
--- a/src/dopplergeist/checkpoint_resume.py
+++ b/src/dopplergeist/checkpoint_resume.py
@@ -33,7 +33,7 @@ def upgrade_legacy_job(directory):
staged batching with the saved first-group arithmetic and fallback checks.
"""
from .automatic import _audio_identity, _digest
- from . import geometry, matching
+ from . import geometry, matching, native_cpa
directory = Path(directory)
search = directory / "search"
@@ -47,6 +47,8 @@ def upgrade_legacy_job(directory):
raise ValueError("Legacy checkpoint continuation requires the Python ridge fitter")
if geometry.ACOUSTIC_BACKEND != "python-acoustic-v1":
raise ValueError("Legacy checkpoint continuation requires the Python acoustic solver")
+ if native_cpa.CPA_BACKEND != "python-cpa-v1":
+ raise ValueError("Legacy checkpoint continuation requires Python closest-approach processing")
if (old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile"
or old["scoring"].get("time_origin") is not None
or old.get("filters", {}).get("association") != "subsets"):
diff --git a/src/dopplergeist/matching.py b/src/dopplergeist/matching.py
index 88afdcd..f3baa7c 100644
--- a/src/dopplergeist/matching.py
+++ b/src/dopplergeist/matching.py
@@ -40,6 +40,8 @@ else:
RIDGE_FIT_BACKEND = f"native-c-irls-v2-pairwise:{_native_binary_hash}"
_native_validate_ridge_events = getattr(_native_ridge_module, "valid_ridge_events", None)
+_native_fit_irls_batch = getattr(_native_ridge_module, "fit_irls_batch", None)
+_CANONICAL_NATIVE_FIT_IRLS = _native_fit_irls
_NATIVE_RIDGE_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}
diff --git a/src/dopplergeist/native_cpa.py b/src/dopplergeist/native_cpa.py
new file mode 100644
index 0000000..6eaad94
--- /dev/null
+++ b/src/dopplergeist/native_cpa.py
@@ -0,0 +1,52 @@
+"""Optional exact-order reduction of a bounded CPA neighbour graph."""
+import os
+
+import numpy as np
+
+
+_backend_choice = os.environ.get("DOPPLERGEIST_CPA_BACKEND", "auto")
+if _backend_choice not in {"auto", "python"}:
+ raise ValueError("DOPPLERGEIST_CPA_BACKEND must be 'auto' or 'python'")
+_native = None
+if _backend_choice == "auto":
+ try:
+ from . import _native_cpa as _native
+ except ImportError:
+ pass
+
+CPA_BACKEND = "python-cpa-v1" if _native is None else "native-c-cpa-v1"
+_ERROR_POLICY = {"divide": "warn", "over": "warn", "under": "ignore", "invalid": "warn"}
+
+
+def try_cpa_edges(position, observers, counts, sample):
+ """Return endpoint candidates and projected minima, or use the reference.
+
+ The caller owns one bounded, sorted radius-neighbour graph. The extension
+ streams its distances and projects only the sampled minima; it does not
+ change the spatial query, spline endpoint solver, or row selection rules.
+ """
+ if _native is None or np.geterr() != _ERROR_POLICY:
+ return None
+ for array, dtype in ((position, np.float64), (observers, np.float64),
+ (counts, np.intp), (sample, np.intp)):
+ if (type(array) is not np.ndarray or array.dtype != dtype
+ or not array.flags.c_contiguous or not array.flags.aligned):
+ return None
+ if (position.ndim != 2 or position.shape[1] != 3
+ or observers.ndim != 2 or observers.shape[1] != 3
+ or counts.ndim != 1 or len(counts) != len(observers)
+ or sample.ndim != 1):
+ return None
+ # Two consecutive samples cannot both satisfy d[k] < d[k+1]. Each
+ # interior minimum therefore consumes at least two neighbour entries.
+ capacity = sample.size // 2
+ endpoints = np.empty((2, len(observers)), dtype=np.intp)
+ indices = np.empty((2, capacity), dtype=np.intp)
+ values = np.empty((2, capacity), dtype=np.float64)
+ sizes = _native.select(position, observers, counts, sample, endpoints, indices, values)
+ if sizes is None:
+ return None
+ first_count, last_count, minimum_count = sizes
+ return ((endpoints[0, :first_count], endpoints[1, :last_count]),
+ indices[0, :minimum_count], indices[1, :minimum_count],
+ values[0, :minimum_count], values[1, :minimum_count])
diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.py
index 37e10df..05c66fb 100644
--- a/src/dopplergeist/parallel_tiles.py
+++ b/src/dopplergeist/parallel_tiles.py
@@ -8,7 +8,7 @@ import pandas as pd
from .data_access import CachedTerrain, DataUnavailable, FlightIndex
from .finite_search import RetentionLedger
-from .tiled_search import (_expanded_bounds, _sample_tile_terrain, _score, _tile_anchor_window,
+from .tiled_search import (_expanded_bounds, _partition_query_time_range, _sample_tile_terrain, _score, _tile_anchor_window,
_tile_coordinates, _tracks_for_points)
@@ -146,10 +146,14 @@ def score_tile(stage, cells, score_floor=None):
radius = max(event["slant_range_m"][1] for event in state["events"])
points = _tile_coordinates(cells, budget.spacing_m, state["bounds"], state["crs"])
tracks = []
+ anchor = _tile_anchor_window(stage.removeprefix("coarse:"), state["start_window"],
+ state["parameters"], state["filters"], state["events"])
+ query_time = _partition_query_time_range(state["time_range"], state["events"], state["parameters"],
+ state["filters"], anchor)
if len(points):
tracks, points = _tracks_for_points(
state["indexes"], _expanded_bounds(cells, budget.spacing_m, radius, state["crs"]),
- state["time_range"], budget, points, terrain, radius, state["parameters"].get("c", 343.0))
+ query_time, budget, points, terrain, radius, state["parameters"].get("c", 343.0))
if tracks and "elev_m" not in points:
points = _sample_tile_terrain(points, terrain)
sampled = "elev_m" in points
@@ -161,8 +165,6 @@ def score_tile(stage, cells, score_floor=None):
ledger = RetentionLedger(path, state["run_hash"], state["score_kind"])
if len(points):
kwargs = {"ledger": ledger, "stage": stage}
- anchor = _tile_anchor_window(stage.removeprefix("coarse:"), state["start_window"],
- state["parameters"], state["filters"], state["events"])
if anchor is not None:
kwargs["anchor_window"] = anchor
if score_floor is not None:
diff --git a/src/dopplergeist/search.py b/src/dopplergeist/search.py
index 54f689f..45430f1 100644
--- a/src/dopplergeist/search.py
+++ b/src/dopplergeist/search.py
@@ -10,6 +10,7 @@ from scipy.spatial import cKDTree
from .geometry import AcousticSupportError, ecef
from .matching import _native_validate_ridge_events
+from .native_cpa import try_cpa_edges
_CPA_COLUMNS = ["point_id", "flight_id", "emission_time", "receive_time", "distance_m", "speed_mps"]
_CPA_DTYPE = np.dtype([(name, "int64" if name == "point_id" else object if name == "flight_id" else "float64")
@@ -146,24 +147,29 @@ class _PreparedCPATrack:
counts=np.fromiter((len(a) for a in neighbours),dtype=int,count=len(neighbours))
if not counts.sum():continue
sample=np.concatenate(neighbours).astype(int)
- observer_index=np.repeat(np.arange(len(obs)),counts)
- distance2=np.sum((pos[sample]-obs[observer_index])**2,axis=1)
- previous=np.r_[False,(observer_index[1:]==observer_index[:-1])&(sample[1:]==sample[:-1]+1)]
- following=np.r_[(observer_index[1:]==observer_index[:-1])&(sample[1:]==sample[:-1]+1),False]
+ projected = try_cpa_edges(pos, obs, counts, sample)
+ if projected is None:
+ observer_index=np.repeat(np.arange(len(obs)),counts)
+ distance2=np.sum((pos[sample]-obs[observer_index])**2,axis=1)
+ previous=np.r_[False,(observer_index[1:]==observer_index[:-1])&(sample[1:]==sample[:-1]+1)]
+ following=np.r_[(observer_index[1:]==observer_index[:-1])&(sample[1:]==sample[:-1]+1),False]
+ eligible = np.flatnonzero(counts >= 2)
+ ends = np.cumsum(counts)[eligible]
+ starts = ends - counts[eligible]
# An endpoint sample can be the sampled minimum even when the
# continuous spline turns strictly inside its adjacent interval.
# Check the endpoint range-rate signs before optimizing the actual
# spline; an endpoint alone is not evidence of a complete pass.
- eligible = np.flatnonzero(counts >= 2)
- ends = np.cumsum(counts)[eligible]
- starts = ends - counts[eligible]
for end in (False, True):
- edge_indices = ends - 1 if end else starts
- adjacent = edge_indices - 1 if end else edge_indices + 1
- mask = ((sample[edge_indices] == (len(t)-1 if end else 0))
- & (previous[edge_indices] if end else following[edge_indices])
- & (distance2[edge_indices] <= distance2[adjacent]))
- oi_edge = eligible[mask]
+ if projected is None:
+ edge_indices = ends - 1 if end else starts
+ adjacent = edge_indices - 1 if end else edge_indices + 1
+ mask = ((sample[edge_indices] == (len(t)-1 if end else 0))
+ & (previous[edge_indices] if end else following[edge_indices])
+ & (distance2[edge_indices] <= distance2[adjacent]))
+ oi_edge = eligible[mask]
+ else:
+ oi_edge = projected[0][end]
if not len(oi_edge):
continue
lo, hi = (t[-2], t[-1]) if end else (t[0], t[1])
@@ -184,25 +190,31 @@ class _PreparedCPATrack:
rows.append((int(point_ids[chosen[oi_edge[j]]]), track.id,
float(when), float(when + distance/c), float(distance),
float(np.linalg.norm(track.speed_vector(when)))))
- minima=(previous&following&(distance2<=np.r_[np.inf,distance2[:-1]])&
- (distance2<np.r_[distance2[1:],np.inf]))
- indices=np.flatnonzero(minima)
- if not len(indices):continue
- k=sample[indices]
- oi=observer_index[indices]
- # Project onto both neighbouring polyline edges, never across a
- # missing interval or between unrelated flight passes.
- first=np.stack((pos[k-1],pos[k]),axis=1)
- last=np.stack((pos[k],pos[k+1]),axis=1)
- delta=last-first
- residual=first-obs[oi,None,:]
- fraction=np.clip(-np.sum(residual*delta,axis=2)/np.maximum(np.sum(delta**2,axis=2),1e-12),0,1)
- d2=np.sum((residual+fraction[:,:,None]*delta)**2,axis=2)
- edge=np.argmin(d2,axis=1)
- n=np.arange(len(k))
- distance=np.sqrt(d2[n,edge])
- start=np.where(edge==0,k-1,k)
- when=t[start]+fraction[n,edge]*(t[start+1]-t[start])
+ if projected is None:
+ minima=(previous&following&(distance2<=np.r_[np.inf,distance2[:-1]])&
+ (distance2<np.r_[distance2[1:],np.inf]))
+ indices=np.flatnonzero(minima)
+ if not len(indices):continue
+ k=sample[indices]
+ oi=observer_index[indices]
+ # Project onto both neighbouring polyline edges, never across a
+ # missing interval or between unrelated flight passes.
+ first=np.stack((pos[k-1],pos[k]),axis=1)
+ last=np.stack((pos[k],pos[k+1]),axis=1)
+ delta=last-first
+ residual=first-obs[oi,None,:]
+ fraction=np.clip(-np.sum(residual*delta,axis=2)/np.maximum(np.sum(delta**2,axis=2),1e-12),0,1)
+ d2=np.sum((residual+fraction[:,:,None]*delta)**2,axis=2)
+ edge=np.argmin(d2,axis=1)
+ n=np.arange(len(k))
+ distance=np.sqrt(d2[n,edge])
+ start=np.where(edge==0,k-1,k)
+ fraction = fraction[n, edge]
+ else:
+ _, oi, start, fraction, d2 = projected
+ if not len(oi):continue
+ distance=np.sqrt(d2)
+ when=t[start]+fraction*(t[start+1]-t[start])
speed=np.linalg.norm(track.speed_vector(when),axis=1)
for j in np.flatnonzero(distance<=max_distance):
rows.append((int(point_ids[chosen[oi[j]]]), track.id,
diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.py
index a502613..7c2ba9d 100644
--- a/src/dopplergeist/tiled_search.py
+++ b/src/dopplergeist/tiled_search.py
@@ -25,6 +25,7 @@ from .finite_search import RetentionLedger, bounded_assessment, classified_asses
from .resource_usage import process_peak_rss_kib
from .geometry import ACOUSTIC_BACKEND, AcousticSupportError, ecef
from .matching import RIDGE_FIT_BACKEND, _cache_ridge_fits
+from .native_cpa import CPA_BACKEND
from .search import (_CPA_DTYPE, _PreparedCPATrack, _ordered_cpa_rows,
filter_association_event, pair_candidates, score_candidate)
@@ -383,6 +384,92 @@ def _tile_anchor_window(tile_id, start_window, parameters, filters, events):
return lo, hi
+def _partition_query_time_range(time_range, events, parameters, filters, anchor_window):
+ """Bound whole-track queries by an association partition, not a clock prior.
+
+ The first matched passage has an implied start inside ``anchor_window``.
+ Every later matched passage must overlap its clock interval. Expanding by
+ both passages' tolerances therefore contains every participating receive
+ time. Subtract the largest possible acoustic delay to bound emission time.
+ FlightIndex uses these bounds only to select complete original segments.
+ """
+ if (anchor_window is None or parameters.get("mode") != "robust_ridge"
+ or filters.get("association", "subsets") != "subsets" or filters.get("silence")):
+ return time_range
+
+ def finite(value):
+ # Keep unusual scalar/container semantics on the established path.
+ if type(value) not in (int, float):
+ return False
+ try:
+ return math.isfinite(value)
+ except OverflowError:
+ return False
+
+ if (not isinstance(anchor_window, (tuple, list)) or len(anchor_window) != 2
+ or not all(finite(value) for value in anchor_window)
+ or anchor_window[0] >= anchor_window[1]
+ or not isinstance(time_range, (tuple, list)) or len(time_range) != 2
+ or not all(finite(value) for value in time_range) or time_range[0] > time_range[1]
+ or not isinstance(events, (tuple, list)) or not events):
+ return time_range
+ c = parameters.get("c", 343.0)
+ offset = filters.get("offset_tolerance_s", 5.0)
+ if not finite(c) or c <= 0 or not finite(offset) or offset < 0:
+ return time_range
+ tolerances, centers, radii = [], [], []
+ for event in events:
+ if not isinstance(event, dict) or not isinstance(event.get("quality", {}), dict):
+ return time_range
+ quality = event.get("quality", {})
+ proposal_tolerances = [quality.get(name, 0) for name in
+ ("cpa_tolerance_s", "speed_tolerance_mps", "slant_tolerance_m")]
+ # Do not let an empty narrowed query hide a filter_event validation error.
+ if not all(finite(value) and value >= 0 for value in proposal_tolerances):
+ return time_range
+ slant = event.get("slant_range_m")
+ speed = event.get("speed_range_mps")
+ center = event.get("cpa_receive_s")
+ if (not isinstance(slant, (tuple, list)) or len(slant) != 2
+ or not all(finite(value) for value in slant) or not 0 <= slant[0] <= slant[1]
+ or not isinstance(speed, (tuple, list)) or len(speed) != 2
+ or not all(finite(value) for value in speed) or not 0 <= speed[0] <= speed[1]
+ or not finite(center)):
+ return time_range
+ tolerance = proposal_tolerances[0] + offset / 2
+ if not finite(tolerance):
+ return time_range
+ tolerances.append(tolerance)
+ centers.append(center)
+ radii.append(slant[1])
+ if max(radii) <= 0:
+ return time_range
+
+ down = lambda value: math.nextafter(value, -math.inf)
+ up = lambda value: math.nextafter(value, math.inf)
+ # Any event can be the first matched one after unmatched proposals. Use
+ # the maximum tolerance over all events, not the first event in the list.
+ margin = max(tolerances)
+ start_lo, start_hi = down(anchor_window[0] - margin), up(anchor_window[1] + margin)
+ receive_lo, receive_hi = math.inf, -math.inf
+ for center, tolerance in zip(centers, tolerances):
+ implied_lo = down(down(start_lo) - tolerance)
+ implied_hi = up(up(start_hi) + tolerance)
+ receive_lo = min(receive_lo, down(down(implied_lo) + center))
+ receive_hi = max(receive_hi, up(up(implied_hi) + center))
+ # Extra outward steps before inverse operations cover rounded start =
+ # receive - event CPA and receive = emission + distance / c arithmetic.
+ delay = up(max(radii) / c)
+ lower = down(down(receive_lo) - delay)
+ upper = up(receive_hi)
+ if not all(math.isfinite(value) for value in (start_lo, start_hi, receive_lo, receive_hi, delay, lower, upper)):
+ return time_range
+ lower, upper = max(time_range[0], lower), min(time_range[1], upper)
+ # A segment can span two disjoint query windows. Keep the original path
+ # rather than treating their empty intersection as no matching segments.
+ return (lower, upper) if lower <= upper else time_range
+
+
def _clock_children(tile_id, cells, start_window, parameters, filters, events):
if len([p for p in tile_id.split("~")[1:] if p.startswith("t")]) >= 24:
return []
@@ -758,6 +845,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
runtime_identity["ridge_fit_backend"] = RIDGE_FIT_BACKEND
if ACOUSTIC_BACKEND != "python-acoustic-v1":
runtime_identity["acoustic_backend"] = ACOUSTIC_BACKEND
+ if CPA_BACKEND != "python-cpa-v1":
+ runtime_identity["cpa_backend"] = CPA_BACKEND
identity = {"indexes": [{"input_hash": i.report["input_hash"], "file_sha256": cached_sha256(i.path)} for i in indexes],
"terrain": terrain_signature, "additional_input_hashes": input_hashes or {},
"events": events, "acoustic_hashes": acoustic, "bounds": list(bounds), "time_range_s": list(time_range_s),
@@ -909,9 +998,11 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
else:
points = _tile_coordinates(cells, budget.spacing_m, bounds, crs)
tracks = []
+ anchor = _tile_anchor_window(tile_id, start_window, parameters, filters, events)
+ query_time = _partition_query_time_range(time_range_s, events, parameters, filters, anchor)
if len(points):
tracks, points = _tracks_for_points(
- indexes, _expanded_bounds(cells, budget.spacing_m, radius, crs), time_range_s,
+ indexes, _expanded_bounds(cells, budget.spacing_m, radius, crs), query_time,
budget, points, terrain, radius, parameters.get("c", 343.0))
if tracks and "elev_m" not in points:
points = _sample_tile_terrain(points, terrain)
@@ -920,7 +1011,6 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
points = pd.DataFrame()
if len(points):
flush_empty_tiles()
- anchor = _tile_anchor_window(tile_id, start_window, parameters, filters, events)
rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,
ledger=ledger, stage=stage,
anchor_window=anchor,
diff --git a/tests/test_real_native_cpa.py b/tests/test_real_native_cpa.py
new file mode 100644
index 0000000..486cb21
--- /dev/null
+++ b/tests/test_real_native_cpa.py
@@ -0,0 +1,185 @@
+"""Native CPA parity on the repository's frozen real ADS-B observations."""
+import importlib.util
+import os
+from pathlib import Path
+import subprocess
+import sys
+from unittest.mock import patch
+
+import numpy as np
+import pytest
+
+from dopplergeist import native_cpa, search
+from dopplergeist.geometry import AcousticSupportError, ecef
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SPEC = importlib.util.spec_from_file_location(
+ "benchmark_real_cpa_rows", ROOT / "scripts/benchmark_real_cpa_rows.py")
+BENCHMARK = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(BENCHMARK)
+
+
[email protected](scope="module")
+def tracks():
+ return BENCHMARK.recorded_tracks()
+
+
+def backend():
+ if native_cpa._native is None:
+ pytest.skip("optional CPA extension is not built")
+ return native_cpa._native
+
+
+def assert_exact(before, after):
+ BENCHMARK.compare(before, after)
+ for column in ("emission_time", "receive_time", "distance_m", "speed_mps"):
+ # Include signed-zero bits, not just pandas' numerical equality.
+ assert before[0][column].to_numpy().tobytes() == after[0][column].to_numpy().tobytes()
+
+
[email protected]("count", [1, 32, 128, 1024])
[email protected]("radius,c,step,chunk", [
+ (1000., 343., 2., 512), (20000., 343., 2., 512),
+ (20000., 338., 1., 7), (20000., 348., 8., 1),
+ (20000., 343., np.float32(2.), 512),
+])
+def test_recorded_tables_and_unsupported_accounting(tracks, backend, count, radius, c, step, chunk):
+ points = BENCHMARK.recorded_points(tracks, count)
+ with patch.object(native_cpa, "_native", None):
+ expected = BENCHMARK.cpa_stage(search, tracks, points, radius, c, step, chunk, True)
+ actual = BENCHMARK.cpa_stage(search, tracks, points, radius, c, step, chunk, True)
+ assert_exact(expected, actual)
+
+
+def test_raw_endpoint_and_interior_order_with_repeated_wide_ids(tracks, backend):
+ points = BENCHMARK.recorded_points(tracks, 1024).iloc[::-1].copy()
+ points["point_id"] = np.arange(len(points), dtype=np.int64)//2 + 2**53
+ observers = ecef(points.lat.to_numpy(), points.lon.to_numpy(), points.elev_m.to_numpy())
+ calls = backend.successful_calls()
+ retained = 0
+ with patch.object(search, "minimize_scalar", wraps=search.minimize_scalar) as endpoint:
+ for track in tracks:
+ try:
+ prepared = search._PreparedCPATrack(track, 343., 2.)
+ except AcousticSupportError:
+ continue
+ with patch.object(native_cpa, "_native", None):
+ expected = prepared.raw_rows(observers, points.point_id.to_numpy(), 20000.)
+ actual = prepared.raw_rows(observers, points.point_id.to_numpy(), 20000.)
+ assert [(row[:2], np.asarray(row[2:]).tobytes()) for row in actual] == [
+ (row[:2], np.asarray(row[2:]).tobytes()) for row in expected]
+ assert search._ordered_cpa_rows(actual, 2.) == search._ordered_cpa_rows(expected, 2.)
+ retained += len(actual)
+ assert retained > 0
+ assert endpoint.call_count > 0
+ assert backend.successful_calls() > calls
+
+
[email protected]("policy", ["raise", "ignore", "warn"])
+def test_nondefault_error_policy_uses_reference(tracks, backend, policy):
+ points = BENCHMARK.recorded_points(tracks, 32)
+ calls = backend.successful_calls()
+ with np.errstate(all=policy):
+ with patch.object(native_cpa, "_native", None):
+ expected = BENCHMARK.cpa_stage(search, tracks, points, row_mode=True)
+ actual = BENCHMARK.cpa_stage(search, tracks, points, row_mode=True)
+ assert_exact(expected, actual)
+ assert backend.successful_calls() == calls
+
+
+def graph(tracks):
+ supported = []
+ for track in tracks:
+ try:
+ track.validate_motion_support()
+ supported.append(track)
+ except AcousticSupportError:
+ pass
+ track = max(supported, key=lambda item: item.time[-1]-item.time[0])
+ prepared = search._PreparedCPATrack(track, 343., 2.)
+ points = BENCHMARK.recorded_points([track], 32)
+ observers = np.ascontiguousarray(ecef(points.lat.to_numpy(), points.lon.to_numpy(),
+ points.elev_m.to_numpy()))
+ neighbours = search.cKDTree(prepared.position).query_ball_point(
+ observers, 20000.+2*prepared.max_step, return_sorted=True)
+ counts = np.fromiter(map(len, neighbours), dtype=np.intp)
+ sample = np.concatenate(neighbours).astype(np.intp)
+ return prepared.position, observers, counts, sample
+
+
[email protected]("index", range(4))
+def test_strided_same_observations_use_reference(graph, backend, index):
+ inputs = list(graph)
+ inputs[index] = np.repeat(inputs[index], 2, axis=0)[::2]
+ assert not inputs[index].flags.c_contiguous
+ calls = backend.successful_calls()
+ assert native_cpa.try_cpa_edges(*inputs) is None
+ assert backend.successful_calls() == calls
+
+
+def test_readonly_buffers_and_empty_neighbour_groups(graph, backend):
+ expected = native_cpa.try_cpa_edges(*graph)
+ assert expected is not None
+ for array in graph:
+ array.flags.writeable = False
+ actual = native_cpa.try_cpa_edges(*graph)
+ for left, right in zip((*actual[0], *actual[1:]), (*expected[0], *expected[1:])):
+ assert left.dtype == right.dtype
+ assert left.shape == right.shape
+ assert left.tobytes() == right.tobytes()
+ # An empty receiver chunk must return no candidate proposals.
+ far_observers = graph[1][:0]
+ empty = native_cpa.try_cpa_edges(graph[0], far_observers, graph[2][:0], graph[3][:0])
+ assert empty is not None
+ assert all(array.size == 0 for array in (*empty[0], *empty[1:]))
+
+
+def test_native_rejects_invalid_graph_metadata_and_output_aliases(graph, backend):
+ position, observers, counts, sample = graph
+ capacity = sample.size//2
+ endpoints = np.empty((2, len(observers)), dtype=np.intp)
+ indices = np.empty((2, capacity), dtype=np.intp)
+ values = np.empty((2, capacity), dtype=np.float64)
+ changed = counts.copy()
+ changed[0] = -1
+ assert backend.select(position, observers, changed, sample, endpoints, indices, values) is None
+ changed = sample.copy()
+ changed[0] = len(position)
+ assert backend.select(position, observers, counts, changed, endpoints, indices, values) is None
+ assert backend.select(position, observers, counts, sample[:-1], endpoints, indices, values) is None
+ assert backend.select(position, observers, counts, sample, endpoints, indices[:, :-1], values) is None
+ # Writable output must never overwrite neighbour indices while C reads them.
+ storage = np.empty(max(sample.size, endpoints.size), dtype=np.intp)
+ storage[:sample.size] = sample
+ alias = storage[:endpoints.size].reshape(endpoints.shape)
+ before = storage.tobytes()
+ assert backend.select(position, observers, counts, storage[:sample.size], alias, indices, values) is None
+ assert storage.tobytes() == before
+ before = values.tobytes()
+ assert backend.select(position, observers, counts, sample, endpoints,
+ values.view(np.intp), values) is None
+ assert values.tobytes() == before
+
+
+def test_backend_selection_and_source_only_import():
+ code = "from dopplergeist.native_cpa import CPA_BACKEND; print(CPA_BACKEND)"
+ environment = dict(os.environ, DOPPLERGEIST_CPA_BACKEND="python")
+ result = subprocess.run([sys.executable, "-c", code], env=environment,
+ text=True, capture_output=True, check=True)
+ assert result.stdout.strip() == "python-cpa-v1"
+ environment["DOPPLERGEIST_CPA_BACKEND"] = "auto"
+ result = subprocess.run([sys.executable, "-c", code], env=environment,
+ text=True, capture_output=True, check=True)
+ assert result.stdout.strip() in {"python-cpa-v1", "native-c-cpa-v1"}
+ missing = "import sys; sys.modules['dopplergeist._native_cpa'] = None; " + code
+ result = subprocess.run([sys.executable, "-c", missing], env=environment,
+ text=True, capture_output=True, check=True)
+ assert result.stdout.strip() == "python-cpa-v1"
+ environment["DOPPLERGEIST_CPA_BACKEND"] = "unknown"
+ result = subprocess.run([sys.executable, "-c", code], env=environment,
+ text=True, capture_output=True)
+ assert result.returncode != 0
+ assert "DOPPLERGEIST_CPA_BACKEND must be" in result.stderr
diff --git a/tests/test_real_native_ridge_batch.py b/tests/test_real_native_ridge_batch.py
new file mode 100644
index 0000000..e8dae3b
--- /dev/null
+++ b/tests/test_real_native_ridge_batch.py
@@ -0,0 +1,152 @@
+"""Batched native fitting on frozen, recorded acoustic factors and ridges."""
+from copy import deepcopy
+import json
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+from dopplergeist import batch_ridge, matching
+
+
+def recorded_batch():
+ if matching._native_fit_irls_batch is None:
+ pytest.skip("optional native batch fitter is not installed")
+ source = Path(__file__).parent / "data/real-ridge-batch-arithmetic.json"
+ data = json.loads(source.read_text())
+ measurement = (matching.RidgeTimes(np.asarray(data["times"], dtype=float)),
+ np.asarray(data["ridge"], dtype=float), np.asarray(data["weight"], dtype=float), {})
+ return measurement, deepcopy(data["event"]), np.asarray(data["factors"]), np.asarray(data["emissions"])
+
+
+def canonical(value):
+ return json.dumps(value, sort_keys=True, allow_nan=True)
+
+
[email protected]("order", ["C", "F", "reversed"])
[email protected]("policy", ["default", "raise"])
+def test_recorded_native_batch_is_exact_and_reduces_dispatch(recorded_batch, monkeypatch, order, policy):
+ measurement, event, factors, emissions = recorded_batch
+ if order == "reversed":
+ factors, emissions = factors[::-1], emissions[::-1]
+ else:
+ factors, emissions = np.array(factors, order=order), np.array(emissions, order=order)
+ settings = {} if policy == "default" else {"all": "raise"}
+ with monkeypatch.context() as patch:
+ patch.setattr(matching, "_native_fit_irls_batch", None)
+ with np.errstate(**settings):
+ expected = batch_ridge._first_event_fits_uncached(measurement, event, factors, emissions)
+ before = matching._native_ridge_module.fit_statistics()
+ with np.errstate(**settings):
+ actual = batch_ridge._first_event_fits_uncached(measurement, event, factors, emissions)
+ after = matching._native_ridge_module.fit_statistics()
+ assert canonical(actual) == canonical(expected)
+ assert after["scalar_calls"] == before["scalar_calls"]
+ assert after["batch_calls"] - before["batch_calls"] == int(order == "C")
+ assert after["batch_successful_rows"] - before["batch_successful_rows"] == (len(factors) if order == "C" else 0)
+
+
[email protected]("failure", ["false", "type_error"])
+def test_recorded_batch_failure_retains_vectorized_fallback(recorded_batch, monkeypatch, failure):
+ with monkeypatch.context() as patch:
+ patch.setattr(matching, "_native_fit_irls", None)
+ expected = batch_ridge._first_event_fits_uncached(*recorded_batch)
+
+ def unavailable(*args):
+ # A partial native output must never enter the final fit diagnostics.
+ args[-1][:] = 12345.
+ if failure == "type_error":
+ raise TypeError("Unsupported native buffers")
+ return False
+
+ monkeypatch.setattr(matching, "_native_fit_irls_batch", unavailable)
+ actual = batch_ridge._first_event_fits_uncached(*recorded_batch)
+ assert canonical(actual) == canonical(expected)
+
+
+def native_arguments(recorded_batch):
+ measurement, event, factors, emissions = recorded_batch
+ times, ridge, weight = matching._ridge_fit_inputs(measurement, matching._frequency_scale(event), .02)
+ measured = np.isfinite(times) & np.isfinite(ridge) & (weight > 0)
+ y, w = ridge[measured], weight[measured]
+ independent = len(np.unique(times[measured]))
+ w = w * independent / w.sum()
+ span = np.maximum(np.ptp(emissions, axis=1), 1e-9)
+ reference = (emissions.min(axis=1) + emissions.max(axis=1)) / 2
+ u = (emissions - reference[:, None]) / span[:, None]
+ low, high = event["f0_range_hz"]
+ cap = np.clip(np.median(y[None, :] / factors, axis=1), low, high) * .02
+ return [factors, u, y, w, cap, low, high, matching._frequency_scale(event),
+ independent, np.empty((len(factors), 5))]
+
+
[email protected]("invalid", ["shape", "dtype", "readonly", "strided"])
+def test_native_output_buffers_are_checked(recorded_batch, invalid):
+ args = native_arguments(recorded_batch)
+ if invalid == "shape":
+ args[-1] = args[-1][:, :4].copy()
+ elif invalid == "dtype":
+ args[-1] = args[-1].astype(np.float32)
+ elif invalid == "readonly":
+ args[-1].setflags(write=False)
+ else:
+ args[-1] = args[-1][::-1]
+ with pytest.raises((TypeError, ValueError, BufferError)):
+ matching._native_fit_irls_batch(*args)
+
+
+def test_native_batch_reuses_scalar_arithmetic_for_recorded_rows(recorded_batch):
+ args = native_arguments(recorded_batch)
+ expected = [matching._native_fit_irls(factor, u, args[2], args[3], args[5], args[6],
+ float(cap), args[7], args[8])
+ for factor, u, cap in zip(args[0], args[1], args[4])]
+ assert matching._native_fit_irls_batch(*args)
+ assert np.array_equal(args[-1], np.asarray(expected))
+
+
[email protected]("source", ["factors", "time", "ridge", "weight", "caps", "reversed_caps"])
+def test_native_batch_rejects_overlapping_output_before_writing(recorded_batch, source):
+ args = native_arguments(recorded_batch)
+ rows = len(args[0])
+ if source in {"caps", "reversed_caps"}:
+ storage = np.empty((rows, 5))
+ caps = storage[:, 0] if source == "caps" else storage[::-1, 0]
+ caps[:] = args[4]
+ args[4], args[-1] = caps, storage
+ else:
+ index = {"factors": 0, "time": 1, "ridge": 2, "weight": 3}[source]
+ args[-1] = args[index].reshape(-1)[:rows * 5].reshape(rows, 5)
+ before = [value.copy() for value in (*args[:5], args[-1])]
+ with pytest.raises(ValueError, match="must not overlap"):
+ matching._native_fit_irls_batch(*args)
+ for actual, saved in zip((*args[:5], args[-1]), before):
+ assert np.array_equal(actual, saved, equal_nan=True)
+
+
+def test_native_batch_accepts_disjoint_strided_recorded_inputs(recorded_batch):
+ args = native_arguments(recorded_batch)
+ expected = [matching._native_fit_irls(factor, u, args[2], args[3], args[5], args[6],
+ float(cap), args[7], args[8])
+ for factor, u, cap in zip(args[0], args[1], args[4])]
+ for index in (2, 3, 4):
+ storage = np.empty(len(args[index]) * 2)
+ vector = storage[-2::-2]
+ vector[:] = args[index]
+ args[index] = vector
+ assert matching._native_fit_irls_batch(*args)
+ assert np.array_equal(args[-1], np.asarray(expected))
+
+
+def test_native_batch_accepts_adjacent_input_and_output(recorded_batch):
+ args = native_arguments(recorded_batch)
+ expected = [matching._native_fit_irls(factor, u, args[2], args[3], args[5], args[6],
+ float(cap), args[7], args[8])
+ for factor, u, cap in zip(args[0], args[1], args[4])]
+ output_size = len(args[0]) * 5
+ storage = np.empty(output_size + len(args[2]))
+ storage[output_size:] = args[2]
+ args[2] = storage[output_size:]
+ args[-1] = storage[:output_size].reshape(len(args[0]), 5)
+ assert matching._native_fit_irls_batch(*args)
+ assert np.array_equal(args[-1], np.asarray(expected))
diff --git a/tests/test_real_partition_query.py b/tests/test_real_partition_query.py
new file mode 100644
index 0000000..bd84f4a
--- /dev/null
+++ b/tests/test_real_partition_query.py
@@ -0,0 +1,187 @@
+"""Clock-query checks on recorded audio settings and historical flight segments.
+
+The numeric boundary controls vary floating-point endpoints, not aircraft/audio.
+The optional dense replay uses the real fixture from benchmark_prepared_measurements.
+"""
+from copy import deepcopy
+import json
+import math
+import os
+from pathlib import Path
+from types import SimpleNamespace
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from dopplergeist.data_access import FlightIndex
+from dopplergeist.matching import association_candidates, prepare_events
+from dopplergeist.tiled_search import _partition_query_time_range
+
+
+def observed_settings():
+ fixture = json.loads((Path(__file__).parent / "data/reference-cpa-real.json").read_text())
+ events, rows = fixture["events"], fixture["cpa_rows"]
+ day = math.floor(rows[0]["emission_time"] / 86400) * 86400
+ anchor = [row["receive_time"] - events[0]["cpa_receive_s"] for row in rows[:2]]
+ return [day, day + 86400], events, {"mode": "robust_ridge"}, {}, anchor
+
+
+def test_recorded_clock_query_defaults_and_fallbacks(observed_settings):
+ original, events, parameters, filters, anchor = observed_settings
+ narrowed = _partition_query_time_range(*observed_settings)
+ assert original[0] < narrowed[0] < narrowed[1] < original[1]
+ assert _partition_query_time_range(original, events, parameters, {"association": "subsets"}, anchor) == narrowed
+ for changes in (
+ {4: None}, {4: [anchor[0], anchor[0]]}, {4: [math.nan, anchor[1]]},
+ {4: [anchor[0], math.inf]}, {4: [anchor[0], 10 ** 1000]},
+ {2: {"mode": "ridge"}}, {2: {"mode": "robust_ridge", "c": 0}},
+ {2: {"mode": "robust_ridge", "c": math.inf}},
+ {3: {"association": "full"}}, {3: {"silence": {"edge_margin_s": -1}}},
+ {3: {"offset_tolerance_s": -1}}, {3: {"offset_tolerance_s": math.inf}},
+ {1: []},
+ ):
+ args = list(observed_settings)
+ for index, value in changes.items():
+ args[index] = value
+ assert _partition_query_time_range(*args) is original
+ for name in ("cpa_tolerance_s", "speed_tolerance_mps", "slant_tolerance_m"):
+ for value in (-1, math.nan, math.inf, "invalid", 10 ** 1000):
+ changed = deepcopy(events)
+ changed[0].setdefault("quality", {})[name] = value
+ assert _partition_query_time_range(original, changed, parameters, filters, anchor) is original
+ malformed = deepcopy(events)
+ del malformed[0]["speed_range_mps"]
+ assert _partition_query_time_range(original, malformed, parameters, filters, anchor) is original
+ # Complete segments can bridge disjoint windows, so retain the original query.
+ disjoint = [original[0], math.nextafter(narrowed[0], -math.inf)]
+ assert _partition_query_time_range(disjoint, events, parameters, filters, anchor) is disjoint
+
+
+def test_recorded_clock_query_contains_rounded_interval_boundaries(observed_settings):
+ original, events, parameters, filters, anchor = observed_settings
+ lower, upper = _partition_query_time_range(*observed_settings)
+ tolerances = [event["quality"]["cpa_tolerance_s"] + 2.5 for event in events]
+ radius = max(event["slant_range_m"][1] for event in events)
+ overlaps = 0
+ for first in (anchor[0], math.nextafter(anchor[0], math.inf), math.nextafter(anchor[1], -math.inf)):
+ for first_tolerance in tolerances:
+ for event, tolerance in zip(events, tolerances):
+ for boundary in (first - first_tolerance - tolerance, first + first_tolerance + tolerance):
+ for start in (math.nextafter(boundary, -math.inf), boundary, math.nextafter(boundary, math.inf)):
+ if max(first - first_tolerance, start - tolerance) > min(first + first_tolerance, start + tolerance):
+ continue
+ for distance in (0.0, radius):
+ received = start + event["cpa_receive_s"]
+ for emitted in (math.nextafter(received - distance / 343.0, -math.inf),
+ received - distance / 343.0,
+ math.nextafter(received - distance / 343.0, math.inf)):
+ assert lower <= emitted <= upper
+ overlaps += 1
+ assert overlaps > 10000
+
+
+def dense_recorded_input():
+ fixture_path = os.environ.get("DOPPLERGEIST_REAL_DENSE_QUERY")
+ index_path = os.environ.get("DOPPLERGEIST_REAL_DEVELOPMENT_INDEX")
+ if not fixture_path or not index_path:
+ pytest.skip("set DOPPLERGEIST_REAL_DENSE_QUERY and DOPPLERGEIST_REAL_DEVELOPMENT_INDEX")
+ fixture = json.loads(Path(fixture_path).read_text())
+ index = FlightIndex(index_path, track_cache_bytes=0)
+ assert index.report["input_hash"] == fixture["index_input_hash"]
+ events, measurements = prepare_events(fixture["events"])
+ return fixture, index, events, measurements
+
+
+def test_dense_recorded_query_preserves_candidates_and_complete_segments(dense_recorded_input):
+ fixture, index, events, _ = dense_recorded_input
+ narrowed = _partition_query_time_range(fixture["query_time_s"], events, fixture["scoring"], {},
+ fixture["anchor_window_s"])
+ tracks = list(index.query(fixture["query_bounds"], narrowed))
+ ids = {track.id for track in tracks}
+ original_count = fixture["track_query_preparation"]["queried_trajectories"]
+ assert 0 < len(tracks) < original_count / 10
+ assert any(track.time[0] < narrowed[0] or track.time[-1] > narrowed[1] for track in tracks)
+ complete = {track.id: track for track in index.lookup(ids)}
+ for track in tracks:
+ np.testing.assert_array_equal(track.time, complete[track.id].time)
+ np.testing.assert_array_equal(track.xyz, complete[track.id].xyz)
+ np.testing.assert_array_equal(track.spline.c, complete[track.id].spline.c)
+ table = pd.DataFrame(fixture["cpa_rows"])
+ selected = table.loc[table.flight_id.isin(ids)]
+ options = dict(anchor_window=fixture["anchor_window_s"], max_candidates=fixture["budget"]["max_candidates_per_tile"])
+ before, before_counts = association_candidates(table, events, **options)
+ after, after_counts = association_candidates(selected, events, **options)
+ assert before and before == after
+ assert after_counts["hypotheses_examined"] == before_counts["hypotheses_examined"]
+ assert after_counts["work_units"] < before_counts["work_units"]
+ missing = {item["track_id"] for item in fixture["unsupported_cpa_tracks"]} - ids
+ assert missing
+ with index.connect() as connection:
+ intervals = dict((name, (start, end)) for name, start, end in
+ connection.execute("SELECT name, start, end FROM segments"))
+ assert all(intervals[name][1] < narrowed[0] or intervals[name][0] > narrowed[1] for name in missing)
+ print(f"real dense query: {original_count} -> {len(tracks)} complete segments; "
+ f"{len(table)} -> {len(selected)} CPA rows; {len(after)} identical association payloads")
+
+
[email protected]("driver", ["serial", "worker"])
+def test_recorded_partition_driver_queries_before_scoring(dense_recorded_input, tmp_path, monkeypatch, driver):
+ """Exercise each driver up to scoring with the recorded receiver and index."""
+ import dopplergeist.parallel_tiles as parallel
+ import dopplergeist.tiled_search as tiled
+ from dopplergeist.native_cpa import CPA_BACKEND
+
+ fixture, index, events, measurements = dense_recorded_input
+ parameters, filters = fixture["scoring"], {"association": "subsets"}
+ start_window = fixture["anchor_window_s"]
+ budget = tiled.SearchBudget(**fixture["budget"])
+ bounds = fixture["query_bounds"]
+ tile, cells = next(tiled.grid_tiles(bounds, budget.spacing_m, budget.tile_cells, "EPSG:5070"))
+ tile += "~t0"
+ anchor = tiled._tile_anchor_window(tile, start_window, parameters, filters, events)
+ expected = tiled._partition_query_time_range(fixture["query_time_s"], events, parameters, filters, anchor)
+ point = pd.DataFrame([fixture["point"]])
+ terrain = SimpleNamespace(last_sample_files=set(), last_sample_ocean_tiles=set(), failures=[])
+ queries = []
+ original_query = index.query
+
+ def query(query_bounds, query_time):
+ queries.append(query_time)
+ yield from original_query(query_bounds, query_time)
+
+ class ReachedScoring(RuntimeError):
+ pass
+
+ def score(points, tracks, *args, **kwargs):
+ assert queries == [expected]
+ assert kwargs["anchor_window"] == anchor
+ assert 0 < len(tracks) < fixture["track_query_preparation"]["queried_trajectories"] / 10
+ assert points.equals(point)
+ raise ReachedScoring
+
+ monkeypatch.setattr(index, "query", query)
+ target = tiled if driver == "serial" else parallel
+ monkeypatch.setattr(target, "_tile_coordinates", lambda *args: point.copy())
+ monkeypatch.setattr(target, "_expanded_bounds", lambda *args: bounds)
+ monkeypatch.setattr(target, "_score", score)
+ if driver == "serial":
+ monkeypatch.setattr(tiled, "grid_tiles", lambda *args: iter([(tile, cells)]))
+ with pytest.raises(ReachedScoring):
+ tiled.search_national([index], terrain, events, measurements,
+ time_range_s=fixture["query_time_s"], output_dir=tmp_path, bounds=bounds, budget=budget,
+ scoring=dict(parameters, recording_start_window_s=start_window), filters=filters, workers=1)
+ runtime = json.loads((tmp_path / "manifest.json").read_text())["configuration"]["runtime"]
+ if CPA_BACKEND == "python-cpa-v1":
+ assert "cpa_backend" not in runtime
+ else:
+ assert runtime["cpa_backend"] == CPA_BACKEND
+ else:
+ monkeypatch.setattr(parallel, "_worker", dict(indexes=[index], terrain=terrain, events=events,
+ measurements=measurements, bounds=bounds, time_range=fixture["query_time_s"], budget=budget,
+ parameters=parameters, filters=filters, start_window=start_window, crs="EPSG:5070",
+ ledger_directory=tmp_path, run_hash="recorded-query-test", score_kind="matched_group_support_minus_normalized_residual"))
+ with pytest.raises(ReachedScoring):
+ parallel.score_tile(f"coarse:{tile}", cells)