Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Integrate exact native search, prepared flight loading, and pipeline reuse

Cinna Davis49c90338f7be
diff --cc .github/workflows/real-performance.yml
index d727775,95b8a1c,a552267..fc1d21f
--- a/.github/workflows/real-performance.yml
+++ b/.github/workflows/real-performance.yml
@@@@ -4,14 -4,14 -4,14 +4,14 @@@@ on
inputs:
baseline:
description: Exact baseline commit for both real-input comparisons
- default: 27da08a878ac2ec90cd7e2c07f43765ceeea609b
- default: b4cd00ad46f0299baa716950fabe062d74f019f0
- default: 994d1ee48e25df6d30b23a356bba8f6502d8b7f6
+++ default: ae7721b7d3099980b884175cde1a3b07fc3fb1b5
required: true
regional:
description: Also compare the complete regional HTTP workflow
type: boolean
default: true
push:
- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103]
- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103, perf/flight-sieve-94a6, perf/flight-sieve-combined-94a6, perf/prepared-flight-scoring-20261103]
- branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103, perf/prepared-flight-search-20261103]
+++ branches: [perf/measured-hotpath-20261103, perf/real-pipeline-throughput-20261103, perf/single-pass-motion-20261103, perf/flight-sieve-reuse-20261103, perf/flight-sieve-94a6, perf/flight-sieve-combined-94a6, perf/prepared-flight-scoring-20261103, perf/prepared-flight-search-20261103, perf/integrated-search-47383f75]
paths:
- .github/workflows/real-performance.yml
- src/dopplergeist/**
@@@@ -26,9 -26,9 -31,11 +31,13 @@@@
- scripts/check_real_upload_arrays.py
- pyproject.toml
- tests/test_real_*.py
+++ - tests/test_refinement_geographic_bounds.py
- tests/data/real-ridge-batch-arithmetic.json
- tests/data/reference-empty-tiles-real.json
- tests/data/reference-retention-real.json
+++ - tests/data/reference-cpa-real.json
++ - tests/data/real-refinement-profile.json
++ - tests/fixtures/real_discovery_normalization.json
permissions:
contents: read
env:
@@@@ -38,7 -38,7 -45,7 +47,7 @@@@
NUMEXPR_NUM_THREADS: '1'
CC: gcc
PYTHONPATH: src
- BASELINE_REF: ${{ inputs.baseline || '27da08a878ac2ec90cd7e2c07f43765ceeea609b' }}
- BASELINE_REF: ${{ inputs.baseline || 'b4cd00ad46f0299baa716950fabe062d74f019f0' }}
- BASELINE_REF: ${{ inputs.baseline || '994d1ee48e25df6d30b23a356bba8f6502d8b7f6' }}
+++ BASELINE_REF: ${{ inputs.baseline || 'ae7721b7d3099980b884175cde1a3b07fc3fb1b5' }}
jobs:
compare:
if: github.event_name != 'push' || github.ref != 'refs/heads/perf/real-pipeline-throughput-20261103'
@@@@ -100,9 -103,11 -113,9 +118,49 @@@@
- name: Real-input regression tests
env:
DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json
+ + DOPPLERGEIST_REAL_DENSE_QUERY: benchmark-inputs/dense-query.json
+ + DOPPLERGEIST_REAL_DEVELOPMENT_INDEX: benchmark-inputs/reference-region.sqlite
DOPPLERGEIST_REAL_REFERENCE_ROOT: benchmark-inputs/reference-checkout
DOPPLERGEIST_REAL_REFERENCE_DISCOVERY: evidence/audio/w1-sensitivity-20261102/reference/discovery.json
- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py
- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_partition_query.py tests/test_real_native_cpa.py tests/test_real_native_ridge.py tests/test_real_native_ridge_batch.py tests/test_real_native_prefix_integration.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py tests/test_real_scalar_terrain.py tests/test_real_scoring_inputs.py tests/test_real_trial_prefix.py tests/test_real_candidate_setup.py tests/test_real_retention_batch.py
- run: python -m pytest -q tests/test_real_prepared_measurements.py tests/test_real_geometry_kinematics.py tests/test_real_batch_scalar_parity.py tests/test_real_motion_support_batch.py tests/test_real_single_pass_sieve.py tests/test_real_refinement_lookup.py tests/test_real_cpa_preparation.py tests/test_real_native_ridge.py tests/test_real_native_ridge_long.py tests/test_real_ridge_backend.py tests/test_real_legacy_batch_cache.py tests/test_real_native_acoustic.py tests/test_real_native_acoustic_pipeline.py tests/test_real_batch_ceilings.py tests/test_real_acoustic_screen_cache.py tests/test_real_cpa_spatial_sieve.py tests/test_real_native_validation.py tests/test_real_index_columns.py tests/test_real_track_cache.py tests/test_real_native_convergence.py tests/test_real_flight_payload.py tests/test_real_prepared_index.py tests/test_real_trial_vectors.py tests/test_real_refinement_reuse.py tests/test_real_discovery_normalization.py
+++ run: >-
+++ python -m pytest -q
+++ tests/test_real_prepared_measurements.py
+++ tests/test_real_geometry_kinematics.py
+++ tests/test_real_batch_scalar_parity.py
+++ tests/test_real_motion_support_batch.py
+++ tests/test_real_single_pass_sieve.py
+++ tests/test_real_refinement_lookup.py
+++ tests/test_real_cpa_preparation.py
+++ tests/test_real_partition_query.py
+++ tests/test_real_native_cpa.py
+++ tests/test_real_native_ridge.py
+++ tests/test_real_native_ridge_batch.py
+++ tests/test_real_native_prefix_integration.py
+++ tests/test_real_native_ridge_long.py
+++ tests/test_real_ridge_backend.py
+++ tests/test_real_legacy_batch_cache.py
+++ tests/test_real_native_acoustic.py
+++ tests/test_real_native_acoustic_pipeline.py
+++ tests/test_real_batch_ceilings.py
+++ tests/test_real_acoustic_screen_cache.py
+++ tests/test_real_cpa_spatial_sieve.py
+++ tests/test_real_native_validation.py
+++ tests/test_real_index_columns.py
+++ tests/test_real_track_cache.py
+++ tests/test_real_scalar_terrain.py
+++ tests/test_real_scoring_inputs.py
+++ tests/test_real_trial_prefix.py
+++ tests/test_real_candidate_setup.py
+++ tests/test_real_retention_batch.py
+++ tests/test_real_native_convergence.py
+++ tests/test_real_flight_payload.py
+++ tests/test_real_prepared_index.py
+++ tests/test_real_trial_vectors.py
+++ tests/test_real_refinement_reuse.py
+++ tests/test_refinement_geographic_bounds.py
+++ tests/test_real_discovery_normalization.py
+++ tests/test_real_association_budget.py
+++ tests/test_real_association_clock_window.py
- name: Verify the explicit Python fallback on recorded arithmetic
env:
DOPPLERGEIST_REAL_CPA_FIXTURE: benchmark-inputs/point-120.json
diff --cc README.md
index 0b843f1,0b843f1,0b843f1..f382184
--- a/README.md
+++ b/README.md
@@@@ -37,7 -37,7 -37,7 +37,7 @@@@ python -m venv .ven
To install the CLI without keeping a checkout, create a virtual environment and run `.venv/bin/python -m pip install "git+https://github.com/1337isnot1337/DOPPLERGEIST.git"`.
---Installation builds optional acoustic and ridge acceleration when a compatible C compiler is available. Python fallbacks remain available. See [native backend checks and reproducible timing](docs/PERFORMANCE.md#reproduce-the-optional-native-backends) to verify which implementations are active and compare complete recorded-input jobs.
+++Installation builds optional acoustic, ridge, and closest-approach (CPA) acceleration when a compatible C compiler is available. Python fallbacks remain available. See [native backend checks and reproducible timing](docs/PERFORMANCE.md#reproduce-the-optional-native-backends) to verify which implementations are active and compare complete recorded-input jobs.
Native Windows execution is unverified.
diff --cc docs/PERFORMANCE.md
index 9d99f2d,f9765f8,492b841..4406e5a
--- a/docs/PERFORMANCE.md
+++ b/docs/PERFORMANCE.md
@@@@ -1,5 -1,5 -1,27 +1,34 @@@@
# Measured pipeline performance
++ ## Prepared flight inputs and exact search reuse
++
++ [PR262](https://github.com/1337isnot1337/DOPPLERGEIST/pull/262) adds an optional
++ prepared trajectory representation, vectorized conservative trial bounds, and
++ exact native acoustic reuse. Against `994d1ee`, which already includes PR258,
-the measured dense query, sieve, scoring, and retention workload falls from
-**33.986 to 26.794 seconds: 1.268×, or 21.16% less elapsed time**.
-Final-runtime CI at `b98a0fa3` repeats the gain at **38.374 to 30.399 seconds,
-or 1.262× / 20.78% less time**, and passes 168 recorded-input and fallback checks.
+++the historical instrumented dense workload falls from
+++**33.986 to 26.794 seconds: a 1.268× diagnostic ratio, or 21.16% less elapsed time**.
+++Final-runtime CI at `b98a0fa3` records **38.374 to 30.399 seconds,
+++or a 1.262× diagnostic ratio / 20.78% less time**, and passes 168 recorded-input and fallback checks.
++ All compared scientific outputs and logical retention records remain equal.
++
+++The original prepared-flight harness wrapped numerical functions during these
+++dense measurements. The wrappers add overhead and can disable optimizations
+++guarded by function identity in the combined runtime. These figures retain
+++diagnostic and exact-output value. They do not measure canonical production throughput.
+++The [historical timing scope](PERFORMANCE_PREPARED_SEARCH.md#historical-timing-scope)
+++explains the qualification and the corrected harness controls.
+++
++ The final implementation also reuses local terrain pixels and profile coordinate
++ transforms, and arranges spectral median inputs contiguously. A completed
++ intermediate HTTP series improves 9.56%. The final source has two complete exact
++ HTTP pairs, both faster, followed by an interrupted third pair. The report keeps
++ that incomplete series separate and makes no three-pair final HTTP claim.
++
++ The [prepared-search guide](PERFORMANCE_PREPARED_SEARCH.md) gives the complete
++ samples, preparation cost, storage tradeoff, HTTP measurements, operator steps,
++ and reproduction commands. Its pinned baseline separates these gains from the
++ earlier changes documented below. Component speedups are not multiplied.
++
## Indexed flight queries and repeated evidence checks
[PR258](https://github.com/1337isnot1337/DOPPLERGEIST/pull/258) reduces repeated work in indexed flight loading and ridge-evidence validation.
@@@@ -1096,8 -1099,8 -1118,8 +1128,8 @@@@ requires its validated checkpoint compa
## Reproduce the optional native backends
---Installation builds the acoustic and ridge extensions when a compatible C compiler is available.
---The package remains usable when either optional extension cannot build.
+++Installation builds the acoustic, ridge, and closest-approach (CPA) extensions when a compatible C compiler is available.
+++The package remains usable when an optional extension cannot build.
A source checkout used for timing must contain its own built extensions; an installed copy elsewhere does not establish native use.
```bash
@@@@ -1106,25 -1109,25 -1128,25 +1138,36 @@@@ CC=gcc python -m pip install --editabl
python - <<'PY'
from dopplergeist.geometry import ACOUSTIC_BACKEND
from dopplergeist.matching import RIDGE_FIT_BACKEND
+++from dopplergeist.native_cpa import CPA_BACKEND
print("Acoustic:", ACOUSTIC_BACKEND)
print("Ridge:", RIDGE_FIT_BACKEND)
+++print("CPA:", CPA_BACKEND)
assert ACOUSTIC_BACKEND.startswith("native-c-")
assert RIDGE_FIT_BACKEND.startswith("native-c-")
+++assert CPA_BACKEND.startswith("native-c-")
PY
```
The native acoustic solver retains polynomial evaluation order, global Newton stopping, and Python motion-support diagnostics.
The ridge extension retains the six IRLS iterations and reduction order.
---Both extensions disable floating-point reassociation and contraction.
+++The CPA extension preserves the existing closest-approach selection and arithmetic order.
+++All three extensions disable floating-point reassociation and contraction.
Incompatible numeric representations use the existing Python implementations.
Acoustic and scalar ridge calls also fall back for unsupported NumPy error policies.
Batched ridge arithmetic retains its existing ignored-error block and restores the caller's policy for diagnostics.
The optional ridge implementation is integrated from [PR228](https://github.com/1337isnot1337/DOPPLERGEIST/pull/228).
---Set `DOPPLERGEIST_ACOUSTIC_BACKEND=python` and `DOPPLERGEIST_RIDGE_BACKEND=python` before starting a process to force both Python implementations.
+++Set all three selectors before starting a process to force the Python implementations:
+++
+++```bash
+++export DOPPLERGEIST_ACOUSTIC_BACKEND=python
+++export DOPPLERGEIST_RIDGE_BACKEND=python
+++export DOPPLERGEIST_CPA_BACKEND=python
+++```
+++
The default for each selector is `auto`; invalid selector values fail explicitly.
---The legacy checkpoint upgrade requires both Python backends and its original runtime and input controls.
---Ordinary checkpoints include the selected native binary identities and reject an implementation mismatch.
+++The legacy checkpoint upgrade requires all three Python backends and its original runtime and input controls.
+++Ordinary checkpoints include the selected backend and implementation identities and reject an implementation mismatch.
### Recorded-input scoring comparison
diff --cc docs/PERFORMANCE_PREPARED_SEARCH.md
index 0000000,0000000,356bf8d..234d18e
mode 000000,000000,100644..100644
--- a/docs/PERFORMANCE_PREPARED_SEARCH.md
+++ b/docs/PERFORMANCE_PREPARED_SEARCH.md
@@@@ -1,0 -1,0 -1,404 +1,426 @@@@
++ # Prepared flight data and exact pipeline optimization
++
-The final runtime's dense-search CI workload falls from **38.374 to 30.399
-seconds**, or **20.78% less elapsed time**. The earlier local search comparison
+++## Historical timing scope
+++
+++The dense-search figures in this document are **instrumented diagnostic timings**.
+++The original `benchmark_prepared_flight_search.py` always wrapped numerical functions,
+++even for its default comparison. Those wrappers add timing overhead and change
+++function identities. In the combined runtime, they can also disable guarded input
+++and prefix reuse and native batch dispatch.
+++
+++The original harness wrapped `_fit_ridge_uncached`, `Track.acoustic`,
+++`_native_fit_irls`, and `flight_payload.decode_track`. The archived dense results
+++retain exact-output evidence, but do not measure canonical production throughput.
+++The separate HTTP, refinement, and discovery comparisons use other harnesses.
+++This qualification does not change their declared measurement scopes.
+++
+++The final runtime's dense-search CI diagnostic falls from **38.374 to 30.399
+++seconds**, or **20.78% less elapsed time**. The earlier local diagnostic
++ falls from **33.986 to 26.794 seconds**, or **21.16% less time**.
++ A completed intermediate HTTP series falls from **58.233 to 52.668
++ seconds**, or **9.56% less time**, after exact refinement reuse is added.
++ These are separate measured workloads. Their speedup ratios must not be multiplied.
++
++ The finalized runtime is `b98a0fa3e9dc8a204ffddf00f0bc411c9206d868`.
++ Earlier measurements retain their original source identities:
++
++ - Baseline: `994d1ee48e25df6d30b23a356bba8f6502d8b7f6`, including PR #258.
++ - Dense search and initial HTTP: `ecb1726fb9872d0060265014aa830e598b4dd3b4`.
++ - Independent CI and portable component baseline: `c0c9e798f47b547ab581fa473912e7218e3b4c3e`.
++
++ The [measurement record](../benchmarks/results/prepared-flight-search-20261103.json)
++ contains individual samples, source identities, preparation costs, rejected
++ experiments, and comparison scope. Later changes on `main` are outside these pins.
++
-## Dense search and independent replication
+++## Historical instrumented dense-search comparisons
++
-| Timed work | Baseline median | Current median | Speedup |
+++| Timed work | Baseline median | Current median | Diagnostic ratio |
++ | --- | ---: | ---: | ---: |
-| Complete measured search, wall | 33.985746 s | 26.793586 s | 1.268× |
-| Complete measured search, CPU | 33.968096 s | 26.784584 s | 1.268× |
+++| Complete instrumented search, wall | 33.985746 s | 26.793586 s | 1.268× |
+++| Complete instrumented search, CPU | 33.968096 s | 26.784584 s | 1.268× |
++ | Flight query and distance sieve, wall | 6.569945 s | 2.237149 s | 2.937× |
++ | CPA, associations, scoring and retention, wall | 27.846495 s | 24.058635 s | 1.157× |
++
++ Stage medians do not sum to the median of the complete search.
++ The query traverses 4,694 observed trajectories and 713,100 samples.
++ The unchanged sieve selects 1,354 trajectories and 357,324 samples.
++ The scorer evaluates 16,101 candidates and retains 100.
++ The recording supplies 16 independent event groups and 724 ridge samples.
++ All six runs return exactly equal complete outputs and logical retention records.
++
++ This workload uses one source-informed receiver point and one association-anchor
++ partition, without a recording-start clock prior. The regional archive has
++ incomplete coverage. The result does not establish independent location accuracy,
++ complete daily search, or national completion time.
++
++ The timer includes the initial flight query, distance sieve, associations,
++ scoring, and retention writes. It excludes imports, index metadata, event
++ preparation, downloads, terrain sampling, audio discovery, and refinement.
++ Both arms read the same prepared database; baseline reads its original payloads.
++
++ [Independent CI](https://github.com/1337isnot1337/DOPPLERGEIST/actions/runs/41701995581)
-repeats the dense comparison on another machine: **38.451 to 30.292 seconds**,
-or **1.269×**. Its three alternating pairs produce exactly equal complete outputs.
+++repeats the instrumented dense comparison on another machine: **38.451 to 30.292 seconds**,
+++or a **1.269× diagnostic ratio**. Its three alternating pairs produce exactly equal complete outputs.
++ CI pins one available CPU and rebuilds the original index with baseline code.
++ That earlier workflow passes 133 recorded-input checks and four fallback checks,
++ without skips. Separate installation and static checks pass on Python 3.11 and 3.14.
++
++ [Final-runtime CI](https://github.com/1337isnot1337/DOPPLERGEIST/actions/runs/41708828025)
-repeats all comparisons at `b98a0fa3`. Prepared dense-search wall medians are
-**38.373940 versus 30.398725 seconds**, or **1.262× / 20.78% less time**.
+++repeats all comparisons at `b98a0fa3`. Instrumented dense-search wall medians are
+++**38.373940 versus 30.398725 seconds**, or a **1.262× diagnostic ratio / 20.78% less time**.
++ CPU medians are 38.329208 versus 30.363599 seconds. Worker-lifetime peak RSS
++ medians are 541.410 versus 518.855 MiB. Query/sieve medians are 8.150010 versus
++ 2.962375 seconds; scoring/retention medians are 30.179885 versus 27.415653 seconds.
++ All three pairs preserve complete outputs. The final workflow passes **164
++ recorded-input checks and four Python-fallback checks**, without skips.
++ [Final installation checks](https://github.com/1337isnot1337/DOPPLERGEIST/actions/runs/41708837646)
++ also pass on Python 3.11 and 3.14. HTTP CI jobs remain outside this branch's push gate.
++
++ ## Full HTTP and refinement measurements
++
++ | Complete HTTP series | Baseline median | Current median | Result |
++ | --- | ---: | ---: | --- |
++ | Initial search changes | 54.020778 s | 54.855469 s | 1.55% more time |
++ | Search plus refinement reuse | 58.233276 s | 52.668349 s | 9.56% less time |
++
++ Each series uses three fresh alternating pairs. The initial series establishes
++ **no complete-pipeline speedup**. Its narrower request has little dense-query cost;
++ producer search timing is essentially unchanged, at 22.759 versus 22.739 seconds.
++ Each current run in the later refinement series is faster than its paired baseline.
++
++ Every pair passes exact scientific JSON, logical SQLite, decoded audio, CSV-array,
++ and PNG-pixel comparisons. All uploads traverse intake, discovery, search, and
++ refinement, returning the same 23 candidates and 17 event records.
++ The request remains `inconclusive / search_incomplete`: 40 refinement points are
++ searched and 80 are skipped. All 15 independent event groups are considered.
++ Faster execution does not establish complete coverage or an accepted location.
++
++ The later series adds these exact files to the earlier runtime:
++
++ - `matching.py`: `8d7c6b8f7bf83f06203640df32136d1dbe625cd558270bc7b4184305f9024826`.
++ - `terrain.py`: `9390f39b25f41ee03e567b3c9c83fce9bf0a0285dc90f842124a39c93e43c8b2`.
++
++ The request, complete recording, index, terrain, and budgets remain unchanged.
++ Each verifier checks source stability. Verifier elapsed time includes service
++ startup, polling, shutdown, and artifact checks. Phase observations have one-second
++ polling uncertainty. Unchanged intake code also varies between runs, so the entire
++ HTTP saving cannot be attributed to terrain alone. Compare paired arms within a
++ series; absolute times across separate experiments have different host conditions.
++
++ A separate complete sensitivity-profile comparison isolates refinement reuse.
++ Median wall time falls from **9.064 to 8.178 seconds**, or **9.78% less time**.
++ Median CPU time falls from 9.054 to 8.175 seconds.
++ All six runs return exactly equal complete profiles and retain all **46 solves**.
++ Each performs 36,835 native acoustic calls. Input loading and earlier finalist
++ refinement are outside this component timer.
++
++ Pixel reuse applies to `terrain.Terrain` with local operator GeoTIFF files.
++ The online provider's `data_access.CachedTerrain` is unchanged. Profile-local
++ transformer reuse applies with either provider. The HTTP measurement uses local
++ rasters and makes no online-provider throughput claim.
++
++ The final `b98a0fa3` HTTP series has **two complete exact pairs and five verified
++ upload receipts**. Its third pair was interrupted before the last receipt.
++
++ | Completed final-runtime pair | Baseline | Current | Paired ratio |
++ | --- | ---: | ---: | ---: |
++ | 1, baseline then current | 58.389875 s | 51.997905 s | 1.122927× |
++ | 2, current then baseline | 57.154603 s | 54.008309 s | 1.058256× |
++
++ The unpaired third baseline took 59.612323 seconds and remains in the record.
++ The interrupted current attempt has no valid elapsed-time sample. The incomplete
++ series has `speedup: null`; no three-pair aggregate is reported.
++ The session's final exit status could not be recovered. These partial observations
++ do not establish a three-pair speedup for the final runtime. The completed
++ refinement series above remains the measured full-HTTP result.
++
++ ## Discovery experiment and final correctness
++
++ The discovery normalization prototype reduces median stage wall time from
++ **6.156 to 5.761 seconds**, or **6.43% less time**, across three alternating pairs.
++ It processes the complete 3,600.1845-second saved PCM recording and writes the
++ normal JSON, 17 CSV files, and summary PNG. Decoding and search are excluded.
++ Pre-call imports are excluded; lazy plotting imports inside discovery are included.
++ Median peak RSS increases from 159.953 to 165.883 MiB, a 5.930 MiB tradeoff.
++
++ This timing precedes the final guards for C-contiguous arrays and canonical NumPy
++ functions. It is **not a separate timing claim for the final runtime**.
++ The final implementation passes a new full-recording comparison: all 60 spectra
++ match in bytes, dtype, shape, strides, and mutability. Complete event records,
++ CSV bytes, and diagnostic pixels also match. Eighteen portable real-PCM tests
++ cover odd/even medians, custom hooks, data types, layouts, and short inputs.
++ The final combined HTTP series supplies two exact integration pairs; its planned
++ three-pair timing series remains incomplete.
++
++ ## What changes
++
++ - **Prepared trajectory inputs:** indexes store numeric constructor inputs beside
++ the original compressed observations. Values follow original decoding and
++ normalization, preserving rounding, inferred velocities, and altitude labels.
++ - **Vectorized trial bounds:** masks and conservative bounds operate across
++ eligible clocks. Event order, upward rounding, strict rejection, diagnostics,
++ and custom-function fallbacks remain unchanged.
++ - **Native acoustic reuse:** the solver reuses exact spline intervals and exact
++ fixed points. Retired samples still contribute to the global stopping condition.
++ Tolerances and iteration limits remain unchanged.
++ - **Refinement reuse:** each profile shares its unchanged coordinate transformers.
++ Each local sampler retains at most 1,024 exact single-pixel masked reads.
++ Pixel selection, raster precedence, missing data, geoid correction, and returned
++ array mutability remain unchanged. Changed or custom readers use original reads.
++ - **Discovery normalization:** contiguous frequency columns accelerate median
++ reductions for eligible finite float32 arrays. Returned layout and every
++ arithmetic expression remain unchanged. Other inputs use the original path.
++
++ Prepared data uses little-endian float64 arrays and bounded JSON metadata, with
++ no executable object format. Tracks remain independently mutable. Original
++ payload integrity checks remain mandatory, including cache hits. Prepared hashes
++ and implementation identity must match; incompatible optional data uses original
++ payloads. Runtime identity includes normalization code, dependencies, Python, and
++ CPU capabilities. Operators can prepare a new copy after changing the runtime.
++ The acoustic scratch allocation is bounded at 2 MiB; larger requests keep the
++ original native calculation. No optimization changes search budgets or resolution.
++
++ Local validation includes 70 focused search tests, 13 refinement/terrain tests,
++ and 18 discovery tests, without skips. The refinement cases cover closed,
++ replaced, changed, and custom readers, masked pixels, bounded eviction, and hooks.
++
++ ## Measurement method
++
++ 1. Freeze real observations, request settings, source hashes, and workload counts.
++ Keep complete outputs so comparisons can detect changed scientific work.
++ 2. Measure the full workflow first. Profile separately to locate substantial costs;
++ exclude profiler instrumentation from reported timing pairs.
++ 3. Optimize repeated work while preserving operations, bounds, and scenario counts.
++ Reject faster results that omit candidates, diagnostics, arrays, or coverage records.
++ 4. Compare full values exactly before accepting timings. Include ordered inventory,
++ candidate fields, logical retention rows, and array bytes and layouts where relevant.
++ 5. Use fresh interpreters or services in alternating baseline/current order.
++ Use one numerical thread and pause other team benchmarks during each comparison.
++ 6. Record wall time, CPU time, memory, preparation, and storage separately.
++ Report workload-specific medians and all samples; never multiply component ratios.
++
+++The current prepared-flight harness leaves numerical functions unchanged during
+++default comparisons. It uses passive native counters when those counters exist.
+++Unmeasured counters are null.
+++`--instrument-numerics`, `--stage-timing`, or active Python profiling set
+++`performance_timing_eligible=false`. These diagnostic reports use `diagnostic_ratio`
+++and leave production speedup and preparation amortization null.
+++
++ Local runs do not pin CPU affinity or flush operating-system caches. Unrelated
++ host load is uncontrolled. CI timing pins one available CPU. Three pairs are a
++ small performance sample, so retain the workload and variability qualifications.
++ Native backends and prepared decoding must actually execute when required.
++ Every measured current dense run records 4,694 successful prepared decodes.
++
++ ## Rejected and deferred experiments
++
++ | Experiment | Observed result | Decision |
++ | --- | --- | --- |
++ | FFmpeg `filter_threads=1`, three pairs | 8.195 → 8.644 s median | Slower; rejected |
++ | FFmpeg `flush_packets=0`, one pair | 8.444 → 7.751 s | Below the declared 10% time-reduction gate; stopped |
++ | FFmpeg direct-file output, one pair | 7.689 → 7.915 s | Slower; rejected |
++ | Native ridge IRLS fixed points, five pairs | 0.212 → 0.226 s median | Slower; rejected |
++
++ Decoder trials preserve all 57,602,952 PCM bytes, sample counts, and peak amplitude.
++ The IRLS trial preserves every result byte from 36,835 actual calls, but this
++ kernel is too small to dominate the full refinement profile. These changes are
++ not integrated. Single-pair results are screening observations, not stable medians.
++ An earlier nominal-solve reuse prototype was simplified to transformer reuse;
++ all 46 final solves remain, with fewer custom-hook guards.
++
++ Raw-STFT overlap reuse is **unimplemented and unmeasured**. The real recording has
++ 14,100 spectral frames but 7,197 distinct aligned windows: 6,903 transforms repeat.
++ However, raw spectrogram work occupies only 0.693 profiled seconds. At equal
++ per-frame cost, perfect reuse of the repeated 48.96% would save about 0.339 seconds.
++ This idealized estimate helped prioritize normalization instead. It is not a cache
++ speedup. A future cache must preserve windows and recompute chunk-dependent statistics.
++ Bounded chunk concurrency also remains unimplemented; results must enter merging
++ and harmonic grouping in original order.
++
++ An earlier decoding-only experiment measured 7.929× on 4,694 trajectories against
++ `8bd72aed`. It excludes SQL reads and cache hits, and does not describe full search.
++
++ ## Preparation cost and operator use
++
++ | One-time preparation | Result |
++ | --- | ---: |
++ | Wall / CPU time | 7.418266 / 7.370144 s |
++ | Original SQLite file | 41,385,984 bytes |
++ | Prepared SQLite copy | 85,929,984 bytes |
++ | Added storage | 44,544,000 bytes |
++
++ Preparation copies the index, decodes canonical observations, writes the optional
++ representation, and commits. The original remains unchanged. Input identity,
++ source records, and coverage declarations stay equal. Original payloads remain
++ available for compatibility and fallback.
++ The timer excludes imports and separate whole-file integrity verification.
++ That verification reads the source before timing, warming its file pages.
-Two comparable searches recover the measured preparation cost. This estimate
-uses the combined code-and-format saving, not an isolated format benefit.
+++The historical two-search break-even estimate used the instrumented dense timings.
+++It does not establish production amortization. A new estimate requires
+++uninstrumented comparisons in the integrated runtime.
++
++ Prepare an existing index in the serving runtime. Choose a new destination:
++
++ ```bash
++ python scripts/prepare_flight_index.py \
++ --index /absolute/path/original.sqlite \
++ --output /absolute/path/prepared.sqlite
++ ```
++
++ Set the operator configuration's index path:
++
++ ```json
++ {"flight_indexes": ["/absolute/path/prepared.sqlite"]}
++ ```
++
++ Keep the existing terrain, coverage, and search configuration. Start the service
++ as described in [PRODUCT.md](PRODUCT.md). New indexes include prepared inputs by
++ default. Unprepared indexes remain usable; preparation does not expand coverage.
++
++ ## Reproduce the comparisons
++
++ Use the final checkout containing these scripts, Python 3.12, GCC, and the same
++ dependencies for all source trees. Build the pinned dense runtime and baseline:
++
++ ```bash
++ export PYTHONPATH=src
++ export CC=gcc
++ export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1
++ export MKL_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1
++ python -m pip install 'setuptools>=68'
++ python -m pip install --editable '.[test]'
++ git worktree add --detach ../dopplergeist-baseline \
++ 994d1ee48e25df6d30b23a356bba8f6502d8b7f6
++ git worktree add --detach ../dopplergeist-dense \
++ ecb1726fb9872d0060265014aa830e598b4dd3b4
++ (cd ../dopplergeist-baseline && python setup.py build_ext --inplace)
++ (cd ../dopplergeist-dense && python setup.py build_ext --inplace)
++ ```
++
++ Restore public observations. Build the original index with baseline code, then
++ prepare a common copy with the measured dense runtime:
++
++ ```bash
++ python scripts/restore_artifacts.py --bundle data-adsb-derived --bundle data-terrain
++ PYTHONPATH=../dopplergeist-baseline/src \
++ python scripts/prepare_reference_upload.py --data-root data --output benchmark-inputs
++ python scripts/benchmark_prepared_measurements.py prepare \
++ --discovery evidence/audio/w1-sensitivity-20261102/reference/discovery.json \
++ --index benchmark-inputs/reference-region.sqlite \
++ --reference-report evidence/results/reference-local/report.json \
++ --full-day --max-tracks 2000 --max-samples 500000 --max-candidates 20000 \
++ --anchor-window-s 1788864888.7151732 1788865236.7214916 \
++ --include-track-query --output benchmark-inputs/dense-query.json
++ python scripts/benchmark_prepared_flight_search.py prepare-index \
++ --source ../dopplergeist-dense/src --index benchmark-inputs/reference-region.sqlite \
++ --destination benchmark-inputs/reference-region-prepared.sqlite \
++ --output results/flight-preparation.json
++ python scripts/benchmark_prepared_flight_search.py compare \
++ --fixture benchmark-inputs/dense-query.json \
++ --baseline-source ../dopplergeist-baseline/src --current-source ../dopplergeist-dense/src \
++ --baseline-index benchmark-inputs/reference-region-prepared.sqlite \
++ --current-index benchmark-inputs/reference-region-prepared.sqlite \
++ --preparation-report results/flight-preparation.json \
++ --repeats 3 --timeout-s 600 --output results/performance-prepared-flight-search \
++ --require-native --require-native-acoustic --require-current-prepared
++ ```
++
++ ### Bounded HTTP replay
++
++ Use the existing `benchmark-inputs` directory from the preparation steps above.
++ The measured upload is the archived **43,202,925-byte MP3**, with SHA-256
++ `707083f0344823eeea7f130faec6c22b6628b8b9fea04d026915d93df9a07b15`.
++ Place that exact file at `benchmark-inputs/recording.mp3`. The public WAV or a new
++ MP3 encoding is a **new workload**: decoding cost, decoded samples, and downstream
++ results can differ. Do not attribute the archived timing results to that substitute.
++
++ Set the exact bounded request and one-worker operator configuration. Both arms
++ must read the same prepared index. Keep the restored terrain paths and the
++ preparation helper's search budgets unchanged:
++
++ ```bash
++ cp /absolute/path/archived-recording.mp3 benchmark-inputs/recording.mp3
++ python - <<'PY'
++ import hashlib
++ import json
++ from pathlib import Path
++
++ inputs = Path("benchmark-inputs")
++ recording = inputs / "recording.mp3"
++ assert recording.stat().st_size == 43202925
++ with recording.open("rb") as stream:
++ assert hashlib.file_digest(stream, "sha256").hexdigest() == "707083f0344823eeea7f130faec6c22b6628b8b9fea04d026915d93df9a07b15"
++ request = {
++ "schema_version": 1,
++ "search": {"coverage": "conus", "bounds_wsen": [-73.545, 40.865, -73.505, 40.895]},
++ "time_window": {
++ "start_utc": "2026-09-08T20:17:20Z", "end_utc": "2026-09-08T20:17:40Z",
++ "source": "user",
++ "source_detail": "Source-informed regional runtime control using the historical fitted clock and the published four-point receiver region; not independent timing evidence or a location-accuracy evaluation.",
++ },
++ }
++ operator = json.loads((inputs / "operator.json").read_text())
++ operator["flight_indexes"] = [str((inputs / "reference-region-prepared.sqlite").resolve(strict=True))]
++ operator["search_workers"] = 1
++ assert operator["search_budget"] == {"spacing_m": 2000, "tile_cells": 16, "top_k": 20,
++ "refine_spacing_m": 500, "refine_finalists": 3}
++ assert operator["refine_finalists"] == 3 and operator["offline"] is True
++ for name, value in (("request.json", request), ("operator.json", operator)):
++ (inputs / name).write_text(json.dumps(value, indent=2) + "\n")
++ PY
++ python scripts/benchmark_real_bounded_comparison.py \
++ --baseline-source ../dopplergeist-baseline/src --current-source src \
++ --inputs benchmark-inputs --output results/performance-bounded-http \
++ --repeats 3 --timeout-s 900
++ ```
++
++ This command runs three alternating pairs with fresh services and jobs. It
++ requires `inconclusive / search_incomplete`, complete workflow receipts, and exact
++ scientific and array comparisons. Preparation is outside the timer. The controller
++ preserves logs and receipts, and reports a speedup only after all requested pairs
++ complete successfully. An interrupted run must remain an incomplete series.
++
++ ### Refinement and discovery
++
++ For refinement, build the public component baseline. The committed fixture
++ contains the real finalist, unchanged discovery observations, and provenance:
++
++ ```bash
++ git worktree add --detach ../dopplergeist-component-baseline \
++ c0c9e798f47b547ab581fa473912e7218e3b4c3e
++ (cd ../dopplergeist-component-baseline && python setup.py build_ext --inplace)
++ python scripts/benchmark_real_refinement.py compare \
++ --baseline-source ../dopplergeist-component-baseline/src --current-source src \
++ --index benchmark-inputs/reference-region-prepared.sqlite --terrain-root . \
++ --repeats 3 --require-native --require-native-acoustic \
++ --output results/performance-refinement
++ ```
++
++ This timer covers the complete profile and its exact-fit cache, excluding
++ imports, index lookup, terrain construction, event preparation, and earlier
++ finalist refinement. Every arm writes its full result and source hashes.
++ The portable wrapper has AST, CLI-help, and static validation. The reported
++ component timings came from the earlier tested harness on these same recorded
++ inputs; the optional portable-wrapper timing smoke was canceled.
++
++ Discovery requires a saved real service job with decoded PCM, intake metadata,
++ and discovery artifacts. Replace the job path with that existing directory:
++
++ ```bash
++ python scripts/benchmark_real_discovery_normalization.py compare \
++ --baseline-source ../dopplergeist-component-baseline/src --current-source src \
++ --job /absolute/path/saved-job --pairs 3 \
++ --output results/performance-discovery
++ ```
++
++ The discovery script checks all spectra outside timing. It continues three-pair
++ screening only when its first exact pair clears the declared improvement threshold.
++ The completed HTTP comparisons supply separate integration checks.
++ Use new output directories. Commands fail on unequal outputs or unused required
++ backends. Their reports contain samples and medians; workers preserve complete
++ comparison outputs. Keep profiling separate and avoid competing benchmarks.
++
++ The [performance workflow](../.github/workflows/real-performance.yml) contains
++ recorded-input regression commands and the dense comparison. It uploads the
++ preparation report and pins one CPU for timing. Local full HTTP replays remain
++ separate from the branch's automatic dense and static CI gates.
diff --cc scripts/benchmark_prepared_flight_search.py
index 0000000,0000000,b70e45e..67aa440
mode 000000,000000,100644..100644
--- a/scripts/benchmark_prepared_flight_search.py
+++ b/scripts/benchmark_prepared_flight_search.py
@@@@ -1,0 -1,0 -1,484 +1,529 @@@@
++ """Measure real flight-query, overload-sieve and scoring work with exact parity.
++
++ Use a frozen recorded-data fixture and two pinned src/ directories. Each arm runs
++ in a fresh interpreter. An optional prepared index changes representation only;
++ its one-time construction is measured separately by the prepare-index command.
++ The source-informed regional receiver and anchor partition are performance
++ workloads, not independent localization or complete national-search evidence.
+++Default timings leave numerical and decoder functions unchanged. Use
+++--instrument-numerics or --stage-timing for separate diagnostics; their ratios
+++are never reported as performance speedups or preparation amortization.
++ """
++ from __future__ import annotations
++
++ import argparse
++ import hashlib
++ import json
++ import math
++ import os
++ from pathlib import Path
++ import statistics
++ import subprocess
++ import sys
++ import time
++
++ try:
++ import resource
++ except ImportError:
++ resource = None
++
++ ROOT = Path(__file__).resolve().parents[1]
++
++ # Query and scientific-output capture adapted from the project's PR #258
++ # benchmark_prepared_measurements.py; keep the original scoring harness intact.
++
++
++ def json_scalar(value):
++ if hasattr(value, "tolist"):
++ return value.tolist()
++ raise TypeError(f"Cannot serialize {type(value).__name__}")
++
++ def write_new(path, value):
++ path = Path(path)
++ path.parent.mkdir(parents=True, exist_ok=True)
++ with path.open("x") as output:
++ json.dump(value, output, indent=2, allow_nan=True, default=json_scalar)
++ output.write("\n")
++
++ def canonical(value):
++ return json.dumps(value, sort_keys=True, separators=(",", ":"),
++ allow_nan=True, default=json_scalar)
++
++ def query_tracks(index, points, events, budget, scoring, bounds, time_range):
++ """Count actual index traversal while exercising the production overload path."""
++ from dopplergeist.tiled_search import _tracks_for_points
++
++ if "elev_m" not in points:
++ raise ValueError("The recorded receiver hypothesis must include its elevation")
++ operations = {"query_calls": 0, "decoded_track_yields": 0, "decoded_sample_yields": 0}
++ inventory = {}
++ ordered_inventory = []
++
++ class CountedIndex:
++ def query(self, query_bounds, query_time):
++ operations["query_calls"] += 1
++ for track in index.query(query_bounds, query_time):
++ operations["decoded_track_yields"] += 1
++ operations["decoded_sample_yields"] += len(track.time)
++ inventory[track.id] = len(track.time)
++ ordered_inventory.append([track.id, len(track.time)])
++ yield track
++
++ radius = max(event["slant_range_m"][1] for event in events)
++ tracks, points = _tracks_for_points(
++ [CountedIndex()], bounds, time_range, budget, points, None, radius,
++ scoring.get("c", 343.0))
++ accounting = {
++ "queried_tracks_in_order": ordered_inventory,
++ "queried_track_ids": sorted(inventory),
++ "queried_trajectory_samples": sum(inventory.values()),
++ "selected_track_ids": [track.id for track in tracks],
++ "selected_trajectory_samples": sum(len(track.time) for track in tracks),
++ }
++ return tracks, points, operations, accounting
++
++ def worker(args):
++ source = args.source.resolve()
++ sys.path.insert(0, str(source))
++ import dopplergeist
++ import pandas as pd
++ from dopplergeist import geometry, matching, tiled_search
++ from dopplergeist.data_access import FlightIndex
++ from dopplergeist.finite_search import RetentionLedger
++ from dopplergeist.geometry import AcousticSupportError, Track
++
++ if Path(dopplergeist.__file__).resolve().parent != source / "dopplergeist":
++ raise RuntimeError("Comparison imported a different source tree")
++ native_fit = getattr(matching, "_native_fit_irls", None)
+++ native_module = getattr(matching, "_native_ridge_module", None)
+++ native_statistics = getattr(native_module, "fit_statistics", None)
++ ridge_backend = getattr(matching, "RIDGE_FIT_BACKEND", "python-irls-v1")
++ acoustic_backend = getattr(geometry, "ACOUSTIC_BACKEND", "python-acoustic-v1")
++ try:
++ from dopplergeist.native_acoustic import _native as native_acoustic
++ except ImportError:
++ native_acoustic = None
++ if args.require_native and not callable(native_fit):
++ raise RuntimeError("The selected source's native ridge fitter is unavailable")
++ if args.require_native_acoustic and native_acoustic is None:
++ raise RuntimeError("The selected source's native acoustic solver is unavailable")
++ fixture = json.loads(args.fixture.read_text())
++ index = FlightIndex(args.index or fixture["real_index"],
++ track_cache_bytes=args.track_cache_bytes)
++ if index.report["input_hash"] != fixture["index_input_hash"]:
++ raise ValueError("The real trajectory index no longer matches the fixture")
++ if args.require_prepared and not getattr(index, "_prepared_tracks", False):
++ raise RuntimeError("The selected source cannot use this prepared index")
++ budget = tiled_search.SearchBudget(**fixture["budget"])
++ loaded_at = time.perf_counter()
++ tracks = None
++ if not args.include_track_query:
++ tracks = list(index.lookup(fixture["track_ids"], max_tracks=budget.max_tracks_per_tile,
++ max_samples=budget.max_track_samples_per_tile))
++ if sorted(track.id for track in tracks) != sorted(fixture["track_ids"]):
++ raise ValueError("The index is missing a recorded trajectory")
++ events, measurements = matching.prepare_events(fixture["events"])
++ points = pd.DataFrame([fixture["point"]])
++ load_seconds = time.perf_counter() - loaded_at
++ args.output.mkdir(parents=True, exist_ok=False)
++ ledger = RetentionLedger(args.output / "retention.sqlite", "recorded-point-benchmark",
++ "matched_group_support_minus_normalized_residual")
++ stage = "coarse:recorded-point"
++ operations = {"scalar_fit_calls": 0, "acoustic_calls": 0, "scored_candidates": 0,
++ "native_fit_calls": 0, "native_fit_successes": 0,
++ "native_acoustic_successes": 0, "prepared_decode_calls": 0,
++ "prepared_decode_successes": 0}
++ all_rows = []
++ original_fit, original_acoustic = matching._fit_ridge_uncached, Track.acoustic
++ original_score = tiled_search.score_candidate
++ phase_seconds, phase_patches = {}, []
++ original_decoder, payload_module = None, None
- if getattr(index, "_prepared_tracks", False):
+++ if args.instrument_numerics and getattr(index, "_prepared_tracks", False):
+++ # Leave data_access._frame_track unchanged even in diagnostics: its
+++ # identity guards the prepared decoder. Count the selected decoder only.
++ from dopplergeist import flight_payload as payload_module
++ original_decoder = payload_module.decode_track
++
++ def counted_decode(*values, **options):
++ operations["prepared_decode_calls"] += 1
++ track = original_decoder(*values, **options)
++ operations["prepared_decode_successes"] += 1
++ return track
++
++ payload_module.decode_track = counted_decode
++
++ def timed_phase(name, function):
++ def run(*values, **options):
++ began, cpu = time.perf_counter(), time.process_time()
++ try:
++ return function(*values, **options)
++ finally:
++ phase = phase_seconds.setdefault(name, {"wall_seconds": 0.0, "cpu_seconds": 0.0, "calls": 0})
++ phase["wall_seconds"] += time.perf_counter() - began
++ phase["cpu_seconds"] += time.process_time() - cpu
++ phase["calls"] += 1
++ return run
++
++ if args.stage_timing:
++ phase_targets = [
++ (Track, "outside_observer_radius", "distance_sieve"),
++ (matching, "association_candidates", "association_enumeration"),
++ (tiled_search, "merge_candidates", "retention"),
++ ]
++ if callable(getattr(tiled_search, "cpa_table", None)):
++ phase_targets.append((tiled_search, "cpa_table", "closest_approach"))
++ else:
++ prepared_cpa = getattr(tiled_search, "_PreparedCPATrack", None)
++ if prepared_cpa is not None:
++ phase_targets.extend([
++ (prepared_cpa, "__init__", "closest_approach_preparation"),
++ (prepared_cpa, "raw_rows", "closest_approach"),
++ ])
++ for owner, attribute, name in phase_targets:
++ function = getattr(owner, attribute, None)
++ if function is not None:
++ phase_patches.append((owner, attribute, function))
++ setattr(owner, attribute, timed_phase(name, function))
++ score_function = timed_phase("candidate_scoring", original_score) if args.stage_timing else original_score
++
++ def counted_fit(*values, **options):
++ operations["scalar_fit_calls"] += 1
++ return original_fit(*values, **options)
++
++ def counted_acoustic(*values, **options):
++ operations["acoustic_calls"] += 1
++ return original_acoustic(*values, **options)
++
++ def counted_native(*values, **options):
++ operations["native_fit_calls"] += 1
++ result = native_fit(*values, **options)
++ if result is not None and all(math.isfinite(value) for value in result):
++ operations["native_fit_successes"] += 1
++ return result
++
++ def captured_score(*values, **options):
++ operations["scored_candidates"] += 1
++ try:
++ result = score_function(*values, **options)
++ except AcousticSupportError as error:
++ all_rows.append({"support_failure": tiled_search._support_error(error)})
++ raise
++ all_rows.append(result)
++ return result
++
- matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic
+++ if args.instrument_numerics:
+++ matching._fit_ridge_uncached, Track.acoustic = counted_fit, counted_acoustic
+++ else:
+++ for name in ("scalar_fit_calls", "acoustic_calls", "native_fit_calls", "native_fit_successes",
+++ "prepared_decode_calls", "prepared_decode_successes"):
+++ operations[name] = None
++ tiled_search.score_candidate = captured_score
- if callable(native_fit):
+++ native_before = native_statistics() if callable(native_statistics) else None
+++ if args.instrument_numerics and callable(native_fit) and native_before is None:
++ matching._native_fit_irls = counted_native
++ native_acoustic_before = native_acoustic.successful_calls() if native_acoustic is not None else 0
++ query_accounting, query_operations, execution_stages = None, None, {}
++ began, cpu = time.perf_counter(), time.process_time()
++ try:
++ if args.include_track_query:
++ query_began, query_cpu = time.perf_counter(), time.process_time()
++ tracks, points, query_operations, query_accounting = query_tracks(
++ index, points, events, budget, fixture["scoring"],
++ fixture["query_bounds"], fixture["query_time_s"])
++ execution_stages["track_query_and_distance_sieve"] = {
++ "wall_seconds": time.perf_counter() - query_began,
++ "cpu_seconds": time.process_time() - query_cpu,
++ }
++ if query_accounting["selected_track_ids"] != fixture["track_ids"]:
++ raise AssertionError("The real query changed the ordered selected-track inventory")
++ if query_accounting["selected_trajectory_samples"] != fixture["counts"]["trajectory_samples"]:
++ raise AssertionError("The real query changed the selected trajectory sample count")
++ prepared_query = fixture.get("track_query_preparation")
++ if prepared_query is not None:
++ if (len(query_accounting["queried_track_ids"]) != prepared_query["queried_trajectories"]
++ or query_accounting["queried_trajectory_samples"] != prepared_query["queried_trajectory_samples"]):
++ raise AssertionError("The real query changed the complete observed-track inventory")
++ scoring_began, scoring_cpu = time.perf_counter(), time.process_time()
++ partition = ({"anchor_window": fixture["anchor_window_s"]}
++ if "anchor_window_s" in fixture else {})
++ ranked, counts = tiled_search._score(
++ points, tracks, events, measurements, budget, fixture["scoring"], {},
++ start_window=fixture["recording_start_window_s"], ledger=ledger, stage=stage, **partition)
++ if args.include_track_query:
++ execution_stages["cpa_association_scoring_and_retention"] = {
++ "wall_seconds": time.perf_counter() - scoring_began,
++ "cpu_seconds": time.process_time() - scoring_cpu,
++ }
++ finally:
++ elapsed, cpu_seconds = time.perf_counter() - began, time.process_time() - cpu
- matching._fit_ridge_uncached, Track.acoustic = original_fit, original_acoustic
++ tiled_search.score_candidate = original_score
- if callable(native_fit):
- matching._native_fit_irls = native_fit
+++ if args.instrument_numerics:
+++ matching._fit_ridge_uncached, Track.acoustic = original_fit, original_acoustic
+++ if callable(native_fit) and native_before is None:
+++ matching._native_fit_irls = native_fit
+++ if native_before is not None:
+++ native_after = native_statistics()
+++ delta = {key: native_after[key] - native_before[key] for key in native_before}
+++ operations["native_fit_calls"] = delta["scalar_calls"] + delta["batch_rows"]
+++ operations["native_fit_successes"] = delta["scalar_successes"] + delta["batch_successful_rows"]
+++ operations["native_batch_calls"] = delta["batch_calls"]
+++ operations["native_batch_rows"] = delta["batch_rows"]
++ if native_acoustic is not None:
++ operations["native_acoustic_successes"] = native_acoustic.successful_calls() - native_acoustic_before
++ for owner, attribute, function in reversed(phase_patches):
++ setattr(owner, attribute, function)
++ if payload_module is not None:
++ payload_module.decode_track = original_decoder
++ expected_counts = fixture.get("expected_counts")
++ if expected_counts is not None:
++ actual_counts = {"cpa_rows": counts["cpa_rows"],
++ "associations": counts["candidates"], "retained": len(ranked)}
++ if actual_counts != expected_counts:
++ raise AssertionError(f"Recorded workload counts changed: {actual_counts}")
++ # macOS reports bytes and Linux reports KiB. Keep scoring usable where
++ # resource or its unit convention is unavailable, without guessing a value.
++ peak_rss_mib = None
++ if resource is not None and (sys.platform == "darwin" or sys.platform.startswith("linux")):
++ rss_divisor = 1024 * 1024 if sys.platform == "darwin" else 1024
++ peak_rss_mib = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rss_divisor
++ checksum = ledger.checksum(stage)
++ if not ledger.stage_valid(stage, checksum):
++ raise AssertionError("The scoring pass produced an invalid retention ledger")
++ if not operations["scored_candidates"]:
++ raise AssertionError("The benchmark did not score any real associations")
- if args.require_native and not operations["native_fit_successes"]:
+++ if (args.require_native and operations["native_fit_successes"] is not None
+++ and not operations["native_fit_successes"]):
++ raise AssertionError("No successful native ridge fit occurred inside the timed scoring call")
++ if args.require_native_acoustic and not operations["native_acoustic_successes"]:
++ raise AssertionError("No successful native acoustic solve occurred inside the timed scoring call")
- if args.require_prepared and not operations["prepared_decode_successes"]:
+++ if (args.require_prepared and operations["prepared_decode_successes"] is not None
+++ and not operations["prepared_decode_successes"]):
++ raise AssertionError("No successful prepared decode occurred inside the timed query")
++ with ledger.connect() as database:
++ retention = {name: database.execute(f"SELECT * FROM {name} ORDER BY 1,2").fetchall()
++ for name in ("records", "bounds", "stages")}
++ results = {"ranked": ranked, "counts": counts,
++ "all_scored_candidates": all_rows, "retention": retention}
++ if query_accounting is not None:
++ results["track_query_accounting"] = query_accounting
++ results["workload"] = {key: fixture.get(key) for key in (
++ "index_input_hash", "point", "events", "query_bounds", "query_time_s",
++ "recording_start_window_s", "anchor_window_s", "budget", "scoring")}
++ results["index_coverage"] = index.report["coverage"]
++ write_new(args.output / "results.json", results)
+++ diagnostic = args.instrument_numerics or args.stage_timing or sys.getprofile() is not None
++ summary = {"wall_seconds": elapsed, "cpu_seconds": cpu_seconds,
++ "load_seconds_excluded": load_seconds, "operations": operations,
+++ "numerical_functions_canonical": not (args.instrument_numerics or args.stage_timing),
+++ "performance_timing_eligible": not diagnostic,
+++ "measurement_mode": "diagnostic" if diagnostic else "canonical_timing",
+++ "ridge_execution_check": ("In-module successful-call counters" if native_before is not None else
+++ "Wrapped successful-call counters" if args.instrument_numerics else
+++ "Backend availability; successful-call count unmeasured"),
+++ "prepared_execution_check": ("Wrapped successful-call counters" if args.instrument_numerics else
+++ "Compatible representation availability; successful-call count unmeasured"),
++ "ridge_fit_backend": ridge_backend, "acoustic_backend": acoustic_backend,
++ "native_extension_available": callable(native_fit),
++ "peak_rss_mib": peak_rss_mib,
++ "peak_rss_scope": ("Worker lifetime through scoring, including imports and input loading"
++ if peak_rss_mib is not None else "Unavailable on this platform"),
++ "retained": len(ranked), "independent_groups": len(events), "scope": fixture["scope"]}
++ summary["index_prepared_tracks"] = index.report.get("prepared_tracks")
++ summary["track_cache_bytes"] = args.track_cache_bytes
++ if "anchor_window_s" in fixture:
++ summary["anchor_window_s"] = fixture["anchor_window_s"]
++ if args.include_track_query:
++ summary["track_query_operations"] = query_operations
++ summary["execution_stage_seconds"] = execution_stages
++ summary["timing_scope"] = (
++ "Initial local index traversal and conservative distance sieve, followed by complete "
++ "CPA/association scoring and retention. Excludes imports, index metadata, event "
++ "preparation, downloads, terrain sampling, audio discovery, and refinement.")
++ if args.stage_timing:
++ summary["phase_seconds"] = phase_seconds
++ summary["phase_timing_scope"] = "Inclusive instrumented call times; overlapping phases must not be summed."
++ write_new(args.output / "summary.json", summary)
++ print(json.dumps(summary), flush=True)
++
++
++ def file_digest(path):
++ with Path(path).open("rb") as stream:
++ return hashlib.file_digest(stream, "sha256").hexdigest()
++
++
++ def prepare_index(args):
++ """Time a new optional representation without changing the original index."""
++ sys.path.insert(0, str(args.source.resolve()))
++ import dopplergeist
++ from dopplergeist.data_access import FlightIndex
++
++ if Path(dopplergeist.__file__).resolve().parent != args.source.resolve() / "dopplergeist":
++ raise RuntimeError("Preparation imported a different source tree")
++ if args.output.exists() or args.destination.exists():
++ raise FileExistsError("Choose new preparation output and index paths")
++ before = file_digest(args.index)
++ began, cpu = time.perf_counter(), time.process_time()
++ original = FlightIndex(args.index)
++ prepared = original.prepare(args.destination)
++ elapsed, cpu_seconds = time.perf_counter() - began, time.process_time() - cpu
++ if before != file_digest(args.index):
++ raise AssertionError("Preparation modified the original observed index")
++ for key in ("input_hash", "coverage", "sources"):
++ if original.report.get(key) != prepared.report.get(key):
++ raise AssertionError(f"Preparation changed the original {key}")
++ if not getattr(prepared, "_prepared_tracks", False):
++ raise AssertionError("The optional representation is unavailable to its builder")
++ report = {
++ "wall_seconds": elapsed, "cpu_seconds": cpu_seconds,
++ "source": str(args.source.resolve()), "index": str(args.index.resolve()),
++ "destination": str(args.destination.resolve()),
++ "input_hash": original.report["input_hash"],
++ "original_unchanged": True,
++ "source_bytes": args.index.stat().st_size,
++ "prepared_bytes": args.destination.stat().st_size,
++ "prepared_tracks": prepared.report["prepared_tracks"],
++ "timing_scope": "One-time index copy, canonical decoding, optional representation and commit; excludes imports and verification.",
++ "cache_scope": "Source integrity verification reads the original index before timing; operating-system file caches are not flushed.",
++ }
++ write_new(args.output, report)
++ print(json.dumps(report), flush=True)
++
++
++ def compare(args):
++ args.output.mkdir(parents=True, exist_ok=False)
++ sources = {"baseline": args.baseline_source.resolve(), "current": args.current_source.resolve()}
++ fixture = json.loads(args.fixture.read_text())
++ indexes = {"baseline": Path(args.baseline_index or fixture["real_index"]).resolve(),
++ "current": Path(args.current_index or fixture["real_index"]).resolve()}
++ environment = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
++ for variable in ("OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS"):
++ environment[variable] = "1"
++ runs, expected = [], None
++ for repetition in range(args.repeats):
++ order = ("baseline", "current") if repetition % 2 == 0 else ("current", "baseline")
++ for name in order:
++ output = args.output / f"{repetition + 1}-{name}"
++ command = [sys.executable, str(Path(__file__).resolve()), "_worker",
++ "--source", str(sources[name]), "--fixture", str(args.fixture.resolve()),
++ "--index", str(indexes[name]), "--output", str(output.resolve()),
++ "--track-cache-bytes", str(args.track_cache_bytes)]
- for flag in ("stage_timing", "require_native", "require_native_acoustic"):
+++ for flag in ("stage_timing", "instrument_numerics", "require_native", "require_native_acoustic"):
++ if getattr(args, flag):
++ command.append("--" + flag.replace("_", "-"))
++ if name == "current" and args.require_current_prepared:
++ command.append("--require-prepared")
++ print(f"Starting pair {repetition + 1}: {name}", flush=True)
++ subprocess.run(command, check=True, env=environment, timeout=args.timeout_s)
++ payload = canonical(json.loads((output / "results.json").read_text()))
++ if expected is None:
++ expected = payload
++ elif payload != expected:
++ raise AssertionError(f"Exact inventory, scientific-output or retention mismatch: {output}")
++ summary = json.loads((output / "summary.json").read_text())
++ runs.append({"source": name, "repetition": repetition + 1, **summary})
++ medians = {name: {} for name in sources}
++ for name in sources:
++ for metric in ("wall_seconds", "cpu_seconds", "peak_rss_mib"):
++ values = [row[metric] for row in runs if row["source"] == name and row[metric] is not None]
++ medians[name][metric] = statistics.median(values) if values else None
++ report = {
++ "identical_results": True, "repeats": args.repeats,
++ "fixture": str(args.fixture.resolve()),
++ "sources": {name: str(path) for name, path in sources.items()},
++ "indexes": {name: str(path) for name, path in indexes.items()},
++ "medians": medians,
++ "speedup": {metric: medians["baseline"][metric] / medians["current"][metric]
++ for metric in ("wall_seconds", "cpu_seconds")},
++ "execution_stage_medians": {
++ name: {stage: {metric: statistics.median(
++ row["execution_stage_seconds"][stage][metric]
++ for row in runs if row["source"] == name)
++ for metric in ("wall_seconds", "cpu_seconds")}
++ for stage in runs[0]["execution_stage_seconds"]}
++ for name in sources},
++ "scope": runs[0]["scope"], "timing_scope": runs[0]["timing_scope"],
++ "cache_scope": "Fresh interpreter and FlightIndex per arm; operating-system file caches are not flushed.",
++ "runs": runs,
++ }
+++ report["performance_timing_eligible"] = (
+++ not (args.instrument_numerics or args.stage_timing or sys.getprofile() is not None)
+++ and all(row["performance_timing_eligible"] for row in runs))
+++ if not report["performance_timing_eligible"]:
+++ report["diagnostic_ratio"] = report["speedup"]
+++ report["speedup"] = None
++ if args.preparation_report:
++ preparation = json.loads(args.preparation_report.read_text())
++ if (Path(preparation["destination"]).resolve() != indexes["current"]
++ or preparation["input_hash"] != fixture["index_input_hash"]):
++ raise ValueError("Preparation report does not describe the compared current index")
++ report["preparation"] = preparation
++ report["preparation_amortization_scope"] = (
++ "Preparation cost divided by the complete baseline/current query-and-score saving. "
++ "This includes other code improvements and measures the combined upgrade, "
++ "not the isolated prepared-format contribution.")
++ report["queries_to_amortize_preparation"] = {}
++ for metric in ("wall_seconds", "cpu_seconds"):
++ saved = medians["baseline"][metric] - medians["current"][metric]
++ report["queries_to_amortize_preparation"][metric] = (
- max(1, math.ceil(preparation[metric] / saved)) if saved > 0 else None)
+++ max(1, math.ceil(preparation[metric] / saved))
+++ if report["performance_timing_eligible"] and saved > 0 else None)
+++ if not report["performance_timing_eligible"]:
+++ report["preparation_amortization_scope"] += " Unavailable for diagnostic timings."
++ write_new(args.output / "comparison.json", report)
++ print(json.dumps({key: report[key] for key in (
++ "identical_results", "medians", "speedup", "execution_stage_medians", "scope")}), flush=True)
++
++
++ def main():
++ parser = argparse.ArgumentParser(description=__doc__)
++ commands = parser.add_subparsers(dest="command", required=True)
++ preparation = commands.add_parser("prepare-index", help="Measure one-time creation of an optional index representation")
++ preparation.add_argument("--source", type=Path, default=ROOT / "src")
++ preparation.add_argument("--index", type=Path, required=True)
++ preparation.add_argument("--destination", type=Path, required=True)
++ preparation.add_argument("--output", type=Path, required=True)
++ comparison = commands.add_parser("compare", help="Compare equal real query/sieve/scoring workloads")
++ comparison.add_argument("--fixture", required=True, type=Path)
++ comparison.add_argument("--baseline-source", required=True, type=Path)
++ comparison.add_argument("--current-source", type=Path, default=ROOT / "src")
++ comparison.add_argument("--baseline-index", type=Path)
++ comparison.add_argument("--current-index", type=Path)
++ comparison.add_argument("--preparation-report", type=Path)
++ comparison.add_argument("--repeats", type=int, default=3)
++ comparison.add_argument("--timeout-s", type=float, default=900.0)
++ comparison.add_argument("--output", required=True, type=Path)
- comparison.add_argument("--require-current-prepared", action="store_true")
+++ comparison.add_argument("--require-current-prepared", action="store_true",
+++ help="Require a compatible prepared index; count successful decodes in diagnostics")
++ internal = commands.add_parser("_worker", help=argparse.SUPPRESS)
++ internal.add_argument("--source", required=True, type=Path)
++ internal.add_argument("--fixture", required=True, type=Path)
++ internal.add_argument("--index", type=Path)
++ internal.add_argument("--output", required=True, type=Path)
- internal.add_argument("--require-prepared", action="store_true")
+++ internal.add_argument("--require-prepared", action="store_true",
+++ help="Require a compatible prepared index; count successful decodes in diagnostics")
++ for command in (comparison, internal):
++ command.add_argument("--track-cache-bytes", type=int, default=32 * 1024 * 1024)
++ command.add_argument("--stage-timing", action="store_true",
- help="Collect inclusive call times in a separate diagnostic run")
+++ help="Diagnostic phase timings; do not report a performance speedup")
+++ command.add_argument("--instrument-numerics", action="store_true",
+++ help="Diagnostic numerical/decoder counters; wrappers may disable guarded optimizations")
++ command.add_argument("--require-native", action="store_true",
- help="Require successful native ridge fits in every compared source")
+++ help="Require native ridge backends; verify successful calls when counters are available")
++ command.add_argument("--require-native-acoustic", action="store_true",
++ help="Require successful native acoustic solves in every compared source")
++ command.set_defaults(include_track_query=True)
++ args = parser.parse_args()
++ if hasattr(args, "track_cache_bytes") and args.track_cache_bytes < 0:
++ parser.error("--track-cache-bytes must be nonnegative")
++ if args.command == "prepare-index":
++ prepare_index(args)
++ elif args.command == "compare":
++ if args.repeats < 1 or not math.isfinite(args.timeout_s) or args.timeout_s <= 0:
++ parser.error("--repeats and --timeout-s must be positive")
++ compare(args)
++ else:
++ worker(args)
++
++
++ if __name__ == "__main__":
++ main()
diff --cc scripts/benchmark_real_association.py
index e0d3f37,e0d3f37,e0d3f37..866423c
--- a/scripts/benchmark_real_association.py
+++ b/scripts/benchmark_real_association.py
@@@@ -5,6 -5,6 -5,6 +5,7 @@@@ flight observations or proposal measure
join, not whole-search runtime or localization accuracy.
"""
import argparse
+++import gc
import json
from pathlib import Path
import statistics
@@@@ -39,7 -39,7 -39,7 +40,13 @@@@ def main()
sys.modules[baseline.__name__] = baseline
exec(compile(source, "<baseline matching.py>", "exec"), baseline.__dict__)
fixture = json.loads(args.fixture.read_text())
--- events, _ = prepare_events(fixture["events"])
+++ # Archived CPA fixtures contain either original discovery records or their
+++ # already prepared association settings. Preserve both observed formats.
+++ recorded_events = fixture["events"]
+++ if recorded_events and "ridge" in recorded_events[0]:
+++ events, _ = prepare_events(recorded_events)
+++ else:
+++ events = recorded_events
table = pd.DataFrame(fixture["cpa_rows"])
reference, counts = baseline.association_candidates(table, events)
if not reference:
@@@@ -92,6 -92,6 -92,6 +99,11 @@@@
for repeat in range(args.repetitions):
order = ("baseline", "optimized") if repeat % 2 == 0 else ("optimized", "baseline")
for name in order:
+++ # Release the previous result and begin each measured call at the
+++ # same garbage-collector state. Otherwise a prior run can choose
+++ # which arm pays for collecting thousands of candidate objects.
+++ actual = None
+++ gc.collect()
wall, cpu = time.perf_counter(), time.process_time()
actual = functions[name](table, events)
run = {"implementation": name, "cpu_s": time.process_time()-cpu,
@@@@ -103,6 -103,6 -103,6 +115,7 @@@@
medians = {name: statistics.median(r["cpu_s"] for r in runs if r["implementation"] == name)
for name in functions}
report = {"fixture": str(args.fixture), "baseline_ref": args.baseline_ref,
+++ "garbage_collection_reset_before_each_run": True,
"events": len(events), "cpa_rows": len(table), "candidates": len(reference),
"exact_candidate_order_and_work_accounting": True, "checks": checks, "runs": runs,
"median_cpu_s": medians, "cpu_speedup": medians["baseline"]/medians["optimized"]}
diff --cc scripts/compare_real_upload_performance.py
index 38075ae,38075ae,38075ae..0dfe5ac
--- a/scripts/compare_real_upload_performance.py
+++ b/scripts/compare_real_upload_performance.py
@@@@ -146,7 -146,7 -146,7 +146,8 @@@@ class Run
reason = "search implementation files"
elif not root and relative == "search/manifest.json" and here in {
("configuration", "runtime", "ridge_fit_backend"),
--- ("configuration", "runtime", "acoustic_backend")}:
+++ ("configuration", "runtime", "acoustic_backend"),
+++ ("configuration", "runtime", "cpa_backend")}:
reason = "search native implementation metadata:" + key
elif not root and relative == "provenance.json" and here == ("resumed_from",):
reason = "explicit resume metadata: resumed_from"
@@@@ -330,7 -330,7 -330,7 +331,8 @@@@ def compare_runs(baseline, current, max
if passed and left.continuation is not None else None),
"normalization": {"implementation_fields": sorted(IMPLEMENTATION_FIELDS),
"native_backend_metadata": ["search/manifest.json/configuration/runtime/ridge_fit_backend",
--- "search/manifest.json/configuration/runtime/acoustic_backend"],
+++ "search/manifest.json/configuration/runtime/acoustic_backend",
+++ "search/manifest.json/configuration/runtime/cpa_backend"],
"performance_fields": sorted(PERFORMANCE_FIELDS),
"path_rule": "Only each run directory, job directory and exact job identifier are replaced.",
"sqlite_rule": "Compare all logical tables, columns and rows; ignore only run.identity and physical SQLite/index layout.",
diff --cc src/dopplergeist/matching.py
index a36d049,869c9a7,2e38359..246c02e
--- a/src/dopplergeist/matching.py
+++ b/src/dopplergeist/matching.py
@@@@ -332,7 -344,7 -332,7 +344,10 @@@@ def association_candidates(table, event
selected = max(0, right - left)
if selected:
reserve(selected * selected.bit_length())
--- return sorted(indices[left:right])
+++ return sorted(indices[left:right])
+++ # Most skipped events have no overlapping row. Avoid allocating and
+++ # sorting an empty slice while preserving the same branch accounting.
+++ return ()
def row_options(indices):
return chain((records[i] for i in indices), [None])
@@@@ -707,8 -713,8 -692,22 +731,22 @@@@ def refine_candidate(candidate, tracks
Returned radius describes the explored alternatives only. A boundary optimum
warns callers that this local search has not enclosed the possible solution.
"""
-- forward = Transformer.from_crs("EPSG:4326", crs, always_xy=True)
-- inverse = Transformer.from_crs(crs, "EPSG:4326", always_xy=True)
++ return _refine_candidate(candidate, tracks, events, measurements, terrain,
++ radius_m=radius_m, clock_radius_s=clock_radius_s,
++ altitude_offsets_m=altitude_offsets_m, crs=crs, c=c,
- recording_start_window_s=recording_start_window_s)
+++ recording_start_window_s=recording_start_window_s, bounds_wsen=bounds_wsen)
++
++
++ def _refine_candidate(candidate, tracks, events, measurements, terrain, radius_m=500.0,
++ clock_radius_s=5.0, altitude_offsets_m=(-150.0, 0.0, 150.0),
- crs="EPSG:5070", c=343.0, recording_start_window_s=None, *,
- _transformers=None):
+++ crs="EPSG:5070", c=343.0, recording_start_window_s=None,
+++ bounds_wsen=None, *, _transformers=None):
++ """Share coordinate operations only within one declared sensitivity profile."""
++ if _transformers is None:
++ forward = Transformer.from_crs("EPSG:4326", crs, always_xy=True)
++ inverse = Transformer.from_crs(crs, "EPSG:4326", always_xy=True)
++ else:
++ forward, inverse = _transformers
x0, y0 = forward.transform(candidate["lon"], candidate["lat"])
start = candidate["recording_start_time_s"]
selected = [tracks[f] for f in candidate["flight_ids"]]
@@@@ -802,8 -796,8 -792,22 +843,22 @@@@ def profile_candidate(candidate, tracks
raise ValueError("Sensitivity ranges must be nonnegative and sound speed must remain positive")
kwargs = dict(radius_m=radius_m, clock_radius_s=clock_radius_s,
altitude_offsets_m=(0.0,), crs=crs, c=c,
-- recording_start_window_s=recording_start_window_s)
- nominal = refine_candidate(candidate, tracks, events, measurements, terrain, **kwargs)
++ recording_start_window_s=recording_start_window_s, bounds_wsen=bounds_wsen)
- nominal = refine_candidate(candidate, tracks, events, measurements, terrain, **kwargs)
++ transformers = None
++
++ def refine(*args, **options):
++ nonlocal transformers
++ # Custom refiners keep the original dispatch and receive no private
++ # arguments. The standard path owns exactly one pair for this profile.
++ current_refiner = refine_candidate
++ if current_refiner is not _CANONICAL_REFINE_CANDIDATE:
++ return current_refiner(*args, **options)
++ if transformers is None:
++ transformers = (Transformer.from_crs("EPSG:4326", crs, always_xy=True),
++ Transformer.from_crs(crs, "EPSG:4326", always_xy=True))
++ return _refine_candidate(*args, _transformers=transformers, **options)
++
++ nominal = refine(candidate, tracks, events, measurements, terrain, **kwargs)
scenarios = [("nominal", "nominal", events, measurements, {})]
for sign in (-1, 1):
scenarios.append(("altitude", f"height_{sign:+d}", events, measurements,
diff --cc src/dopplergeist/terrain.py
index 608a803,6703db0,934a435..dac01d3
--- a/src/dopplergeist/terrain.py
+++ b/src/dopplergeist/terrain.py
@@@@ -1,6 -1,7 -1,10 +1,10 @@@@
"""Terrain sampling and common projected grids."""
+ from collections import OrderedDict
from functools import lru_cache
from pathlib import Path
++ import stat
++ from threading import RLock
++ from weakref import ref
import numpy as np
from pyproj import Transformer
@@@@ -16,11 -17,93 -54,58 +54,58 @@@@ class Terrain
def __init__(self, paths=None, constant_m=None, geoid_path=None):
import rasterio
self.rasters = [rasterio.open(p) for p in (paths or [])]
++ self._pixel_cache = OrderedDict()
++ self._pixel_cache_lock = RLock()
++ self._pixel_sources = {}
++ for raster in self.rasters:
++ source = _raster_pixel_source(raster)
++ if source is not None:
++ self._pixel_sources[id(raster)] = source
self.constant = constant_m
self.geoid = Terrain([geoid_path]) if geoid_path else None
- # Each sampler, including the optional geoid, retains at most 4 MiB
- # of exact pixels and masks for scalar optimizer evaluations.
- self._scalar_blocks = OrderedDict()
- self._scalar_block_bytes = 0
- self._scalar_cache_limit = 4 * 1024 * 1024
- self._scalar_sources = {}
if not self.rasters and constant_m is None:
raise ValueError("Supply DEM raster paths or an explicit constant ground elevation")
- @staticmethod
- def _source_signature(paths):
- result = []
- for path in paths:
- stat = path.stat()
- result.append((stat.st_dev, stat.st_ino, stat.st_size,
- stat.st_mtime_ns, stat.st_ctime_ns))
- return tuple(result)
-
- def _scalar_pixel(self, raster, row, col):
- """Return an unchanged pixel and mask, with bounded local-file reuse."""
- from rasterio.io import DatasetReader
- from rasterio.windows import Window
-
- # Custom readers, writable/closed datasets and virtual sources retain
- # their normal read behavior. A closed reader must not return old pixels.
- cacheable = (self._scalar_cache_limit > 0 and
- type(raster) is DatasetReader and not raster.closed and
- raster.mode == "r" and raster.driver == "GTiff")
- if cacheable and raster not in self._scalar_sources:
- paths = tuple(Path(path) for path in raster.files)
- try:
- signature = self._source_signature(paths) if paths else None
- except OSError:
- signature = None
- self._scalar_sources[raster] = ((paths, signature)
- if signature is not None else None)
- source = self._scalar_sources.get(raster) if cacheable else None
- if source is None:
- data = raster.read(1, window=Window(col, row, 1, 1), masked=True)
- return np.asarray(data)[0, 0], np.ma.getmaskarray(data)[0, 0]
-
- paths, signature = source
- try:
- unchanged = self._source_signature(paths) == signature
- except OSError:
- unchanged = False
- if not unchanged:
- raise ValueError(f"Terrain raster changed after caching: {raster.name}")
-
- # Keep windows inside the same underlying storage block as the requested
- # pixel, including striped TIFFs and blocks not divisible by 64.
- size = 64
- block_h, block_w = raster.block_shapes[0]
- block_r, block_c = row // block_h * block_h, col // block_w * block_w
- r0 = block_r + (row-block_r) // size * size
- c0 = block_c + (col-block_c) // size * size
- key = (raster, r0, c0)
- entry = self._scalar_blocks.get(key)
- if entry is None:
- height = min(size, block_r+block_h-r0, raster.height-r0)
- width = min(size, block_c+block_w-c0, raster.width-c0)
- data = raster.read(1, window=Window(c0, r0, width, height), masked=True)
- pixels, mask = np.asarray(data), np.ma.getmaskarray(data)
- try:
- unchanged = self._source_signature(paths) == signature
- except OSError:
- unchanged = False
- if not unchanged:
- raise ValueError(f"Terrain raster changed while reading: {raster.name}")
- byte_count = pixels.nbytes + mask.nbytes
- if byte_count <= self._scalar_cache_limit:
- while self._scalar_blocks and (
- self._scalar_block_bytes + byte_count > self._scalar_cache_limit or
- len(self._scalar_blocks) >= 256):
- _, old = self._scalar_blocks.popitem(last=False)
- self._scalar_block_bytes -= old[0].nbytes + old[1].nbytes
- pixels.flags.writeable = False
- mask.flags.writeable = False
- self._scalar_blocks[key] = (pixels, mask)
- self._scalar_block_bytes += byte_count
- else:
- self._scalar_blocks.move_to_end(key)
- pixels, mask = entry
- return pixels[row-r0, col-c0], mask[row-r0, col-c0]
++ def _read_window(self, raster, window):
++ # Only reuse the exact 1x1 reads already requested by the original path.
++ # Array coordinates, floor arithmetic, masks and raster order stay there.
++ sources = getattr(self, "_pixel_sources", None)
++ source = sources.get(id(raster)) if sources is not None else None
++ if source is None or window.width != 1 or window.height != 1:
++ return raster.read(1, window=window, masked=True)
++ with self._pixel_cache_lock:
++ signature = _raster_file_signature(source[2])
++ if (raster is not source[0]() or raster.closed or raster.mode != "r"
++ or type(raster).read is not source[1]
++ or "read" in getattr(raster, "__dict__", {}) or signature != source[3]):
++ # Do not re-enable a changed reader: an open handle can still
++ # refer to the old inode after its path is replaced.
++ self._pixel_sources.pop(id(raster), None)
++ for key in list(self._pixel_cache):
++ if key[0] == id(raster):
++ del self._pixel_cache[key]
++ return raster.read(1, window=window, masked=True)
++ key = (id(raster), window.row_off, window.col_off)
++ if key in self._pixel_cache:
++ self._pixel_cache.move_to_end(key)
++ return self._pixel_cache[key]
++ data = raster.read(1, window=window, masked=True)
++ if _raster_file_signature(source[2]) != signature:
++ self._pixel_sources.pop(id(raster), None)
++ for old in list(self._pixel_cache):
++ if old[0] == id(raster):
++ del self._pixel_cache[old]
++ return raster.read(1, window=window, masked=True)
++ # The cache never escapes sample(). Returned elevations stay fresh
++ # and writable. Both numeric values and masks belong to this entry.
++ data.flags.writeable = False
++ if isinstance(data.mask, np.ndarray):
++ data.mask.flags.writeable = False
++ self._pixel_cache[key] = data
++ if len(self._pixel_cache) > _PIXEL_CACHE_MAX_ENTRIES:
++ self._pixel_cache.popitem(last=False)
++ return data
+
def sample(self, lat, lon):
from rasterio.windows import Window
@@@@ -31,6 -114,13 -116,6 +116,15 @@@@
x, y = transform.transform(lon.ravel(), lat.ravel())
col,row=(~raster.transform) @ (np.asarray(x),np.asarray(y))
col,row=np.floor(col).astype(int),np.floor(row).astype(int)
+ + if lat.size == 1:
+ + c, r = int(col[0]), int(row[0])
+ + if 0 <= c < raster.width and 0 <= r < raster.height:
- value, masked = self._scalar_pixel(raster, r, c)
- if not masked:
- elevation.ravel()[0] = value
+++ # Preserve the exact read window and all reader/source
+++ # guards while avoiding vector mask assembly for one pixel.
+++ data = self._read_window(raster, Window(c, r, 1, 1))
+++ if not np.ma.getmaskarray(data)[0, 0]:
+++ elevation.ravel()[0] = np.asarray(data)[0, 0]
+ + continue
inside=(col>=0)&(col<raster.width)&(row>=0)&(row<raster.height)
values=np.ma.masked_all(len(col))
if np.any(inside):
diff --cc tests/test_real_refinement_reuse.py
index 0000000,0000000,3c6b041..f075dc1
mode 000000,000000,100644..100644
--- a/tests/test_real_refinement_reuse.py
+++ b/tests/test_real_refinement_reuse.py
@@@@ -1,0 -1,0 -1,242 +1,248 @@@@
++ """Exact refinement and terrain.Terrain pixel reuse on archived real inputs."""
++ from contextlib import contextmanager
++ from copy import deepcopy
++ import json
++ import os
++ from pathlib import Path
++ import shutil
++
++ import numpy as np
++ import pytest
++
++ from dopplergeist import matching, terrain
++ from dopplergeist.data_access import FlightIndex
++
++
++ ROOT = Path(__file__).resolve().parents[1]
++
++
++ @pytest.fixture(scope="module")
++ def recorded_profile():
++ return json.loads((ROOT / "tests/data/real-refinement-profile.json").read_text())
++
++
++ @pytest.fixture(scope="module")
++ def real_rasters(recorded_profile):
++ root = Path(os.environ.get("DOPPLERGEIST_REAL_REFERENCE_ROOT", ROOT))
++ paths = tuple(root / name for name in recorded_profile["source"]["terrain_data"])
++ if not all(path.is_file() for path in paths):
++ pytest.skip("Restore the real data-terrain bundle or set DOPPLERGEIST_REAL_REFERENCE_ROOT")
++ return paths
++
++
++ def close(sampler):
++ for raster in sampler.rasters:
++ raster.close()
++ if sampler.geoid is not None:
++ close(sampler.geoid)
++
++
++ def uncached(sampler):
++ sampler._pixel_sources.clear()
++ sampler._pixel_cache.clear()
++ if sampler.geoid is not None:
++ uncached(sampler.geoid)
++
++
++ @contextmanager
++ def samplers(paths, **options):
++ fast, original = terrain.Terrain(paths, **options), terrain.Terrain(paths, **options)
++ uncached(original)
++ try:
++ yield fast, original
++ finally:
++ close(fast)
++ close(original)
++
++
++ def exact_height(actual, expected):
++ assert actual.shape == expected.shape and actual.dtype == expected.dtype
++ assert actual.tobytes() == expected.tobytes()
++ assert actual.flags.writeable and expected.flags.writeable
++
++
++ @pytest.mark.parametrize("configuration", ["dem", "ellipsoidal", "geoid_last", "dem_last"])
++ def test_recorded_positions_keep_exact_heights(real_rasters, recorded_profile, configuration):
++ dem, geoid = real_rasters
++ paths = {"dem": [dem], "ellipsoidal": [dem],
++ "geoid_last": [dem, geoid], "dem_last": [geoid, dem]}[configuration]
++ options = {"geoid_path": geoid} if configuration == "ellipsoidal" else {}
++ lat, lon = np.asarray(recorded_profile["positions"]).T
++ with samplers(paths, **options) as (fast, original):
++ assert fast._pixel_sources, "The actual GeoTIFF must exercise pixel reuse"
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++ for a, b in zip(lat, lon):
++ exact_height(fast.sample(a, b), original.sample(a, b))
++ exact_height(fast.sample(a, b), original.sample(a, b))
++ assert fast._pixel_cache
++ changed = fast.sample(lat[0], lon[0])
++ changed[...] += 1.
++ exact_height(fast.sample(lat[0], lon[0]), original.sample(lat[0], lon[0]))
++
++
++ def test_pixel_cache_bound_uses_original_heights(real_rasters, recorded_profile, monkeypatch):
++ monkeypatch.setattr(terrain, "_PIXEL_CACHE_MAX_ENTRIES", 2)
++ with samplers([real_rasters[0]]) as (fast, original):
++ for lat, lon in recorded_profile["positions"]:
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++ assert len(fast._pixel_cache) <= 2
++ assert len(fast._pixel_cache) == 2
++ lat, lon = recorded_profile["positions"][0]
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++
++
++ def test_closed_real_reader_keeps_original_error(real_rasters, recorded_profile):
++ lat, lon = recorded_profile["positions"][0]
++ with samplers([real_rasters[0]]) as (fast, original):
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++ close(fast)
++ close(original)
++ with pytest.raises(Exception) as expected:
++ original.sample(lat, lon)
++ with pytest.raises(type(expected.value)) as actual:
++ fast.sample(lat, lon)
++ assert str(actual.value) == str(expected.value)
++
++
++ @pytest.mark.parametrize("change", ["mtime", "replacement", "sidecar"])
++ def test_changed_real_raster_discards_cached_pixels(real_rasters, recorded_profile, tmp_path, change):
++ copied = tmp_path / "observed.tif"
++ shutil.copyfile(real_rasters[0], copied)
++ lat, lon = recorded_profile["positions"][0]
++ with samplers([copied]) as (fast, original):
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++ assert fast._pixel_cache
++ if change == "mtime":
++ info = copied.stat()
++ os.utime(copied, ns=(info.st_atime_ns, info.st_mtime_ns + 1_000_000))
++ elif change == "replacement":
++ replacement = tmp_path / "replacement.tif"
++ shutil.copyfile(real_rasters[0], replacement)
++ replacement.replace(copied)
++ else:
++ copied.with_name(copied.name + ".aux.xml").write_text("<PAMDataset/>\n")
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++ assert not fast._pixel_cache and not fast._pixel_sources
++
++
++ def test_custom_reader_keeps_each_real_read(real_rasters, recorded_profile):
++ class ReaderProxy:
++ def __init__(self, reader):
++ self.reader, self.calls = reader, 0
++
++ def __getattr__(self, name):
++ return getattr(self.reader, name)
++
++ def read(self, *args, **kwargs):
++ self.calls += 1
++ return self.reader.read(*args, **kwargs)
++
++ lat, lon = recorded_profile["positions"][0]
++ with samplers([real_rasters[0]]) as (fast, original):
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++ proxy = ReaderProxy(fast.rasters[0])
++ fast.rasters = [proxy]
++ for _ in range(2):
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++ assert proxy.calls == 2
++
++
++ def test_replaced_dataset_keeps_new_real_values(real_rasters, recorded_profile):
++ import rasterio
++
++ lat, lon = recorded_profile["positions"][0]
++ with samplers([real_rasters[0]]) as (fast, original):
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++ close(fast)
++ close(original)
++ fast.rasters = [rasterio.open(real_rasters[1])]
++ original.rasters = [rasterio.open(real_rasters[1])]
++ for a, b in recorded_profile["positions"][:8]:
++ exact_height(fast.sample(a, b), original.sample(a, b))
++
++
++ def test_real_pixel_nodata_metadata_and_constant(real_rasters, recorded_profile, tmp_path):
++ import rasterio
++
++ # Change only nodata metadata on a disposable copy of the real raster.
++ # Every observed pixel value stays unchanged.
++ copied = tmp_path / "nodata-metadata.tif"
++ shutil.copyfile(real_rasters[0], copied)
++ lat, lon = recorded_profile["positions"][0]
++ reference = terrain.Terrain([real_rasters[0]])
++ try:
++ observed_height = reference.sample(lat, lon).item()
++ finally:
++ close(reference)
++ with rasterio.open(copied, "r+", IGNORE_COG_LAYOUT_BREAK="YES") as writer:
++ writer.nodata = observed_height
++ with samplers([copied]) as (fast, original):
++ for _ in range(2):
++ with pytest.raises(ValueError) as expected:
++ original.sample(lat, lon)
++ with pytest.raises(ValueError) as actual:
++ fast.sample(lat, lon)
++ assert str(actual.value) == str(expected.value)
++ assert fast._pixel_cache
++ assert all(np.ma.getmaskarray(value).all() for value in fast._pixel_cache.values())
++ with samplers([copied], constant_m=observed_height) as (fast, original):
++ for _ in range(2):
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++ assert fast.sample(lat, lon).item() == observed_height
++ with samplers([real_rasters[1], copied]) as (fast, original):
++ exact_height(fast.sample(lat, lon), original.sample(lat, lon))
++
++
++ def encoded(value):
++ def scalar(item):
++ return item.tolist() if isinstance(item, np.ndarray) else item.item()
++ return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=True, default=scalar)
++
++
[email protected]("geographic_bounds", [False, True])
++ def test_real_complete_profile_keeps_all_fields_and_custom_dispatch(
- real_rasters, recorded_profile, monkeypatch):
+++ real_rasters, recorded_profile, monkeypatch, geographic_bounds):
++ source = os.environ.get("DOPPLERGEIST_REAL_CPA_FIXTURE")
++ if not source:
++ pytest.skip("Set DOPPLERGEIST_REAL_CPA_FIXTURE to the restored real-index fixture")
++ diagnostic = json.loads(Path(source).read_text())
++ candidate = recorded_profile["candidate"]
++ index = FlightIndex(diagnostic["real_index"])
++ assert index.report["input_hash"] == diagnostic["index_input_hash"]
++ tracks = {track.id: track for track in index.lookup(candidate["flight_ids"])}
++ assert set(tracks) == set(candidate["flight_ids"])
++ events, measurements = matching.prepare_events(recorded_profile["events"], c=343., max_distance_m=20000.)
++ options = {"radius_m": 500., "recording_start_window_s": recorded_profile["recording_start_window_s"], "c": 343.}
+++ if geographic_bounds:
+++ request = json.loads((ROOT / "benchmarks/reference-regional-upload-request.json").read_text())
+++ options["bounds_wsen"] = request["search"]["bounds_wsen"]
++ refiner = matching.refine_candidate
++ calls = []
++
++ def later(*args, **kwargs):
++ assert len(args) == 5
- assert set(kwargs) == {"radius_m", "clock_radius_s", "altitude_offsets_m", "crs", "c", "recording_start_window_s"}
+++ assert set(kwargs) == {"radius_m", "clock_radius_s", "altitude_offsets_m", "crs", "c",
+++ "recording_start_window_s", "bounds_wsen"}
+++ assert kwargs["bounds_wsen"] == options.get("bounds_wsen")
++ calls.append("later")
++ return refiner(*args, **kwargs)
++
++ def first(*args, **kwargs):
++ calls.append("first")
++ assert len(args) == 5
++ assert not any(key.startswith("_") for key in kwargs)
++ monkeypatch.setattr(matching, "refine_candidate", later)
++ return refiner(*args, **kwargs)
++
++ with samplers([real_rasters[0]], geoid_path=real_rasters[1]) as (fast, original):
++ actual = matching._cache_ridge_fits(matching.profile_candidate)(
++ deepcopy(candidate), tracks, events, measurements, fast, **options)
++ assert len(actual["sensitivity"]["profiles"]) == recorded_profile["expected_profile_count"]
++ assert fast._pixel_cache and fast.geoid._pixel_cache
++ # This public wrapper forces the existing unshared-transform path and
++ # changes global dispatch after the nominal call, using real solves.
++ monkeypatch.setattr(matching, "refine_candidate", first)
++ expected = matching._cache_ridge_fits(matching.profile_candidate)(
++ deepcopy(candidate), tracks, events, measurements, original, **options)
++ assert encoded(actual) == encoded(expected)
++ assert calls == ["first"] + ["later"] * 43
diff --cc tests/test_real_scalar_terrain.py
index 0000000,5dcb65c,0000000..dc2548d
mode 000000,100644,000000..100644
--- a/tests/test_real_scalar_terrain.py
+++ b/tests/test_real_scalar_terrain.py
@@@@ -1,0 -1,139 -1,0 +1,143 @@@@
+ +"""Exact scalar terrain reuse on restored rasters and recorded candidate points."""
+ +import json
+ +import os
+ +from pathlib import Path
+ +import shutil
+ +
+ +import numpy as np
+ +import pytest
+ +import rasterio
+ +
+++from dopplergeist import terrain
+ +from dopplergeist.terrain import Terrain
+ +
+ +
+ [email protected](scope="module")
+ +def observed_terrain():
+ + root = Path(os.environ.get("DOPPLERGEIST_REAL_REFERENCE_ROOT",
+ + Path(__file__).resolve().parents[1]))
+ + dem = root / "data/terrain/Copernicus_DSM_COG_10_N40_00_W074_00_DEM.tif"
+ + geoid = root / "data/terrain/us_nga_egm96_15.tif"
+ + if not dem.exists() or not geoid.exists():
+ + pytest.skip("Restore the real reference DEM and geoid rasters")
+ + trace = os.environ.get("DOPPLERGEIST_REAL_TERRAIN_TRACE")
+ + if trace:
+ + rows = json.loads(Path(trace).read_text())["rows"]
+ + points = np.asarray(rows, dtype=float)[:, :2]
+ + # Retain actual requests throughout the optimizer trace, without
+ + # constructing new coordinates or a simulated search workload.
+ + points = points[np.linspace(0, len(points)-1, min(256, len(points)), dtype=int)]
+ + else:
+ + choices = [root / "results/reference-local-fast/ranked.json",
+ + root / "evidence/results/reference-local-fast/ranked.json"]
+ + source = next((path for path in choices if path.exists()), None)
+ + if source is None:
+ + pytest.skip("Restore the archived real reference candidate results")
+ + rows = json.loads(source.read_text())
+ + points = np.asarray([(row["lat"], row["lon"]) for row in rows])
+ + assert len(points) > 1
+ + return dem, geoid, points
+ +
+ +
+ +def _close(sampler):
+ + for raster in sampler.rasters:
+ + raster.close()
+ + if sampler.geoid is not None:
+ + _close(sampler.geoid)
+ +
+ +
+ [email protected]("with_geoid", [False, True])
+ +def test_scalar_cache_preserves_real_vector_samples_and_result_ownership(observed_terrain, with_geoid):
+ + dem, geoid, points = observed_terrain
+ + sampler = Terrain([dem], geoid_path=geoid if with_geoid else None)
+ + try:
+ + expected = sampler.sample(points[:, 0], points[:, 1])
+ + for _ in range(2):
+ + actual = np.asarray([sampler.sample(lat, lon).item() for lat, lon in points])
+ + np.testing.assert_array_equal(actual, expected)
- assert sampler._scalar_blocks
+++ assert sampler._pixel_cache
+ + lat, lon = points[0]
+ + result = sampler.sample([[lat]], [[lon]])
+ + assert result.shape == (1, 1)
+ + assert result.item() == expected[0]
+ + result[...] = np.nan
+ + assert sampler.sample(lat, lon).item() == expected[0]
- for pixels, mask in sampler._scalar_blocks.values():
- assert not pixels.flags.writeable and not mask.flags.writeable
+++ for pixels in sampler._pixel_cache.values():
+++ assert not pixels.flags.writeable
+++ if isinstance(pixels.mask, np.ndarray):
+++ assert not pixels.mask.flags.writeable
+ + finally:
+ + _close(sampler)
+ +
+ +
- def test_scalar_cache_stays_bounded_on_actual_points(observed_terrain):
+++def test_scalar_cache_stays_bounded_on_actual_points(observed_terrain, monkeypatch):
+ + dem, _, points = observed_terrain
+ + sampler = Terrain([dem])
- sampler._scalar_cache_limit = 21_000
+++ monkeypatch.setattr(terrain, "_PIXEL_CACHE_MAX_ENTRIES", 2)
+ + try:
+ + expected = sampler.sample(points[:, 0], points[:, 1])
+ + for point, height in zip(points, expected):
+ + assert sampler.sample(*point).item() == height
- assert sampler._scalar_block_bytes <= sampler._scalar_cache_limit
- assert sampler._scalar_block_bytes == sum(
- pixels.nbytes + mask.nbytes for pixels, mask in sampler._scalar_blocks.values())
+++ assert len(sampler._pixel_cache) <= 2
+++ assert all(pixels.shape == (1, 1) for pixels in sampler._pixel_cache.values())
+ + # The same observed points also take the explicit uncached read path.
- sampler._scalar_cache_limit = 0
+++ sampler._pixel_sources.clear()
+++ sampler._pixel_cache.clear()
+ + np.testing.assert_array_equal(
+ + [sampler.sample(*point).item() for point in points], expected)
+ + finally:
+ + _close(sampler)
+ +
+ +
+ +def test_changed_real_raster_is_not_served_from_cache(observed_terrain, tmp_path):
+ + dem, _, points = observed_terrain
+ + copy = tmp_path / dem.name
+ + shutil.copy2(dem, copy)
+ + sampler = Terrain([copy])
+ + try:
- sampler.sample(*points[0])
+++ expected = sampler.sample(*points[0])
+++ assert sampler._pixel_cache
+ + stat = copy.stat()
+ + # Change file metadata, keeping the real measured terrain untouched.
+ + os.utime(copy, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000))
- with pytest.raises(ValueError, match="Terrain raster changed after caching"):
- sampler.sample(*points[0])
+++ np.testing.assert_array_equal(sampler.sample(*points[0]), expected)
+++ assert not sampler._pixel_cache and not sampler._pixel_sources
+ + finally:
+ + _close(sampler)
+ +
+ +
+ +def test_closed_real_reader_does_not_serve_cached_pixels(observed_terrain):
+ + dem, _, points = observed_terrain
+ + sampler = Terrain([dem])
+ + sampler.sample(*points[0])
+ + sampler.rasters[0].close()
+ + with pytest.raises(rasterio.errors.RasterioIOError):
+ + sampler.sample(*points[0])
+ +
+ +
+ +def test_custom_reader_keeps_its_read_behavior(observed_terrain):
+ + dem, _, points = observed_terrain
+ +
+ + class ObservedReader:
+ + def __init__(self, reader):
+ + self.reader, self.windows = reader, []
+ +
+ + def __getattr__(self, name):
+ + return getattr(self.reader, name)
+ +
+ + def read(self, *args, **kwargs):
+ + self.windows.append(kwargs["window"])
+ + return self.reader.read(*args, **kwargs)
+ +
+ + sampler = Terrain([dem])
+ + reader = ObservedReader(sampler.rasters[0])
+ + sampler.rasters = [reader]
+ + try:
+ + first = sampler.sample(*points[0])
+ + second = sampler.sample(*points[0])
+ + np.testing.assert_array_equal(first, second)
+ + assert len(reader.windows) == 2
+ + assert all(window.width == window.height == 1 for window in reader.windows)
- assert not sampler._scalar_blocks
+++ assert not sampler._pixel_cache
+ + finally:
+ + _close(sampler)