Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Reuse verified terrain hashes during checkpoint replay (#243)
Cinna Davisedfe1df1aba7
diff --git a/benchmarks/results/terrain-resume-hash-20261103.json b/benchmarks/results/terrain-resume-hash-20261103.jsonnew file mode 100644index 0000000..c051d56--- /dev/null+++ b/benchmarks/results/terrain-resume-hash-20261103.json@@ -0,0 +1,17 @@+{+ "raster_name": "us_nga_egm08_25.tif",+ "bytes": 80585622,+ "repeats": 4,+ "sha256": "4191d471eefebf24091b56dbc604353cb3b8cf8cc70e448bb9ae56a272bef17a",+ "ordinary": {+ "cpu_s": 0.467821013,+ "elapsed_s": 4.075744799978565,+ "full_hash_reads": 4+ },+ "reused": {+ "cpu_s": 0.11279131700000011,+ "elapsed_s": 0.12048817300819792,+ "full_hash_reads": 1+ },+ "limitation": "Single-raster hash timing, not a complete resume or search benchmark."+}diff --git a/docs/DATA_AND_SCALE.md b/docs/DATA_AND_SCALE.mdindex c403522..19ba5ef 100644--- a/docs/DATA_AND_SCALE.md+++ b/docs/DATA_AND_SCALE.md@@ -222,6 +222,18 @@ Manifest and tile receipts hash configuration, input index files, eventmeasurements, implementation and terrain files. Interrupted jobs reuse completedtiles with the same identity. Changed inputs require a new output directory.Failed tiles are retried on resume; `max_tiles` is a fixed configuration budget.+Within one search process, terrain receipt validation reuses up to 256 verified+file digests while device, inode, size, modification time and change time remain+unchanged. A changed signature causes a full rehash; a file that changes during+hashing fails the tile check. This avoids re-reading the same geoid for every+saved tile. The reuse relies on filesystem metadata identifying changes; it+cannot detect content changes that leave the entire signature unchanged. It does+not make checkpoints from a different implementation automatically compatible.+On the real 80,585,622-byte geoid, four hash checks took 0.468 CPU seconds with+four full reads versus 0.113 seconds with one read; this single-raster test is+not a complete resume or nationwide timing. See the+[measurement](../benchmarks/results/terrain-resume-hash-20261103.json) and+`scripts/benchmark_terrain_resume_hash.py`.Reports distinguish complete/partial computation, source scope, missing tiles,refinement failure, runtime and Linux process peak RSS. A complete tile is neverlabelled a complete national search. The product/confidence workers translatediff --git a/scripts/benchmark_terrain_resume_hash.py b/scripts/benchmark_terrain_resume_hash.pynew file mode 100644index 0000000..d9fe60f--- /dev/null+++ b/scripts/benchmark_terrain_resume_hash.py@@ -0,0 +1,40 @@+"""Measure repeated verification of an existing real terrain raster."""+import argparse+from collections import OrderedDict+import json+from pathlib import Path+import time++from dopplergeist.data_access import sha256+from dopplergeist.tiled_search import _verified_terrain_hash+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--raster', type=Path, required=True)+ parser.add_argument('--repeats', type=int, default=4)+ parser.add_argument('--output', type=Path, required=True)+ args = parser.parse_args()+ if args.repeats < 1:+ parser.error('repeats must be positive')+ cpu, wall = time.process_time(), time.perf_counter()+ ordinary = [sha256(args.raster) for _ in range(args.repeats)]+ before = {'cpu_s':time.process_time()-cpu, 'elapsed_s':time.perf_counter()-wall,+ 'full_hash_reads':args.repeats}+ cache = OrderedDict()+ cpu, wall = time.process_time(), time.perf_counter()+ reused = [_verified_terrain_hash(args.raster, cache) for _ in range(args.repeats)]+ after = {'cpu_s':time.process_time()-cpu, 'elapsed_s':time.perf_counter()-wall,+ 'full_hash_reads':1}+ if ordinary != reused:+ raise ValueError('Repeated terrain hashes differ')+ result = {'raster_name':args.raster.name, 'bytes':args.raster.stat().st_size,+ 'repeats':args.repeats, 'sha256':ordinary[0], 'ordinary':before, 'reused':after,+ 'limitation':'Single-raster hash timing, not a complete resume or search benchmark.'}+ args.output.parent.mkdir(parents=True, exist_ok=True)+ args.output.write_text(json.dumps(result, indent=2)+'\n')+ print(json.dumps(result, indent=2))+++if __name__ == '__main__':+ main()diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.pyindex bb5b55f..04acebe 100644--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@ -4,6 +4,7 @@ Every tile sees all overlapping original flight segments and the same eventclocks, propagation model and scoring units. Overloaded single cells may alsopartition association anchors without narrowing the physical fit intervals."""+from collections import OrderedDictfrom dataclasses import asdict, dataclassfrom functools import lru_cachefrom importlib.metadata import version@@ -27,6 +28,29 @@ from .matching import _cache_ridge_fitsfrom .search import cpa_table, filter_association_event, pair_candidates, score_candidate+def _verified_terrain_hash(path, cache):+ """Reuse a digest only while the same file identity and timestamps hold."""+ path = Path(path)++ def identity():+ stat = path.stat()+ return stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns++ before = identity()+ entry = cache.get(path)+ if entry is not None and entry[0] == before:+ cache.move_to_end(path)+ return entry[1]+ digest = sha256(path)+ if identity() != before:+ raise DataUnavailable(f"Terrain changed while hashing: {path}")+ cache[path] = (before, digest)+ cache.move_to_end(path)+ if len(cache) > 256:+ cache.popitem(last=False)+ return digest++def _support_error(error):result = {"track_id": error.track_id, "reason": error.reason}for name in ("emission_intervals_s", "receive_intervals_s"):@@ -709,6 +733,7 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outwrite_json(manifest, {"run_hash": run_hash, "configuration": identity})from .checkpoint_resume import compatible_checkpoint_runscompatible_runs = compatible_checkpoint_runs(output, run_hash)+ terrain_hash_cache = OrderedDict()mode = parameters.get("mode", "brightness")score_kind = ("matched_group_support_minus_normalized_residual" if mode == "robust_ridge" and filters["association"] == "subsets"else {"robust_ridge": "negative_robust_ridge_loss_hz", "ridge": "negative_ridge_rmse_hz",@@ -739,7 +764,7 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outraise DataUnavailable("Corrupt tile checkpoint")prior = record["payload"]for source in prior.get("terrain_files", []):- if sha256(source["path"]) != source["sha256"]:+ if _verified_terrain_hash(source["path"], terrain_hash_cache) != source["sha256"]:raise DataUnavailable("Terrain changed since tile checkpoint")if ledger.stage_valid(stage, prior.get("retention_checksum")):cached = checkpointdiff --git a/tests/test_real_terrain_resume_hash.py b/tests/test_real_terrain_resume_hash.pynew file mode 100644index 0000000..d507d58--- /dev/null+++ b/tests/test_real_terrain_resume_hash.py@@ -0,0 +1,79 @@+"""Resume hash reuse and mutation checks using a real cached terrain raster."""+from collections import OrderedDict+import json+import os+from pathlib import Path+import shutil++import pytest++from dopplergeist import tiled_search+from dopplergeist.data_access import DataUnavailable+++def real_geoid_copy(tmp_path):+ cache = os.environ.get('DOPPLERGEIST_REAL_CACHE_ROOT')+ if not cache:+ pytest.skip('set DOPPLERGEIST_REAL_CACHE_ROOT to the real terrain cache')+ source = Path(cache)/'terrain/us_nga_egm08_25.tif'+ target = tmp_path/source.name+ shutil.copyfile(source, target)+ return target+++def test_real_terrain_hash_is_reused_and_changed_content_is_detected(real_geoid_copy, monkeypatch):+ path = real_geoid_copy+ original = tiled_search.sha256+ calls = []+ def counted(file):+ calls.append(file)+ return original(file)+ monkeypatch.setattr(tiled_search, 'sha256', counted)+ cache = OrderedDict()+ first = tiled_search._verified_terrain_hash(path, cache)+ assert tiled_search._verified_terrain_hash(path, cache) == first+ assert len(calls) == 1+ stat = path.stat()+ with path.open('r+b') as file:+ file.seek(-1, os.SEEK_END)+ original_byte = file.read(1)+ file.seek(-1, os.SEEK_END)+ file.write(bytes([original_byte[0] ^ 1]))+ os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns))+ assert path.stat().st_size == stat.st_size and path.stat().st_mtime_ns == stat.st_mtime_ns+ assert tiled_search._verified_terrain_hash(path, cache) != first+ assert len(calls) == 2+++def test_real_terrain_change_during_hash_fails_closed(real_geoid_copy, monkeypatch):+ path = real_geoid_copy+ original = tiled_search.sha256+ def change_after_hash(file):+ digest = original(file)+ with Path(file).open('ab') as target:+ target.write(b'\0')+ return digest+ monkeypatch.setattr(tiled_search, 'sha256', change_after_hash)+ with pytest.raises(DataUnavailable, match='changed while hashing'):+ tiled_search._verified_terrain_hash(path, OrderedDict())+++def test_real_saved_tile_terrain_receipts_validate_and_reuse(monkeypatch):+ job = os.environ.get('DOPPLERGEIST_REAL_JOB_ROOT')+ if not job:+ pytest.skip('set DOPPLERGEIST_REAL_JOB_ROOT to the real saved job')+ receipt = json.loads((Path(job)/'search/tile--15_7.json').read_text())['payload']+ files = receipt['terrain_files']+ assert files+ original = tiled_search.sha256+ calls = []+ def counted(file):+ calls.append(file)+ return original(file)+ monkeypatch.setattr(tiled_search, 'sha256', counted)+ cache = OrderedDict()+ for _ in range(2):+ for source in files:+ assert tiled_search._verified_terrain_hash(source['path'], cache) == source['sha256']+ assert len(calls) == len({source['path'] for source in files})