Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Bound fit reuse at 32 MiB and compact internal search checkpoints
Cinna Davis5699a657d57d
diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.pyindex 4c2353b..95fef82 100644--- a/src/dopplergeist/data_access.py+++ b/src/dopplergeist/data_access.py@@ -114,10 +114,13 @@ def content_hash(value):return hashlib.sha256(json.dumps(value, sort_keys=True, allow_nan=False).encode()).hexdigest()-def write_json(path, value):+def write_json(path, value, *, compact=False):+ """Atomically write JSON, with optional compact encoding for checkpoints."""path = Path(path)temporary = path.with_name(path.name + ".tmp")- temporary.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n")+ temporary.write_text(json.dumps(value, indent=None if compact else 2,+ separators=(",", ":") if compact else None,+ allow_nan=False) + "\n")temporary.replace(path)diff --git a/src/dopplergeist/matching.py b/src/dopplergeist/matching.pyindex d6f38ab..3ccfeed 100644--- a/src/dopplergeist/matching.py+++ b/src/dopplergeist/matching.py@@ -57,7 +57,7 @@ class _RidgeFitCache:self.measurement_limit = 4 * 1024 * 1024self.batch_entries = OrderedDict()self.batch_bytes = 0- self.batch_limit = 16 * 1024 * 1024+ self.batch_limit = 32 * 1024 * 1024self.ceiling_entries = OrderedDict()def prepared_measurement(self, measurement, sigma_hz, drift_fraction, key):@@ -151,9 +151,11 @@ class _RidgeFitCache:def _cache_ridge_fits(function, *, max_entries=4096):"""Reuse fits only while this batch's tracks stay unchanged.- Each call has at most 4,096 entries by default. Context-local state prevents- reuse between concurrent jobs, nested batches or later calls. Cached results- are copied so candidate diagnostics remain independently mutable.+ Scalar fits, batched fits, measurements and validated ceilings each retain+ at most 4,096 entries by default. Batched fits and stored arrays also have+ byte limits. Context-local state prevents reuse between concurrent jobs,+ nested batches or later calls. Cached results are copied so candidate+ diagnostics remain independently mutable."""if type(max_entries) is not int or max_entries < 1:raise ValueError("Fit-cache capacity must be a positive integer")diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.pyindex f3e5a4d..4508022 100644--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@ -565,7 +565,8 @@ def _record_support(output, stage, counts, summary):(output / "support").mkdir(exist_ok=True)write_json(output / "support" / (stage.replace(":", "-") + ".json"),{"stage": stage, "unsupported_cpa_tracks": counts.get("unsupported_cpa_tracks", []),- "unclassified_candidate_assessments": counts.get("unclassified_candidate_assessments", [])})+ "unclassified_candidate_assessments": counts.get("unclassified_candidate_assessments", [])},+ compact=True)def search_national(indexes, terrain, events, measurements, *, time_range_s, output_dir,@@ -745,7 +746,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outtry:payload, worker_path = future.result()import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])- write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload})+ write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload},+ compact=True)try:Path(worker_path).unlink()except OSError:@@ -814,7 +816,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outpayload = {"ranked": rows, "counts": counts, "grid_points": len(points), "tracks": len(tracks),"cells": list(cells), "terrain_files": terrain_files, "terrain_ocean_tiles": oceans,"terrain_failures": [], "retention_checksum": ledger.checksum(stage)}- write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload})+ write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload},+ compact=True)done += 1if "~t" not in tile_id:points_searched += payload["grid_points"]diff --git a/tests/test_real_compact_checkpoints.py b/tests/test_real_compact_checkpoints.pynew file mode 100644index 0000000..0b85085--- /dev/null+++ b/tests/test_real_compact_checkpoints.py@@ -0,0 +1,63 @@+"""Checkpoint encoding parity on completed real search evidence."""+import json+import os+from pathlib import Path++import pytest++from dopplergeist.data_access import content_hash, write_json+from dopplergeist.tiled_search import _record_support++[email protected](scope="module")+def recorded_tile():+ source = os.environ.get("DOPPLERGEIST_REAL_TILE_CHECKPOINT")+ if not source:+ pytest.skip("set DOPPLERGEIST_REAL_TILE_CHECKPOINT to a completed real tile JSON")+ path = Path(source)+ assert path.name.startswith("tile-"), "Supply the original completed tile filename"+ value = json.loads(path.read_text())+ assert value["payload"]["counts"]["candidates"] > 0+ assert content_hash(value["payload"]) == value["payload_hash"]+ return path, value+++def test_compact_real_checkpoint_preserves_payload_and_resume_identifiers(recorded_tile, tmp_path):+ source, expected = recorded_tile+ destination = tmp_path / source.name+ write_json(destination, expected)+ readable = destination.read_text()+ assert readable.count("\n") > 1+ assert json.loads(readable) == expected++ # Replace the same checkpoint atomically; filenames and reader inputs stay+ # unchanged. The payload hash also detects signed-zero or number-type changes.+ write_json(destination, expected, compact=True)+ compact = destination.read_text()+ actual = json.loads(compact)+ assert actual == expected+ assert actual["run_hash"] == expected["run_hash"]+ assert content_hash(actual["payload"]) == actual["payload_hash"] == expected["payload_hash"]+ assert actual["payload"]["retention_checksum"] == expected["payload"]["retention_checksum"]+ assert compact.count("\n") == 1+ assert len(compact) < len(readable)+ assert not destination.with_name(destination.name + ".tmp").exists()+++def test_compact_real_support_file_matches_existing_accounting(recorded_tile, tmp_path):+ source, tile = recorded_tile+ stage = "coarse:" + source.stem.removeprefix("tile-")+ relative = Path("support") / (stage.replace(":", "-") + ".json")+ previous = source.parent / relative+ if not previous.is_file():+ pytest.skip("The selected recorded tile has no saved support-accounting sidecar")+ expected = json.loads(previous.read_text())+ summary = dict.fromkeys(("unsupported_cpa_track_evaluations", "unclassified_score_candidates",+ "bounded_score_candidates", "unbounded_score_candidates", "evaluated_trials",+ "rejected_trials", "unknown_trials", "unreported_trial_envelopes"), 0)+ _record_support(tmp_path, stage, tile["payload"]["counts"], summary)+ compact = (tmp_path / relative).read_text()+ actual = json.loads(compact)+ assert actual == expected+ assert content_hash(actual) == content_hash(expected)+ assert compact.count("\n") == 1