Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Bound fit reuse at 32 MiB and compact internal search checkpoints

Cinna Davis5699a657d57d
diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.py
index 4c2353b..95fef82 100644
--- a/src/dopplergeist/data_access.py
+++ b/src/dopplergeist/data_access.py
@@ -114,10 +114,13 @@ def content_hash(value):
return hashlib.sha256(json.dumps(value, sort_keys=True, allow_nan=False).encode()).hexdigest()
-def write_json(path, value):
+def write_json(path, value, *, compact=False):
+ """Atomically write JSON, with optional compact encoding for checkpoints."""
path = Path(path)
temporary = path.with_name(path.name + ".tmp")
- temporary.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n")
+ temporary.write_text(json.dumps(value, indent=None if compact else 2,
+ separators=(",", ":") if compact else None,
+ allow_nan=False) + "\n")
temporary.replace(path)
diff --git a/src/dopplergeist/matching.py b/src/dopplergeist/matching.py
index d6f38ab..3ccfeed 100644
--- a/src/dopplergeist/matching.py
+++ b/src/dopplergeist/matching.py
@@ -57,7 +57,7 @@ class _RidgeFitCache:
self.measurement_limit = 4 * 1024 * 1024
self.batch_entries = OrderedDict()
self.batch_bytes = 0
- self.batch_limit = 16 * 1024 * 1024
+ self.batch_limit = 32 * 1024 * 1024
self.ceiling_entries = OrderedDict()
def prepared_measurement(self, measurement, sigma_hz, drift_fraction, key):
@@ -151,9 +151,11 @@ class _RidgeFitCache:
def _cache_ridge_fits(function, *, max_entries=4096):
"""Reuse fits only while this batch's tracks stay unchanged.
- Each call has at most 4,096 entries by default. Context-local state prevents
- reuse between concurrent jobs, nested batches or later calls. Cached results
- are copied so candidate diagnostics remain independently mutable.
+ Scalar fits, batched fits, measurements and validated ceilings each retain
+ at most 4,096 entries by default. Batched fits and stored arrays also have
+ byte limits. Context-local state prevents reuse between concurrent jobs,
+ nested batches or later calls. Cached results are copied so candidate
+ diagnostics remain independently mutable.
"""
if type(max_entries) is not int or max_entries < 1:
raise ValueError("Fit-cache capacity must be a positive integer")
diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.py
index f3e5a4d..4508022 100644
--- a/src/dopplergeist/tiled_search.py
+++ b/src/dopplergeist/tiled_search.py
@@ -565,7 +565,8 @@ def _record_support(output, stage, counts, summary):
(output / "support").mkdir(exist_ok=True)
write_json(output / "support" / (stage.replace(":", "-") + ".json"),
{"stage": stage, "unsupported_cpa_tracks": counts.get("unsupported_cpa_tracks", []),
- "unclassified_candidate_assessments": counts.get("unclassified_candidate_assessments", [])})
+ "unclassified_candidate_assessments": counts.get("unclassified_candidate_assessments", [])},
+ compact=True)
def search_national(indexes, terrain, events, measurements, *, time_range_s, output_dir,
@@ -745,7 +746,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
try:
payload, worker_path = future.result()
import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])
- write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload})
+ write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload},
+ compact=True)
try:
Path(worker_path).unlink()
except OSError:
@@ -814,7 +816,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
payload = {"ranked": rows, "counts": counts, "grid_points": len(points), "tracks": len(tracks),
"cells": list(cells), "terrain_files": terrain_files, "terrain_ocean_tiles": oceans,
"terrain_failures": [], "retention_checksum": ledger.checksum(stage)}
- write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload})
+ write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload},
+ compact=True)
done += 1
if "~t" not in tile_id:
points_searched += payload["grid_points"]
diff --git a/tests/test_real_compact_checkpoints.py b/tests/test_real_compact_checkpoints.py
new file mode 100644
index 0000000..0b85085
--- /dev/null
+++ b/tests/test_real_compact_checkpoints.py
@@ -0,0 +1,63 @@
+"""Checkpoint encoding parity on completed real search evidence."""
+import json
+import os
+from pathlib import Path
+
+import pytest
+
+from dopplergeist.data_access import content_hash, write_json
+from dopplergeist.tiled_search import _record_support
+
+
[email protected](scope="module")
+def recorded_tile():
+ source = os.environ.get("DOPPLERGEIST_REAL_TILE_CHECKPOINT")
+ if not source:
+ pytest.skip("set DOPPLERGEIST_REAL_TILE_CHECKPOINT to a completed real tile JSON")
+ path = Path(source)
+ assert path.name.startswith("tile-"), "Supply the original completed tile filename"
+ value = json.loads(path.read_text())
+ assert value["payload"]["counts"]["candidates"] > 0
+ assert content_hash(value["payload"]) == value["payload_hash"]
+ return path, value
+
+
+def test_compact_real_checkpoint_preserves_payload_and_resume_identifiers(recorded_tile, tmp_path):
+ source, expected = recorded_tile
+ destination = tmp_path / source.name
+ write_json(destination, expected)
+ readable = destination.read_text()
+ assert readable.count("\n") > 1
+ assert json.loads(readable) == expected
+
+ # Replace the same checkpoint atomically; filenames and reader inputs stay
+ # unchanged. The payload hash also detects signed-zero or number-type changes.
+ write_json(destination, expected, compact=True)
+ compact = destination.read_text()
+ actual = json.loads(compact)
+ assert actual == expected
+ assert actual["run_hash"] == expected["run_hash"]
+ assert content_hash(actual["payload"]) == actual["payload_hash"] == expected["payload_hash"]
+ assert actual["payload"]["retention_checksum"] == expected["payload"]["retention_checksum"]
+ assert compact.count("\n") == 1
+ assert len(compact) < len(readable)
+ assert not destination.with_name(destination.name + ".tmp").exists()
+
+
+def test_compact_real_support_file_matches_existing_accounting(recorded_tile, tmp_path):
+ source, tile = recorded_tile
+ stage = "coarse:" + source.stem.removeprefix("tile-")
+ relative = Path("support") / (stage.replace(":", "-") + ".json")
+ previous = source.parent / relative
+ if not previous.is_file():
+ pytest.skip("The selected recorded tile has no saved support-accounting sidecar")
+ expected = json.loads(previous.read_text())
+ summary = dict.fromkeys(("unsupported_cpa_track_evaluations", "unclassified_score_candidates",
+ "bounded_score_candidates", "unbounded_score_candidates", "evaluated_trials",
+ "rejected_trials", "unknown_trials", "unreported_trial_envelopes"), 0)
+ _record_support(tmp_path, stage, tile["payload"]["counts"], summary)
+ compact = (tmp_path / relative).read_text()
+ actual = json.loads(compact)
+ assert actual == expected
+ assert content_hash(actual) == content_hash(expected)
+ assert compact.count("\n") == 1