Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

perf: preserve immutable checkpoint payloads during upgrade (#7)

Cinna Davis871c57eeead7
diff --git a/src/dopplergeist/checkpoint_resume.py b/src/dopplergeist/checkpoint_resume.py
index 68faee0..a3a3e13 100644
--- a/src/dopplergeist/checkpoint_resume.py
+++ b/src/dopplergeist/checkpoint_resume.py
@@ -45,7 +45,9 @@ def upgrade_legacy_job(directory):
return None
if getattr(matching, "_native_fit_irls", None) is not None:
raise ValueError("Legacy checkpoint continuation requires the Python ridge fitter")
- if old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile":
+ if (old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile"
+ or old["scoring"].get("time_origin") is not None
+ or old.get("filters", {}).get("association") != "subsets"):
raise ValueError("This legacy checkpoint mode has no compatible upgrade")
runtime = {"python": platform.python_version(), "machine": platform.machine(),
"packages": {name: version(name) for name in old["runtime"]["packages"]}}
@@ -93,15 +95,10 @@ def upgrade_legacy_job(directory):
with sqlite3.connect(staged / "retention.sqlite") as target:
source.backup(target)
target.execute("UPDATE run SET identity=?", (new_hash,))
- tiles = 0
- for pattern in ("tile-*.json", "split-*.json"):
- for path in staged.glob(pattern):
- record = json.loads(path.read_text())
- if record["run_hash"] != manifest["run_hash"]:
- raise ValueError(f"Saved checkpoint has a different run identity: {path.name}")
- record["run_hash"] = new_hash
- write_json(path, record)
- tiles += pattern == "tile-*.json"
+ # Keep saved payloads and their producer identity byte-for-byte. The search
+ # reader accepts this one explicitly compatible producer and still checks
+ # each payload and retention stage before reuse.
+ tiles = len(list(staged.glob("tile-*.json")))
write_json(staged / "manifest.json", {"run_hash": new_hash, "configuration": current})
report = {"version": 1, "created_utc": datetime.now(timezone.utc).isoformat(),
"old_run_hash": manifest["run_hash"], "new_run_hash": new_hash,
@@ -126,3 +123,15 @@ def upgrade_legacy_job(directory):
shutil.copy2(backup / name, directory / name)
raise
return report
+
+
+def compatible_checkpoint_runs(output, run_hash):
+ """Accept the validated legacy producer only for this exact upgraded run."""
+ accepted = {run_hash}
+ path = Path(output).parent / "checkpoint-upgrade.json"
+ if path.exists():
+ report = json.loads(path.read_text())
+ if (report.get("version") == 1 and report.get("new_run_hash") == run_hash
+ and report.get("legacy_first_batch") is True):
+ accepted.add(report["old_run_hash"])
+ return accepted
diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.py
index f3e5a4d..34d01d7 100644
--- a/src/dopplergeist/tiled_search.py
+++ b/src/dopplergeist/tiled_search.py
@@ -667,6 +667,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
if manifest.exists() and json.loads(manifest.read_text())["run_hash"] != run_hash:
raise DataUnavailable("Checkpoint configuration/input/implementation mismatch; use a new output directory")
write_json(manifest, {"run_hash": run_hash, "configuration": identity})
+ from .checkpoint_resume import compatible_checkpoint_runs
+ compatible_runs = compatible_checkpoint_runs(output, run_hash)
mode = parameters.get("mode", "brightness")
score_kind = ("matched_group_support_minus_normalized_residual" if mode == "robust_ridge" and filters["association"] == "subsets"
else {"robust_ridge": "negative_robust_ridge_loss_hz", "ridge": "negative_ridge_rmse_hz",
@@ -693,7 +695,7 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
try:
if checkpoint.exists():
record = json.loads(checkpoint.read_text())
- if record["run_hash"] != run_hash or record["payload_hash"] != content_hash(record["payload"]):
+ if record["run_hash"] not in compatible_runs or record["payload_hash"] != content_hash(record["payload"]):
raise DataUnavailable("Corrupt tile checkpoint")
prior = record["payload"]
for source in prior.get("terrain_files", []):
@@ -714,6 +716,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
continue
try:
record = json.loads(marker.read_text())
+ if record.get("run_hash") in compatible_runs:
+ record["run_hash"] = run_hash
parts = _split_tile_cells(cells)
children = ([[f"{tile_id}~{i}", list(part)] for i, part in enumerate(parts)] if parts else
_clock_children(tile_id, cells, start_window, parameters, filters, events))
diff --git a/tests/test_real_checkpoint_upgrade.py b/tests/test_real_checkpoint_upgrade.py
index a3d13dc..e57c2d3 100644
--- a/tests/test_real_checkpoint_upgrade.py
+++ b/tests/test_real_checkpoint_upgrade.py
@@ -9,7 +9,7 @@ import tempfile
import pytest
from dopplergeist.automatic import _load_audio_checkpoint
-from dopplergeist.checkpoint_resume import upgrade_legacy_job
+from dopplergeist.checkpoint_resume import compatible_checkpoint_runs, upgrade_legacy_job
from dopplergeist.finite_search import RetentionLedger
@@ -40,6 +40,9 @@ def test_saved_scored_tile_and_ledger_survive_upgrade(request):
for name in ("stages", "records", "bounds")}
report = upgrade_legacy_job(tmp_path)
assert report["tiles_preserved"] == 1
+ assert (tmp_path / "search" / tile.name).read_bytes() == original
+ assert compatible_checkpoint_runs(tmp_path / "search", report["new_run_hash"]) == {
+ report["old_run_hash"], report["new_run_hash"]}
assert json.loads((tmp_path / "search" / tile.name).read_text())["payload"] == payload
assert (Path(report["previous_search"]) / tile.name).read_bytes() == original
assert tile.read_bytes() == original