Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
perf: preserve immutable checkpoint payloads during upgrade (#7)
Cinna Davis871c57eeead7
diff --git a/src/dopplergeist/checkpoint_resume.py b/src/dopplergeist/checkpoint_resume.pyindex 68faee0..a3a3e13 100644--- a/src/dopplergeist/checkpoint_resume.py+++ b/src/dopplergeist/checkpoint_resume.py@@ -45,7 +45,9 @@ def upgrade_legacy_job(directory):return Noneif getattr(matching, "_native_fit_irls", None) is not None:raise ValueError("Legacy checkpoint continuation requires the Python ridge fitter")- if old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile":+ if (old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile"+ or old["scoring"].get("time_origin") is not None+ or old.get("filters", {}).get("association") != "subsets"):raise ValueError("This legacy checkpoint mode has no compatible upgrade")runtime = {"python": platform.python_version(), "machine": platform.machine(),"packages": {name: version(name) for name in old["runtime"]["packages"]}}@@ -93,15 +95,10 @@ def upgrade_legacy_job(directory):with sqlite3.connect(staged / "retention.sqlite") as target:source.backup(target)target.execute("UPDATE run SET identity=?", (new_hash,))- tiles = 0- for pattern in ("tile-*.json", "split-*.json"):- for path in staged.glob(pattern):- record = json.loads(path.read_text())- if record["run_hash"] != manifest["run_hash"]:- raise ValueError(f"Saved checkpoint has a different run identity: {path.name}")- record["run_hash"] = new_hash- write_json(path, record)- tiles += pattern == "tile-*.json"+ # Keep saved payloads and their producer identity byte-for-byte. The search+ # reader accepts this one explicitly compatible producer and still checks+ # each payload and retention stage before reuse.+ tiles = len(list(staged.glob("tile-*.json")))write_json(staged / "manifest.json", {"run_hash": new_hash, "configuration": current})report = {"version": 1, "created_utc": datetime.now(timezone.utc).isoformat(),"old_run_hash": manifest["run_hash"], "new_run_hash": new_hash,@@ -126,3 +123,15 @@ def upgrade_legacy_job(directory):shutil.copy2(backup / name, directory / name)raisereturn report+++def compatible_checkpoint_runs(output, run_hash):+ """Accept the validated legacy producer only for this exact upgraded run."""+ accepted = {run_hash}+ path = Path(output).parent / "checkpoint-upgrade.json"+ if path.exists():+ report = json.loads(path.read_text())+ if (report.get("version") == 1 and report.get("new_run_hash") == run_hash+ and report.get("legacy_first_batch") is True):+ accepted.add(report["old_run_hash"])+ return accepteddiff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.pyindex f3e5a4d..34d01d7 100644--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@ -667,6 +667,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outif manifest.exists() and json.loads(manifest.read_text())["run_hash"] != run_hash:raise DataUnavailable("Checkpoint configuration/input/implementation mismatch; use a new output directory")write_json(manifest, {"run_hash": run_hash, "configuration": identity})+ from .checkpoint_resume import compatible_checkpoint_runs+ compatible_runs = compatible_checkpoint_runs(output, run_hash)mode = parameters.get("mode", "brightness")score_kind = ("matched_group_support_minus_normalized_residual" if mode == "robust_ridge" and filters["association"] == "subsets"else {"robust_ridge": "negative_robust_ridge_loss_hz", "ridge": "negative_ridge_rmse_hz",@@ -693,7 +695,7 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outtry:if checkpoint.exists():record = json.loads(checkpoint.read_text())- if record["run_hash"] != run_hash or record["payload_hash"] != content_hash(record["payload"]):+ if record["run_hash"] not in compatible_runs or record["payload_hash"] != content_hash(record["payload"]):raise DataUnavailable("Corrupt tile checkpoint")prior = record["payload"]for source in prior.get("terrain_files", []):@@ -714,6 +716,8 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outcontinuetry:record = json.loads(marker.read_text())+ if record.get("run_hash") in compatible_runs:+ record["run_hash"] = run_hashparts = _split_tile_cells(cells)children = ([[f"{tile_id}~{i}", list(part)] for i, part in enumerate(parts)] if parts else_clock_children(tile_id, cells, start_window, parameters, filters, events))diff --git a/tests/test_real_checkpoint_upgrade.py b/tests/test_real_checkpoint_upgrade.pyindex a3d13dc..e57c2d3 100644--- a/tests/test_real_checkpoint_upgrade.py+++ b/tests/test_real_checkpoint_upgrade.py@@ -9,7 +9,7 @@ import tempfileimport pytestfrom dopplergeist.automatic import _load_audio_checkpoint-from dopplergeist.checkpoint_resume import upgrade_legacy_job+from dopplergeist.checkpoint_resume import compatible_checkpoint_runs, upgrade_legacy_jobfrom dopplergeist.finite_search import RetentionLedger@@ -40,6 +40,9 @@ def test_saved_scored_tile_and_ledger_survive_upgrade(request):for name in ("stages", "records", "bounds")}report = upgrade_legacy_job(tmp_path)assert report["tiles_preserved"] == 1+ assert (tmp_path / "search" / tile.name).read_bytes() == original+ assert compatible_checkpoint_runs(tmp_path / "search", report["new_run_hash"]) == {+ report["old_run_hash"], report["new_run_hash"]}assert json.loads((tmp_path / "search" / tile.name).read_text())["payload"] == payloadassert (Path(report["previous_search"]) / tile.name).read_bytes() == originalassert tile.read_bytes() == original