Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
fix(#13): preserve cb996dc saved search on upgrade
Cinna Davis635713b5bb55
diff --git a/src/dopplergeist/checkpoint_resume.py b/src/dopplergeist/checkpoint_resume.pyindex 5ff63b4..3a11ac6 100644--- a/src/dopplergeist/checkpoint_resume.py+++ b/src/dopplergeist/checkpoint_resume.py@@ -3,6 +3,7 @@ from copy import deepcopyfrom datetime import datetime, timezoneimport jsonfrom importlib.metadata import version+import mathfrom pathlib import Pathimport platformimport shutil@@ -24,13 +25,40 @@ LEGACY_SOURCES = {"finite_search.py": "78ed9fca6cb13c9754ea0f4438f1bb54bbcdc488ead2a4d77a15f149b6d77b1c",}+# Complete code-map identity for the real combined scorer/scheduler used by+# the 7ae saved continuation. Unlike the older legacy scorer above, this code+# already used the batched scoring arithmetic now on main.+CB996DC_CODE_FINGERPRINT = "c66d402b750381303bd0c7d9c828d2294942227176ccb3cb157f0ef6299cbe40"+++def _legacy_source_profile(configuration):+ code = configuration.get("code", {})+ if all(code.get(name) == digest for name, digest in LEGACY_SOURCES.items()):+ return "legacy-first-batch"+ if content_hash(code) == CB996DC_CODE_FINGERPRINT:+ return "combined-cb996dc"+ return None+++def _combined_min_events(configuration):+ """Preserve cb996dc's broad-clock default when current main resumes it."""+ filters = configuration.get("filters", {})+ if "min_events" in filters:+ return filters["min_events"]+ events = configuration["events"]+ max_events = filters.get("max_events") or len(events)+ start_window = configuration["scoring"].get("recording_start_window_s")+ clock_window_width = math.inf if start_window is None else start_window[1] - start_window[0]+ default = min(2, max_events) if len(events) > 1 and clock_window_width > 10.0 else 1+ return default+def upgrade_legacy_job(directory):- """Prepare a replacement search before swapping it into a stopped job.+ """Prepare a compatible replacement search before swapping a stopped job.Completed payloads and retention records stay unchanged. The old directory- and audio/operator identities remain available for rollback. New work uses- staged batching with the saved first-group arithmetic and fallback checks.+ and audio/operator identities remain available for rollback. The old+ first-batch arithmetic is retained only for the matching legacy source."""from .automatic import _audio_identity, _digestfrom . import geometry, matching@@ -41,12 +69,14 @@ def upgrade_legacy_job(directory):return Nonemanifest = json.loads((search / "manifest.json").read_text())old = manifest["configuration"]- if any(old["code"].get(name) != digest for name, digest in LEGACY_SOURCES.items()):+ source_profile = _legacy_source_profile(old)+ if source_profile is None:return None+ legacy_first_batch = source_profile == "legacy-first-batch"if getattr(matching, "_native_fit_irls", None) is not None:- raise ValueError("Legacy checkpoint continuation requires the Python ridge fitter")+ raise ValueError("Checkpoint compatibility requires the Python ridge fitter")if geometry.ACOUSTIC_BACKEND != "python-acoustic-v1":- raise ValueError("Legacy checkpoint continuation requires the Python acoustic solver")+ raise ValueError("Checkpoint compatibility requires the Python acoustic solver")if (old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile"or old["scoring"].get("time_origin") is not Noneor old.get("filters", {}).get("association") != "subsets"):@@ -72,11 +102,14 @@ def upgrade_legacy_job(directory):if events != old["events"]:raise ValueError("Prepared events no longer match the saved search")current = deepcopy(old)+ if source_profile == "combined-cb996dc":+ current.setdefault("filters", {})["min_events"] = _combined_min_events(old)from .provenance import implementation_filespackage_root = Path(__file__).parentcurrent["code"] = {p.relative_to(package_root).as_posix(): _digest(p)for p in implementation_files(package_root, recursive=False)}- current["scoring"]["legacy_first_batch"] = True+ if legacy_first_batch:+ current["scoring"]["legacy_first_batch"] = Truenew_hash = content_hash(current)suffix = uuid.uuid4().hex[:12]staged = directory / f".search-upgrade-{suffix}"@@ -107,14 +140,18 @@ def upgrade_legacy_job(directory):write_json(staged / "manifest.json", {"run_hash": new_hash, "configuration": current})report = {"version": 1, "created_utc": datetime.now(timezone.utc).isoformat(),"old_run_hash": manifest["run_hash"], "new_run_hash": new_hash,- "tiles_preserved": tiles, "legacy_first_batch": True,+ "tiles_preserved": tiles, "legacy_first_batch": legacy_first_batch,+ "compatible_source": source_profile,"previous_search": str(previous), "identity_backup": str(backup)}write_json(backup / "upgrade.json", report)search.rename(previous)try:staged.rename(search)audio["identity"] = identity- operator["resume_legacy_first_batch"] = True+ if legacy_first_batch:+ operator["resume_legacy_first_batch"] = True+ else:+ operator.pop("resume_legacy_first_batch", None)write_json(directory / "resume.json", audio)write_json(directory / "operator.json", operator)write_json(directory / "checkpoint-upgrade.json", report)@@ -137,6 +174,7 @@ def compatible_checkpoint_runs(output, run_hash):if path.exists():report = json.loads(path.read_text())if (report.get("version") == 1 and report.get("new_run_hash") == run_hash- and report.get("legacy_first_batch") is True):+ and (report.get("legacy_first_batch") is True+ or report.get("compatible_source") == "combined-cb996dc")):accepted.add(report["old_run_hash"])return accepteddiff --git a/tests/test_real_checkpoint_upgrade.py b/tests/test_real_checkpoint_upgrade.pyindex e57c2d3..47581d8 100644--- a/tests/test_real_checkpoint_upgrade.py+++ b/tests/test_real_checkpoint_upgrade.py@@ -1,4 +1,4 @@-"""Upgrade a subset of an actual stopped job; no recording or tracks generated."""+"""Upgrade real stopped jobs without generating recordings or tracks."""import jsonimport osfrom pathlib import Path@@ -16,30 +16,42 @@ from dopplergeist.finite_search import RetentionLedgerdef test_saved_scored_tile_and_ledger_survive_upgrade(request):location = os.environ.get("DOPPLERGEIST_REAL_SAVED_JOB")if not location:- pytest.skip("Supply the actual stopped pre-batching job")+ pytest.skip("Supply an actual stopped checkpoint-compatible job")source = Path(location)+ state = json.loads((source / "result.json").read_text())+ if state.get("status") not in {"cancelled", "failed", "inconclusive"}:+ pytest.skip("The real job must be stopped before testing checkpoint upgrade")temporary = tempfile.TemporaryDirectory(prefix="checkpoint-upgrade-test-", dir=source.parent.parent)request.addfinalizer(temporary.cleanup)tmp_path = Path(temporary.name)for name in ("request.json", "operator.json", "resume.json", "result.json"):shutil.copy2(source / name, tmp_path / name)- for name in ("intake/recording.f32", "intake/recording.json", "intake/intake.json",- "discovery/discovery.json", "search/manifest.json", "search/retention.sqlite"):- path = tmp_path / name- path.parent.mkdir(parents=True, exist_ok=True)- path.hardlink_to(source / name)- tile = source / "search/tile-30_7.json"+ tile_name = os.environ.get("DOPPLERGEIST_REAL_SAVED_TILE", "tile-30_7.json")+ tile = source / "search" / tile_nameif not tile.exists():- pytest.skip("The saved real scored tile is unavailable")+ pytest.skip(f"The saved real scored tile {tile_name} is unavailable")original = tile.read_bytes()payload = json.loads(original)["payload"]assert payload["counts"]["candidates"] > 0- (tmp_path / "search" / tile.name).hardlink_to(tile)+ for directory in ("intake", "discovery"):+ shutil.copytree(source / directory, tmp_path / directory,+ copy_function=lambda _src, dst: Path(dst).hardlink_to(_src))++ def ignore_search(_path, names):+ return [name for name in names if name == ".tile-workers" or name.endswith(".tmp")+ or name.startswith("retention.sqlite")]++ shutil.copytree(source / "search", tmp_path / "search",+ copy_function=lambda _src, dst: Path(dst).hardlink_to(_src), ignore=ignore_search)+ (tmp_path / "search" / "retention.sqlite").hardlink_to(source / "search" / "retention.sqlite")+ original_tiles = {path.name for path in (tmp_path / "search").glob("tile-*.json")}with sqlite3.connect(source / "search/retention.sqlite") as db:counts = {name: db.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]for name in ("stages", "records", "bounds")}report = upgrade_legacy_job(tmp_path)- assert report["tiles_preserved"] == 1+ assert report["tiles_preserved"] == len(original_tiles)+ assert {path.name for path in (tmp_path / "search").glob("tile-*.json")} == original_tiles+ assert {path.name for path in (Path(report["previous_search"]).glob("tile-*.json"))} == original_tilesassert (tmp_path / "search" / tile.name).read_bytes() == originalassert compatible_checkpoint_runs(tmp_path / "search", report["new_run_hash"]) == {report["old_run_hash"], report["new_run_hash"]}@@ -52,8 +64,32 @@ def test_saved_scored_tile_and_ledger_survive_upgrade(request):for name in counts} == countsledger = RetentionLedger(tmp_path / "search/retention.sqlite", report["new_run_hash"],"matched_group_support_minus_normalized_residual")- assert ledger.stage_valid("coarse:30_7", payload["retention_checksum"])+ tile_id = tile.stem.removeprefix("tile-")+ assert ledger.stage_valid(f"coarse:{tile_id}", payload["retention_checksum"])operator = json.loads((tmp_path / "operator.json").read_text())- assert operator["resume_legacy_first_batch"] is True+ if report["compatible_source"] == "legacy-first-batch":+ assert report["legacy_first_batch"] is True+ assert operator["resume_legacy_first_batch"] is True+ else:+ upgraded = json.loads((tmp_path / "search/manifest.json").read_text())+ assert report["compatible_source"] == "combined-cb996dc"+ assert report["legacy_first_batch"] is False+ assert (upgraded["configuration"]["filters"]["min_events"]+ == payload["counts"]["minimum_matched_events"])+ assert "resume_legacy_first_batch" not in operator_load_audio_checkpoint(tmp_path, json.loads((tmp_path / "request.json").read_text()), operator)assert upgrade_legacy_job(tmp_path) is None+++def test_combined_minimum_event_default_matches_real_checkpoint():+ location = os.environ.get("DOPPLERGEIST_REAL_SAVED_JOB")+ if not location:+ pytest.skip("Supply the real cb996dc checkpoint")+ source = Path(location)+ manifest = json.loads((source / "search/manifest.json").read_text())+ from dopplergeist.checkpoint_resume import _combined_min_events, _legacy_source_profile++ configuration = manifest["configuration"]+ assert _legacy_source_profile(configuration) == "combined-cb996dc"+ assert "min_events" not in configuration["filters"]+ assert _combined_min_events(configuration) == 2