Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

fix(#13): preserve cb996dc saved search on upgrade

Cinna Davis635713b5bb55
diff --git a/src/dopplergeist/checkpoint_resume.py b/src/dopplergeist/checkpoint_resume.py
index 5ff63b4..3a11ac6 100644
--- a/src/dopplergeist/checkpoint_resume.py
+++ b/src/dopplergeist/checkpoint_resume.py
@@ -3,6 +3,7 @@ from copy import deepcopy
from datetime import datetime, timezone
import json
from importlib.metadata import version
+import math
from pathlib import Path
import platform
import shutil
@@ -24,13 +25,40 @@ LEGACY_SOURCES = {
"finite_search.py": "78ed9fca6cb13c9754ea0f4438f1bb54bbcdc488ead2a4d77a15f149b6d77b1c",
}
+# Complete code-map identity for the real combined scorer/scheduler used by
+# the 7ae saved continuation. Unlike the older legacy scorer above, this code
+# already used the batched scoring arithmetic now on main.
+CB996DC_CODE_FINGERPRINT = "c66d402b750381303bd0c7d9c828d2294942227176ccb3cb157f0ef6299cbe40"
+
+
+def _legacy_source_profile(configuration):
+ code = configuration.get("code", {})
+ if all(code.get(name) == digest for name, digest in LEGACY_SOURCES.items()):
+ return "legacy-first-batch"
+ if content_hash(code) == CB996DC_CODE_FINGERPRINT:
+ return "combined-cb996dc"
+ return None
+
+
+def _combined_min_events(configuration):
+ """Preserve cb996dc's broad-clock default when current main resumes it."""
+ filters = configuration.get("filters", {})
+ if "min_events" in filters:
+ return filters["min_events"]
+ events = configuration["events"]
+ max_events = filters.get("max_events") or len(events)
+ start_window = configuration["scoring"].get("recording_start_window_s")
+ clock_window_width = math.inf if start_window is None else start_window[1] - start_window[0]
+ default = min(2, max_events) if len(events) > 1 and clock_window_width > 10.0 else 1
+ return default
+
def upgrade_legacy_job(directory):
- """Prepare a replacement search before swapping it into a stopped job.
+ """Prepare a compatible replacement search before swapping a stopped job.
Completed payloads and retention records stay unchanged. The old directory
- and audio/operator identities remain available for rollback. New work uses
- staged batching with the saved first-group arithmetic and fallback checks.
+ and audio/operator identities remain available for rollback. The old
+ first-batch arithmetic is retained only for the matching legacy source.
"""
from .automatic import _audio_identity, _digest
from . import geometry, matching
@@ -41,12 +69,14 @@ def upgrade_legacy_job(directory):
return None
manifest = json.loads((search / "manifest.json").read_text())
old = manifest["configuration"]
- if any(old["code"].get(name) != digest for name, digest in LEGACY_SOURCES.items()):
+ source_profile = _legacy_source_profile(old)
+ if source_profile is None:
return None
+ legacy_first_batch = source_profile == "legacy-first-batch"
if getattr(matching, "_native_fit_irls", None) is not None:
- raise ValueError("Legacy checkpoint continuation requires the Python ridge fitter")
+ raise ValueError("Checkpoint compatibility requires the Python ridge fitter")
if geometry.ACOUSTIC_BACKEND != "python-acoustic-v1":
- raise ValueError("Legacy checkpoint continuation requires the Python acoustic solver")
+ raise ValueError("Checkpoint compatibility requires the Python acoustic solver")
if (old["scoring"].get("mode") != "robust_ridge" or old.get("coarse_floor_policy") != "per_tile"
or old["scoring"].get("time_origin") is not None
or old.get("filters", {}).get("association") != "subsets"):
@@ -72,11 +102,14 @@ def upgrade_legacy_job(directory):
if events != old["events"]:
raise ValueError("Prepared events no longer match the saved search")
current = deepcopy(old)
+ if source_profile == "combined-cb996dc":
+ current.setdefault("filters", {})["min_events"] = _combined_min_events(old)
from .provenance import implementation_files
package_root = Path(__file__).parent
current["code"] = {p.relative_to(package_root).as_posix(): _digest(p)
for p in implementation_files(package_root, recursive=False)}
- current["scoring"]["legacy_first_batch"] = True
+ if legacy_first_batch:
+ current["scoring"]["legacy_first_batch"] = True
new_hash = content_hash(current)
suffix = uuid.uuid4().hex[:12]
staged = directory / f".search-upgrade-{suffix}"
@@ -107,14 +140,18 @@ def upgrade_legacy_job(directory):
write_json(staged / "manifest.json", {"run_hash": new_hash, "configuration": current})
report = {"version": 1, "created_utc": datetime.now(timezone.utc).isoformat(),
"old_run_hash": manifest["run_hash"], "new_run_hash": new_hash,
- "tiles_preserved": tiles, "legacy_first_batch": True,
+ "tiles_preserved": tiles, "legacy_first_batch": legacy_first_batch,
+ "compatible_source": source_profile,
"previous_search": str(previous), "identity_backup": str(backup)}
write_json(backup / "upgrade.json", report)
search.rename(previous)
try:
staged.rename(search)
audio["identity"] = identity
- operator["resume_legacy_first_batch"] = True
+ if legacy_first_batch:
+ operator["resume_legacy_first_batch"] = True
+ else:
+ operator.pop("resume_legacy_first_batch", None)
write_json(directory / "resume.json", audio)
write_json(directory / "operator.json", operator)
write_json(directory / "checkpoint-upgrade.json", report)
@@ -137,6 +174,7 @@ def compatible_checkpoint_runs(output, run_hash):
if path.exists():
report = json.loads(path.read_text())
if (report.get("version") == 1 and report.get("new_run_hash") == run_hash
- and report.get("legacy_first_batch") is True):
+ and (report.get("legacy_first_batch") is True
+ or report.get("compatible_source") == "combined-cb996dc")):
accepted.add(report["old_run_hash"])
return accepted
diff --git a/tests/test_real_checkpoint_upgrade.py b/tests/test_real_checkpoint_upgrade.py
index e57c2d3..47581d8 100644
--- a/tests/test_real_checkpoint_upgrade.py
+++ b/tests/test_real_checkpoint_upgrade.py
@@ -1,4 +1,4 @@
-"""Upgrade a subset of an actual stopped job; no recording or tracks generated."""
+"""Upgrade real stopped jobs without generating recordings or tracks."""
import json
import os
from pathlib import Path
@@ -16,30 +16,42 @@ from dopplergeist.finite_search import RetentionLedger
def test_saved_scored_tile_and_ledger_survive_upgrade(request):
location = os.environ.get("DOPPLERGEIST_REAL_SAVED_JOB")
if not location:
- pytest.skip("Supply the actual stopped pre-batching job")
+ pytest.skip("Supply an actual stopped checkpoint-compatible job")
source = Path(location)
+ state = json.loads((source / "result.json").read_text())
+ if state.get("status") not in {"cancelled", "failed", "inconclusive"}:
+ pytest.skip("The real job must be stopped before testing checkpoint upgrade")
temporary = tempfile.TemporaryDirectory(prefix="checkpoint-upgrade-test-", dir=source.parent.parent)
request.addfinalizer(temporary.cleanup)
tmp_path = Path(temporary.name)
for name in ("request.json", "operator.json", "resume.json", "result.json"):
shutil.copy2(source / name, tmp_path / name)
- for name in ("intake/recording.f32", "intake/recording.json", "intake/intake.json",
- "discovery/discovery.json", "search/manifest.json", "search/retention.sqlite"):
- path = tmp_path / name
- path.parent.mkdir(parents=True, exist_ok=True)
- path.hardlink_to(source / name)
- tile = source / "search/tile-30_7.json"
+ tile_name = os.environ.get("DOPPLERGEIST_REAL_SAVED_TILE", "tile-30_7.json")
+ tile = source / "search" / tile_name
if not tile.exists():
- pytest.skip("The saved real scored tile is unavailable")
+ pytest.skip(f"The saved real scored tile {tile_name} is unavailable")
original = tile.read_bytes()
payload = json.loads(original)["payload"]
assert payload["counts"]["candidates"] > 0
- (tmp_path / "search" / tile.name).hardlink_to(tile)
+ for directory in ("intake", "discovery"):
+ shutil.copytree(source / directory, tmp_path / directory,
+ copy_function=lambda _src, dst: Path(dst).hardlink_to(_src))
+
+ def ignore_search(_path, names):
+ return [name for name in names if name == ".tile-workers" or name.endswith(".tmp")
+ or name.startswith("retention.sqlite")]
+
+ shutil.copytree(source / "search", tmp_path / "search",
+ copy_function=lambda _src, dst: Path(dst).hardlink_to(_src), ignore=ignore_search)
+ (tmp_path / "search" / "retention.sqlite").hardlink_to(source / "search" / "retention.sqlite")
+ original_tiles = {path.name for path in (tmp_path / "search").glob("tile-*.json")}
with sqlite3.connect(source / "search/retention.sqlite") as db:
counts = {name: db.execute(f"SELECT COUNT(*) FROM {name}").fetchone()[0]
for name in ("stages", "records", "bounds")}
report = upgrade_legacy_job(tmp_path)
- assert report["tiles_preserved"] == 1
+ assert report["tiles_preserved"] == len(original_tiles)
+ assert {path.name for path in (tmp_path / "search").glob("tile-*.json")} == original_tiles
+ assert {path.name for path in (Path(report["previous_search"]).glob("tile-*.json"))} == original_tiles
assert (tmp_path / "search" / tile.name).read_bytes() == original
assert compatible_checkpoint_runs(tmp_path / "search", report["new_run_hash"]) == {
report["old_run_hash"], report["new_run_hash"]}
@@ -52,8 +64,32 @@ def test_saved_scored_tile_and_ledger_survive_upgrade(request):
for name in counts} == counts
ledger = RetentionLedger(tmp_path / "search/retention.sqlite", report["new_run_hash"],
"matched_group_support_minus_normalized_residual")
- assert ledger.stage_valid("coarse:30_7", payload["retention_checksum"])
+ tile_id = tile.stem.removeprefix("tile-")
+ assert ledger.stage_valid(f"coarse:{tile_id}", payload["retention_checksum"])
operator = json.loads((tmp_path / "operator.json").read_text())
- assert operator["resume_legacy_first_batch"] is True
+ if report["compatible_source"] == "legacy-first-batch":
+ assert report["legacy_first_batch"] is True
+ assert operator["resume_legacy_first_batch"] is True
+ else:
+ upgraded = json.loads((tmp_path / "search/manifest.json").read_text())
+ assert report["compatible_source"] == "combined-cb996dc"
+ assert report["legacy_first_batch"] is False
+ assert (upgraded["configuration"]["filters"]["min_events"]
+ == payload["counts"]["minimum_matched_events"])
+ assert "resume_legacy_first_batch" not in operator
_load_audio_checkpoint(tmp_path, json.loads((tmp_path / "request.json").read_text()), operator)
assert upgrade_legacy_job(tmp_path) is None
+
+
+def test_combined_minimum_event_default_matches_real_checkpoint():
+ location = os.environ.get("DOPPLERGEIST_REAL_SAVED_JOB")
+ if not location:
+ pytest.skip("Supply the real cb996dc checkpoint")
+ source = Path(location)
+ manifest = json.loads((source / "search/manifest.json").read_text())
+ from dopplergeist.checkpoint_resume import _combined_min_events, _legacy_source_profile
+
+ configuration = manifest["configuration"]
+ assert _legacy_source_profile(configuration) == "combined-cb996dc"
+ assert "min_events" not in configuration["filters"]
+ assert _combined_min_events(configuration) == 2