Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Resume completed fine-grid scoring pieces (#291)
Cinna Davis57e21d7675a7
diff --git a/docs/PRODUCT.md b/docs/PRODUCT.mdindex 9238e9d..a4898ac 100644--- a/docs/PRODUCT.md+++ b/docs/PRODUCT.md@@ -29,6 +29,10 @@ audio discovery is reused, as are valid completed search tiles; a partiallyprocessed stage or tile is repeated. Input, detector and checkpoint integrityare checked before reuse; incompatible search checkpoints fail explicitly.Stopping before discovery completes cannot resume from that stage.+Completed scored fine-grid pieces are also saved and reused when their scoring and+retention evidence match. Fine-stage checkpoint reuse currently requires the+same effective search worker count; use a new output directory if that count+must change after fine checkpoints have been written.To back up or move a complete job, stop it, wait for its terminal status andworker exit, then copy its entire job directory. If a service owns the store anddiff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.pyindex 0320a84..e458c6d 100644--- a/src/dopplergeist/parallel_tiles.py+++ b/src/dopplergeist/parallel_tiles.py@@ -206,7 +206,8 @@ def score_tile(stage, cells, score_floor=None):def discard_pending_fine_ledgers(inflight):"""Remove successful worker ledgers that were not consumed after an abort."""- for _, _, future in inflight:+ for item in inflight:+ future = item[2]try:_, path = future.result()except BaseException:diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.pyindex 738925b..67efa30 100644--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@ -877,6 +877,20 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outelse {"robust_ridge": "negative_robust_ridge_loss_hz", "ridge": "negative_ridge_rmse_hz","brightness": "summed_spectral_contrast"}[mode])ledger = RetentionLedger(output / "retention.sqlite", run_hash, score_kind)+ fine_manifest = output / "fine-checkpoint-config.json"+ fine_config = {"run_hash": run_hash, "workers": workers if parallel else 1}+ has_fine_checkpoints = any(output.glob("fine-tile-*.json"))+ fine_config_written = False+ if fine_manifest.exists():+ try:+ previous_fine_config = json.loads(fine_manifest.read_text())+ except (OSError, ValueError) as error:+ raise DataUnavailable("Corrupt fine-stage checkpoint configuration") from error+ if previous_fine_config != fine_config and has_fine_checkpoints:+ raise DataUnavailable("Fine-stage checkpoints require the same search worker count")+ fine_config_written = previous_fine_config == fine_config+ elif has_fine_checkpoints:+ raise DataUnavailable("Fine-stage checkpoint configuration is missing")began, ranked, failures, done, processed_cells, resumed, points_searched = time.monotonic(), [], [], 0, 0, 0, 0counted_clock_cells = set()support = dict(unsupported_cpa_track_evaluations=0, unclassified_score_candidates=0,@@ -891,6 +905,38 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outterrain.ocean_tiles.update(provenance.get("terrain_ocean_tiles", []))terrain.failures.extend(provenance.get("terrain_failures", []))+ def fine_checkpoint(stage, cells, exclusions, score_floor, payload=None):+ """Persist or validate a complete fine-piece payload and its ledger."""+ nonlocal fine_config_written+ coordinates = "_".join(str(value) for value in cells)+ path = output / f"fine-tile-{stage.replace(':', '-')}-{coordinates}.json"+ parameters = {"stage": stage, "cells": list(cells),+ "exclusions": [list(rectangle) for rectangle in exclusions], "score_floor": score_floor}+ if payload is not None:+ if not fine_config_written:+ write_json(fine_manifest, fine_config)+ fine_config_written = True+ record = {"run_hash": run_hash, "parameters": parameters,+ "payload_hash": content_hash([parameters, payload]), "payload": payload}+ write_json(path, record, compact=True)+ return None+ if not path.exists():+ return None+ try:+ record = json.loads(path.read_text())+ saved = record["payload"]+ if (record["run_hash"] != run_hash+ or record["payload_hash"] != content_hash([record["parameters"], saved])):+ raise DataUnavailable("Corrupt fine-tile checkpoint")+ if record["parameters"] != parameters:+ return None+ for source in saved.get("terrain_files", []):+ if _verified_terrain_hash(source["path"], terrain_hash_cache) != source["sha256"]:+ raise DataUnavailable("Terrain changed since fine-tile checkpoint")+ return saved if ledger.stage_valid(stage, saved["retention_checksum"]) else None+ except (KeyError, TypeError, ValueError, OSError) as error:+ raise DataUnavailable("Corrupt fine-tile checkpoint") from error+def tile_request(tile_id, cells):stage = f"coarse:{tile_id}"checkpoint = output / f"tile-{tile_id}.json"@@ -1091,6 +1137,7 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outrefinement_failure = Nonerefinement_points_searched, refinement_points_skipped = 0, 0fine_stages = 0+ fine_resumed = 0if ranked and budget.refine_spacing_m < budget.spacing_m:chosen, seen = [], set()for row in ranked:@@ -1106,6 +1153,7 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outcompleted_rectangles = []if parallel:from collections import deque+ from concurrent.futures import Futurefrom .data_access import _index_poolfrom .parallel_tiles import discard_pending_fine_ledgers, score_fine_tile@@ -1125,10 +1173,15 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out# The currently retained floor is conservative for every# submitted piece, even if later merges raise it.floor = _retained_score_floor(ranked, budget.top_k)- future = fine_pool.submit(score_fine_tile, stage, cells, fine,- tuple(completed_rectangles), floor)- inflight.append((cells, stage, future))- cells, stage, future = inflight.popleft()+ cached = fine_checkpoint(stage, cells, completed_rectangles, floor)+ if cached is None:+ future = fine_pool.submit(score_fine_tile, stage, cells, fine,+ tuple(completed_rectangles), floor)+ else:+ future = Future()+ future.set_result((cached, None))+ inflight.append((cells, stage, future, floor))+ cells, stage, future, floor = inflight.popleft()try:payload, worker_path = future.result()except (DataUnavailable, ValueError, OSError) as error:@@ -1141,13 +1194,17 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outcontinuerestore_terrain(payload)refinement_points_skipped += payload["skipped"]- if worker_path is None:+ if worker_path is None and "retention_checksum" not in payload:continue- import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])- try:- Path(worker_path).unlink()- except OSError:- pass+ if worker_path is not None:+ import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])+ try:+ Path(worker_path).unlink()+ except OSError:+ pass+ fine_checkpoint(stage, cells, completed_rectangles, floor, payload)+ else:+ fine_resumed += 1_record_support(output, stage, payload["counts"], support)fine_stages += 1refinement_points_searched += payload["grid_points"]@@ -1168,35 +1225,50 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outpending = [root]while pending:cells = pending.pop()- points = tile_points(cells, fine, bounds, terrain, crs)- keep = np.ones(len(points), dtype=bool)- for left, bottom, right, top in completed_rectangles:- keep &= ~((points.x >= left * fine) & (points.x < right * fine)- & (points.y >= bottom * fine) & (points.y < top * fine))- skipped = int((~keep).sum())- points = points[keep]- if not len(points):- refinement_points_skipped += skipped- continue- try:- tracks, points = _tracks_for_points(- indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s,- budget, points, terrain, radius, parameters.get("c", 343.0))- stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"- rows, counts = _score(points, tracks, events, measurements, budget, parameters, filters, start_window,- ledger=ledger, stage=stage,- score_floor=_retained_score_floor(ranked, budget.top_k))- except (DataUnavailable, ValueError, OSError) as error:- parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []- if not parts:- raise- pending.extend(reversed(parts))- continue- refinement_points_skipped += skipped- _record_support(output, stage, counts, support)+ stage = f"fine:{seed_index}:{cells[0]}:{cells[1]}"+ floor = _retained_score_floor(ranked, budget.top_k)+ payload = fine_checkpoint(stage, cells, completed_rectangles, floor)+ if payload is None:+ failures_before = len(getattr(terrain, "failures", ()))+ points = tile_points(cells, fine, bounds, terrain, crs)+ keep = np.ones(len(points), dtype=bool)+ for left, bottom, right, top in completed_rectangles:+ keep &= ~((points.x >= left * fine) & (points.x < right * fine)+ & (points.y >= bottom * fine) & (points.y < top * fine))+ skipped = int((~keep).sum())+ points = points[keep]+ if not len(points):+ refinement_points_skipped += skipped+ continue+ try:+ tracks, points = _tracks_for_points(+ indexes, _expanded_bounds(cells, fine, radius, crs), time_range_s,+ budget, points, terrain, radius, parameters.get("c", 343.0))+ rows, counts = _score(points, tracks, events, measurements, budget, parameters,+ filters, start_window, ledger=ledger, stage=stage,+ score_floor=floor)+ except (DataUnavailable, ValueError, OSError) as error:+ parts = _split_tile_cells(cells) if _tile_capacity_error(error) else []+ if not parts:+ raise+ pending.extend(reversed(parts))+ continue+ files = getattr(terrain, "files", {})+ names = getattr(terrain, "last_sample_files", set())+ payload = {"ranked": rows, "counts": counts, "grid_points": len(points),+ "skipped": skipped, "retention_checksum": ledger.checksum(stage),+ "terrain_files": [files[name] for name in sorted(names) if name in files],+ "terrain_ocean_tiles": sorted(getattr(terrain, "last_sample_ocean_tiles", set())),+ "terrain_failures": getattr(terrain, "failures", [])[failures_before:]}+ fine_checkpoint(stage, cells, completed_rectangles, floor, payload)+ else:+ fine_resumed += 1+ restore_terrain(payload)+ refinement_points_skipped += payload["skipped"]+ _record_support(output, stage, payload["counts"], support)fine_stages += 1- refinement_points_searched += len(points)- ranked = merge_candidates(itertools.chain(ranked, rows), budget.top_k,+ refinement_points_searched += payload["grid_points"]+ ranked = merge_candidates(itertools.chain(ranked, payload["ranked"]), budget.top_k,ledger=ledger, stage=f"fine-global:{seed_index}:{cells[0]}:{cells[1]}")completed_rectangles.append((cx - width, cy - width, cx + width + 1, cy + width + 1))report_search_progress(0.8 + 0.2 * (seed_index + 1) / len(chosen))@@ -1254,6 +1326,7 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out"model_support_complete": model_complete, "model_support": dict(support, diagnostics_directory="support"),"model_assessment_complete": model_assessed,"refinement_points_searched": refinement_points_searched, "refinement_points_skipped": refinement_points_skipped,+ "refinement_stages_resumed": fine_resumed,"required_flight_bounds": flight_bounds,"candidate_retention_limit": budget.top_k, "alternative_modes_exhaustively_assessed": False,"finite_search_evidence": finite,diff --git a/tests/test_real_fine_resume.py b/tests/test_real_fine_resume.pynew file mode 100644index 0000000..ab2487f--- /dev/null+++ b/tests/test_real_fine_resume.py@@ -0,0 +1,97 @@+"""Optional fine-checkpoint restart checks using recorded audio and aircraft."""+import json+import os+from pathlib import Path+import shutil++import pytest++from dopplergeist.data_access import DataUnavailable, FlightIndex+from dopplergeist.matching import prepare_events+from dopplergeist.terrain import Terrain+from dopplergeist.tiled_search import SearchBudget, search_national++[email protected]('workers', [1, 2])+def test_real_fine_checkpoints_resume_and_reject_corruption(tmp_path, monkeypatch, workers):+ names = ('DOPPLERGEIST_REAL_TERRAIN_DEM', 'DOPPLERGEIST_REAL_TERRAIN_GEOID',+ 'DOPPLERGEIST_REAL_TERRAIN_FLIGHT_INDEX', 'DOPPLERGEIST_REAL_TILE_JOB')+ values = [os.environ.get(name) for name in names]+ if not all(values):+ pytest.skip('set real terrain, index and ESID 762 job paths')+ dem, geoid, index_path, saved_job = values+ job = Path(saved_job)+ config = json.loads((job / 'search/manifest.json').read_text())['configuration']+ raw = json.loads((job / 'discovery/discovery.json').read_text())['events']+ events, measurements = prepare_events(raw, c=343.0, max_distance_m=20000.0)++ def run(output, count):+ terrain = Terrain([dem], geoid_path=geoid)+ try:+ return search_national([FlightIndex(index_path)], terrain, events, measurements,+ time_range_s=config['time_range_s'], output_dir=output, bounds=config['bounds'],+ budget=SearchBudget(**config['budget']), scoring=config['scoring'], filters=config['filters'],+ crs=config['crs'], input_hashes=config['additional_input_hashes'], workers=count)+ finally:+ for raster in terrain.rasters + terrain.geoid.rasters:+ raster.close()++ baseline = run(tmp_path / 'baseline', workers)+ import dopplergeist.tiled_search as tiled+ write = tiled.write_json+ calls = []++ def interrupt_after_three(path, value, **kwargs):+ write(path, value, **kwargs)+ if Path(path).name.startswith('fine-tile-'):+ calls.append(path)+ if len(calls) == 3:+ raise KeyboardInterrupt++ output = tmp_path / 'interrupted'+ with monkeypatch.context() as patch:+ patch.setattr(tiled, 'write_json', interrupt_after_three)+ with pytest.raises(KeyboardInterrupt):+ run(output, workers)+ assert len(list(output.glob('fine-tile-*.json'))) == 3+ resumed = run(output, workers)+ assert resumed['refinement_stages_resumed'] >= 3+ assert resumed['ranked'] == baseline['ranked']+ assert resumed['model_support'] == baseline['model_support']+ assert resumed['finite_search_evidence'] == baseline['finite_search_evidence']+ assert resumed['refinement_failure'] is None+ with pytest.raises(DataUnavailable, match='worker count|configuration/input/implementation mismatch'):+ run(output, 1 if workers == 2 else 2)++ if workers == 2:+ before_fine = tmp_path / 'before-fine-checkpoint'++ def interrupt_before_first_fine(path, value, **kwargs):+ if Path(path).name == 'fine-checkpoint-config.json':+ raise KeyboardInterrupt+ write(path, value, **kwargs)++ with monkeypatch.context() as patch:+ patch.setattr(tiled, 'write_json', interrupt_before_first_fine)+ with pytest.raises(KeyboardInterrupt):+ run(before_fine, 2)+ assert not list(before_fine.glob('fine-tile-*.json'))+ more_workers = run(before_fine, 3)+ assert more_workers['tiles_resumed'] > 0+ assert more_workers['refinement_failure'] is None+ assert more_workers['ranked'] == baseline['ranked']+ fresh_more_workers = run(tmp_path / 'fresh-more-workers', 3)+ assert more_workers['ranked'] == fresh_more_workers['ranked']+ assert more_workers['model_support'] == fresh_more_workers['model_support']+ assert more_workers['finite_search_evidence'] == fresh_more_workers['finite_search_evidence']++ corrupt = tmp_path / 'corrupt'+ shutil.copytree(output, corrupt)+ checkpoint = next(corrupt.glob('fine-tile-*.json'))+ data = json.loads(checkpoint.read_text())+ floor = data['parameters']['score_floor']+ data['parameters']['score_floor'] = 1.0 if floor is None else floor + 1.0+ checkpoint.write_text(json.dumps(data))+ damaged = run(corrupt, workers)+ assert damaged['status'] == 'partial'+ assert 'Corrupt fine-tile checkpoint' in damaged['refinement_failure']