Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Evaluate real search tiles in isolated worker processes
Cinna Davis2e0feb0d5438
diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.pynew file mode 100644index 0000000..0fd7e95--- /dev/null+++ b/src/dopplergeist/parallel_tiles.py@@ -0,0 +1,90 @@+"""Isolated coarse-tile workers with private retention ledgers."""+from collections import deque+from pathlib import Path+import tempfile++import pandas as pd++from .data_access import CachedTerrain, FlightIndex+from .finite_search import RetentionLedger+from .tiled_search import _expanded_bounds, _score, _tracks, tile_points+++_worker = None+++def initialize_tile_worker(index_paths, terrain_root, terrain_options, events, measurements,+ bounds, time_range, budget, parameters, filters, start_window,+ crs, ledger_directory, run_hash, score_kind):+ """Initialize read-only indexes and reusable terrain state inside one worker."""+ global _worker+ _worker = {+ "indexes": [FlightIndex(path) for path in index_paths],+ "terrain": CachedTerrain(terrain_root, **terrain_options),+ "events": events, "measurements": measurements, "bounds": bounds,+ "time_range": time_range, "budget": budget, "parameters": parameters,+ "filters": filters, "start_window": start_window, "crs": crs,+ "ledger_directory": Path(ledger_directory), "run_hash": run_hash,+ "score_kind": score_kind,+ }+ _worker["ledger_directory"].mkdir(parents=True, exist_ok=True)+++def score_tile(stage, cells, score_floor=None):+ """Evaluate one tile and return its payload and completed private ledger."""+ if _worker is None:+ raise RuntimeError("Tile worker was not initialized")+ state = _worker+ budget = state["budget"]+ radius = max(event["slant_range_m"][1] for event in state["events"])+ tracks = _tracks(state["indexes"], _expanded_bounds(cells, budget.spacing_m, radius, state["crs"]),+ state["time_range"], budget)+ points = (tile_points(cells, budget.spacing_m, state["bounds"], state["terrain"], state["crs"])+ if tracks else pd.DataFrame())+ with tempfile.NamedTemporaryFile(prefix="retention-", suffix=".sqlite",+ dir=state["ledger_directory"], delete=False) as handle:+ path = Path(handle.name)+ try:+ ledger = RetentionLedger(path, state["run_hash"], state["score_kind"])+ if len(points):+ kwargs = {"ledger": ledger, "stage": stage}+ if score_floor is not None:+ kwargs["score_floor"] = score_floor+ rows, counts = _score(points, tracks, state["events"], state["measurements"], budget,+ state["parameters"], state["filters"], state["start_window"], **kwargs)+ else:+ rows, counts = [], {}+ ledger.record(stage, [], [], 0)+ used = getattr(state["terrain"], "last_sample_files", set()) if len(points) else set()+ terrain_files = [state["terrain"].files[name] for name in sorted(used)] if used else []+ payload = {"ranked": rows, "counts": counts, "grid_points": len(points), "tracks": len(tracks),+ "cells": list(cells), "terrain_files": terrain_files,+ "retention_checksum": ledger.checksum(stage)}+ return payload, str(path)+ except BaseException:+ path.unlink(missing_ok=True)+ raise+++def ordered_tile_futures(pool, jobs, workers):+ """Prefetch at most two tiles per worker, yielding in search order.++ Each job is ``(stage, cells, cached_payload, score_floor)``. Cached+ payloads require no worker. For other jobs the caller receives a future,+ imports its verified ledger, and checkpoints it before advancing.+ """+ if type(workers) is not int or workers < 1:+ raise ValueError("workers must be a positive integer")+ pending = deque()++ def enqueue(job):+ stage, cells, cached, floor = job+ future = None if cached is not None else pool.submit(score_tile, stage, cells, floor)+ pending.append((stage, cells, cached, future))++ for job in jobs:+ enqueue(job)+ if len(pending) >= 2 * workers:+ yield pending.popleft()+ while pending:+ yield pending.popleft()diff --git a/tests/test_real_parallel_retention.py b/tests/test_real_parallel_retention.pyindex 15aa756..af0726c 100644--- a/tests/test_real_parallel_retention.py+++ b/tests/test_real_parallel_retention.py@@ -38,3 +38,63 @@ def test_real_retention_stages_survive_worker_import(tmp_path):expected = worker.execute(f"SELECT * FROM {table} WHERE {column}=? ORDER BY 1,2", (stage,)).fetchall()actual = parent.execute(f"SELECT * FROM {table} WHERE {column}=? ORDER BY 1,2", (stage,)).fetchall()assert actual == expected+++def test_real_tiles_match_sequential_scoring_and_preserve_ledgers(tmp_path):+ import json+ import multiprocessing+ from concurrent.futures import ProcessPoolExecutor++ from dopplergeist.data_access import FlightIndex, content_hash+ from dopplergeist.matching import prepare_events+ from dopplergeist.parallel_tiles import initialize_tile_worker, ordered_tile_futures, score_tile+ from dopplergeist.tiled_search import SearchBudget++ source = os.environ.get("DOPPLERGEIST_REAL_JOB_ROOT")+ cache = os.environ.get("DOPPLERGEIST_REAL_CACHE_ROOT")+ if not source or not cache:+ pytest.skip("set the real job and cache roots")+ job, cache = Path(source), Path(cache)+ manifest = json.loads((job / "search/manifest.json").read_text())+ config = manifest["configuration"]+ tiles = [(name, json.loads((job / f"search/tile-{name}.json").read_text())["payload"])+ for name in ("-10_1", "-9_1")]+ # A previously scored clock makes the development replay small. It is not+ # supplied to the real automatic job or treated as location truth.+ clock = tiles[0][1]["ranked"][0]["recording_start_time_s"]+ events, measurements = prepare_events(json.loads((job / "discovery/discovery.json").read_text())["events"])+ assert content_hash(events) == content_hash(config["events"])+ available = {FlightIndex(path).report["input_hash"]: str(path)+ for path in (cache / "flights").glob("*/staging-0/*.sqlite")}+ index_paths = [available[index["input_hash"]] for index in config["indexes"]]+ budget = SearchBudget(**config["budget"])+ params = dict(config["scoring"], time_origin=clock, time_sweep_s=0.)+ params.pop("recording_start_window_s", None)+ run_hash = content_hash({"source_run": manifest["run_hash"], "development_clock_s": clock})+ score_kind = "matched_group_support_minus_normalized_residual"+ args = (index_paths, cache / "terrain", {"offline": True}, events, measurements,+ config["bounds"], config["time_range_s"], budget, params, config["filters"],+ None, config["crs"], tmp_path / "workers", run_hash, score_kind)+ stages = [(f"coarse:{name}", payload["cells"]) for name, payload in tiles]++ initialize_tile_worker(*args)+ serial = [score_tile(stage, cells) for stage, cells in stages]+ jobs = [(stage, cells, None, None) for stage, cells in stages]+ jobs.append((stages[0][0], stages[0][1], serial[0][0], None))+ parallel = []+ with ProcessPoolExecutor(max_workers=2, mp_context=multiprocessing.get_context("spawn"),+ initializer=initialize_tile_worker, initargs=args) as pool:+ for stage, cells, cached, future in ordered_tile_futures(pool, jobs, 2):+ if cached is not None:+ assert stage == stages[0][0] and cached == serial[0][0] and future is None+ else:+ assert stage == stages[len(parallel)][0] and cells == stages[len(parallel)][1]+ parallel.append(future.result())+ parent = RetentionLedger(tmp_path / "parallel-parent.sqlite", run_hash, score_kind)+ assert sum(p[0]["counts"].get("candidates", 0) for p in serial) > 0+ for (stage, _), (expected, expected_db), (actual, worker_db) in zip(stages, serial, parallel):+ assert json.dumps(actual, sort_keys=True, allow_nan=True) == json.dumps(expected, sort_keys=True, allow_nan=True)+ checksum = actual["retention_checksum"]+ assert RetentionLedger(expected_db, run_hash, score_kind).stage_valid(stage, checksum)+ import_retention_stage(parent, worker_db, stage, checksum)+ assert parent.stage_valid(stage, checksum)