Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
perf(#7): add bounded DEM-affine tile scheduling (#241)
Cinna Davisa4d27b2cacfe
diff --git a/benchmarks/results/dem-worker-affinity-20261103.json b/benchmarks/results/dem-worker-affinity-20261103.jsonnew file mode 100644index 0000000..1cc52eb--- /dev/null+++ b/benchmarks/results/dem-worker-affinity-20261103.json@@ -0,0 +1,91 @@+{+ "tiles": [+ "-19_7",+ "-18_7",+ "-17_7",+ "-16_7",+ "-15_7",+ "-14_7"+ ],+ "points_per_tile": [+ 1024,+ 1024,+ 1024,+ 1024,+ 1024,+ 1024+ ],+ "repetitions": 2,+ "workers": 2,+ "chunk_size": 2,+ "shared_pool": {+ "elapsed_s": 13.646711387002142,+ "ordered_hashes": [+ "22387a651f2f204b1c5e904d39d7f31678571806f085792641f9ddaabb08cf9a",+ "30b4ba42d862fc9db9a6df780ab9b53e5fef3baa793bc40c964e5678b6ebe90f",+ "0ab2c28d818dc32d39a1f4ba5acf0e20535744353b20f8a93a28a4d6161e81db",+ "c30cfe92b379377344657a069b0d1bbeba536029c2ec33c6862fe8125e75daa6",+ "34624a4dba49acfe0d5da054b143c841599a3c175715ee94187f1c0446cd2b9f",+ "ab89c86583610a4420a75925a239925797e38b9779de83ca8614aeb4b0112ad0",+ "22387a651f2f204b1c5e904d39d7f31678571806f085792641f9ddaabb08cf9a",+ "30b4ba42d862fc9db9a6df780ab9b53e5fef3baa793bc40c964e5678b6ebe90f",+ "0ab2c28d818dc32d39a1f4ba5acf0e20535744353b20f8a93a28a4d6161e81db",+ "c30cfe92b379377344657a069b0d1bbeba536029c2ec33c6862fe8125e75daa6",+ "34624a4dba49acfe0d5da054b143c841599a3c175715ee94187f1c0446cd2b9f",+ "ab89c86583610a4420a75925a239925797e38b9779de83ca8614aeb4b0112ad0"+ ],+ "worker_count": 2,+ "workers": [+ {+ "dem_raster_opens": 19,+ "cache_hits": 0,+ "cache_misses": 11,+ "cpu_s": 1.9719897010000014,+ "resident_dem_tiles": 4+ },+ {+ "dem_raster_opens": 24,+ "cache_hits": 11,+ "cache_misses": 16,+ "cpu_s": 2.2134649629999994,+ "resident_dem_tiles": 4+ }+ ]+ },+ "worker_affine": {+ "elapsed_s": 10.155085653997958,+ "ordered_hashes": [+ "22387a651f2f204b1c5e904d39d7f31678571806f085792641f9ddaabb08cf9a",+ "30b4ba42d862fc9db9a6df780ab9b53e5fef3baa793bc40c964e5678b6ebe90f",+ "0ab2c28d818dc32d39a1f4ba5acf0e20535744353b20f8a93a28a4d6161e81db",+ "c30cfe92b379377344657a069b0d1bbeba536029c2ec33c6862fe8125e75daa6",+ "34624a4dba49acfe0d5da054b143c841599a3c175715ee94187f1c0446cd2b9f",+ "ab89c86583610a4420a75925a239925797e38b9779de83ca8614aeb4b0112ad0",+ "22387a651f2f204b1c5e904d39d7f31678571806f085792641f9ddaabb08cf9a",+ "30b4ba42d862fc9db9a6df780ab9b53e5fef3baa793bc40c964e5678b6ebe90f",+ "0ab2c28d818dc32d39a1f4ba5acf0e20535744353b20f8a93a28a4d6161e81db",+ "c30cfe92b379377344657a069b0d1bbeba536029c2ec33c6862fe8125e75daa6",+ "34624a4dba49acfe0d5da054b143c841599a3c175715ee94187f1c0446cd2b9f",+ "ab89c86583610a4420a75925a239925797e38b9779de83ca8614aeb4b0112ad0"+ ],+ "worker_count": 2,+ "workers": [+ {+ "dem_raster_opens": 22,+ "cache_hits": 7,+ "cache_misses": 13,+ "cpu_s": 2.2020988649999986,+ "resident_dem_tiles": 4+ },+ {+ "dem_raster_opens": 16,+ "cache_hits": 8,+ "cache_misses": 10,+ "cpu_s": 1.6895734730000003,+ "resident_dem_tiles": 4+ }+ ]+ },+ "elevation_arrays_equal": true,+ "note": "Real saved-grid terrain replay; not a full-search timing or accuracy result."+}diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.mdindex ca3b24d..3af8a95 100644--- a/docs/PERFORMANCE.md+++ b/docs/PERFORMANCE.md@@ -796,3 +796,45 @@ Each search worker retains its own bounded cache; row-major tile scheduling doesnot guarantee that neighboring work reaches the same worker. Grouping work bygeography remains a separate scheduling opportunity requiring a full search andload-balance comparison.++On a separate ordering-only replay of 12 saved real tile footprints (up to 1,024+points each, repeated three times), contiguous per-tile sampling reduced DEM+raster opens from 78 to 26 and sampler CPU from 5.89 s to 2.73 s (2.16x).+Elevation arrays matched exactly. Wall time was slower in the clustered order+(13.36 s versus 11.26 s), so the result supports cache locality but not end-to-end+speedup. This offline replay excludes download latency; reproduce with the same+[`benchmark_dem_locality.py`](../scripts/benchmark_dem_locality.py) script.++### Bounded worker-affine scheduling prototype++The scheduler prototype keeps the lazy row-major request stream and ordered+parent merge. It pins four consecutive uncached tile requests to one worker,+choosing a worker from a conservative geographic DEM footprint and its reported+in-memory DEM keys. A bounded lookahead of four tiles per worker keeps request+materialization independent of total search size; results and checkpoints remain+per tile. The worker cache keys are returned only as scheduling metadata and do+not enter tile payloads or retention records.++On the saved 745-tile checkpoint sequence, 130 payloads referenced terrain and+recorded 280 tile-to-DEM accesses across 66 unique rasters. A 32 MiB LRU model+replayed those observed footprints: round-robin assignment hit 28/280 accesses+(10.0%), four-tile worker chunks hit 117/280 (41.8%), and static contiguous+lanes hit 133/280 (47.5%). Bounding-box DEM prediction covered the recorded DEM+set for all 130 terrain-using tiles; mean per-tile precision was 90.6%, with+three predicted DEM footprints on otherwise empty tiles. These cache figures+are simulated from saved payload footprints, not timed worker measurements;+they do not establish search runtime or load balance.++A real worker replay used six contiguous saved tiles with 1,024 grid points each,+sampled twice through two workers. Both the shared pool and affinity scheduler+started two worker processes, and all elevation hashes matched. The affinity+scheduler reduced DEM raster opens from 43 to 38, decoded-cache misses from 27 to+23, and increased cache hits from 11 to 15. Summed terrain-sampler CPU fell from+4.19 to 3.89 seconds (1.08x) in this single replay. Wall time was not treated as+a result because it was one run under changing host load. Reproduce with+[`benchmark_dem_worker_affinity.py`](../scripts/benchmark_dem_worker_affinity.py);+the [run record](../benchmarks/results/dem-worker-affinity-20261103.json) is+terrain-stage evidence, not a full-search speed estimate.++Applying this scheduler to a saved search changes the package identity and+requires its validated checkpoint compatibility path before resuming that job.diff --git a/scripts/benchmark_dem_worker_affinity.py b/scripts/benchmark_dem_worker_affinity.pynew file mode 100644index 0000000..60d4ba8--- /dev/null+++ b/scripts/benchmark_dem_worker_affinity.py@@ -0,0 +1,193 @@+"""Compare shared-pool and worker-affine DEM access on saved real tiles."""+import argparse+from collections import Counter+import hashlib+import json+import multiprocessing+import os+from pathlib import Path+import time++import numpy as np++from dopplergeist import parallel_tiles+from dopplergeist.data_access import CachedTerrain, _index_pool+from dopplergeist.parallel_tiles import ordered_tile_results, tile_worker_lanes+from dopplergeist.tiled_search import _tile_coordinates, _tile_dem_keys+++_terrain = None+_points = None+_counters = None+_ready_barrier = None+++def _initialize_worker(terrain_root, points, ready_barrier):+ global _terrain, _points, _counters, _ready_barrier+ _terrain = CachedTerrain(terrain_root, offline=True)+ _points = points+ _counters = {"opens": 0, "hits": 0, "misses": 0}+ _ready_barrier = ready_barrier++ import rasterio++ original_open = rasterio.open++ def counted_open(path, *args, **kwargs):+ if Path(path).name.startswith("Copernicus_DSM_COG_30_"):+ _counters["opens"] += 1+ return original_open(path, *args, **kwargs)++ rasterio.open = counted_open+ original_sample = CachedTerrain._raster_sample++ def counted_sample(terrain, path, lat, lon):+ if Path(path).name.startswith("Copernicus_DSM_COG_30_"):+ _counters["hits" if Path(path) in terrain._tile_cache else "misses"] += 1+ return original_sample(terrain, path, lat, lon)++ CachedTerrain._raster_sample = counted_sample+++def _warmup_worker():+ _ready_barrier.wait(timeout=120)+ return os.getpid()+++def _sample_tile(stage, cells, score_floor=None):+ lat, lon = _points[tuple(cells)]+ before = dict(_counters)+ cpu_started = time.process_time()+ elevations = _terrain.sample(lat, lon)+ cpu_s = time.process_time() - cpu_started+ delta = {key: _counters[key] - before[key] for key in _counters}+ cache_keys = tuple(path.stem for path in _terrain._tile_cache)+ digest = hashlib.sha256(np.asarray(elevations, dtype="<f8").tobytes()).hexdigest()+ return digest, delta, cache_keys, os.getpid(), cpu_s+++def _eligible_tiles(job, count):+ rows = []+ for path in (job / "search").glob("tile-*.json"):+ try:+ payload = json.loads(path.read_text())["payload"]+ cells = tuple(payload["cells"])+ if payload.get("grid_points", 0) < 500 or len(payload.get("terrain_files", [])) < 3:+ continue+ dem_names = {Path(source["path"]).name for source in payload["terrain_files"]+ if Path(source["path"]).name.startswith("Copernicus_DSM_COG_30_")}+ if dem_names:+ rows.append((cells[1], cells[0], path.name, cells, dem_names))+ except (KeyError, OSError, ValueError, TypeError):+ continue+ rows.sort()+ if len(rows) < count:+ raise ValueError(f"Need at least {count} saved terrain tiles; found {len(rows)}")++ best_start, best_score = 0, -1+ for start in range(len(rows) - count + 1):+ frequencies = Counter(name for row in rows[start:start + count] for name in row[4])+ score = sum(amount * (amount - 1) for amount in frequencies.values())+ if score > best_score:+ best_start, best_score = start, score+ selected = rows[best_start:best_start + count]+ return [(name.removeprefix("tile-").removesuffix(".json"), cells)+ for _, _, name, cells, _ in selected]+++def _run(pool_context, jobs, workers, lookahead=None):+ results = []+ with pool_context as pool:+ if isinstance(pool, parallel_tiles.WorkerAffinePool):+ warmups = [lane.submit(_warmup_worker) for lane in pool.pools]+ else:+ warmups = [pool.submit(_warmup_worker) for _ in range(workers)]+ warmup_pids = [future.result(timeout=120) for future in warmups]+ if len(set(warmup_pids)) != workers:+ raise RuntimeError(f"Expected {workers} workers, started {len(set(warmup_pids))}")+ if lookahead is not None and not hasattr(pool, "lookahead"):+ pool.lookahead = lookahead+ results.extend(ordered_tile_results(+ pool, jobs, workers, lambda stage, cells, future: future.result()))+ by_worker = {}+ for _, _, result, _ in results:+ digest, delta, cache_keys, pid, cpu_s = result+ record = by_worker.setdefault(pid, {"dem_raster_opens": 0, "cache_hits": 0,+ "cache_misses": 0, "cpu_s": 0.0,+ "resident_dem_tiles": 0})+ record["dem_raster_opens"] += delta["opens"]+ record["cache_hits"] += delta["hits"]+ record["cache_misses"] += delta["misses"]+ record["cpu_s"] += cpu_s+ record["resident_dem_tiles"] = sum(+ name.startswith("Copernicus_DSM_COG_30_") for name in cache_keys)+ hashes = [prepared[0] for _, _, prepared, _ in results]+ return {"ordered_hashes": hashes, "worker_count": len(warmup_pids),+ "workers": [record for _, record in sorted(by_worker.items())]}+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--job", type=Path, required=True, help="saved real job with a search manifest")+ parser.add_argument("--cache", type=Path, required=True, help="populated real cache containing terrain/")+ parser.add_argument("--output", type=Path, required=True)+ parser.add_argument("--workers", type=int, default=4)+ parser.add_argument("--tiles", type=int, default=12)+ parser.add_argument("--repetitions", type=int, default=3)+ parser.add_argument("--chunk-size", type=int, default=4)+ args = parser.parse_args()+ if min(args.workers, args.tiles, args.repetitions, args.chunk_size) < 1:+ parser.error("workers, tiles, repetitions, and chunk-size must be positive")++ config = json.loads((args.job / "search/manifest.json").read_text())["configuration"]+ selected = _eligible_tiles(args.job, args.tiles)+ points = {}+ for _, cells in selected:+ frame = _tile_coordinates(cells, config["budget"]["spacing_m"], config["bounds"], config["crs"])+ points[cells] = (frame.lat.to_numpy(), frame.lon.to_numpy())+ jobs = [(f"coarse:{name}:pass-{rep}", cells, None, None)+ for rep in range(args.repetitions) for name, cells in selected]+ terrain_root = args.cache / "terrain"+ catalog_path = args.cache / "terrain/tileList.txt"+ catalog = {line.rstrip("/") for line in catalog_path.read_text().splitlines()}+ affinity_keys = lambda cells: _tile_dem_keys(+ cells, config["budget"]["spacing_m"], config["bounds"], config["crs"], catalog)+ original_score_tile = parallel_tiles.score_tile+ original_affine_score = parallel_tiles.score_tile_with_cache_state+ parallel_tiles.score_tile = _sample_tile+ parallel_tiles.score_tile_with_cache_state = _sample_tile+ try:+ began = time.perf_counter()+ barrier = multiprocessing.get_context("spawn").Barrier(args.workers)+ shared_context = _index_pool(args.workers, _initialize_worker,+ (terrain_root, points, barrier))+ lookahead = args.workers * args.chunk_size+ shared = _run(shared_context, jobs, args.workers, lookahead=lookahead)+ shared_elapsed = time.perf_counter() - began++ began = time.perf_counter()+ barrier = multiprocessing.get_context("spawn").Barrier(args.workers)+ affinity_context = tile_worker_lanes(+ args.workers, _initialize_worker, (terrain_root, points, barrier), affinity_keys,+ chunk_size=args.chunk_size)+ affinity = _run(affinity_context, jobs, args.workers, lookahead=lookahead)+ affinity_elapsed = time.perf_counter() - began+ finally:+ parallel_tiles.score_tile = original_score_tile+ parallel_tiles.score_tile_with_cache_state = original_affine_score++ if shared["ordered_hashes"] != affinity["ordered_hashes"]:+ raise AssertionError("shared-pool and worker-affine elevation samples differ")+ output = {"tiles": [name for name, _ in selected], "points_per_tile": [len(v[0]) for v in points.values()],+ "repetitions": args.repetitions, "workers": args.workers, "chunk_size": args.chunk_size,+ "shared_pool": {"elapsed_s": shared_elapsed, **shared},+ "worker_affine": {"elapsed_s": affinity_elapsed, **affinity},+ "elevation_arrays_equal": True,+ "note": "Real saved-grid terrain replay; not a full-search timing or accuracy result."}+ args.output.parent.mkdir(parents=True, exist_ok=True)+ args.output.write_text(json.dumps(output, indent=2) + "\n")+ print(json.dumps(output, indent=2), flush=True)+++if __name__ == "__main__":+ main()diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.pyindex bb6b7e5..37e10df 100644--- a/src/dopplergeist/parallel_tiles.py+++ b/src/dopplergeist/parallel_tiles.py@@ -1,6 +1,8 @@"""Isolated coarse-tile workers with private retention ledgers."""+from contextlib import ExitStack, contextmanagerfrom pathlib import Pathimport tempfile+from threading import Lockimport pandas as pd@@ -21,6 +23,87 @@ class TileFailure(DataUnavailable):return str(self.args[0])+class WorkerAffinePool:+ """Keep tile work on the worker whose DEM cache best matches its area."""++ def __init__(self, pools, tile_dem_keys, cache_keys, chunk_size=4):+ if not pools:+ raise ValueError("At least one tile worker is required")+ if type(chunk_size) is not int or chunk_size < 1:+ raise ValueError("chunk_size must be a positive integer")+ self.pools = tuple(pools)+ self.tile_dem_keys = tile_dem_keys+ self.cache_keys = cache_keys+ self.chunk_size = chunk_size+ self.lookahead = len(pools) * chunk_size+ self._resident = [frozenset() for _ in pools]+ self._pending = [[] for _ in pools]+ self._used = [False for _ in pools]+ self._next_lane = 0+ self._chunk_lane = None+ self._chunk_remaining = 0+ self._lock = Lock()++ def submit(self, function, stage, cells, score_floor):+ predicted = frozenset(self.tile_dem_keys(cells))+ with self._lock:+ if not self._chunk_remaining:+ eligible = [i for i, used in enumerate(self._used) if not used]+ if not eligible:+ eligible = list(range(len(self.pools)))+ loads = [len(self._pending[i]) for i in eligible]+ least_loaded = min(loads)+ candidates = [i for i in eligible if len(self._pending[i]) == least_loaded]++ def affinity(worker):+ nearby = self._resident[worker].union(*self._pending[worker])+ return (-len(predicted.intersection(nearby)),+ (worker - self._next_lane) % len(self.pools))++ self._chunk_lane = min(candidates, key=affinity)+ self._chunk_remaining = self.chunk_size+ self._used[self._chunk_lane] = True+ lane = self._chunk_lane+ self._chunk_remaining -= 1+ self._pending[lane].append(predicted)+ self._next_lane = (lane + 1) % len(self.pools)+ try:+ future = self.pools[lane].submit(function, stage, cells, score_floor)+ except BaseException:+ with self._lock:+ self._pending[lane].pop()+ raise+ future.add_done_callback(lambda completed, worker=lane: self._completed(worker, completed))+ return future++ def finish_wave(self):+ """Keep a short final chunk from leaking into the next search wave."""+ with self._lock:+ self._chunk_lane = None+ self._chunk_remaining = 0+ self._used = [False for _ in self.pools]++ def _completed(self, lane, future):+ try:+ resident = frozenset(self.cache_keys(future.result()))+ except BaseException:+ resident = frozenset()+ with self._lock:+ if self._pending[lane]:+ self._pending[lane].pop(0)+ self._resident[lane] = resident+++@contextmanager+def tile_worker_lanes(workers, initializer, initargs, tile_dem_keys, chunk_size=4):+ """Create pinned worker processes so spatial cache affinity is real."""+ from .data_access import _index_pool++ with ExitStack() as stack:+ pools = [stack.enter_context(_index_pool(1, initializer, initargs)) for _ in range(workers)]+ yield WorkerAffinePool(pools, tile_dem_keys, lambda result: result[2], chunk_size=chunk_size)++_worker = None@@ -104,8 +187,15 @@ def score_tile(stage, cells, score_floor=None):raise+def score_tile_with_cache_state(stage, cells, score_floor=None):+ """Return the worker's resident DEM keys for scheduling; never persist them."""+ payload, worker_path = score_tile(stage, cells, score_floor)+ cache_keys = tuple(path.stem for path in _worker["terrain"]._tile_cache)+ return payload, worker_path, cache_keys++def ordered_tile_results(pool, jobs, workers, finish):- """Persist completed work promptly but yield lightweight handles in order."""+ """Persist promptly, bound out-of-order results, and yield in input order."""from concurrent.futures import FIRST_COMPLETED, waitif type(workers) is not int or workers < 1:@@ -114,8 +204,10 @@ def ordered_tile_results(pool, jobs, workers, finish):pending, ready = {}, {}assigned = next_result = 0exhausted = False+ lookahead = getattr(pool, "lookahead", 2 * workers)while not exhausted or pending or ready:- while not exhausted and len(pending) < 2 * workers and next_result not in ready:+ while (not exhausted and len(pending) + len(ready) < lookahead+ and next_result not in ready):try:stage, cells, cached, floor = next(source)except StopIteration:@@ -124,7 +216,8 @@ def ordered_tile_results(pool, jobs, workers, finish):index = assignedassigned += 1if cached is None:- future = pool.submit(score_tile, stage, cells, floor)+ scorer = score_tile_with_cache_state if isinstance(pool, WorkerAffinePool) else score_tile+ future = pool.submit(scorer, stage, cells, floor)pending[future] = (index, stage, cells)else:ready[index] = (stage, cells, cached, True)@@ -139,3 +232,5 @@ def ordered_tile_results(pool, jobs, workers, finish):ready[index] = (stage, cells, finish(stage, cells, future), False)elif exhausted and ready:raise RuntimeError("Tile result ordering is inconsistent")+ if isinstance(pool, WorkerAffinePool):+ pool.finish_wave()diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.pyindex 8ad2639..bb5b55f 100644--- a/src/dopplergeist/tiled_search.py+++ b/src/dopplergeist/tiled_search.py@@ -188,6 +188,27 @@ def _sample_tile_terrain(points, terrain):return points+def _tile_dem_keys(cells, spacing, bounds, crs, catalog):+ """Conservatively predict DEMs touched by a tile for worker affinity."""+ x0, y0, x1, y1 = cells+ west, south, east, north = _to_geographic(crs).transform_bounds(+ x0 * spacing, y0 * spacing, x1 * spacing, y1 * spacing, densify_pts=21)+ west, south = max(west, bounds[0]), max(south, bounds[1])+ east, north = min(east, bounds[2]), min(north, bounds[3])+ if west > east or south > north:+ return ()+ south_key, north_key = math.floor(south - 1 / 2400), math.floor(north - 1 / 2400)+ west_key, east_key = math.floor(west + 1 / 2400), math.floor(east + 1 / 2400)+ keys = []+ for latitude in range(south_key, north_key + 1):+ for longitude in range(west_key, east_key + 1):+ tag = (f"Copernicus_DSM_COG_30_{'N' if latitude >= 0 else 'S'}{abs(latitude):02d}_00_"+ f"{'E' if longitude >= 0 else 'W'}{abs(longitude):03d}_00_DEM")+ if tag in catalog:+ keys.append(tag)+ return tuple(keys)++def tile_points(cells, spacing, bounds, terrain, crs):return _sample_tile_terrain(_tile_coordinates(cells, spacing, bounds, crs), terrain)@@ -622,9 +643,9 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outevent_limit_applied = filters["association"] == "subsets" and event_limit < len(events)import osfrom contextlib import ExitStack, nullcontext- from .data_access import CachedTerrain, _index_pool+ from .data_access import CachedTerrainfrom .parallel_retention import import_retention_stage- from .parallel_tiles import TileFailure, initialize_tile_worker, ordered_tile_results+ from .parallel_tiles import TileFailure, initialize_tile_worker, ordered_tile_results, tile_worker_lanesxmin, ymin, xmax, ymax = _grid_extent(bounds, budget.spacing_m, crs)total = (xmax // budget.tile_cells - xmin // budget.tile_cells + 1) * (ymax // budget.tile_cells - ymin // budget.tile_cells + 1)if budget.max_tiles is None:@@ -759,14 +780,16 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, outinitargs = ([index.path for index in indexes], terrain.root, terrain_options, events, measurements,bounds, time_range_s, budget, parameters, filters, start_window, crs,output / ".tile-workers", run_hash, score_kind)- context = _index_pool(workers, initialize_tile_worker, initargs)+ context = tile_worker_lanes(+ workers, initialize_tile_worker, initargs,+ lambda cells: _tile_dem_keys(cells, budget.spacing_m, bounds, crs, terrain.catalog))else:context = nullcontext(None)def finish_tile(stage, cells, future):checkpoint = output / f"tile-{stage[len('coarse:'):]}.json"payload = Nonetry:- payload, worker_path = future.result()+ payload, worker_path, *_ = future.result()import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload},compact=True)diff --git a/tests/test_real_parallel_retention.py b/tests/test_real_parallel_retention.pyindex 48f389f..08c7f19 100644--- a/tests/test_real_parallel_retention.py+++ b/tests/test_real_parallel_retention.py@@ -1,4 +1,6 @@"""Verify worker-ledger imports using an actual running search's evidence."""+from concurrent.futures import ProcessPoolExecutor+import multiprocessingimport osfrom pathlib import Pathimport sqlite3@@ -58,14 +60,13 @@ def test_real_retention_stages_survive_worker_import(tmp_path):def test_real_tiles_match_sequential_scoring_and_preserve_ledgers(tmp_path):import json- import multiprocessingimport shutil- from concurrent.futures import ProcessPoolExecutorfrom dopplergeist.data_access import FlightIndex, content_hashfrom dopplergeist.matching import prepare_events- from dopplergeist.parallel_tiles import TileFailure, initialize_tile_worker, ordered_tile_results, score_tile- from dopplergeist.tiled_search import SearchBudget+ from dopplergeist.parallel_tiles import (TileFailure, initialize_tile_worker, ordered_tile_results,+ score_tile, tile_worker_lanes)+ from dopplergeist.tiled_search import SearchBudget, _tile_dem_keyssource = os.environ.get("DOPPLERGEIST_REAL_JOB_ROOT")cache = os.environ.get("DOPPLERGEIST_REAL_CACHE_ROOT")@@ -99,10 +100,11 @@ def test_real_tiles_match_sequential_scoring_and_preserve_ledgers(tmp_path):jobs = [(stage, cells, None, None) for stage, cells in stages]jobs.append((stages[0][0], stages[0][1], serial[0][0], None))parallel = []- with ProcessPoolExecutor(max_workers=2, mp_context=multiprocessing.get_context("spawn"),- initializer=initialize_tile_worker, initargs=args) as pool:+ catalog = {line.rstrip("/") for line in (cache / "terrain/tileList.txt").read_text().splitlines()}+ tile_keys = lambda cells: _tile_dem_keys(cells, budget.spacing_m, config["bounds"], config["crs"], catalog)+ with tile_worker_lanes(2, initialize_tile_worker, args, tile_keys, chunk_size=1) as pool:for stage, cells, result, was_cached in ordered_tile_results(- pool, jobs, 2, lambda stage, cells, future: future.result()):+ pool, jobs, 2, lambda stage, cells, future: future.result()[:2]):if was_cached:assert stage == stages[0][0] and result == serial[0][0]else:diff --git a/tests/test_tile_worker_affinity.py b/tests/test_tile_worker_affinity.pynew file mode 100644index 0000000..5da3ef5--- /dev/null+++ b/tests/test_tile_worker_affinity.py@@ -0,0 +1,115 @@+import concurrent.futures+from concurrent.futures import Future++from dopplergeist import parallel_tiles+from dopplergeist.parallel_tiles import WorkerAffinePool+++class ImmediatePool:+ def __init__(self):+ self.stages = []++ def submit(self, function, stage, cells, score_floor):+ self.stages.append(stage)+ future = Future()+ try:+ future.set_result(function(stage, cells, score_floor))+ except BaseException as error:+ future.set_exception(error)+ return future+++def _scored_tile(stage, cells, score_floor):+ return {"stage": stage}, None, tuple(cells)+++def test_affinity_routes_to_resident_dem_and_preserves_result_order(monkeypatch):+ lanes = [ImmediatePool(), ImmediatePool()]+ pool = WorkerAffinePool(lanes, lambda cells: cells, lambda result: result[2], chunk_size=1)+ monkeypatch.setattr(parallel_tiles, "score_tile_with_cache_state", _scored_tile)+ jobs = [(f"coarse:{name}", (dem,), None, None)+ for name, dem in (("first", "A"), ("second", "B"), ("third", "A"))]++ results = list(parallel_tiles.ordered_tile_results(+ pool, jobs, 2, lambda stage, cells, future: future.result()[0]))++ assert [(stage, result, cached) for stage, _, result, cached in results] == [+ (stage, {"stage": stage}, False) for stage, _, _, _ in jobs]+ assert lanes[0].stages == ["coarse:first", "coarse:third"]+ assert lanes[1].stages == ["coarse:second"]+++def test_tiles_in_a_small_spatial_chunk_stay_on_one_worker(monkeypatch):+ lanes = [ImmediatePool(), ImmediatePool()]+ pool = WorkerAffinePool(lanes, lambda cells: cells, lambda result: result[2], chunk_size=2)+ monkeypatch.setattr(parallel_tiles, "score_tile_with_cache_state", _scored_tile)+ jobs = [(f"coarse:{name}", (dem,), None, None)+ for name, dem in (("first", "A"), ("second", "A"), ("third", "B"), ("fourth", "B"))]++ results = list(parallel_tiles.ordered_tile_results(+ pool, jobs, 2, lambda stage, cells, future: future.result()[0]))++ assert [result[0] for result in results] == [stage for stage, _, _, _ in jobs]+ assert lanes[0].stages == ["coarse:first", "coarse:second"]+ assert lanes[1].stages == ["coarse:third", "coarse:fourth"]+++def test_ordered_tile_results_reads_only_a_bounded_window_before_first_yield(monkeypatch):+ lanes = [ImmediatePool(), ImmediatePool()]+ pool = WorkerAffinePool(lanes, lambda cells: cells, lambda result: result[2], chunk_size=2)+ monkeypatch.setattr(parallel_tiles, "score_tile_with_cache_state", _scored_tile)+ consumed = []++ def jobs():+ for i in range(20):+ consumed.append(i)+ yield f"coarse:{i}", (str(i),), None, None++ results = parallel_tiles.ordered_tile_results(pool, jobs(), 2,+ lambda stage, cells, future: future.result()[0])+ first = next(results)++ assert first[0] == "coarse:0"+ assert len(consumed) <= 4+++def test_completed_later_tiles_do_not_expand_the_window(monkeypatch):+ lanes = [ImmediatePool(), ImmediatePool()]+ pool = WorkerAffinePool(lanes, lambda cells: cells, lambda result: result[2], chunk_size=2)+ monkeypatch.setattr(parallel_tiles, "score_tile_with_cache_state", _scored_tile)+ consumed = []+ waits = 0+ first_future = None++ def submit(function, stage, cells, score_floor):+ nonlocal first_future+ future = Future()+ if first_future is None:+ first_future = future+ else:+ future.set_result(function(stage, cells, score_floor))+ return future++ monkeypatch.setattr(pool.pools[0], "submit", submit)+ monkeypatch.setattr(pool.pools[1], "submit", submit)++ def wait(futures, return_when):+ nonlocal waits+ waits += 1+ if waits == 1:+ completed = {future for future in futures if future is not first_future}+ return completed, set(futures) - completed+ first_future.set_result(_scored_tile("coarse:0", ("0",), None))+ return {first_future}, set(futures) - {first_future}++ monkeypatch.setattr(concurrent.futures, "wait", wait)++ def jobs():+ for i in range(20):+ consumed.append(i)+ yield f"coarse:{i}", (str(i),), None, None++ results = parallel_tiles.ordered_tile_results(pool, jobs(), 2,+ lambda stage, cells, future: future.result()[0])+ assert next(results)[0] == "coarse:0"+ assert consumed == [0, 1, 2, 3]