Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

perf(#7): add bounded DEM-affine tile scheduling (#241)

Cinna Davisa4d27b2cacfe
diff --git a/benchmarks/results/dem-worker-affinity-20261103.json b/benchmarks/results/dem-worker-affinity-20261103.json
new file mode 100644
index 0000000..1cc52eb
--- /dev/null
+++ b/benchmarks/results/dem-worker-affinity-20261103.json
@@ -0,0 +1,91 @@
+{
+ "tiles": [
+ "-19_7",
+ "-18_7",
+ "-17_7",
+ "-16_7",
+ "-15_7",
+ "-14_7"
+ ],
+ "points_per_tile": [
+ 1024,
+ 1024,
+ 1024,
+ 1024,
+ 1024,
+ 1024
+ ],
+ "repetitions": 2,
+ "workers": 2,
+ "chunk_size": 2,
+ "shared_pool": {
+ "elapsed_s": 13.646711387002142,
+ "ordered_hashes": [
+ "22387a651f2f204b1c5e904d39d7f31678571806f085792641f9ddaabb08cf9a",
+ "30b4ba42d862fc9db9a6df780ab9b53e5fef3baa793bc40c964e5678b6ebe90f",
+ "0ab2c28d818dc32d39a1f4ba5acf0e20535744353b20f8a93a28a4d6161e81db",
+ "c30cfe92b379377344657a069b0d1bbeba536029c2ec33c6862fe8125e75daa6",
+ "34624a4dba49acfe0d5da054b143c841599a3c175715ee94187f1c0446cd2b9f",
+ "ab89c86583610a4420a75925a239925797e38b9779de83ca8614aeb4b0112ad0",
+ "22387a651f2f204b1c5e904d39d7f31678571806f085792641f9ddaabb08cf9a",
+ "30b4ba42d862fc9db9a6df780ab9b53e5fef3baa793bc40c964e5678b6ebe90f",
+ "0ab2c28d818dc32d39a1f4ba5acf0e20535744353b20f8a93a28a4d6161e81db",
+ "c30cfe92b379377344657a069b0d1bbeba536029c2ec33c6862fe8125e75daa6",
+ "34624a4dba49acfe0d5da054b143c841599a3c175715ee94187f1c0446cd2b9f",
+ "ab89c86583610a4420a75925a239925797e38b9779de83ca8614aeb4b0112ad0"
+ ],
+ "worker_count": 2,
+ "workers": [
+ {
+ "dem_raster_opens": 19,
+ "cache_hits": 0,
+ "cache_misses": 11,
+ "cpu_s": 1.9719897010000014,
+ "resident_dem_tiles": 4
+ },
+ {
+ "dem_raster_opens": 24,
+ "cache_hits": 11,
+ "cache_misses": 16,
+ "cpu_s": 2.2134649629999994,
+ "resident_dem_tiles": 4
+ }
+ ]
+ },
+ "worker_affine": {
+ "elapsed_s": 10.155085653997958,
+ "ordered_hashes": [
+ "22387a651f2f204b1c5e904d39d7f31678571806f085792641f9ddaabb08cf9a",
+ "30b4ba42d862fc9db9a6df780ab9b53e5fef3baa793bc40c964e5678b6ebe90f",
+ "0ab2c28d818dc32d39a1f4ba5acf0e20535744353b20f8a93a28a4d6161e81db",
+ "c30cfe92b379377344657a069b0d1bbeba536029c2ec33c6862fe8125e75daa6",
+ "34624a4dba49acfe0d5da054b143c841599a3c175715ee94187f1c0446cd2b9f",
+ "ab89c86583610a4420a75925a239925797e38b9779de83ca8614aeb4b0112ad0",
+ "22387a651f2f204b1c5e904d39d7f31678571806f085792641f9ddaabb08cf9a",
+ "30b4ba42d862fc9db9a6df780ab9b53e5fef3baa793bc40c964e5678b6ebe90f",
+ "0ab2c28d818dc32d39a1f4ba5acf0e20535744353b20f8a93a28a4d6161e81db",
+ "c30cfe92b379377344657a069b0d1bbeba536029c2ec33c6862fe8125e75daa6",
+ "34624a4dba49acfe0d5da054b143c841599a3c175715ee94187f1c0446cd2b9f",
+ "ab89c86583610a4420a75925a239925797e38b9779de83ca8614aeb4b0112ad0"
+ ],
+ "worker_count": 2,
+ "workers": [
+ {
+ "dem_raster_opens": 22,
+ "cache_hits": 7,
+ "cache_misses": 13,
+ "cpu_s": 2.2020988649999986,
+ "resident_dem_tiles": 4
+ },
+ {
+ "dem_raster_opens": 16,
+ "cache_hits": 8,
+ "cache_misses": 10,
+ "cpu_s": 1.6895734730000003,
+ "resident_dem_tiles": 4
+ }
+ ]
+ },
+ "elevation_arrays_equal": true,
+ "note": "Real saved-grid terrain replay; not a full-search timing or accuracy result."
+}
diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md
index ca3b24d..3af8a95 100644
--- a/docs/PERFORMANCE.md
+++ b/docs/PERFORMANCE.md
@@ -796,3 +796,45 @@ Each search worker retains its own bounded cache; row-major tile scheduling does
not guarantee that neighboring work reaches the same worker. Grouping work by
geography remains a separate scheduling opportunity requiring a full search and
load-balance comparison.
+
+On a separate ordering-only replay of 12 saved real tile footprints (up to 1,024
+points each, repeated three times), contiguous per-tile sampling reduced DEM
+raster opens from 78 to 26 and sampler CPU from 5.89 s to 2.73 s (2.16x).
+Elevation arrays matched exactly. Wall time was slower in the clustered order
+(13.36 s versus 11.26 s), so the result supports cache locality but not end-to-end
+speedup. This offline replay excludes download latency; reproduce with the same
+[`benchmark_dem_locality.py`](../scripts/benchmark_dem_locality.py) script.
+
+### Bounded worker-affine scheduling prototype
+
+The scheduler prototype keeps the lazy row-major request stream and ordered
+parent merge. It pins four consecutive uncached tile requests to one worker,
+choosing a worker from a conservative geographic DEM footprint and its reported
+in-memory DEM keys. A bounded lookahead of four tiles per worker keeps request
+materialization independent of total search size; results and checkpoints remain
+per tile. The worker cache keys are returned only as scheduling metadata and do
+not enter tile payloads or retention records.
+
+On the saved 745-tile checkpoint sequence, 130 payloads referenced terrain and
+recorded 280 tile-to-DEM accesses across 66 unique rasters. A 32 MiB LRU model
+replayed those observed footprints: round-robin assignment hit 28/280 accesses
+(10.0%), four-tile worker chunks hit 117/280 (41.8%), and static contiguous
+lanes hit 133/280 (47.5%). Bounding-box DEM prediction covered the recorded DEM
+set for all 130 terrain-using tiles; mean per-tile precision was 90.6%, with
+three predicted DEM footprints on otherwise empty tiles. These cache figures
+are simulated from saved payload footprints, not timed worker measurements;
+they do not establish search runtime or load balance.
+
+A real worker replay used six contiguous saved tiles with 1,024 grid points each,
+sampled twice through two workers. Both the shared pool and affinity scheduler
+started two worker processes, and all elevation hashes matched. The affinity
+scheduler reduced DEM raster opens from 43 to 38, decoded-cache misses from 27 to
+23, and increased cache hits from 11 to 15. Summed terrain-sampler CPU fell from
+4.19 to 3.89 seconds (1.08x) in this single replay. Wall time was not treated as
+a result because it was one run under changing host load. Reproduce with
+[`benchmark_dem_worker_affinity.py`](../scripts/benchmark_dem_worker_affinity.py);
+the [run record](../benchmarks/results/dem-worker-affinity-20261103.json) is
+terrain-stage evidence, not a full-search speed estimate.
+
+Applying this scheduler to a saved search changes the package identity and
+requires its validated checkpoint compatibility path before resuming that job.
diff --git a/scripts/benchmark_dem_worker_affinity.py b/scripts/benchmark_dem_worker_affinity.py
new file mode 100644
index 0000000..60d4ba8
--- /dev/null
+++ b/scripts/benchmark_dem_worker_affinity.py
@@ -0,0 +1,193 @@
+"""Compare shared-pool and worker-affine DEM access on saved real tiles."""
+import argparse
+from collections import Counter
+import hashlib
+import json
+import multiprocessing
+import os
+from pathlib import Path
+import time
+
+import numpy as np
+
+from dopplergeist import parallel_tiles
+from dopplergeist.data_access import CachedTerrain, _index_pool
+from dopplergeist.parallel_tiles import ordered_tile_results, tile_worker_lanes
+from dopplergeist.tiled_search import _tile_coordinates, _tile_dem_keys
+
+
+_terrain = None
+_points = None
+_counters = None
+_ready_barrier = None
+
+
+def _initialize_worker(terrain_root, points, ready_barrier):
+ global _terrain, _points, _counters, _ready_barrier
+ _terrain = CachedTerrain(terrain_root, offline=True)
+ _points = points
+ _counters = {"opens": 0, "hits": 0, "misses": 0}
+ _ready_barrier = ready_barrier
+
+ import rasterio
+
+ original_open = rasterio.open
+
+ def counted_open(path, *args, **kwargs):
+ if Path(path).name.startswith("Copernicus_DSM_COG_30_"):
+ _counters["opens"] += 1
+ return original_open(path, *args, **kwargs)
+
+ rasterio.open = counted_open
+ original_sample = CachedTerrain._raster_sample
+
+ def counted_sample(terrain, path, lat, lon):
+ if Path(path).name.startswith("Copernicus_DSM_COG_30_"):
+ _counters["hits" if Path(path) in terrain._tile_cache else "misses"] += 1
+ return original_sample(terrain, path, lat, lon)
+
+ CachedTerrain._raster_sample = counted_sample
+
+
+def _warmup_worker():
+ _ready_barrier.wait(timeout=120)
+ return os.getpid()
+
+
+def _sample_tile(stage, cells, score_floor=None):
+ lat, lon = _points[tuple(cells)]
+ before = dict(_counters)
+ cpu_started = time.process_time()
+ elevations = _terrain.sample(lat, lon)
+ cpu_s = time.process_time() - cpu_started
+ delta = {key: _counters[key] - before[key] for key in _counters}
+ cache_keys = tuple(path.stem for path in _terrain._tile_cache)
+ digest = hashlib.sha256(np.asarray(elevations, dtype="<f8").tobytes()).hexdigest()
+ return digest, delta, cache_keys, os.getpid(), cpu_s
+
+
+def _eligible_tiles(job, count):
+ rows = []
+ for path in (job / "search").glob("tile-*.json"):
+ try:
+ payload = json.loads(path.read_text())["payload"]
+ cells = tuple(payload["cells"])
+ if payload.get("grid_points", 0) < 500 or len(payload.get("terrain_files", [])) < 3:
+ continue
+ dem_names = {Path(source["path"]).name for source in payload["terrain_files"]
+ if Path(source["path"]).name.startswith("Copernicus_DSM_COG_30_")}
+ if dem_names:
+ rows.append((cells[1], cells[0], path.name, cells, dem_names))
+ except (KeyError, OSError, ValueError, TypeError):
+ continue
+ rows.sort()
+ if len(rows) < count:
+ raise ValueError(f"Need at least {count} saved terrain tiles; found {len(rows)}")
+
+ best_start, best_score = 0, -1
+ for start in range(len(rows) - count + 1):
+ frequencies = Counter(name for row in rows[start:start + count] for name in row[4])
+ score = sum(amount * (amount - 1) for amount in frequencies.values())
+ if score > best_score:
+ best_start, best_score = start, score
+ selected = rows[best_start:best_start + count]
+ return [(name.removeprefix("tile-").removesuffix(".json"), cells)
+ for _, _, name, cells, _ in selected]
+
+
+def _run(pool_context, jobs, workers, lookahead=None):
+ results = []
+ with pool_context as pool:
+ if isinstance(pool, parallel_tiles.WorkerAffinePool):
+ warmups = [lane.submit(_warmup_worker) for lane in pool.pools]
+ else:
+ warmups = [pool.submit(_warmup_worker) for _ in range(workers)]
+ warmup_pids = [future.result(timeout=120) for future in warmups]
+ if len(set(warmup_pids)) != workers:
+ raise RuntimeError(f"Expected {workers} workers, started {len(set(warmup_pids))}")
+ if lookahead is not None and not hasattr(pool, "lookahead"):
+ pool.lookahead = lookahead
+ results.extend(ordered_tile_results(
+ pool, jobs, workers, lambda stage, cells, future: future.result()))
+ by_worker = {}
+ for _, _, result, _ in results:
+ digest, delta, cache_keys, pid, cpu_s = result
+ record = by_worker.setdefault(pid, {"dem_raster_opens": 0, "cache_hits": 0,
+ "cache_misses": 0, "cpu_s": 0.0,
+ "resident_dem_tiles": 0})
+ record["dem_raster_opens"] += delta["opens"]
+ record["cache_hits"] += delta["hits"]
+ record["cache_misses"] += delta["misses"]
+ record["cpu_s"] += cpu_s
+ record["resident_dem_tiles"] = sum(
+ name.startswith("Copernicus_DSM_COG_30_") for name in cache_keys)
+ hashes = [prepared[0] for _, _, prepared, _ in results]
+ return {"ordered_hashes": hashes, "worker_count": len(warmup_pids),
+ "workers": [record for _, record in sorted(by_worker.items())]}
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--job", type=Path, required=True, help="saved real job with a search manifest")
+ parser.add_argument("--cache", type=Path, required=True, help="populated real cache containing terrain/")
+ parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument("--workers", type=int, default=4)
+ parser.add_argument("--tiles", type=int, default=12)
+ parser.add_argument("--repetitions", type=int, default=3)
+ parser.add_argument("--chunk-size", type=int, default=4)
+ args = parser.parse_args()
+ if min(args.workers, args.tiles, args.repetitions, args.chunk_size) < 1:
+ parser.error("workers, tiles, repetitions, and chunk-size must be positive")
+
+ config = json.loads((args.job / "search/manifest.json").read_text())["configuration"]
+ selected = _eligible_tiles(args.job, args.tiles)
+ points = {}
+ for _, cells in selected:
+ frame = _tile_coordinates(cells, config["budget"]["spacing_m"], config["bounds"], config["crs"])
+ points[cells] = (frame.lat.to_numpy(), frame.lon.to_numpy())
+ jobs = [(f"coarse:{name}:pass-{rep}", cells, None, None)
+ for rep in range(args.repetitions) for name, cells in selected]
+ terrain_root = args.cache / "terrain"
+ catalog_path = args.cache / "terrain/tileList.txt"
+ catalog = {line.rstrip("/") for line in catalog_path.read_text().splitlines()}
+ affinity_keys = lambda cells: _tile_dem_keys(
+ cells, config["budget"]["spacing_m"], config["bounds"], config["crs"], catalog)
+ original_score_tile = parallel_tiles.score_tile
+ original_affine_score = parallel_tiles.score_tile_with_cache_state
+ parallel_tiles.score_tile = _sample_tile
+ parallel_tiles.score_tile_with_cache_state = _sample_tile
+ try:
+ began = time.perf_counter()
+ barrier = multiprocessing.get_context("spawn").Barrier(args.workers)
+ shared_context = _index_pool(args.workers, _initialize_worker,
+ (terrain_root, points, barrier))
+ lookahead = args.workers * args.chunk_size
+ shared = _run(shared_context, jobs, args.workers, lookahead=lookahead)
+ shared_elapsed = time.perf_counter() - began
+
+ began = time.perf_counter()
+ barrier = multiprocessing.get_context("spawn").Barrier(args.workers)
+ affinity_context = tile_worker_lanes(
+ args.workers, _initialize_worker, (terrain_root, points, barrier), affinity_keys,
+ chunk_size=args.chunk_size)
+ affinity = _run(affinity_context, jobs, args.workers, lookahead=lookahead)
+ affinity_elapsed = time.perf_counter() - began
+ finally:
+ parallel_tiles.score_tile = original_score_tile
+ parallel_tiles.score_tile_with_cache_state = original_affine_score
+
+ if shared["ordered_hashes"] != affinity["ordered_hashes"]:
+ raise AssertionError("shared-pool and worker-affine elevation samples differ")
+ output = {"tiles": [name for name, _ in selected], "points_per_tile": [len(v[0]) for v in points.values()],
+ "repetitions": args.repetitions, "workers": args.workers, "chunk_size": args.chunk_size,
+ "shared_pool": {"elapsed_s": shared_elapsed, **shared},
+ "worker_affine": {"elapsed_s": affinity_elapsed, **affinity},
+ "elevation_arrays_equal": True,
+ "note": "Real saved-grid terrain replay; not a full-search timing or accuracy result."}
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ args.output.write_text(json.dumps(output, indent=2) + "\n")
+ print(json.dumps(output, indent=2), flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/dopplergeist/parallel_tiles.py b/src/dopplergeist/parallel_tiles.py
index bb6b7e5..37e10df 100644
--- a/src/dopplergeist/parallel_tiles.py
+++ b/src/dopplergeist/parallel_tiles.py
@@ -1,6 +1,8 @@
"""Isolated coarse-tile workers with private retention ledgers."""
+from contextlib import ExitStack, contextmanager
from pathlib import Path
import tempfile
+from threading import Lock
import pandas as pd
@@ -21,6 +23,87 @@ class TileFailure(DataUnavailable):
return str(self.args[0])
+class WorkerAffinePool:
+ """Keep tile work on the worker whose DEM cache best matches its area."""
+
+ def __init__(self, pools, tile_dem_keys, cache_keys, chunk_size=4):
+ if not pools:
+ raise ValueError("At least one tile worker is required")
+ if type(chunk_size) is not int or chunk_size < 1:
+ raise ValueError("chunk_size must be a positive integer")
+ self.pools = tuple(pools)
+ self.tile_dem_keys = tile_dem_keys
+ self.cache_keys = cache_keys
+ self.chunk_size = chunk_size
+ self.lookahead = len(pools) * chunk_size
+ self._resident = [frozenset() for _ in pools]
+ self._pending = [[] for _ in pools]
+ self._used = [False for _ in pools]
+ self._next_lane = 0
+ self._chunk_lane = None
+ self._chunk_remaining = 0
+ self._lock = Lock()
+
+ def submit(self, function, stage, cells, score_floor):
+ predicted = frozenset(self.tile_dem_keys(cells))
+ with self._lock:
+ if not self._chunk_remaining:
+ eligible = [i for i, used in enumerate(self._used) if not used]
+ if not eligible:
+ eligible = list(range(len(self.pools)))
+ loads = [len(self._pending[i]) for i in eligible]
+ least_loaded = min(loads)
+ candidates = [i for i in eligible if len(self._pending[i]) == least_loaded]
+
+ def affinity(worker):
+ nearby = self._resident[worker].union(*self._pending[worker])
+ return (-len(predicted.intersection(nearby)),
+ (worker - self._next_lane) % len(self.pools))
+
+ self._chunk_lane = min(candidates, key=affinity)
+ self._chunk_remaining = self.chunk_size
+ self._used[self._chunk_lane] = True
+ lane = self._chunk_lane
+ self._chunk_remaining -= 1
+ self._pending[lane].append(predicted)
+ self._next_lane = (lane + 1) % len(self.pools)
+ try:
+ future = self.pools[lane].submit(function, stage, cells, score_floor)
+ except BaseException:
+ with self._lock:
+ self._pending[lane].pop()
+ raise
+ future.add_done_callback(lambda completed, worker=lane: self._completed(worker, completed))
+ return future
+
+ def finish_wave(self):
+ """Keep a short final chunk from leaking into the next search wave."""
+ with self._lock:
+ self._chunk_lane = None
+ self._chunk_remaining = 0
+ self._used = [False for _ in self.pools]
+
+ def _completed(self, lane, future):
+ try:
+ resident = frozenset(self.cache_keys(future.result()))
+ except BaseException:
+ resident = frozenset()
+ with self._lock:
+ if self._pending[lane]:
+ self._pending[lane].pop(0)
+ self._resident[lane] = resident
+
+
+@contextmanager
+def tile_worker_lanes(workers, initializer, initargs, tile_dem_keys, chunk_size=4):
+ """Create pinned worker processes so spatial cache affinity is real."""
+ from .data_access import _index_pool
+
+ with ExitStack() as stack:
+ pools = [stack.enter_context(_index_pool(1, initializer, initargs)) for _ in range(workers)]
+ yield WorkerAffinePool(pools, tile_dem_keys, lambda result: result[2], chunk_size=chunk_size)
+
+
_worker = None
@@ -104,8 +187,15 @@ def score_tile(stage, cells, score_floor=None):
raise
+def score_tile_with_cache_state(stage, cells, score_floor=None):
+ """Return the worker's resident DEM keys for scheduling; never persist them."""
+ payload, worker_path = score_tile(stage, cells, score_floor)
+ cache_keys = tuple(path.stem for path in _worker["terrain"]._tile_cache)
+ return payload, worker_path, cache_keys
+
+
def ordered_tile_results(pool, jobs, workers, finish):
- """Persist completed work promptly but yield lightweight handles in order."""
+ """Persist promptly, bound out-of-order results, and yield in input order."""
from concurrent.futures import FIRST_COMPLETED, wait
if type(workers) is not int or workers < 1:
@@ -114,8 +204,10 @@ def ordered_tile_results(pool, jobs, workers, finish):
pending, ready = {}, {}
assigned = next_result = 0
exhausted = False
+ lookahead = getattr(pool, "lookahead", 2 * workers)
while not exhausted or pending or ready:
- while not exhausted and len(pending) < 2 * workers and next_result not in ready:
+ while (not exhausted and len(pending) + len(ready) < lookahead
+ and next_result not in ready):
try:
stage, cells, cached, floor = next(source)
except StopIteration:
@@ -124,7 +216,8 @@ def ordered_tile_results(pool, jobs, workers, finish):
index = assigned
assigned += 1
if cached is None:
- future = pool.submit(score_tile, stage, cells, floor)
+ scorer = score_tile_with_cache_state if isinstance(pool, WorkerAffinePool) else score_tile
+ future = pool.submit(scorer, stage, cells, floor)
pending[future] = (index, stage, cells)
else:
ready[index] = (stage, cells, cached, True)
@@ -139,3 +232,5 @@ def ordered_tile_results(pool, jobs, workers, finish):
ready[index] = (stage, cells, finish(stage, cells, future), False)
elif exhausted and ready:
raise RuntimeError("Tile result ordering is inconsistent")
+ if isinstance(pool, WorkerAffinePool):
+ pool.finish_wave()
diff --git a/src/dopplergeist/tiled_search.py b/src/dopplergeist/tiled_search.py
index 8ad2639..bb5b55f 100644
--- a/src/dopplergeist/tiled_search.py
+++ b/src/dopplergeist/tiled_search.py
@@ -188,6 +188,27 @@ def _sample_tile_terrain(points, terrain):
return points
+def _tile_dem_keys(cells, spacing, bounds, crs, catalog):
+ """Conservatively predict DEMs touched by a tile for worker affinity."""
+ x0, y0, x1, y1 = cells
+ west, south, east, north = _to_geographic(crs).transform_bounds(
+ x0 * spacing, y0 * spacing, x1 * spacing, y1 * spacing, densify_pts=21)
+ west, south = max(west, bounds[0]), max(south, bounds[1])
+ east, north = min(east, bounds[2]), min(north, bounds[3])
+ if west > east or south > north:
+ return ()
+ south_key, north_key = math.floor(south - 1 / 2400), math.floor(north - 1 / 2400)
+ west_key, east_key = math.floor(west + 1 / 2400), math.floor(east + 1 / 2400)
+ keys = []
+ for latitude in range(south_key, north_key + 1):
+ for longitude in range(west_key, east_key + 1):
+ tag = (f"Copernicus_DSM_COG_30_{'N' if latitude >= 0 else 'S'}{abs(latitude):02d}_00_"
+ f"{'E' if longitude >= 0 else 'W'}{abs(longitude):03d}_00_DEM")
+ if tag in catalog:
+ keys.append(tag)
+ return tuple(keys)
+
+
def tile_points(cells, spacing, bounds, terrain, crs):
return _sample_tile_terrain(_tile_coordinates(cells, spacing, bounds, crs), terrain)
@@ -622,9 +643,9 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
event_limit_applied = filters["association"] == "subsets" and event_limit < len(events)
import os
from contextlib import ExitStack, nullcontext
- from .data_access import CachedTerrain, _index_pool
+ from .data_access import CachedTerrain
from .parallel_retention import import_retention_stage
- from .parallel_tiles import TileFailure, initialize_tile_worker, ordered_tile_results
+ from .parallel_tiles import TileFailure, initialize_tile_worker, ordered_tile_results, tile_worker_lanes
xmin, ymin, xmax, ymax = _grid_extent(bounds, budget.spacing_m, crs)
total = (xmax // budget.tile_cells - xmin // budget.tile_cells + 1) * (ymax // budget.tile_cells - ymin // budget.tile_cells + 1)
if budget.max_tiles is None:
@@ -759,14 +780,16 @@ def search_national(indexes, terrain, events, measurements, *, time_range_s, out
initargs = ([index.path for index in indexes], terrain.root, terrain_options, events, measurements,
bounds, time_range_s, budget, parameters, filters, start_window, crs,
output / ".tile-workers", run_hash, score_kind)
- context = _index_pool(workers, initialize_tile_worker, initargs)
+ context = tile_worker_lanes(
+ workers, initialize_tile_worker, initargs,
+ lambda cells: _tile_dem_keys(cells, budget.spacing_m, bounds, crs, terrain.catalog))
else:
context = nullcontext(None)
def finish_tile(stage, cells, future):
checkpoint = output / f"tile-{stage[len('coarse:'):]}.json"
payload = None
try:
- payload, worker_path = future.result()
+ payload, worker_path, *_ = future.result()
import_retention_stage(ledger, worker_path, stage, payload["retention_checksum"])
write_json(checkpoint, {"run_hash": run_hash, "payload_hash": content_hash(payload), "payload": payload},
compact=True)
diff --git a/tests/test_real_parallel_retention.py b/tests/test_real_parallel_retention.py
index 48f389f..08c7f19 100644
--- a/tests/test_real_parallel_retention.py
+++ b/tests/test_real_parallel_retention.py
@@ -1,4 +1,6 @@
"""Verify worker-ledger imports using an actual running search's evidence."""
+from concurrent.futures import ProcessPoolExecutor
+import multiprocessing
import os
from pathlib import Path
import sqlite3
@@ -58,14 +60,13 @@ def test_real_retention_stages_survive_worker_import(tmp_path):
def test_real_tiles_match_sequential_scoring_and_preserve_ledgers(tmp_path):
import json
- import multiprocessing
import shutil
- from concurrent.futures import ProcessPoolExecutor
from dopplergeist.data_access import FlightIndex, content_hash
from dopplergeist.matching import prepare_events
- from dopplergeist.parallel_tiles import TileFailure, initialize_tile_worker, ordered_tile_results, score_tile
- from dopplergeist.tiled_search import SearchBudget
+ from dopplergeist.parallel_tiles import (TileFailure, initialize_tile_worker, ordered_tile_results,
+ score_tile, tile_worker_lanes)
+ from dopplergeist.tiled_search import SearchBudget, _tile_dem_keys
source = os.environ.get("DOPPLERGEIST_REAL_JOB_ROOT")
cache = os.environ.get("DOPPLERGEIST_REAL_CACHE_ROOT")
@@ -99,10 +100,11 @@ def test_real_tiles_match_sequential_scoring_and_preserve_ledgers(tmp_path):
jobs = [(stage, cells, None, None) for stage, cells in stages]
jobs.append((stages[0][0], stages[0][1], serial[0][0], None))
parallel = []
- with ProcessPoolExecutor(max_workers=2, mp_context=multiprocessing.get_context("spawn"),
- initializer=initialize_tile_worker, initargs=args) as pool:
+ catalog = {line.rstrip("/") for line in (cache / "terrain/tileList.txt").read_text().splitlines()}
+ tile_keys = lambda cells: _tile_dem_keys(cells, budget.spacing_m, config["bounds"], config["crs"], catalog)
+ with tile_worker_lanes(2, initialize_tile_worker, args, tile_keys, chunk_size=1) as pool:
for stage, cells, result, was_cached in ordered_tile_results(
- pool, jobs, 2, lambda stage, cells, future: future.result()):
+ pool, jobs, 2, lambda stage, cells, future: future.result()[:2]):
if was_cached:
assert stage == stages[0][0] and result == serial[0][0]
else:
diff --git a/tests/test_tile_worker_affinity.py b/tests/test_tile_worker_affinity.py
new file mode 100644
index 0000000..5da3ef5
--- /dev/null
+++ b/tests/test_tile_worker_affinity.py
@@ -0,0 +1,115 @@
+import concurrent.futures
+from concurrent.futures import Future
+
+from dopplergeist import parallel_tiles
+from dopplergeist.parallel_tiles import WorkerAffinePool
+
+
+class ImmediatePool:
+ def __init__(self):
+ self.stages = []
+
+ def submit(self, function, stage, cells, score_floor):
+ self.stages.append(stage)
+ future = Future()
+ try:
+ future.set_result(function(stage, cells, score_floor))
+ except BaseException as error:
+ future.set_exception(error)
+ return future
+
+
+def _scored_tile(stage, cells, score_floor):
+ return {"stage": stage}, None, tuple(cells)
+
+
+def test_affinity_routes_to_resident_dem_and_preserves_result_order(monkeypatch):
+ lanes = [ImmediatePool(), ImmediatePool()]
+ pool = WorkerAffinePool(lanes, lambda cells: cells, lambda result: result[2], chunk_size=1)
+ monkeypatch.setattr(parallel_tiles, "score_tile_with_cache_state", _scored_tile)
+ jobs = [(f"coarse:{name}", (dem,), None, None)
+ for name, dem in (("first", "A"), ("second", "B"), ("third", "A"))]
+
+ results = list(parallel_tiles.ordered_tile_results(
+ pool, jobs, 2, lambda stage, cells, future: future.result()[0]))
+
+ assert [(stage, result, cached) for stage, _, result, cached in results] == [
+ (stage, {"stage": stage}, False) for stage, _, _, _ in jobs]
+ assert lanes[0].stages == ["coarse:first", "coarse:third"]
+ assert lanes[1].stages == ["coarse:second"]
+
+
+def test_tiles_in_a_small_spatial_chunk_stay_on_one_worker(monkeypatch):
+ lanes = [ImmediatePool(), ImmediatePool()]
+ pool = WorkerAffinePool(lanes, lambda cells: cells, lambda result: result[2], chunk_size=2)
+ monkeypatch.setattr(parallel_tiles, "score_tile_with_cache_state", _scored_tile)
+ jobs = [(f"coarse:{name}", (dem,), None, None)
+ for name, dem in (("first", "A"), ("second", "A"), ("third", "B"), ("fourth", "B"))]
+
+ results = list(parallel_tiles.ordered_tile_results(
+ pool, jobs, 2, lambda stage, cells, future: future.result()[0]))
+
+ assert [result[0] for result in results] == [stage for stage, _, _, _ in jobs]
+ assert lanes[0].stages == ["coarse:first", "coarse:second"]
+ assert lanes[1].stages == ["coarse:third", "coarse:fourth"]
+
+
+def test_ordered_tile_results_reads_only_a_bounded_window_before_first_yield(monkeypatch):
+ lanes = [ImmediatePool(), ImmediatePool()]
+ pool = WorkerAffinePool(lanes, lambda cells: cells, lambda result: result[2], chunk_size=2)
+ monkeypatch.setattr(parallel_tiles, "score_tile_with_cache_state", _scored_tile)
+ consumed = []
+
+ def jobs():
+ for i in range(20):
+ consumed.append(i)
+ yield f"coarse:{i}", (str(i),), None, None
+
+ results = parallel_tiles.ordered_tile_results(pool, jobs(), 2,
+ lambda stage, cells, future: future.result()[0])
+ first = next(results)
+
+ assert first[0] == "coarse:0"
+ assert len(consumed) <= 4
+
+
+def test_completed_later_tiles_do_not_expand_the_window(monkeypatch):
+ lanes = [ImmediatePool(), ImmediatePool()]
+ pool = WorkerAffinePool(lanes, lambda cells: cells, lambda result: result[2], chunk_size=2)
+ monkeypatch.setattr(parallel_tiles, "score_tile_with_cache_state", _scored_tile)
+ consumed = []
+ waits = 0
+ first_future = None
+
+ def submit(function, stage, cells, score_floor):
+ nonlocal first_future
+ future = Future()
+ if first_future is None:
+ first_future = future
+ else:
+ future.set_result(function(stage, cells, score_floor))
+ return future
+
+ monkeypatch.setattr(pool.pools[0], "submit", submit)
+ monkeypatch.setattr(pool.pools[1], "submit", submit)
+
+ def wait(futures, return_when):
+ nonlocal waits
+ waits += 1
+ if waits == 1:
+ completed = {future for future in futures if future is not first_future}
+ return completed, set(futures) - completed
+ first_future.set_result(_scored_tile("coarse:0", ("0",), None))
+ return {first_future}, set(futures) - {first_future}
+
+ monkeypatch.setattr(concurrent.futures, "wait", wait)
+
+ def jobs():
+ for i in range(20):
+ consumed.append(i)
+ yield f"coarse:{i}", (str(i),), None, None
+
+ results = parallel_tiles.ordered_tile_results(pool, jobs(), 2,
+ lambda stage, cells, future: future.result()[0])
+ assert next(results)[0] == "coarse:0"
+ assert consumed == [0, 1, 2, 3]