Skip to content

Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)

Serialize adoption of cached terrain by parallel workers

Cinna Davisc3a6afff96f7
diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.py
index 47cb195..eee6dfd 100644
--- a/src/dopplergeist/data_access.py
+++ b/src/dopplergeist/data_access.py
@@ -672,12 +672,16 @@ class CachedTerrain:
if path.name in self.files:
return path
receipt = path.with_name(path.name + ".sha256.json")
- if path.exists() and not receipt.exists() and self.adopt_existing:
- if raster:
- import rasterio
- with rasterio.open(path) as dataset:
- dataset.read(1, window=((0, 1), (0, 1)))
- write_json(receipt, {"sha256": sha256(path), "origin": "operator-supplied-cache", "url": url})
+ if self.adopt_existing:
+ # Adoption and download share the same file lock, so workers cannot
+ # race while creating the receipt for an existing terrain file.
+ with cache_lock(path.with_name(path.name + ".lock")):
+ if path.exists() and not receipt.exists():
+ if raster:
+ import rasterio
+ with rasterio.open(path) as dataset:
+ dataset.read(1, window=((0, 1), (0, 1)))
+ write_json(receipt, {"sha256": sha256(path), "origin": "operator-supplied-cache", "url": url})
result = download(url, path, offline=self.offline)
if raster:
import rasterio
diff --git a/tests/test_real_parallel_terrain_adoption.py b/tests/test_real_parallel_terrain_adoption.py
new file mode 100644
index 0000000..f54d158
--- /dev/null
+++ b/tests/test_real_parallel_terrain_adoption.py
@@ -0,0 +1,42 @@
+"""Concurrent adoption of a real cached terrain raster."""
+from concurrent.futures import ThreadPoolExecutor
+import os
+from pathlib import Path
+import shutil
+import threading
+
+import pytest
+
+from dopplergeist.data_access import CachedTerrain, sha256
+
+
+def test_real_raster_adoption_is_serialized(tmp_path, monkeypatch):
+ source = os.environ.get("DOPPLERGEIST_REAL_TERRAIN_RASTER")
+ if not source:
+ pytest.skip("set DOPPLERGEIST_REAL_TERRAIN_RASTER to a real cached DEM")
+ source = Path(source)
+ raster = tmp_path / source.name
+ shutil.copyfile(source, raster)
+ receipt = raster.with_name(raster.name + ".sha256.json")
+ barrier = threading.Barrier(4)
+ original_replace = Path.replace
+
+ def synchronized_replace(path, target):
+ if target == receipt:
+ try:
+ barrier.wait(timeout=1)
+ except threading.BrokenBarrierError:
+ pass
+ return original_replace(path, target)
+
+ monkeypatch.setattr(Path, "replace", synchronized_replace)
+
+ def adopt(_):
+ terrain = object.__new__(CachedTerrain)
+ terrain.files, terrain.adopt_existing, terrain.offline = {}, True, True
+ return terrain._file("https://example.invalid/real-dem", raster, raster=True)
+
+ with ThreadPoolExecutor(max_workers=4) as pool:
+ assert list(pool.map(adopt, range(4))) == [raster] * 4
+ import json
+ assert json.loads(receipt.read_text())["sha256"] == sha256(source)