Python tool for aircraft Doppler audio geolocation using historical ADS-B flight data (Source-available)
Python acoustic-geolocationads-baircraft-trackingairplaneaudio-forensicsdoppler-effectdoppler-shiftgeolocationosintsignal-processing
Serialize adoption of cached terrain by parallel workers
Cinna Davisc3a6afff96f7
diff --git a/src/dopplergeist/data_access.py b/src/dopplergeist/data_access.pyindex 47cb195..eee6dfd 100644--- a/src/dopplergeist/data_access.py+++ b/src/dopplergeist/data_access.py@@ -672,12 +672,16 @@ class CachedTerrain:if path.name in self.files:return pathreceipt = path.with_name(path.name + ".sha256.json")- if path.exists() and not receipt.exists() and self.adopt_existing:- if raster:- import rasterio- with rasterio.open(path) as dataset:- dataset.read(1, window=((0, 1), (0, 1)))- write_json(receipt, {"sha256": sha256(path), "origin": "operator-supplied-cache", "url": url})+ if self.adopt_existing:+ # Adoption and download share the same file lock, so workers cannot+ # race while creating the receipt for an existing terrain file.+ with cache_lock(path.with_name(path.name + ".lock")):+ if path.exists() and not receipt.exists():+ if raster:+ import rasterio+ with rasterio.open(path) as dataset:+ dataset.read(1, window=((0, 1), (0, 1)))+ write_json(receipt, {"sha256": sha256(path), "origin": "operator-supplied-cache", "url": url})result = download(url, path, offline=self.offline)if raster:import rasteriodiff --git a/tests/test_real_parallel_terrain_adoption.py b/tests/test_real_parallel_terrain_adoption.pynew file mode 100644index 0000000..f54d158--- /dev/null+++ b/tests/test_real_parallel_terrain_adoption.py@@ -0,0 +1,42 @@+"""Concurrent adoption of a real cached terrain raster."""+from concurrent.futures import ThreadPoolExecutor+import os+from pathlib import Path+import shutil+import threading++import pytest++from dopplergeist.data_access import CachedTerrain, sha256+++def test_real_raster_adoption_is_serialized(tmp_path, monkeypatch):+ source = os.environ.get("DOPPLERGEIST_REAL_TERRAIN_RASTER")+ if not source:+ pytest.skip("set DOPPLERGEIST_REAL_TERRAIN_RASTER to a real cached DEM")+ source = Path(source)+ raster = tmp_path / source.name+ shutil.copyfile(source, raster)+ receipt = raster.with_name(raster.name + ".sha256.json")+ barrier = threading.Barrier(4)+ original_replace = Path.replace++ def synchronized_replace(path, target):+ if target == receipt:+ try:+ barrier.wait(timeout=1)+ except threading.BrokenBarrierError:+ pass+ return original_replace(path, target)++ monkeypatch.setattr(Path, "replace", synchronized_replace)++ def adopt(_):+ terrain = object.__new__(CachedTerrain)+ terrain.files, terrain.adopt_existing, terrain.offline = {}, True, True+ return terrain._file("https://example.invalid/real-dem", raster, raster=True)++ with ThreadPoolExecutor(max_workers=4) as pool:+ assert list(pool.map(adopt, range(4))) == [raster] * 4+ import json+ assert json.loads(receipt.read_text())["sha256"] == sha256(source)