The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.
JavaScript
"""The scraper self-updater: shadowing, fetching, and refusing bad wheels.
Nothing here touches the network or the real yt-dlp/gallery-dl: a synthetic
distribution ("fake-scraper") is registered in SCRAPERS and served from a fake
PyPI, which lets the import-shadowing be tested for real — an actual import
statement resolving to an actual file — without a multi-MB download.
"""
from __future__ import annotations
import hashlib
import importlib
import io
import json
import sys
import zipfile
import pytest
from azimut import config
from azimut.engine import scrapers
MODULE = "azimut_fake_scraper"
DIST = "fake-scraper"
def make_wheel(version: str, module: str = MODULE, package: bool = True) -> bytes:
"""A minimal but structurally real pure-Python wheel."""
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as archive:
if package:
archive.writestr(f"{module}/__init__.py", f"__version__ = {version!r}\n")
archive.writestr(f"{module}/deep/__init__.py", "MARKER = 'submodule'\n")
archive.writestr(
f"{module}-{version}.dist-info/METADATA",
f"Metadata-Version: 2.1\nName: {DIST}\nVersion: {version}\n",
)
archive.writestr(f"{module}-{version}.dist-info/WHEEL", "Wheel-Version: 1.0\n")
return buffer.getvalue()
class FakeResponse:
def __init__(self, payload=None, data=b""):
self._payload = payload
self._data = data
def raise_for_status(self):
return None
def json(self):
return self._payload
def iter_bytes(self):
yield self._data
def __enter__(self):
return self
def __exit__(self, *exc):
return False
@pytest.fixture()
def fake_dist(monkeypatch, tmp_workspace):
"""Register the synthetic distribution and undo every global it touches."""
monkeypatch.setitem(scrapers.SCRAPERS, DIST, MODULE)
yield
scrapers._remove_finder()
for name in [n for n in sys.modules if n == MODULE or n.startswith(f"{MODULE}.")]:
del sys.modules[name]
importlib.invalidate_caches()
def serve(monkeypatch, version: str, wheel: bytes, sha256: str | None = None):
"""Point the updater at a fake PyPI serving `wheel` as the current release."""
payload = {
"info": {"version": version},
"urls": [
{
"packagetype": "bdist_wheel",
"filename": f"{MODULE}-{version}-py3-none-any.whl",
"url": f"https://example.invalid/{MODULE}-{version}.whl",
"digests": {"sha256": sha256 or hashlib.sha256(wheel).hexdigest()},
}
],
}
monkeypatch.setattr(scrapers.httpx, "get", lambda *a, **k: FakeResponse(payload=payload))
monkeypatch.setattr(scrapers.httpx, "stream", lambda *a, **k: FakeResponse(data=wheel))
return payload
# ---- reading state ----------------------------------------------------------
def test_no_runtime_copy_reports_bundled(fake_dist):
assert scrapers.runtime_version(DIST) is None
version, source = scrapers.active_version(DIST)
assert source == "bundled"
assert version is None # the synthetic dist isn't actually installed
def test_real_scrapers_report_a_bundled_version(tmp_workspace):
"""The versions we ship are discoverable — status() is never blank."""
for dist in ("yt-dlp", "gallery-dl"):
version, source = scrapers.active_version(dist)
assert source == "bundled"
assert version, f"no bundled version found for {dist}"
def test_status_does_not_touch_the_network_by_default(monkeypatch, tmp_workspace):
def explode(*a, **k):
raise AssertionError("status() reached the network without check_pypi")
monkeypatch.setattr(scrapers.httpx, "get", explode)
entries = scrapers.status()
assert {e["dist"] for e in entries} == {"yt-dlp", "gallery-dl"}
assert all("latest" not in e for e in entries)
def test_status_check_survives_being_offline(monkeypatch, tmp_workspace):
def offline(*a, **k):
raise OSError("no route to host")
monkeypatch.setattr(scrapers.httpx, "get", offline)
entries = scrapers.status(check_pypi=True)
assert all(e["latest"] is None and "check_error" in e for e in entries)
assert all(e["version"] for e in entries) # still reports what's in use
# ---- updating ---------------------------------------------------------------
def test_update_fetches_extracts_and_activates(monkeypatch, fake_dist):
serve(monkeypatch, "1.2.3", make_wheel("1.2.3"))
result = scrapers.update(DIST)
assert result["updated"] is True
assert result["version"] == "1.2.3"
assert result["restart_required"] is False # never imported yet
assert scrapers.runtime_version(DIST) == "1.2.3"
assert scrapers.active_version(DIST) == ("1.2.3", "runtime")
def test_updated_copy_is_what_import_resolves_to(monkeypatch, fake_dist):
"""The point of the whole module: a real import gets the fetched code."""
serve(monkeypatch, "9.9.9", make_wheel("9.9.9"))
scrapers.update(DIST)
module = importlib.import_module(MODULE)
assert module.__version__ == "9.9.9"
assert str(scrapers.dist_dir(DIST)) in module.__file__
submodule = importlib.import_module(f"{MODULE}.deep")
assert submodule.MARKER == "submodule"
def test_finder_ignores_everything_else(monkeypatch, fake_dist):
"""The finder is narrow — it must not shadow unrelated imports."""
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
scrapers.update(DIST)
assert importlib.import_module("json") is json
reimported = importlib.import_module("azimut.config")
assert reimported is config
def test_update_is_idempotent(monkeypatch, fake_dist):
serve(monkeypatch, "2.0.0", make_wheel("2.0.0"))
scrapers.update(DIST)
def explode(*a, **k):
raise AssertionError("re-downloaded a version already on disk")
monkeypatch.setattr(scrapers.httpx, "stream", explode)
result = scrapers.update(DIST)
assert result["updated"] is False
assert result["version"] == "2.0.0"
def test_update_replaces_an_older_runtime_copy(monkeypatch, fake_dist):
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
scrapers.update(DIST)
serve(monkeypatch, "1.1.0", make_wheel("1.1.0"))
result = scrapers.update(DIST)
assert result["previous"] == "1.0.0"
assert scrapers.runtime_version(DIST) == "1.1.0"
# exactly one .dist-info — the old one is gone, not merged
infos = list(scrapers.dist_dir(DIST).glob("*.dist-info"))
assert [i.name for i in infos] == [f"{MODULE}-1.1.0.dist-info"]
def test_restart_required_once_the_module_is_imported(monkeypatch, fake_dist):
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
scrapers.update(DIST)
importlib.import_module(MODULE) # now cached in sys.modules
serve(monkeypatch, "1.1.0", make_wheel("1.1.0"))
result = scrapers.update(DIST)
assert result["restart_required"] is True
assert "restart" in result["detail"]
assert sys.modules[MODULE].__version__ == "1.0.0" # honest: still the old one
# ---- refusing bad wheels ----------------------------------------------------
def test_hash_mismatch_is_refused(monkeypatch, fake_dist):
serve(monkeypatch, "6.6.6", make_wheel("6.6.6"), sha256="0" * 64)
with pytest.raises(RuntimeError, match="hash"):
scrapers.update(DIST)
assert scrapers.runtime_version(DIST) is None
def test_a_failed_update_leaves_the_previous_copy_intact(monkeypatch, fake_dist):
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
scrapers.update(DIST)
serve(monkeypatch, "2.0.0", make_wheel("2.0.0"), sha256="0" * 64)
with pytest.raises(RuntimeError):
scrapers.update(DIST)
assert scrapers.runtime_version(DIST) == "1.0.0"
assert not list(scrapers.dist_dir(DIST).parent.glob(".staging-*"))
def test_path_traversal_in_a_wheel_is_refused(monkeypatch, fake_dist):
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as archive:
archive.writestr(f"{MODULE}/__init__.py", "")
archive.writestr(f"{MODULE}-1.0.0.dist-info/METADATA", "Version: 1.0.0\n")
archive.writestr("../../../../evil.py", "pwned = True\n")
evil = buffer.getvalue()
serve(monkeypatch, "1.0.0", evil)
with pytest.raises(RuntimeError, match="unsafe entry"):
scrapers.update(DIST)
assert scrapers.runtime_version(DIST) is None
def test_wheel_without_the_package_is_refused(monkeypatch, fake_dist):
serve(monkeypatch, "1.0.0", make_wheel("1.0.0", package=False))
with pytest.raises(RuntimeError, match="did not contain"):
scrapers.update(DIST)
assert scrapers.runtime_version(DIST) is None
def test_oversized_wheel_is_refused(monkeypatch, fake_dist):
monkeypatch.setattr(scrapers, "WHEEL_MAX_BYTES", 16)
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
with pytest.raises(RuntimeError, match="implausibly large"):
scrapers.update(DIST)
# ---- swapping directories, on every platform --------------------------------
def test_update_over_an_imported_copy(monkeypatch, fake_dist):
"""The Windows-shaped case: replace the package while it's loaded.
Windows can't delete an open file, so the swap renames the old copy aside
instead of deleting it in place. This must not raise anywhere.
"""
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
scrapers.update(DIST)
importlib.import_module(f"{MODULE}.deep") # hold the package open
serve(monkeypatch, "2.0.0", make_wheel("2.0.0"))
result = scrapers.update(DIST)
assert result["updated"] is True
assert scrapers.runtime_version(DIST) == "2.0.0"
assert (scrapers.dist_dir(DIST) / MODULE / "__init__.py").read_text() == "__version__ = '2.0.0'\n"
def test_swap_leaves_no_litter(monkeypatch, fake_dist):
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
scrapers.update(DIST)
serve(monkeypatch, "2.0.0", make_wheel("2.0.0"))
scrapers.update(DIST)
leftovers = [p.name for p in config.runtime_dir().glob(".*")]
assert leftovers == [], f"staging/trash left behind: {leftovers}"
def test_failed_swap_restores_the_working_copy(monkeypatch, fake_dist):
"""If the new copy can't be moved into place, the old one must come back."""
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
scrapers.update(DIST)
real_replace = scrapers.Path.replace
calls = {"n": 0}
def flaky(self, target):
calls["n"] += 1
if calls["n"] == 2: # 1st = retire the old copy, 2nd = move the new one in
raise OSError("disk full")
return real_replace(self, target)
monkeypatch.setattr(scrapers.Path, "replace", flaky)
serve(monkeypatch, "2.0.0", make_wheel("2.0.0"))
with pytest.raises(OSError, match="disk full"):
scrapers.update(DIST)
monkeypatch.setattr(scrapers.Path, "replace", real_replace) # only this one
assert scrapers.runtime_version(DIST) == "1.0.0" # still usable
assert (scrapers.dist_dir(DIST) / MODULE / "__init__.py").exists()
def test_wheel_paths_extract_under_the_dist_dir(monkeypatch, fake_dist):
"""Wheels always use / separators; extraction must land right on Windows too."""
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
scrapers.update(DIST)
nested = scrapers.dist_dir(DIST) / MODULE / "deep" / "__init__.py"
assert nested.is_file()
# ---- reset ------------------------------------------------------------------
def test_reset_falls_back_to_the_bundled_copy(monkeypatch, fake_dist):
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
scrapers.update(DIST)
assert scrapers.dist_dir(DIST).exists()
result = scrapers.reset(DIST)
assert result["removed"] is True
assert not scrapers.dist_dir(DIST).exists()
assert scrapers.active_version(DIST)[1] == "bundled"
def test_reset_when_there_is_nothing_to_reset(fake_dist):
result = scrapers.reset(DIST)
assert result["removed"] is False
def test_activate_is_idempotent(monkeypatch, fake_dist):
serve(monkeypatch, "1.0.0", make_wheel("1.0.0"))
scrapers.update(DIST)
for _ in range(3):
scrapers.activate()
finders = [f for f in sys.meta_path if isinstance(f, scrapers._RuntimeFinder)]
assert len(finders) == 1
def _runtime_copy(version: str) -> None:
root = scrapers.dist_dir(DIST)
(root / MODULE).mkdir(parents=True)
(root / MODULE / "__init__.py").write_text(f'__version__ = "{version}"\n', encoding="utf-8")
(root / f"{MODULE}-{version}.dist-info").mkdir()
@pytest.mark.parametrize(
("bundled", "source", "shadowed"),
[("2026.9.9", "bundled", {}), ("2025.12.31", "runtime", {MODULE: "2026.1.1"})],
)
def test_a_runtime_copy_wins_only_when_newer_than_the_build(monkeypatch, fake_dist, bundled, source, shadowed):
"""An update pressed under an older build must not shadow the newer scraper the
next build ships: the workspace outlives the program."""
_runtime_copy("2026.1.1")
real = importlib.metadata.version
monkeypatch.setattr(
scrapers.importlib.metadata, "version", lambda dist: bundled if dist == DIST else real(dist)
)
assert scrapers.active_version(DIST) == ("2026.1.1" if source == "runtime" else bundled, source)
assert scrapers.activate() == shadowed
def test_a_frozen_build_without_metadata_keeps_the_runtime_copy(fake_dist):
_runtime_copy("2026.1.1")
assert scrapers.active_version(DIST) == ("2026.1.1", "runtime")