Skip to content

The OSINT investigator's workbench. One case, one folder, every tool. Local-first: your media and your investigations never leave your machine.

JavaScript
20.9 KB
Raw Blame
"""Release gate for the storage migration (docs/STORAGE_AND_PERFORMANCE.md
step 7).

The doc's release gate is part manual (record the reference-machine numbers, dry-
run real disposable cases) and part automatable. This file pins the automatable
checks so they run on the CI matrix (three OSes + the 3.11 floor) on every change:

- a legacy json case migrates and every current workflow still answers;
- a *closed* case folder copy opens identically — the "case bundle / copy is
  complete" guarantee (rollback journal, no WAL to checkpoint);
- a large migrated case opens through bounded queries, never shipping its graph;
- the frozen-binary packaging constraints hold (static bundled, dev tooling out,
  no new runtime dependency, case.db under the workspace root).
"""

from __future__ import annotations

import shutil
import tomllib
from pathlib import Path, PurePosixPath

from bigcase import build_big_case
from legacy_case import write_legacy_json_case

from azimut import layout, workspace
from azimut.engine import extinstall
from azimut.engine import links as link_engine
from azimut.engine import media as media_engine
from azimut.engine import thumbnails
from azimut.engine import workqueue
from azimut.workspace import Case

_PROV = {"by": "user", "at": "2026-01-01T00:00:00Z", "status": "confirmed"}


# -- functional + migration -------------------------------------------------


def test_legacy_json_case_migrates_and_every_workflow_answers(tmp_workspace):
    """A legacy json case (unicode, folders, a derivation chain, a media sidecar)
    migrates on open and every read path — snapshot, bounded catalog, neighbour
    chain, media listing, durable jobs — works on the result, with a recoverable
    backup left behind."""
    legacy = write_legacy_json_case(
        "Réunion",  # non-ASCII survives the round trip
        entities=[
            {"id": "e_p", "type": "person", "label": "Ada", "attrs": {"folder": "People"}, "provenance": _PROV},
            {"id": "e_m", "type": "media", "label": "shot", "attrs": {"path": "media/shot.jpg", "sha256": "a" * 64, "kind": "image"}, "provenance": _PROV},
            {"id": "e_pr", "type": "proof", "label": "Proof", "attrs": {"spec": "proofs/p.json"}, "provenance": _PROV},
        ],
        links=[{"id": "l1", "from": "e_pr", "to": "e_m", "type": "derived-from", "provenance": _PROV}],
        folders=["People"],
    )
    # a real media file + sidecar so the media listing has something to read
    (legacy.path / "media" / "shot.jpg").write_bytes(b"\xff\xd8\xff\xe0jpeg")
    (legacy.path / "media" / ("shot.jpg" + layout.PRE_HIDDEN_SIDECAR_SUFFIX)).write_text(
        '{"filename": "shot.jpg", "kind": "image", "sha256": "%s", "size": 5,'
        ' "added_at": "2026-01-01T00:00:00Z", "source": {}, "thumbnail": null}' % ("a" * 64),
        encoding="utf-8",
    )

    case = Case.open(legacy.id)  # migrates json -> sqlite

    # storage flipped, graph preserved, backup recoverable
    assert case.read()["azimut"] == {"schema": workspace.CASE_SCHEMA, "storage": "sqlite"}
    assert (case.tool_root / "case.pre-migrate-v2.json").exists()
    assert {e["id"] for e in case.snapshot()["entities"]} == {"e_p", "e_m", "e_pr"}

    # bounded reads
    assert "entities" not in case.overview() and "links" not in case.overview()
    assert case.catalog_summary()["total"] == 3
    people = case.page_entities(types=["person"])
    assert [e["id"] for e in people["items"]] == ["e_p"]
    chain = link_engine.chain_of(case, "e_pr")
    assert [s["entity"]["id"] for s in chain["sources"]] == ["e_m"]

    # media + durable jobs both live on the migrated case
    assert media_engine.list_media(case)[0]["filename"] == "shot.jpg"
    case.enqueue_job("thumbnail", key="media/shot.jpg")
    assert case.count_jobs() == {"queued": 1}


def test_a_closed_case_folder_copy_opens_identically(tmp_workspace, monkeypatch):
    """Copying a closed case directory yields a complete, openable case — the
    doc's "case bundle / manual copy is guaranteed when the case is closed". The
    rollback journal (no WAL) means no checkpoint is owed, so a plain folder copy
    carries the whole graph."""
    # Closed means nothing is draining it: the jobs are copied as they were queued.
    monkeypatch.setattr(workqueue, "start_workers", False)
    src = Case.create("Bundle")
    p = src.add_entity("person", "Ada", {"handle": "@ada"}, by="user")
    m = src.add_entity("media", "shot", {"path": "media/x.jpg"}, by="user")
    src.add_link(p["id"], m["id"], "depicts", by="user")
    src.add_folder("People/Kyiv")
    src.enqueue_job("thumbnail", key="media/x.jpg")  # a live job travels too
    original = src.snapshot()

    dest = workspace.config.cases_dir() / "bundle-copy"
    shutil.copytree(src.path, dest)
    copy = Case.open("bundle-copy")

    copied = copy.snapshot()
    assert copied["entities"] == original["entities"]
    assert copied["links"] == original["links"]
    assert copied["folders"] == original["folders"]
    # The thumbnail travels, and so does the link pass the media's filing queued.
    assert sorted(job["kind"] for job in copy.list_jobs(state="queued")) == [
        "propose-links", "thumbnail",
    ]


# -- load: bounded loading on a large migrated case -------------------------


def test_large_migrated_case_opens_through_bounded_queries(tmp_workspace):
    case, summary = build_big_case(
        name="Load", entities=600, links=800, media=150, notes=30, artifacts=40,
        write_media_files=False,
    )
    opened = Case.open(summary.case_id)  # migrate json -> sqlite

    # case-open ships the manifest + folders, never the graph arrays
    overview = opened.overview()
    assert "entities" not in overview and "links" not in overview

    # the catalog is read in bounded pages, not one materialised list
    page = opened.page_entities(limit=100)
    assert len(page["items"]) == 100 and page["next_cursor"] is not None
    walked, cursor = 0, None
    while True:
        pg = opened.page_entities(limit=250, cursor=cursor)
        walked += len(pg["items"])
        cursor = pg["next_cursor"]
        if cursor is None:
            break
    assert walked == summary.entities == opened.catalog_summary()["total"]


# -- packaging: frozen-binary constraints -----------------------------------


def _pyproject() -> dict:
    root = Path(__file__).resolve().parent.parent
    return tomllib.loads((root / "pyproject.toml").read_text(encoding="utf-8"))


def test_wheel_bundles_the_frontend_and_leaves_dev_tooling_out():
    cfg = _pyproject()
    assert "src/azimut/static/**" in cfg["tool"]["hatch"]["build"]["artifacts"]
    assert cfg["tool"]["hatch"]["build"]["targets"]["wheel"]["packages"] == ["src/azimut"]
    # The synthetic fixture lives outside the packaged tree, so hatchling never
    # ships it (doc "dev-only tooling stays out of the artifact").
    root = Path(__file__).resolve().parent.parent
    assert (root / "tests" / "bigcase.py").exists()
    assert not (root / "src" / "azimut" / "bigcase.py").exists()


def test_wheel_ships_every_file_the_extension_runs_on():
    """The extension is distributed *by* the wheel: a pip or binary user has no
    checkout, so Settings and ``/api/ingest/extension.zip`` both serve the copy
    packaged here.

    Those two paths list the source folder (``extinstall.shipped_files``), while
    the wheel is assembled from a hand-written ``force-include`` map — so in a
    checkout a new extension file is picked up and every test passes, and the
    release quietly leaves it out. That is how ``handoff.js`` shipped broken:
    ``background.js`` injects it by name into a folder that did not have it.
    """
    root = Path(__file__).resolve().parent.parent
    included = _pyproject()["tool"]["hatch"]["build"]["targets"]["wheel"]["force-include"]
    packaged = {PurePosixPath(name) for name in included}

    def is_packaged(rel: PurePosixPath) -> bool:
        # a directory entry (icons/) covers everything under it
        return any(rel == entry or entry in rel.parents for entry in packaged)

    source = root / "extension"
    missing = [
        f"extension/{name}"
        for name, _ in extinstall.shipped_files(source)
        if not is_packaged(PurePosixPath("extension") / name)
    ]
    assert not missing, f"the extension runs these but the wheel omits them: {missing}"

    # …and nothing is listed that no longer exists, which would fail the build
    assert not [str(entry) for entry in packaged if not (root / entry).exists()]


def test_pdf_font_licenses_are_complete_and_packaged():
    root = Path(__file__).resolve().parent.parent
    fonts = root / "src" / "azimut" / "assets" / "fonts"
    apache = (fonts / "APACHE-2.0.txt").read_text(encoding="utf-8")
    assert "TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION" in apache
    assert "END OF TERMS AND CONDITIONS" in apache
    assert len(apache.splitlines()) >= 190
    for name in ("OFL.txt", "BITSTREAM-VERA.txt", "README.md"):
        assert (fonts / name).is_file()

    spec = (root / "packaging" / "azimut.spec").read_text(encoding="utf-8")
    assert 'collect_data_files("azimut")' in spec


def test_the_binary_carries_the_scrapers_metadata():
    """`engine/scrapers.py` reads the version a build ships from its metadata, never by
    importing it, to decide whether a copy updated into the workspace is newer.
    collect_submodules ships the modules without it, so the spec copies it by name."""
    from azimut.engine import scrapers

    spec = (Path(__file__).resolve().parent.parent / "packaging" / "azimut.spec").read_text()
    for dist in scrapers.SCRAPERS:
        assert f'copy_metadata("{dist}")' in spec


def test_storage_and_jobs_add_no_new_runtime_dependency():
    """The store, the durable queue and the thumbnail worker stay on the standard
    library plus deps already declared — nothing that would need a new wheel on
    the three release binaries."""
    declared = " ".join(_pyproject()["project"]["dependencies"]).lower()
    # sqlite3/threading/subprocess are stdlib; Pillow (declared) does the imaging.
    assert "pillow" in declared
    src = Path(__file__).resolve().parent.parent / "src" / "azimut"
    # The store is a package plus its entry module, so the scan follows it rather
    # than stopping at the file the class happens to live in.
    files = [src / "sqlite_backend.py", src / "engine" / "thumbnails.py"]
    files += sorted((src / "store").glob("*.py"))
    text = "".join(path.read_text(encoding="utf-8") for path in files)
    assert "import sqlite3" in (src / "sqlite_backend.py").read_text(encoding="utf-8")
    for banned in ("import numpy", "import pandas", "import redis", "import celery"):
        assert banned not in text  # no heavyweight queue/DB dependency slipped in


def test_chromium_cookie_decrypt_dependency_ships_on_linux():
    """yt-dlp needs secretstorage to read a Chromium-family browser's cookies on
    Linux (gated downloads). Nothing in src imports it directly — it's used at
    runtime through yt-dlp — so this guards it against an "unused dep" cleanup,
    and pins it Linux-only + bounded so it never touches the Windows/macOS
    binaries or drifts past an untested major."""
    deps = _pyproject()["project"]["dependencies"]
    secret = next((d for d in deps if d.startswith("secretstorage")), None)
    assert secret is not None, "secretstorage must stay declared for Chromium-on-Linux cookies"
    assert "sys_platform == 'linux'" in secret  # never installed on Windows/macOS
    assert "<4" in secret  # bounded at the next untested major, like every other range

    # the frozen Linux binary must bundle it: yt-dlp imports it lazily, so the
    # PyInstaller spec collects it explicitly (macOS/Windows skip this branch).
    spec = (Path(__file__).resolve().parent.parent / "packaging" / "azimut.spec").read_text()
    assert 'collect_submodules("secretstorage")' in spec
    assert 'sys.platform == "linux"' in spec


def test_case_db_lives_under_the_workspace_root(tmp_workspace):
    case = Case.create("Where")
    assert case.path.is_relative_to(workspace.config.workspace_root())
    assert case.db_path.exists()  # beside the case, never beside the binary
    # thumbnails cache under the case's media dir, also inside the workspace
    assert thumbnails._thumb_dir(case).is_relative_to(workspace.config.workspace_root())


def test_release_tooling_is_bounded_and_built_from_the_lock():
    cfg = _pyproject()
    # Version is single-sourced from src/azimut/__init__.py through hatchling's
    # dynamic version, so pyproject carries no literal copy to drift out of sync
    # (the 0.2.2 release shipped a stale __version__ against a bumped pyproject).
    assert "version" not in cfg["project"]
    assert "version" in cfg["project"]["dynamic"]
    assert cfg["tool"]["hatch"]["version"]["path"] == "src/azimut/__init__.py"
    assert cfg["build-system"]["requires"] == ["hatchling>=1.27,<2"]
    release = cfg["dependency-groups"]["release"]
    assert any(dep.startswith("build>=") and "<2" in dep for dep in release)
    assert any(dep.startswith("hatchling>=") and "<2" in dep for dep in release)
    assert any(dep.startswith("pyinstaller>=") and "<7" in dep for dep in release)

    root = Path(__file__).resolve().parent.parent
    workflow = (root / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8")
    assert "pipx run build" not in workflow
    assert "uv pip install . pyinstaller" not in workflow
    assert "python -m build --no-isolation" in workflow
    assert "--group release" in workflow
    assert workflow.count('node-version: "24.19.0"') == 2
    assert "--no-build-isolation" in workflow
    assert "--no-editable" in workflow
    assert "uv run python" not in workflow
    assert "body_path: docs/RELEASE_NOTES.md" in workflow
    for font_asset in (
        "azimut/assets/fonts/APACHE-2.0.txt",
        "azimut/assets/fonts/OFL.txt",
        "azimut/assets/fonts/BITSTREAM-VERA.txt",
    ):
        assert font_asset in workflow
    # A guard job runs before anything builds: it checks the pushed tag equals
    # __version__ and runs the version-parity gate the release used to skip, so
    # a stale version can never ship again (the 0.2.2 false-update-nag bug).
    assert "tag=$tag  __version__=$ver" in workflow
    assert "pytest tests/test_release_gate.py tests/test_updates.py" in workflow
    assert "\n  dist:\n" in workflow and "needs: guard" in workflow
    release_job = workflow.split("  release:\n", maxsplit=1)[1]
    assert release_job.index("actions/checkout@v7") < release_job.index(
        "softprops/action-gh-release@v3"
    )
    assert (root / "docs" / "RELEASE_NOTES.md").is_file()


def test_current_settings_docs_name_the_current_sections():
    root = Path(__file__).resolve().parent.parent
    current = [
        root / "docs" / "SPEC.md",
        root / "docs" / "UI.md",
        root / "src" / "azimut" / "config.py",
        root / "src" / "azimut" / "api" / "settings.py",
        root / "src" / "azimut" / "engine" / "coords.py",
        root / "src" / "azimut" / "engine" / "diagnostics.py",
        root / "src" / "azimut" / "engine" / "satellite.py",
        root / "frontend" / "src" / "components" / "WorkspaceFolder.svelte",
        root / "frontend" / "src" / "tools" / "Settings.svelte",
    ]
    # Settings is a tab host now, so the copy this reads lives in the tabs. Scan
    # the directory rather than the host, or a stale label walks back in unseen.
    current += sorted((root / "frontend" / "src" / "tools" / "settings").glob("*.svelte"))
    text = "\n".join(path.read_text(encoding="utf-8") for path in current)
    for stale in ("Settings → Preferences", "About tab", "Preferences tab"):
        assert stale not in text


def test_release_binary_runs_the_application_smoke_test():
    root = Path(__file__).resolve().parent.parent
    workflow = (root / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8")
    smoke = (root / "scripts" / "smoke_binary.py").read_text(encoding="utf-8")
    vendor = (root / "scripts" / "vendor_ffmpeg.py").read_text(encoding="utf-8")

    assert "scripts/smoke_binary.py" in workflow
    assert "/api/health" in smoke
    assert "/api/settings/ffmpeg" in smoke
    assert 'ffmpeg.get("source") != "bundled"' in smoke
    assert "scripts/vendor_ffmpeg.py" in workflow
    assert "/redirect/" not in vendor
    assert "/latest/" not in vendor
    assert "SHA-256 mismatch" in vendor
    for target in ("windows-x86_64", "linux-x86_64", "macos-arm64"):
        assert f"ffmpeg_target: {target}" in workflow


def test_the_node_floor_for_the_frontend_build_is_the_one_vite_declares():
    """vite and rolldown refuse to start below their range, so the README and the
    frontend's own `engines` state that range rather than a rounder number."""
    import json

    root = Path(__file__).resolve().parent.parent
    readme = (root / "README.md").read_text(encoding="utf-8")
    package = json.loads((root / "frontend" / "package.json").read_text(encoding="utf-8"))
    lock = json.loads((root / "frontend" / "package-lock.json").read_text(encoding="utf-8"))

    vite = lock["packages"]["node_modules/vite"]["engines"]["node"]
    assert vite == "^20.19.0 || >=22.12.0"
    assert package["engines"]["node"] == vite
    assert "Node.js 20.19+ or 22.12+" in readme


def test_intel_macos_package_support_floor_matches_the_lock():
    root = Path(__file__).resolve().parent.parent
    readme = (root / "README.md").read_text(encoding="utf-8")
    lock = (root / "uv.lock").read_text(encoding="utf-8")
    dependencies = _pyproject()["project"]["dependencies"]
    opencv = lock.split(
        '[[package]]\nname = "opencv-python-headless"',
        maxsplit=1,
    )[1]
    opencv = opencv.split("[[package]]", maxsplit=1)[0]

    assert "macOS (Intel, 14+)" in readme
    assert "macosx_14_0_x86_64.whl" in opencv

    crypto = [dependency for dependency in dependencies if dependency.startswith("cryptography")]
    assert len(crypto) == 2
    assert any(
        "<49" in dependency
        and "sys_platform == 'darwin'" in dependency
        and "platform_machine == 'x86_64'" in dependency
        for dependency in crypto
    )
    assert any(
        "<51" in dependency
        and "sys_platform != 'darwin' or platform_machine != 'x86_64'" in dependency
        for dependency in crypto
    )
    assert "cryptography-48.0.1-cp311-abi3-macosx_10_9_universal2.whl" in lock
    assert "cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl" in lock


def _glibc_floor():
    import importlib.util

    path = Path(__file__).resolve().parent.parent / "scripts" / "check_glibc_floor.py"
    spec = importlib.util.spec_from_file_location("check_glibc_floor", path)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def test_the_linux_binary_states_and_holds_its_glibc_floor():
    """The binary carries the runner's own libstdc++, so the runner sets its floor.

    `ubuntu-latest` would move it without a word; the README says the floor, the
    workflow pins the image that gives it, and a step fails a build that asks more.
    """
    floor = _glibc_floor()
    root = Path(__file__).resolve().parent.parent
    workflow = (root / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8")
    readme = (root / "README.md").read_text(encoding="utf-8")
    stated = ".".join(str(part) for part in floor.FLOOR)

    binaries = workflow.split("  binaries:\n", maxsplit=1)[1].split("\n  artifacts:\n")[0]
    assert "- os: ubuntu-24.04" in binaries and "ubuntu-latest" not in binaries
    assert "scripts/check_glibc_floor.py dist-bin/azimut" in binaries
    assert f"| Linux (glibc {stated}+) |" in readme
    assert f"glibc {stated} or newer" in readme


def test_the_linux_binary_leaves_libmvec_to_the_system():
    """The vendored ffmpeg links glibc's libmvec, which PyInstaller packs by default.

    The runner's copy asks the system's libm for GLIBC_2.39, above the stated floor;
    the floor gate caught it on the 0.3.1 dry run, in the 0.3.0 binary as well.
    """
    root = Path(__file__).resolve().parent.parent
    spec = (root / "packaging" / "azimut.spec").read_text(encoding="utf-8")
    kept = 'a.binaries = [entry for entry in a.binaries if not entry[0].startswith("libmvec.so")]'

    assert kept in spec
    assert spec.index(kept) < spec.index("exe = EXE(")


def test_the_glibc_floor_reads_versions_off_an_elf_image():
    floor = _glibc_floor()

    image = b"\x7fELF\0libc.so.6\0GLIBC_2.17\0GLIBC_2.34\0GLIBCXX_3.4.32\0GLIBC_2.3.4\0"
    assert floor.glibc_versions(image) == {(2, 17), (2, 34), (2, 3, 4)}
    newer = b"\x7fELF\0GLIBC_2.39\0"

    assert floor.over_floor(iter([("libstdc++.so.6", image)])) == []
    assert floor.over_floor(iter([("libstdc++.so.6", newer)])) == [("libstdc++.so.6", (2, 39))]


def test_the_macos_first_run_makes_the_download_executable():
    """A browser download carries no exec bit, and right-click → Open left in macOS 15."""
    readme = (Path(__file__).resolve().parent.parent / "README.md").read_text(encoding="utf-8")
    macos = readme.split("- **macOS**:", maxsplit=1)[1].split("- **Windows**:", maxsplit=1)[0]

    assert "chmod +x azimut-macos-arm64" in macos
    assert "right-click" not in macos