AI powered OSINT engine to discover and map 840+ social media identities across the web
Python aiartificial-intelligencedebainhackhackerhackinghackingtoolsinformationinformation-gatheringlinuxlinux-desktoposintpythonpython-scriptpython3social-mediasocialmediahackingtermuxtermux-tool
feat: frozen response corpus for reproducible evaluation
arxhr007c32dbab95d7d
diff --git a/.gitignore b/.gitignoreindex 7220fa2..8a64542 100644--- a/.gitignore+++ b/.gitignore@@ -10,3 +10,7 @@ venv/results/cache/config.json++# Corpus response bodies hold third-party page content: metadata and labels+# are versioned, raw bodies are shared on request (see paper/corpus/README.md).+paper/corpus/*/bodies/diff --git a/src/aliens_eye/core/fingerprints.py b/src/aliens_eye/core/fingerprints.pyindex 83f452f..179a22c 100644--- a/src/aliens_eye/core/fingerprints.py+++ b/src/aliens_eye/core/fingerprints.py@@ -31,11 +31,25 @@ def build_fingerprint(raw: dict[str, Any]) -> dict[str, str]:class FingerprintStore:- """Persist lightweight fingerprints per site."""-- def __init__(self, path: Path, max_entries_per_label: int = 50) -> None:+ """Persist lightweight fingerprints per site.++ ``read_only`` freezes the store: :meth:`add` and :meth:`save` become no-ops+ while :meth:`score` keeps working. This exists for reproducible evaluation.+ A live store accumulates signatures *during* a scan and scores each response+ against whatever earlier responses happened to finish first, so its+ contribution depends on async worker completion order -- which would make+ results vary run to run even when replaying a frozen corpus.+ """++ def __init__(+ self,+ path: Path,+ max_entries_per_label: int = 50,+ read_only: bool = False,+ ) -> None:self.path = pathself.max_entries_per_label = max_entries_per_label+ self.read_only = read_onlyself.data: dict[str, Any] = {"sites": {}}def load(self, logger=None) -> None:@@ -52,6 +66,8 @@ class FingerprintStore:)def save(self) -> None:+ if self.read_only:+ returnself.path.parent.mkdir(parents=True, exist_ok=True)with self.path.open("w", encoding="utf-8") as handle:json.dump(self.data, handle, indent=2, ensure_ascii=True)@@ -61,7 +77,7 @@ class FingerprintStore:return sites.setdefault(site, {"found": [], "not_found": []})def add(self, site: str, label: str, fingerprint: dict[str, str]) -> None:- if label not in {"found", "not_found"}:+ if self.read_only or label not in {"found", "not_found"}:returnsite_entry = self._ensure_site(site)entries = site_entry[label]diff --git a/src/aliens_eye/core/scanner.py b/src/aliens_eye/core/scanner.pyindex a05fc14..c900a99 100644--- a/src/aliens_eye/core/scanner.py+++ b/src/aliens_eye/core/scanner.py@@ -142,7 +142,12 @@ class UsernameScanner:logger,browser_fallback: BrowserFallback | None = None,checkpoint=None,+ fetch=fetch_url,) -> None:+ # ``fetch`` is the single seam between the scanner and the network.+ # Swapping it for aliens_eye.corpus.replay.ReplayFetcher runs the whole+ # pipeline off a frozen corpus with every downstream stage unchanged.+ self.fetch = fetchself.sites_data = sites_dataself.config = configself.extractor = extractor@@ -286,7 +291,7 @@ class UsernameScanner:rate_limiter: DomainRateLimiter,) -> dict[str, Any]:url = self._format_url(site_name, url_template, username)- fetch = await fetch_url(session, url, self.config, rate_limiter, self.logger)+ fetch = await self.fetch(session, url, self.config, rate_limiter, self.logger)if fetch.error and fetch.status == 408:status_text = "Timeout"diff --git a/src/aliens_eye/corpus/__init__.py b/src/aliens_eye/corpus/__init__.pynew file mode 100644index 0000000..135fce2--- /dev/null+++ b/src/aliens_eye/corpus/__init__.py@@ -0,0 +1,22 @@+"""Frozen HTTP response corpus: record once, replay deterministically.++Evaluating a detector against the live web is not reproducible -- platforms+change their markup, rate-limit differently on different days, and block some+networks entirely. A measured change in accuracy cannot be attributed to the+detector rather than to the web.++This package removes the network from the measurement loop:++* :mod:`aliens_eye.corpus.record` captures raw responses to disk.+* :mod:`aliens_eye.corpus.replay` serves them back through the *same*+ ``fetch_url`` interface the scanner already uses.++Because replay substitutes only the fetch, every downstream stage --+``core.analyzer`` feature extraction, ``core.detector`` scoring, ``core.features``+vectorisation -- runs byte-identical code on live and replayed responses. An+evaluation therefore measures the detector, not the harness.+"""++from .store import CorpusError, CorpusRecord, CorpusStore++__all__ = ["CorpusError", "CorpusRecord", "CorpusStore"]diff --git a/src/aliens_eye/corpus/record.py b/src/aliens_eye/corpus/record.pynew file mode 100644index 0000000..aa82821--- /dev/null+++ b/src/aliens_eye/corpus/record.py@@ -0,0 +1,235 @@+"""Capture live responses into a frozen corpus.++The recorder wraps :func:`aliens_eye.core.http.fetch_url` rather than reimplementing+it, so captured responses go through the same retry, backoff, rate-limit and proxy+behaviour as a normal scan. Whatever the scanner would have seen is what lands on+disk.+"""++from __future__ import annotations++import asyncio+import random+import string+from datetime import datetime, timezone+from pathlib import Path+from typing import Any++import aiohttp++from aliens_eye import __version__+from aliens_eye.core.config import DEFAULT_HEADERS, ScannerConfig+from aliens_eye.core.http import FetchResult, fetch_url+from aliens_eye.core.rate_limit import DomainRateLimiter+from aliens_eye.core.scanner import build_connector, format_site_url++from .store import CorpusRecord, CorpusStore++# Ordinary-looking handle morphology, for negatives that are not trivially+# distinguishable from real usernames. A negative pool of random 16-character+# strings is an artificially easy class: a detector can separate it on length and+# character distribution alone, which inflates every metric.+_ADJECTIVES = [+ "quiet", "amber", "rapid", "hollow", "silver", "brave", "narrow", "lunar",+ "cobalt", "gentle", "rustic", "vivid", "solar", "hidden", "copper", "swift",+]+_NOUNS = [+ "falcon", "harbor", "meadow", "lantern", "cipher", "willow", "canyon", "ember",+ "pixel", "otter", "vector", "thicket", "beacon", "marlin", "quartz", "atlas",+]+++def random_negative(rng: random.Random) -> str:+ """A high-entropy handle that is very unlikely to exist anywhere."""+ length = rng.randint(14, 20)+ return "".join(rng.choices(string.ascii_lowercase + string.digits, k=length))+++def plausible_negative(rng: random.Random) -> str:+ """A handle with realistic morphology, which still probably does not exist.++ Plausible negatives are *not* verified absent by construction -- on large+ platforms one may collide with a real account. Records tag their negative+ kind so a later labelling pass can check them; see WORKING.md.+ """+ word = f"{rng.choice(_ADJECTIVES)}{rng.choice(_NOUNS)}"+ style = rng.randint(0, 3)+ if style == 0:+ return word+ if style == 1:+ return f"{word}{rng.randint(1, 9999)}"+ if style == 2:+ return f"{word}_{rng.randint(10, 99)}"+ return f"{word[: len(word) // 2]}_{word[len(word) // 2 :]}"+++class RecordingFetcher:+ """A ``fetch_url``-compatible callable that persists what it fetches.++ Context (site / username / label) cannot be recovered from the URL alone, so+ the driver registers it up front with :meth:`register`. That keeps the call+ signature identical to ``fetch_url``, which is what makes the recorder+ droppable into the scanner without touching the scan path.+ """++ def __init__(self, store: CorpusStore, tool_version: str = __version__) -> None:+ self.store = store+ self.tool_version = tool_version+ self._context: dict[str, dict[str, Any]] = {}+ self._records: list[CorpusRecord] = []+ self._lock = asyncio.Lock()++ def register(+ self,+ url: str,+ site: str,+ username: str,+ label: int | None = None,+ notes: dict[str, Any] | None = None,+ ) -> None:+ self._context[url] = {+ "site": site,+ "username": username,+ "label": label,+ "notes": notes or {},+ }++ @property+ def records(self) -> list[CorpusRecord]:+ return list(self._records)++ async def __call__(+ self,+ session: aiohttp.ClientSession,+ url: str,+ config: ScannerConfig,+ rate_limiter: DomainRateLimiter,+ logger,+ ) -> FetchResult:+ result = await fetch_url(session, url, config, rate_limiter, logger)+ context = self._context.get(url, {})+ async with self._lock:+ digest = self.store.write_body(result.content)+ self._records.append(+ CorpusRecord(+ url=url,+ site=str(context.get("site", "")),+ username=str(context.get("username", "")),+ label=context.get("label"),+ final_url=result.final_url,+ status=result.status,+ headers=dict(result.headers),+ body_sha256=digest,+ body_chars=len(result.content),+ response_time=result.response_time,+ redirect_count=result.redirect_count,+ error=result.error,+ captured_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),+ tool_version=self.tool_version,+ notes=dict(context.get("notes") or {}),+ )+ )+ return result++ def flush(self) -> int:+ """Append buffered records to disk in a stable order. Returns the count."""+ records = sorted(self._records, key=lambda r: (r.site, r.label or 0, r.username))+ self.store.append_records(records)+ count = len(records)+ self._records.clear()+ return count+++def build_jobs(+ sites_data: dict[str, str],+ ground_truth: dict[str, list[str]],+ negatives_per_site: int,+ rng: random.Random,+ plausible_ratio: float = 0.5,+) -> list[tuple[str, str, str, int, dict[str, Any]]]:+ """Expand ground truth into ``(site, url, username, label, notes)`` jobs."""+ jobs: list[tuple[str, str, str, int, dict[str, Any]]] = []+ for site in sorted(ground_truth):+ template = sites_data.get(site)+ if not template:+ continue+ for username in ground_truth[site]:+ jobs.append(+ (site, format_site_url(site, template, username), username, 1,+ {"positive_kind": "ground_truth"})+ )+ n_plausible = int(round(negatives_per_site * plausible_ratio))+ for index in range(negatives_per_site):+ if index < n_plausible:+ username = plausible_negative(rng)+ kind = "plausible"+ else:+ username = random_negative(rng)+ kind = "random"+ jobs.append(+ (site, format_site_url(site, template, username), username, 0,+ {"negative_kind": kind})+ )+ return jobs+++async def record_corpus(+ sites_data: dict[str, str],+ ground_truth: dict[str, list[str]],+ out_dir: Path,+ logger,+ negatives_per_site: int = 4,+ concurrency: int = 20,+ seed: int = 1234,+ split: str = "all",+ plausible_ratio: float = 0.5,+ config: ScannerConfig | None = None,+) -> dict[str, Any]:+ """Capture every ground-truth positive plus generated negatives to ``out_dir``."""+ store = CorpusStore(Path(out_dir))+ store.init()+ rng = random.Random(seed)+ config = config or ScannerConfig(retries=2, timeout=15.0)+ rate_limiter = DomainRateLimiter()+ recorder = RecordingFetcher(store)++ jobs = build_jobs(sites_data, ground_truth, negatives_per_site, rng, plausible_ratio)+ skipped = sorted(set(ground_truth) - set(sites_data))+ for site, url, username, label, notes in jobs:+ recorder.register(url, site, username, label, notes)++ conn_limit = max(1, min(concurrency, len(jobs) or 1))+ semaphore = asyncio.Semaphore(conn_limit)+ connector = build_connector(config, conn_limit)++ async with aiohttp.ClientSession(headers=DEFAULT_HEADERS, connector=connector) as session:++ async def run(url: str) -> None:+ async with semaphore:+ try:+ await recorder(session, url, config, rate_limiter, logger)+ except Exception as exc: # noqa: BLE001 - one bad host must not abort a capture+ logger.debug("Capture failed for %s: %s", url, exc)++ await asyncio.gather(*(run(job[1]) for job in jobs))++ captured = recorder.records+ errors = sum(1 for r in captured if r.error)+ written = recorder.flush()++ manifest = {+ "tool_version": __version__,+ "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),+ "split": split,+ "seed": seed,+ "negatives_per_site": negatives_per_site,+ "plausible_ratio": plausible_ratio,+ "requested": len(jobs),+ "records": written,+ "errors": errors,+ "sites": sorted({r.site for r in captured if r.site}),+ "skipped_sites": skipped,+ }+ store.write_manifest(manifest)+ logger.info("Captured %d records (%d errors) to %s", written, errors, out_dir)+ return manifestdiff --git a/src/aliens_eye/corpus/replay.py b/src/aliens_eye/corpus/replay.pynew file mode 100644index 0000000..dec84ab--- /dev/null+++ b/src/aliens_eye/corpus/replay.py@@ -0,0 +1,140 @@+"""Serve a frozen corpus through the ``fetch_url`` interface.++Replay makes an evaluation reproducible: the same corpus yields the same report,+today and in a year, whatever the platforms have since done to their markup.++Determinism requires more than returning stored bytes:++* No network, no rate limiting, no retries -- the rate limiter is not consulted,+ so replay does not sleep and wall-clock timing cannot leak into results.+* ``response_time`` is the recorded value, not a fresh measurement, so the+ ``response_time`` feature is stable across runs.+* Fingerprints must be neutralised by the caller. ``core.fingerprints.FingerprintStore``+ accumulates signatures *during* a scan and scores against what it has seen so+ far, which makes its contribution depend on worker completion order. Pass a+ read-only store (see :class:`aliens_eye.core.fingerprints.FingerprintStore`) or+ results will vary run to run even off a frozen corpus.+"""++from __future__ import annotations++from pathlib import Path+from typing import Any++import aiohttp++from aliens_eye.core.config import ScannerConfig+from aliens_eye.core.http import FetchResult+from aliens_eye.core.rate_limit import DomainRateLimiter++from .store import CorpusError, CorpusRecord, CorpusStore++MISSING_ERROR = "not in corpus"+++class ReplayFetcher:+ """A ``fetch_url``-compatible callable backed by a recorded corpus.++ ``strict=True`` raises on a URL the corpus does not contain, which is what an+ evaluation wants: a silently-missing response would be scored as a network+ error and quietly bias the metrics. ``strict=False`` returns an error+ ``FetchResult`` instead, matching how a real failed fetch behaves.+ """++ def __init__(self, root: Path, strict: bool = True, cache_bodies: bool = True) -> None:+ self.store = CorpusStore(Path(root))+ self.manifest = self.store.read_manifest()+ self.strict = strict+ self.cache_bodies = cache_bodies+ self.records: dict[str, CorpusRecord] = self.store.index_by_url()+ self._bodies: dict[str, str] = {}+ self.hits = 0+ self.misses: list[str] = []++ # -- corpus introspection --------------------------------------------++ def __len__(self) -> int:+ return len(self.records)++ @property+ def sites(self) -> list[str]:+ return sorted({r.site for r in self.records.values() if r.site})++ def labels_by_url(self) -> dict[str, int | None]:+ return {url: record.label for url, record in self.records.items()}++ def eval_jobs(self, sites: set[str] | None = None) -> list[tuple[str, str, str, int]]:+ """The labelled evaluation set held by this corpus.++ Returns ``(site, url, username, label)`` for every labelled record, in a+ stable order. Callers must evaluate *these* rows rather than regenerating+ usernames: freshly generated negatives were never captured, so they would+ all miss the corpus and leave the negative class empty -- yielding a+ false-positive rate computed over nothing.+ """+ jobs = [+ (r.site, r.url, r.username, int(r.label))+ for r in self.records.values()+ if r.label is not None and (sites is None or r.site in sites)+ ]+ return sorted(jobs, key=lambda j: (j[0], j[3], j[2]))++ def stats(self) -> dict[str, Any]:+ records = list(self.records.values())+ positives = sum(1 for r in records if r.label == 1)+ negatives = sum(1 for r in records if r.label == 0)+ errors = sum(1 for r in records if r.error)+ return {+ "records": len(records),+ "sites": len(self.sites),+ "positives": positives,+ "negatives": negatives,+ "unlabeled": len(records) - positives - negatives,+ "capture_errors": errors,+ "distinct_bodies": len({r.body_sha256 for r in records if r.body_sha256}),+ "tool_version": self.manifest.get("tool_version"),+ "created_at": self.manifest.get("created_at"),+ }++ # -- fetch ------------------------------------------------------------++ def _body(self, record: CorpusRecord) -> str:+ if not self.cache_bodies:+ return self.store.read_body(record.body_sha256)+ if record.body_sha256 not in self._bodies:+ self._bodies[record.body_sha256] = self.store.read_body(record.body_sha256)+ return self._bodies[record.body_sha256]++ async def __call__(+ self,+ session: aiohttp.ClientSession | None,+ url: str,+ config: ScannerConfig,+ rate_limiter: DomainRateLimiter,+ logger,+ ) -> FetchResult:+ record = self.records.get(url)+ if record is None:+ self.misses.append(url)+ if self.strict:+ raise CorpusError(+ f"{url} is not in the corpus at {self.store.root}. "+ "Re-record, or pass strict=False to score it as a fetch error."+ )+ logger.debug("Corpus miss for %s", url)+ return FetchResult(+ url=url, final_url=url, status=0, content="", response_time=0.0,+ headers={}, error=MISSING_ERROR, redirect_count=0,+ )++ self.hits += 1+ return FetchResult(+ url=record.url,+ final_url=record.final_url,+ status=record.status,+ content="" if record.error else self._body(record),+ response_time=record.response_time,+ headers=dict(record.headers),+ error=record.error,+ redirect_count=record.redirect_count,+ )diff --git a/src/aliens_eye/corpus/store.py b/src/aliens_eye/corpus/store.pynew file mode 100644index 0000000..ea056ad--- /dev/null+++ b/src/aliens_eye/corpus/store.py@@ -0,0 +1,189 @@+"""On-disk layout for a frozen response corpus.++::++ <corpus>/+ manifest.json metadata: tool version, counts, capture run+ records.jsonl one JSON object per captured request+ bodies/<aa>/<sha256>.gz gzipped response bodies, content-addressed++Bodies are content-addressed because error pages repeat heavily: a platform+serves one "no such user" page for every non-existent username, so a corpus of+N negatives per site stores that body once rather than N times.+"""++from __future__ import annotations++import gzip+import hashlib+import json+from collections.abc import Iterator+from dataclasses import asdict, dataclass, field+from pathlib import Path+from typing import Any++CORPUS_VERSION = 1+MANIFEST_NAME = "manifest.json"+RECORDS_NAME = "records.jsonl"+BODIES_DIR = "bodies"+++class CorpusError(RuntimeError):+ """Raised when a corpus is missing, malformed, or version-mismatched."""+++@dataclass+class CorpusRecord:+ """One captured request/response pair.++ ``url`` is the lookup key: it is derived deterministically from+ ``(site, template, username)`` by ``core.scanner.format_site_url``, and it is+ the only identifier ``fetch_url`` receives, so replay must key on it.+ """++ url: str+ site: str+ username: str+ label: int | None+ final_url: str+ status: int+ headers: dict[str, str]+ body_sha256: str+ body_chars: int+ response_time: float+ redirect_count: int+ error: str | None+ captured_at: str+ tool_version: str+ notes: dict[str, Any] = field(default_factory=dict)++ def to_json(self) -> str:+ return json.dumps(asdict(self), ensure_ascii=False, sort_keys=True)++ @classmethod+ def from_dict(cls, data: dict[str, Any]) -> CorpusRecord:+ try:+ return cls(+ url=str(data["url"]),+ site=str(data["site"]),+ username=str(data["username"]),+ label=None if data.get("label") is None else int(data["label"]),+ final_url=str(data.get("final_url") or data["url"]),+ status=int(data.get("status", 0)),+ headers={str(k): str(v) for k, v in (data.get("headers") or {}).items()},+ body_sha256=str(data.get("body_sha256", "")),+ body_chars=int(data.get("body_chars", 0)),+ response_time=float(data.get("response_time", 0.0)),+ redirect_count=int(data.get("redirect_count", 0)),+ error=data.get("error"),+ captured_at=str(data.get("captured_at", "")),+ tool_version=str(data.get("tool_version", "unknown")),+ notes=dict(data.get("notes") or {}),+ )+ except (KeyError, TypeError, ValueError) as exc:+ raise CorpusError(f"Malformed corpus record: {exc}") from exc+++def body_digest(content: str) -> str:+ """Content address for a response body. An empty body has no digest."""+ if not content:+ return ""+ return hashlib.sha256(content.encode("utf-8")).hexdigest()+++class CorpusStore:+ """Read/write access to a corpus directory."""++ def __init__(self, root: Path) -> None:+ self.root = Path(root)++ # -- paths ------------------------------------------------------------++ @property+ def manifest_path(self) -> Path:+ return self.root / MANIFEST_NAME++ @property+ def records_path(self) -> Path:+ return self.root / RECORDS_NAME++ def body_path(self, digest: str) -> Path:+ return self.root / BODIES_DIR / digest[:2] / f"{digest}.gz"++ # -- writing ----------------------------------------------------------++ def init(self) -> None:+ (self.root / BODIES_DIR).mkdir(parents=True, exist_ok=True)++ def write_body(self, content: str) -> str:+ """Store a body and return its digest. Idempotent for repeated bodies."""+ digest = body_digest(content)+ if not digest:+ return ""+ path = self.body_path(digest)+ if path.exists():+ return digest+ path.parent.mkdir(parents=True, exist_ok=True)+ # mtime=0 so identical bodies produce identical bytes on disk, which+ # keeps a corpus directory itself checksummable across capture runs.+ with path.open("wb") as raw:+ with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as handle:+ handle.write(content.encode("utf-8"))+ return digest++ def append_records(self, records: list[CorpusRecord]) -> None:+ self.records_path.parent.mkdir(parents=True, exist_ok=True)+ with self.records_path.open("a", encoding="utf-8", newline="\n") as handle:+ for record in records:+ handle.write(record.to_json() + "\n")++ def write_manifest(self, manifest: dict[str, Any]) -> None:+ payload = {"corpus_version": CORPUS_VERSION, **manifest}+ self.manifest_path.write_text(+ json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"+ )++ # -- reading ----------------------------------------------------------++ def read_manifest(self) -> dict[str, Any]:+ if not self.manifest_path.exists():+ raise CorpusError(f"No corpus manifest at {self.manifest_path}")+ try:+ manifest = json.loads(self.manifest_path.read_text("utf-8"))+ except json.JSONDecodeError as exc:+ raise CorpusError(f"Corpus manifest is not valid JSON: {exc}") from exc+ version = manifest.get("corpus_version")+ if version != CORPUS_VERSION:+ raise CorpusError(+ f"Corpus version {version!r} != supported {CORPUS_VERSION}; re-record."+ )+ return manifest++ def iter_records(self) -> Iterator[CorpusRecord]:+ if not self.records_path.exists():+ raise CorpusError(f"No corpus records at {self.records_path}")+ with self.records_path.open("r", encoding="utf-8") as handle:+ for line_no, line in enumerate(handle, 1):+ stripped = line.strip()+ if not stripped:+ continue+ try:+ data = json.loads(stripped)+ except json.JSONDecodeError as exc:+ raise CorpusError(+ f"{self.records_path}:{line_no} is not valid JSON: {exc}"+ ) from exc+ yield CorpusRecord.from_dict(data)++ def read_body(self, digest: str) -> str:+ if not digest:+ return ""+ path = self.body_path(digest)+ if not path.exists():+ raise CorpusError(f"Corpus body {digest} missing at {path}")+ with gzip.open(path, "rb") as handle:+ return handle.read().decode("utf-8")++ def index_by_url(self) -> dict[str, CorpusRecord]:+ """Map url -> record. A later capture of the same url wins."""+ return {record.url: record for record in self.iter_records()}diff --git a/tests/test_corpus.py b/tests/test_corpus.pynew file mode 100644index 0000000..7b01681--- /dev/null+++ b/tests/test_corpus.py@@ -0,0 +1,390 @@+"""Frozen-corpus record/replay: round-trip, determinism, and the guards that+keep a replayed evaluation from silently drifting."""++import asyncio+import json++import pytest++from aliens_eye.core.config import ScannerConfig+from aliens_eye.core.detector import Detector+from aliens_eye.core.fingerprints import FingerprintStore, build_fingerprint+from aliens_eye.core.http import FetchResult+from aliens_eye.corpus.record import (+ RecordingFetcher,+ build_jobs,+ plausible_negative,+ random_negative,+)+from aliens_eye.corpus.replay import MISSING_ERROR, ReplayFetcher+from aliens_eye.corpus.store import CorpusError, CorpusStore, body_digest++SITES = {+ "github": "https://github.com/{}",+ "reddit": "https://www.reddit.com/user/{}",+}+++class FakeNetwork:+ """Stands in for fetch_url so tests never touch the network."""++ def __init__(self, responses: dict[str, tuple[int, str]]) -> None:+ self.responses = responses+ self.calls: list[str] = []++ async def __call__(self, session, url, config, rate_limiter, logger):+ self.calls.append(url)+ status, body = self.responses.get(url, (404, "<html>no such user</html>"))+ return FetchResult(+ url=url, final_url=url, status=status, content=body,+ response_time=0.125, headers={"Server": "test"}, error=None,+ redirect_count=0,+ )+++def make_corpus(tmp_path, monkeypatch, responses, jobs):+ """Record `jobs` against a fake network and return the corpus root."""+ from aliens_eye.corpus import record as record_mod++ network = FakeNetwork(responses)+ monkeypatch.setattr(record_mod, "fetch_url", network)++ root = tmp_path / "corpus"+ store = CorpusStore(root)+ store.init()+ recorder = RecordingFetcher(store, tool_version="test")+ for site, url, username, label in jobs:+ recorder.register(url, site, username, label)++ async def drive():+ for _, url, _, _ in jobs:+ await recorder(None, url, ScannerConfig(), None, _NullLogger())++ asyncio.run(drive())+ written = recorder.flush()+ store.write_manifest({"tool_version": "test", "records": written})+ return root, network+++class _NullLogger:+ def debug(self, *a, **k):+ pass++ def info(self, *a, **k):+ pass++ def warning(self, *a, **k):+ pass+++# --- round trip -----------------------------------------------------------+++def test_record_then_replay_round_trips(tmp_path, monkeypatch, found_html):+ url = "https://github.com/torvalds"+ root, _ = make_corpus(+ tmp_path, monkeypatch, {url: (200, found_html)},+ [("github", url, "torvalds", 1)],+ )++ replay = ReplayFetcher(root)+ result = asyncio.run(replay(None, url, ScannerConfig(), None, _NullLogger()))+ assert result.status == 200+ assert result.content == found_html+ assert result.response_time == 0.125+ assert result.headers["Server"] == "test"+ assert result.error is None+++def test_replay_makes_no_network_calls(tmp_path, monkeypatch, found_html):+ url = "https://github.com/torvalds"+ root, network = make_corpus(+ tmp_path, monkeypatch, {url: (200, found_html)},+ [("github", url, "torvalds", 1)],+ )+ before = len(network.calls)+ replay = ReplayFetcher(root)+ asyncio.run(replay(None, url, ScannerConfig(), None, _NullLogger()))+ assert len(network.calls) == before+++# --- determinism ----------------------------------------------------------+++def test_replay_is_deterministic_across_runs_and_orders(+ tmp_path, monkeypatch, found_html, not_found_html+):+ """Same corpus -> identical detections, regardless of visit order.++ This is the property the whole corpus exists for: a metric change must be+ attributable to the detector, never to which worker finished first.+ """+ urls = {+ "https://github.com/torvalds": (200, found_html),+ "https://github.com/nosuchuser99": (404, not_found_html),+ "https://www.reddit.com/user/spez": (200, found_html),+ }+ jobs = [+ ("github", "https://github.com/torvalds", "torvalds", 1),+ ("github", "https://github.com/nosuchuser99", "nosuchuser99", 0),+ ("reddit", "https://www.reddit.com/user/spez", "spez", 1),+ ]+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs)++ def score(order):+ from aliens_eye.core.analyzer import FeatureExtractor++ replay = ReplayFetcher(root)+ extractor = FeatureExtractor()+ detector = Detector()+ # Read-only, empty store: no cross-response accumulation, so scoring+ # cannot depend on completion order.+ fingerprints = FingerprintStore(tmp_path / "fp.json", read_only=True)+ out = {}+ for site, url, username, _ in order:+ fetch = asyncio.run(replay(None, url, ScannerConfig(), None, _NullLogger()))+ bundle = extractor.extract(+ fetch.content, fetch.final_url, username, site,+ fetch.status, fetch.response_time, fetch.headers, fetch.redirect_count,+ )+ fp = build_fingerprint(bundle.fingerprint)+ scored = fingerprints.score(site, fp)+ bundle.features["fingerprint_match_found"] = float(scored["match_found"])+ bundle.features["fingerprint_match_not_found"] = float(scored["match_not_found"])+ fingerprints.add(site, "found", fp)+ detection = detector.predict(bundle.features)+ out[url] = (detection.status, detection.confidence, detection.probability)+ return out++ first = score(jobs)+ assert first == score(jobs), "same order must reproduce"+ assert first == score(list(reversed(jobs))), "reversed order must reproduce"+++def test_readonly_fingerprint_store_does_not_accumulate(tmp_path, found_html):+ store = FingerprintStore(tmp_path / "fp.json", read_only=True)+ fp = {"title_hash": "a", "meta_hash": "b", "dom_signature": "c", "server": "d"}+ store.add("github", "found", fp)+ assert store.score("github", fp)["match_found"] == 0+ store.save()+ assert not (tmp_path / "fp.json").exists()+++def test_writable_fingerprint_store_still_accumulates(tmp_path):+ store = FingerprintStore(tmp_path / "fp.json")+ fp = {"title_hash": "a", "meta_hash": "b", "dom_signature": "c", "server": "d"}+ store.add("github", "found", fp)+ assert store.score("github", fp)["match_found"] > 0+++# --- storage --------------------------------------------------------------+++def test_identical_bodies_are_stored_once(tmp_path, monkeypatch, not_found_html):+ """Error pages repeat across every negative; the corpus must not duplicate them."""+ urls = {+ f"https://github.com/ghost{i}": (404, not_found_html) for i in range(5)+ }+ jobs = [("github", url, f"ghost{i}", 0) for i, url in enumerate(urls)]+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs)++ bodies = list((root / "bodies").rglob("*.gz"))+ assert len(bodies) == 1+ assert ReplayFetcher(root).stats()["records"] == 5+++def test_body_digest_is_content_addressed():+ assert body_digest("abc") == body_digest("abc")+ assert body_digest("abc") != body_digest("abd")+ assert body_digest("") == ""+++def test_records_are_written_in_stable_order(tmp_path, monkeypatch, found_html):+ urls = {f"https://github.com/u{i}": (200, found_html + str(i)) for i in range(4)}+ jobs = [("github", f"https://github.com/u{i}", f"u{i}", i % 2) for i in range(4)]+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs)+ lines = (root / "records.jsonl").read_text("utf-8").strip().splitlines()+ keys = [(json.loads(x)["site"], json.loads(x)["label"], json.loads(x)["username"]) for x in lines]+ assert keys == sorted(keys)+++# --- guards ---------------------------------------------------------------+++def test_strict_replay_raises_on_missing_url(tmp_path, monkeypatch, found_html):+ url = "https://github.com/torvalds"+ root, _ = make_corpus(+ tmp_path, monkeypatch, {url: (200, found_html)},+ [("github", url, "torvalds", 1)],+ )+ replay = ReplayFetcher(root, strict=True)+ with pytest.raises(CorpusError, match="not in the corpus"):+ asyncio.run(replay(None, "https://github.com/absent", ScannerConfig(), None, _NullLogger()))+++def test_lenient_replay_reports_miss_as_fetch_error(tmp_path, monkeypatch, found_html):+ url = "https://github.com/torvalds"+ root, _ = make_corpus(+ tmp_path, monkeypatch, {url: (200, found_html)},+ [("github", url, "torvalds", 1)],+ )+ replay = ReplayFetcher(root, strict=False)+ result = asyncio.run(+ replay(None, "https://github.com/absent", ScannerConfig(), None, _NullLogger())+ )+ assert result.error == MISSING_ERROR+ assert replay.misses == ["https://github.com/absent"]+++def test_missing_corpus_raises(tmp_path):+ with pytest.raises(CorpusError, match="No corpus manifest"):+ ReplayFetcher(tmp_path / "nope")+++def test_version_mismatch_raises(tmp_path, monkeypatch, found_html):+ url = "https://github.com/torvalds"+ root, _ = make_corpus(+ tmp_path, monkeypatch, {url: (200, found_html)},+ [("github", url, "torvalds", 1)],+ )+ manifest = json.loads((root / "manifest.json").read_text("utf-8"))+ manifest["corpus_version"] = 999+ (root / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")+ with pytest.raises(CorpusError, match="re-record"):+ ReplayFetcher(root)+++# --- job construction -----------------------------------------------------+++def test_build_jobs_covers_positives_and_negatives():+ import random++ ground_truth = {"github": ["torvalds", "mojombo"], "reddit": ["spez"]}+ jobs = build_jobs(SITES, ground_truth, negatives_per_site=4, rng=random.Random(1))+ positives = [j for j in jobs if j[3] == 1]+ negatives = [j for j in jobs if j[3] == 0]+ assert len(positives) == 3+ assert len(negatives) == 8+ assert {j[0] for j in jobs} == {"github", "reddit"}+++def test_build_jobs_skips_sites_absent_from_sites_json():+ import random++ jobs = build_jobs(SITES, {"nowhere": ["someone"]}, 2, random.Random(1))+ assert jobs == []+++def test_build_jobs_is_seed_reproducible():+ import random++ gt = {"github": ["torvalds"]}+ a = build_jobs(SITES, gt, 4, random.Random(7))+ b = build_jobs(SITES, gt, 4, random.Random(7))+ assert a == b+++def test_negative_pools_have_different_morphology():+ """Random negatives are separable on shape alone; plausible ones are not.++ A negative class of 16-char random strings is an artificially easy problem+ and inflates every metric, which is why the corpus mixes in handles built+ from ordinary word morphology.+ """+ import random++ rng = random.Random(3)+ randoms = [random_negative(rng) for _ in range(50)]+ plausibles = [plausible_negative(rng) for _ in range(50)]+ assert all(len(u) >= 14 for u in randoms)+ assert min(len(u) for u in plausibles) < 14+ # Random strings are ~uniform over the alphabet; word-built handles are not.+ assert sum(c.isdigit() for u in randoms for c in u) > sum(+ c.isdigit() for u in plausibles for c in u+ )+++def test_plausible_negatives_vary():+ import random++ rng = random.Random(11)+ assert len({plausible_negative(rng) for _ in range(40)}) > 20+++# --- corpus as the evaluation set -----------------------------------------+++def test_eval_jobs_returns_labelled_rows(tmp_path, monkeypatch, found_html, not_found_html):+ urls = {+ "https://github.com/torvalds": (200, found_html),+ "https://github.com/ghost1": (404, not_found_html),+ "https://www.reddit.com/user/spez": (200, found_html),+ }+ jobs_in = [+ ("github", "https://github.com/torvalds", "torvalds", 1),+ ("github", "https://github.com/ghost1", "ghost1", 0),+ ("reddit", "https://www.reddit.com/user/spez", "spez", 1),+ ]+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs_in)++ jobs = ReplayFetcher(root).eval_jobs()+ assert sorted(jobs) == sorted(jobs_in)+ assert jobs == sorted(jobs, key=lambda j: (j[0], j[3], j[2])), "must be stably ordered"+++def test_eval_jobs_filters_by_site(tmp_path, monkeypatch, found_html):+ urls = {+ "https://github.com/torvalds": (200, found_html),+ "https://www.reddit.com/user/spez": (200, found_html),+ }+ jobs_in = [+ ("github", "https://github.com/torvalds", "torvalds", 1),+ ("reddit", "https://www.reddit.com/user/spez", "spez", 1),+ ]+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs_in)++ jobs = ReplayFetcher(root).eval_jobs({"github"})+ assert [j[0] for j in jobs] == ["github"]+++def test_eval_jobs_hit_the_corpus_exactly(tmp_path, monkeypatch, found_html, not_found_html):+ """Regression: the eval set must come from the corpus, not be regenerated.++ Regenerating negatives at eval time yields URLs the corpus never captured,+ so every negative misses and the negative class is empty -- which silently+ reports a false-positive rate computed over nothing.+ """+ urls = {+ "https://github.com/torvalds": (200, found_html),+ "https://github.com/ghost1": (404, not_found_html),+ "https://github.com/ghost2": (404, not_found_html),+ }+ jobs_in = [+ ("github", "https://github.com/torvalds", "torvalds", 1),+ ("github", "https://github.com/ghost1", "ghost1", 0),+ ("github", "https://github.com/ghost2", "ghost2", 0),+ ]+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs_in)++ replay = ReplayFetcher(root, strict=True)+ jobs = replay.eval_jobs()++ async def drive():+ for _, url, _, _ in jobs:+ await replay(None, url, ScannerConfig(), None, _NullLogger())++ asyncio.run(drive())+ assert replay.misses == []+ assert replay.hits == len(jobs)+ # The negative class is non-empty, so FPR is computed over something.+ assert sum(1 for j in jobs if j[3] == 0) == 2+++def test_eval_jobs_skips_unlabelled_records(tmp_path, monkeypatch, found_html):+ url = "https://github.com/torvalds"+ root, _ = make_corpus(+ tmp_path, monkeypatch, {url: (200, found_html)},+ [("github", url, "torvalds", None)],+ )+ assert ReplayFetcher(root).eval_jobs() == []