Skip to content

AI powered OSINT engine to discover and map 840+ social media identities across the web

feat: frozen response corpus for reproducible evaluation

arxhr007c32dbab95d7d
diff --git a/.gitignore b/.gitignore
index 7220fa2..8a64542 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,7 @@ venv/
results/
cache/
config.json
+
+# Corpus response bodies hold third-party page content: metadata and labels
+# are versioned, raw bodies are shared on request (see paper/corpus/README.md).
+paper/corpus/*/bodies/
diff --git a/src/aliens_eye/core/fingerprints.py b/src/aliens_eye/core/fingerprints.py
index 83f452f..179a22c 100644
--- a/src/aliens_eye/core/fingerprints.py
+++ b/src/aliens_eye/core/fingerprints.py
@@ -31,11 +31,25 @@ def build_fingerprint(raw: dict[str, Any]) -> dict[str, str]:
class FingerprintStore:
- """Persist lightweight fingerprints per site."""
-
- def __init__(self, path: Path, max_entries_per_label: int = 50) -> None:
+ """Persist lightweight fingerprints per site.
+
+ ``read_only`` freezes the store: :meth:`add` and :meth:`save` become no-ops
+ while :meth:`score` keeps working. This exists for reproducible evaluation.
+ A live store accumulates signatures *during* a scan and scores each response
+ against whatever earlier responses happened to finish first, so its
+ contribution depends on async worker completion order -- which would make
+ results vary run to run even when replaying a frozen corpus.
+ """
+
+ def __init__(
+ self,
+ path: Path,
+ max_entries_per_label: int = 50,
+ read_only: bool = False,
+ ) -> None:
self.path = path
self.max_entries_per_label = max_entries_per_label
+ self.read_only = read_only
self.data: dict[str, Any] = {"sites": {}}
def load(self, logger=None) -> None:
@@ -52,6 +66,8 @@ class FingerprintStore:
)
def save(self) -> None:
+ if self.read_only:
+ return
self.path.parent.mkdir(parents=True, exist_ok=True)
with self.path.open("w", encoding="utf-8") as handle:
json.dump(self.data, handle, indent=2, ensure_ascii=True)
@@ -61,7 +77,7 @@ class FingerprintStore:
return sites.setdefault(site, {"found": [], "not_found": []})
def add(self, site: str, label: str, fingerprint: dict[str, str]) -> None:
- if label not in {"found", "not_found"}:
+ if self.read_only or label not in {"found", "not_found"}:
return
site_entry = self._ensure_site(site)
entries = site_entry[label]
diff --git a/src/aliens_eye/core/scanner.py b/src/aliens_eye/core/scanner.py
index a05fc14..c900a99 100644
--- a/src/aliens_eye/core/scanner.py
+++ b/src/aliens_eye/core/scanner.py
@@ -142,7 +142,12 @@ class UsernameScanner:
logger,
browser_fallback: BrowserFallback | None = None,
checkpoint=None,
+ fetch=fetch_url,
) -> None:
+ # ``fetch`` is the single seam between the scanner and the network.
+ # Swapping it for aliens_eye.corpus.replay.ReplayFetcher runs the whole
+ # pipeline off a frozen corpus with every downstream stage unchanged.
+ self.fetch = fetch
self.sites_data = sites_data
self.config = config
self.extractor = extractor
@@ -286,7 +291,7 @@ class UsernameScanner:
rate_limiter: DomainRateLimiter,
) -> dict[str, Any]:
url = self._format_url(site_name, url_template, username)
- fetch = await fetch_url(session, url, self.config, rate_limiter, self.logger)
+ fetch = await self.fetch(session, url, self.config, rate_limiter, self.logger)
if fetch.error and fetch.status == 408:
status_text = "Timeout"
diff --git a/src/aliens_eye/corpus/__init__.py b/src/aliens_eye/corpus/__init__.py
new file mode 100644
index 0000000..135fce2
--- /dev/null
+++ b/src/aliens_eye/corpus/__init__.py
@@ -0,0 +1,22 @@
+"""Frozen HTTP response corpus: record once, replay deterministically.
+
+Evaluating a detector against the live web is not reproducible -- platforms
+change their markup, rate-limit differently on different days, and block some
+networks entirely. A measured change in accuracy cannot be attributed to the
+detector rather than to the web.
+
+This package removes the network from the measurement loop:
+
+* :mod:`aliens_eye.corpus.record` captures raw responses to disk.
+* :mod:`aliens_eye.corpus.replay` serves them back through the *same*
+ ``fetch_url`` interface the scanner already uses.
+
+Because replay substitutes only the fetch, every downstream stage --
+``core.analyzer`` feature extraction, ``core.detector`` scoring, ``core.features``
+vectorisation -- runs byte-identical code on live and replayed responses. An
+evaluation therefore measures the detector, not the harness.
+"""
+
+from .store import CorpusError, CorpusRecord, CorpusStore
+
+__all__ = ["CorpusError", "CorpusRecord", "CorpusStore"]
diff --git a/src/aliens_eye/corpus/record.py b/src/aliens_eye/corpus/record.py
new file mode 100644
index 0000000..aa82821
--- /dev/null
+++ b/src/aliens_eye/corpus/record.py
@@ -0,0 +1,235 @@
+"""Capture live responses into a frozen corpus.
+
+The recorder wraps :func:`aliens_eye.core.http.fetch_url` rather than reimplementing
+it, so captured responses go through the same retry, backoff, rate-limit and proxy
+behaviour as a normal scan. Whatever the scanner would have seen is what lands on
+disk.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import random
+import string
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import aiohttp
+
+from aliens_eye import __version__
+from aliens_eye.core.config import DEFAULT_HEADERS, ScannerConfig
+from aliens_eye.core.http import FetchResult, fetch_url
+from aliens_eye.core.rate_limit import DomainRateLimiter
+from aliens_eye.core.scanner import build_connector, format_site_url
+
+from .store import CorpusRecord, CorpusStore
+
+# Ordinary-looking handle morphology, for negatives that are not trivially
+# distinguishable from real usernames. A negative pool of random 16-character
+# strings is an artificially easy class: a detector can separate it on length and
+# character distribution alone, which inflates every metric.
+_ADJECTIVES = [
+ "quiet", "amber", "rapid", "hollow", "silver", "brave", "narrow", "lunar",
+ "cobalt", "gentle", "rustic", "vivid", "solar", "hidden", "copper", "swift",
+]
+_NOUNS = [
+ "falcon", "harbor", "meadow", "lantern", "cipher", "willow", "canyon", "ember",
+ "pixel", "otter", "vector", "thicket", "beacon", "marlin", "quartz", "atlas",
+]
+
+
+def random_negative(rng: random.Random) -> str:
+ """A high-entropy handle that is very unlikely to exist anywhere."""
+ length = rng.randint(14, 20)
+ return "".join(rng.choices(string.ascii_lowercase + string.digits, k=length))
+
+
+def plausible_negative(rng: random.Random) -> str:
+ """A handle with realistic morphology, which still probably does not exist.
+
+ Plausible negatives are *not* verified absent by construction -- on large
+ platforms one may collide with a real account. Records tag their negative
+ kind so a later labelling pass can check them; see WORKING.md.
+ """
+ word = f"{rng.choice(_ADJECTIVES)}{rng.choice(_NOUNS)}"
+ style = rng.randint(0, 3)
+ if style == 0:
+ return word
+ if style == 1:
+ return f"{word}{rng.randint(1, 9999)}"
+ if style == 2:
+ return f"{word}_{rng.randint(10, 99)}"
+ return f"{word[: len(word) // 2]}_{word[len(word) // 2 :]}"
+
+
+class RecordingFetcher:
+ """A ``fetch_url``-compatible callable that persists what it fetches.
+
+ Context (site / username / label) cannot be recovered from the URL alone, so
+ the driver registers it up front with :meth:`register`. That keeps the call
+ signature identical to ``fetch_url``, which is what makes the recorder
+ droppable into the scanner without touching the scan path.
+ """
+
+ def __init__(self, store: CorpusStore, tool_version: str = __version__) -> None:
+ self.store = store
+ self.tool_version = tool_version
+ self._context: dict[str, dict[str, Any]] = {}
+ self._records: list[CorpusRecord] = []
+ self._lock = asyncio.Lock()
+
+ def register(
+ self,
+ url: str,
+ site: str,
+ username: str,
+ label: int | None = None,
+ notes: dict[str, Any] | None = None,
+ ) -> None:
+ self._context[url] = {
+ "site": site,
+ "username": username,
+ "label": label,
+ "notes": notes or {},
+ }
+
+ @property
+ def records(self) -> list[CorpusRecord]:
+ return list(self._records)
+
+ async def __call__(
+ self,
+ session: aiohttp.ClientSession,
+ url: str,
+ config: ScannerConfig,
+ rate_limiter: DomainRateLimiter,
+ logger,
+ ) -> FetchResult:
+ result = await fetch_url(session, url, config, rate_limiter, logger)
+ context = self._context.get(url, {})
+ async with self._lock:
+ digest = self.store.write_body(result.content)
+ self._records.append(
+ CorpusRecord(
+ url=url,
+ site=str(context.get("site", "")),
+ username=str(context.get("username", "")),
+ label=context.get("label"),
+ final_url=result.final_url,
+ status=result.status,
+ headers=dict(result.headers),
+ body_sha256=digest,
+ body_chars=len(result.content),
+ response_time=result.response_time,
+ redirect_count=result.redirect_count,
+ error=result.error,
+ captured_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+ tool_version=self.tool_version,
+ notes=dict(context.get("notes") or {}),
+ )
+ )
+ return result
+
+ def flush(self) -> int:
+ """Append buffered records to disk in a stable order. Returns the count."""
+ records = sorted(self._records, key=lambda r: (r.site, r.label or 0, r.username))
+ self.store.append_records(records)
+ count = len(records)
+ self._records.clear()
+ return count
+
+
+def build_jobs(
+ sites_data: dict[str, str],
+ ground_truth: dict[str, list[str]],
+ negatives_per_site: int,
+ rng: random.Random,
+ plausible_ratio: float = 0.5,
+) -> list[tuple[str, str, str, int, dict[str, Any]]]:
+ """Expand ground truth into ``(site, url, username, label, notes)`` jobs."""
+ jobs: list[tuple[str, str, str, int, dict[str, Any]]] = []
+ for site in sorted(ground_truth):
+ template = sites_data.get(site)
+ if not template:
+ continue
+ for username in ground_truth[site]:
+ jobs.append(
+ (site, format_site_url(site, template, username), username, 1,
+ {"positive_kind": "ground_truth"})
+ )
+ n_plausible = int(round(negatives_per_site * plausible_ratio))
+ for index in range(negatives_per_site):
+ if index < n_plausible:
+ username = plausible_negative(rng)
+ kind = "plausible"
+ else:
+ username = random_negative(rng)
+ kind = "random"
+ jobs.append(
+ (site, format_site_url(site, template, username), username, 0,
+ {"negative_kind": kind})
+ )
+ return jobs
+
+
+async def record_corpus(
+ sites_data: dict[str, str],
+ ground_truth: dict[str, list[str]],
+ out_dir: Path,
+ logger,
+ negatives_per_site: int = 4,
+ concurrency: int = 20,
+ seed: int = 1234,
+ split: str = "all",
+ plausible_ratio: float = 0.5,
+ config: ScannerConfig | None = None,
+) -> dict[str, Any]:
+ """Capture every ground-truth positive plus generated negatives to ``out_dir``."""
+ store = CorpusStore(Path(out_dir))
+ store.init()
+ rng = random.Random(seed)
+ config = config or ScannerConfig(retries=2, timeout=15.0)
+ rate_limiter = DomainRateLimiter()
+ recorder = RecordingFetcher(store)
+
+ jobs = build_jobs(sites_data, ground_truth, negatives_per_site, rng, plausible_ratio)
+ skipped = sorted(set(ground_truth) - set(sites_data))
+ for site, url, username, label, notes in jobs:
+ recorder.register(url, site, username, label, notes)
+
+ conn_limit = max(1, min(concurrency, len(jobs) or 1))
+ semaphore = asyncio.Semaphore(conn_limit)
+ connector = build_connector(config, conn_limit)
+
+ async with aiohttp.ClientSession(headers=DEFAULT_HEADERS, connector=connector) as session:
+
+ async def run(url: str) -> None:
+ async with semaphore:
+ try:
+ await recorder(session, url, config, rate_limiter, logger)
+ except Exception as exc: # noqa: BLE001 - one bad host must not abort a capture
+ logger.debug("Capture failed for %s: %s", url, exc)
+
+ await asyncio.gather(*(run(job[1]) for job in jobs))
+
+ captured = recorder.records
+ errors = sum(1 for r in captured if r.error)
+ written = recorder.flush()
+
+ manifest = {
+ "tool_version": __version__,
+ "created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+ "split": split,
+ "seed": seed,
+ "negatives_per_site": negatives_per_site,
+ "plausible_ratio": plausible_ratio,
+ "requested": len(jobs),
+ "records": written,
+ "errors": errors,
+ "sites": sorted({r.site for r in captured if r.site}),
+ "skipped_sites": skipped,
+ }
+ store.write_manifest(manifest)
+ logger.info("Captured %d records (%d errors) to %s", written, errors, out_dir)
+ return manifest
diff --git a/src/aliens_eye/corpus/replay.py b/src/aliens_eye/corpus/replay.py
new file mode 100644
index 0000000..dec84ab
--- /dev/null
+++ b/src/aliens_eye/corpus/replay.py
@@ -0,0 +1,140 @@
+"""Serve a frozen corpus through the ``fetch_url`` interface.
+
+Replay makes an evaluation reproducible: the same corpus yields the same report,
+today and in a year, whatever the platforms have since done to their markup.
+
+Determinism requires more than returning stored bytes:
+
+* No network, no rate limiting, no retries -- the rate limiter is not consulted,
+ so replay does not sleep and wall-clock timing cannot leak into results.
+* ``response_time`` is the recorded value, not a fresh measurement, so the
+ ``response_time`` feature is stable across runs.
+* Fingerprints must be neutralised by the caller. ``core.fingerprints.FingerprintStore``
+ accumulates signatures *during* a scan and scores against what it has seen so
+ far, which makes its contribution depend on worker completion order. Pass a
+ read-only store (see :class:`aliens_eye.core.fingerprints.FingerprintStore`) or
+ results will vary run to run even off a frozen corpus.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import aiohttp
+
+from aliens_eye.core.config import ScannerConfig
+from aliens_eye.core.http import FetchResult
+from aliens_eye.core.rate_limit import DomainRateLimiter
+
+from .store import CorpusError, CorpusRecord, CorpusStore
+
+MISSING_ERROR = "not in corpus"
+
+
+class ReplayFetcher:
+ """A ``fetch_url``-compatible callable backed by a recorded corpus.
+
+ ``strict=True`` raises on a URL the corpus does not contain, which is what an
+ evaluation wants: a silently-missing response would be scored as a network
+ error and quietly bias the metrics. ``strict=False`` returns an error
+ ``FetchResult`` instead, matching how a real failed fetch behaves.
+ """
+
+ def __init__(self, root: Path, strict: bool = True, cache_bodies: bool = True) -> None:
+ self.store = CorpusStore(Path(root))
+ self.manifest = self.store.read_manifest()
+ self.strict = strict
+ self.cache_bodies = cache_bodies
+ self.records: dict[str, CorpusRecord] = self.store.index_by_url()
+ self._bodies: dict[str, str] = {}
+ self.hits = 0
+ self.misses: list[str] = []
+
+ # -- corpus introspection --------------------------------------------
+
+ def __len__(self) -> int:
+ return len(self.records)
+
+ @property
+ def sites(self) -> list[str]:
+ return sorted({r.site for r in self.records.values() if r.site})
+
+ def labels_by_url(self) -> dict[str, int | None]:
+ return {url: record.label for url, record in self.records.items()}
+
+ def eval_jobs(self, sites: set[str] | None = None) -> list[tuple[str, str, str, int]]:
+ """The labelled evaluation set held by this corpus.
+
+ Returns ``(site, url, username, label)`` for every labelled record, in a
+ stable order. Callers must evaluate *these* rows rather than regenerating
+ usernames: freshly generated negatives were never captured, so they would
+ all miss the corpus and leave the negative class empty -- yielding a
+ false-positive rate computed over nothing.
+ """
+ jobs = [
+ (r.site, r.url, r.username, int(r.label))
+ for r in self.records.values()
+ if r.label is not None and (sites is None or r.site in sites)
+ ]
+ return sorted(jobs, key=lambda j: (j[0], j[3], j[2]))
+
+ def stats(self) -> dict[str, Any]:
+ records = list(self.records.values())
+ positives = sum(1 for r in records if r.label == 1)
+ negatives = sum(1 for r in records if r.label == 0)
+ errors = sum(1 for r in records if r.error)
+ return {
+ "records": len(records),
+ "sites": len(self.sites),
+ "positives": positives,
+ "negatives": negatives,
+ "unlabeled": len(records) - positives - negatives,
+ "capture_errors": errors,
+ "distinct_bodies": len({r.body_sha256 for r in records if r.body_sha256}),
+ "tool_version": self.manifest.get("tool_version"),
+ "created_at": self.manifest.get("created_at"),
+ }
+
+ # -- fetch ------------------------------------------------------------
+
+ def _body(self, record: CorpusRecord) -> str:
+ if not self.cache_bodies:
+ return self.store.read_body(record.body_sha256)
+ if record.body_sha256 not in self._bodies:
+ self._bodies[record.body_sha256] = self.store.read_body(record.body_sha256)
+ return self._bodies[record.body_sha256]
+
+ async def __call__(
+ self,
+ session: aiohttp.ClientSession | None,
+ url: str,
+ config: ScannerConfig,
+ rate_limiter: DomainRateLimiter,
+ logger,
+ ) -> FetchResult:
+ record = self.records.get(url)
+ if record is None:
+ self.misses.append(url)
+ if self.strict:
+ raise CorpusError(
+ f"{url} is not in the corpus at {self.store.root}. "
+ "Re-record, or pass strict=False to score it as a fetch error."
+ )
+ logger.debug("Corpus miss for %s", url)
+ return FetchResult(
+ url=url, final_url=url, status=0, content="", response_time=0.0,
+ headers={}, error=MISSING_ERROR, redirect_count=0,
+ )
+
+ self.hits += 1
+ return FetchResult(
+ url=record.url,
+ final_url=record.final_url,
+ status=record.status,
+ content="" if record.error else self._body(record),
+ response_time=record.response_time,
+ headers=dict(record.headers),
+ error=record.error,
+ redirect_count=record.redirect_count,
+ )
diff --git a/src/aliens_eye/corpus/store.py b/src/aliens_eye/corpus/store.py
new file mode 100644
index 0000000..ea056ad
--- /dev/null
+++ b/src/aliens_eye/corpus/store.py
@@ -0,0 +1,189 @@
+"""On-disk layout for a frozen response corpus.
+
+::
+
+ <corpus>/
+ manifest.json metadata: tool version, counts, capture run
+ records.jsonl one JSON object per captured request
+ bodies/<aa>/<sha256>.gz gzipped response bodies, content-addressed
+
+Bodies are content-addressed because error pages repeat heavily: a platform
+serves one "no such user" page for every non-existent username, so a corpus of
+N negatives per site stores that body once rather than N times.
+"""
+
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+from collections.abc import Iterator
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+CORPUS_VERSION = 1
+MANIFEST_NAME = "manifest.json"
+RECORDS_NAME = "records.jsonl"
+BODIES_DIR = "bodies"
+
+
+class CorpusError(RuntimeError):
+ """Raised when a corpus is missing, malformed, or version-mismatched."""
+
+
+@dataclass
+class CorpusRecord:
+ """One captured request/response pair.
+
+ ``url`` is the lookup key: it is derived deterministically from
+ ``(site, template, username)`` by ``core.scanner.format_site_url``, and it is
+ the only identifier ``fetch_url`` receives, so replay must key on it.
+ """
+
+ url: str
+ site: str
+ username: str
+ label: int | None
+ final_url: str
+ status: int
+ headers: dict[str, str]
+ body_sha256: str
+ body_chars: int
+ response_time: float
+ redirect_count: int
+ error: str | None
+ captured_at: str
+ tool_version: str
+ notes: dict[str, Any] = field(default_factory=dict)
+
+ def to_json(self) -> str:
+ return json.dumps(asdict(self), ensure_ascii=False, sort_keys=True)
+
+ @classmethod
+ def from_dict(cls, data: dict[str, Any]) -> CorpusRecord:
+ try:
+ return cls(
+ url=str(data["url"]),
+ site=str(data["site"]),
+ username=str(data["username"]),
+ label=None if data.get("label") is None else int(data["label"]),
+ final_url=str(data.get("final_url") or data["url"]),
+ status=int(data.get("status", 0)),
+ headers={str(k): str(v) for k, v in (data.get("headers") or {}).items()},
+ body_sha256=str(data.get("body_sha256", "")),
+ body_chars=int(data.get("body_chars", 0)),
+ response_time=float(data.get("response_time", 0.0)),
+ redirect_count=int(data.get("redirect_count", 0)),
+ error=data.get("error"),
+ captured_at=str(data.get("captured_at", "")),
+ tool_version=str(data.get("tool_version", "unknown")),
+ notes=dict(data.get("notes") or {}),
+ )
+ except (KeyError, TypeError, ValueError) as exc:
+ raise CorpusError(f"Malformed corpus record: {exc}") from exc
+
+
+def body_digest(content: str) -> str:
+ """Content address for a response body. An empty body has no digest."""
+ if not content:
+ return ""
+ return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+class CorpusStore:
+ """Read/write access to a corpus directory."""
+
+ def __init__(self, root: Path) -> None:
+ self.root = Path(root)
+
+ # -- paths ------------------------------------------------------------
+
+ @property
+ def manifest_path(self) -> Path:
+ return self.root / MANIFEST_NAME
+
+ @property
+ def records_path(self) -> Path:
+ return self.root / RECORDS_NAME
+
+ def body_path(self, digest: str) -> Path:
+ return self.root / BODIES_DIR / digest[:2] / f"{digest}.gz"
+
+ # -- writing ----------------------------------------------------------
+
+ def init(self) -> None:
+ (self.root / BODIES_DIR).mkdir(parents=True, exist_ok=True)
+
+ def write_body(self, content: str) -> str:
+ """Store a body and return its digest. Idempotent for repeated bodies."""
+ digest = body_digest(content)
+ if not digest:
+ return ""
+ path = self.body_path(digest)
+ if path.exists():
+ return digest
+ path.parent.mkdir(parents=True, exist_ok=True)
+ # mtime=0 so identical bodies produce identical bytes on disk, which
+ # keeps a corpus directory itself checksummable across capture runs.
+ with path.open("wb") as raw:
+ with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as handle:
+ handle.write(content.encode("utf-8"))
+ return digest
+
+ def append_records(self, records: list[CorpusRecord]) -> None:
+ self.records_path.parent.mkdir(parents=True, exist_ok=True)
+ with self.records_path.open("a", encoding="utf-8", newline="\n") as handle:
+ for record in records:
+ handle.write(record.to_json() + "\n")
+
+ def write_manifest(self, manifest: dict[str, Any]) -> None:
+ payload = {"corpus_version": CORPUS_VERSION, **manifest}
+ self.manifest_path.write_text(
+ json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+ )
+
+ # -- reading ----------------------------------------------------------
+
+ def read_manifest(self) -> dict[str, Any]:
+ if not self.manifest_path.exists():
+ raise CorpusError(f"No corpus manifest at {self.manifest_path}")
+ try:
+ manifest = json.loads(self.manifest_path.read_text("utf-8"))
+ except json.JSONDecodeError as exc:
+ raise CorpusError(f"Corpus manifest is not valid JSON: {exc}") from exc
+ version = manifest.get("corpus_version")
+ if version != CORPUS_VERSION:
+ raise CorpusError(
+ f"Corpus version {version!r} != supported {CORPUS_VERSION}; re-record."
+ )
+ return manifest
+
+ def iter_records(self) -> Iterator[CorpusRecord]:
+ if not self.records_path.exists():
+ raise CorpusError(f"No corpus records at {self.records_path}")
+ with self.records_path.open("r", encoding="utf-8") as handle:
+ for line_no, line in enumerate(handle, 1):
+ stripped = line.strip()
+ if not stripped:
+ continue
+ try:
+ data = json.loads(stripped)
+ except json.JSONDecodeError as exc:
+ raise CorpusError(
+ f"{self.records_path}:{line_no} is not valid JSON: {exc}"
+ ) from exc
+ yield CorpusRecord.from_dict(data)
+
+ def read_body(self, digest: str) -> str:
+ if not digest:
+ return ""
+ path = self.body_path(digest)
+ if not path.exists():
+ raise CorpusError(f"Corpus body {digest} missing at {path}")
+ with gzip.open(path, "rb") as handle:
+ return handle.read().decode("utf-8")
+
+ def index_by_url(self) -> dict[str, CorpusRecord]:
+ """Map url -> record. A later capture of the same url wins."""
+ return {record.url: record for record in self.iter_records()}
diff --git a/tests/test_corpus.py b/tests/test_corpus.py
new file mode 100644
index 0000000..7b01681
--- /dev/null
+++ b/tests/test_corpus.py
@@ -0,0 +1,390 @@
+"""Frozen-corpus record/replay: round-trip, determinism, and the guards that
+keep a replayed evaluation from silently drifting."""
+
+import asyncio
+import json
+
+import pytest
+
+from aliens_eye.core.config import ScannerConfig
+from aliens_eye.core.detector import Detector
+from aliens_eye.core.fingerprints import FingerprintStore, build_fingerprint
+from aliens_eye.core.http import FetchResult
+from aliens_eye.corpus.record import (
+ RecordingFetcher,
+ build_jobs,
+ plausible_negative,
+ random_negative,
+)
+from aliens_eye.corpus.replay import MISSING_ERROR, ReplayFetcher
+from aliens_eye.corpus.store import CorpusError, CorpusStore, body_digest
+
+SITES = {
+ "github": "https://github.com/{}",
+ "reddit": "https://www.reddit.com/user/{}",
+}
+
+
+class FakeNetwork:
+ """Stands in for fetch_url so tests never touch the network."""
+
+ def __init__(self, responses: dict[str, tuple[int, str]]) -> None:
+ self.responses = responses
+ self.calls: list[str] = []
+
+ async def __call__(self, session, url, config, rate_limiter, logger):
+ self.calls.append(url)
+ status, body = self.responses.get(url, (404, "<html>no such user</html>"))
+ return FetchResult(
+ url=url, final_url=url, status=status, content=body,
+ response_time=0.125, headers={"Server": "test"}, error=None,
+ redirect_count=0,
+ )
+
+
+def make_corpus(tmp_path, monkeypatch, responses, jobs):
+ """Record `jobs` against a fake network and return the corpus root."""
+ from aliens_eye.corpus import record as record_mod
+
+ network = FakeNetwork(responses)
+ monkeypatch.setattr(record_mod, "fetch_url", network)
+
+ root = tmp_path / "corpus"
+ store = CorpusStore(root)
+ store.init()
+ recorder = RecordingFetcher(store, tool_version="test")
+ for site, url, username, label in jobs:
+ recorder.register(url, site, username, label)
+
+ async def drive():
+ for _, url, _, _ in jobs:
+ await recorder(None, url, ScannerConfig(), None, _NullLogger())
+
+ asyncio.run(drive())
+ written = recorder.flush()
+ store.write_manifest({"tool_version": "test", "records": written})
+ return root, network
+
+
+class _NullLogger:
+ def debug(self, *a, **k):
+ pass
+
+ def info(self, *a, **k):
+ pass
+
+ def warning(self, *a, **k):
+ pass
+
+
+# --- round trip -----------------------------------------------------------
+
+
+def test_record_then_replay_round_trips(tmp_path, monkeypatch, found_html):
+ url = "https://github.com/torvalds"
+ root, _ = make_corpus(
+ tmp_path, monkeypatch, {url: (200, found_html)},
+ [("github", url, "torvalds", 1)],
+ )
+
+ replay = ReplayFetcher(root)
+ result = asyncio.run(replay(None, url, ScannerConfig(), None, _NullLogger()))
+ assert result.status == 200
+ assert result.content == found_html
+ assert result.response_time == 0.125
+ assert result.headers["Server"] == "test"
+ assert result.error is None
+
+
+def test_replay_makes_no_network_calls(tmp_path, monkeypatch, found_html):
+ url = "https://github.com/torvalds"
+ root, network = make_corpus(
+ tmp_path, monkeypatch, {url: (200, found_html)},
+ [("github", url, "torvalds", 1)],
+ )
+ before = len(network.calls)
+ replay = ReplayFetcher(root)
+ asyncio.run(replay(None, url, ScannerConfig(), None, _NullLogger()))
+ assert len(network.calls) == before
+
+
+# --- determinism ----------------------------------------------------------
+
+
+def test_replay_is_deterministic_across_runs_and_orders(
+ tmp_path, monkeypatch, found_html, not_found_html
+):
+ """Same corpus -> identical detections, regardless of visit order.
+
+ This is the property the whole corpus exists for: a metric change must be
+ attributable to the detector, never to which worker finished first.
+ """
+ urls = {
+ "https://github.com/torvalds": (200, found_html),
+ "https://github.com/nosuchuser99": (404, not_found_html),
+ "https://www.reddit.com/user/spez": (200, found_html),
+ }
+ jobs = [
+ ("github", "https://github.com/torvalds", "torvalds", 1),
+ ("github", "https://github.com/nosuchuser99", "nosuchuser99", 0),
+ ("reddit", "https://www.reddit.com/user/spez", "spez", 1),
+ ]
+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs)
+
+ def score(order):
+ from aliens_eye.core.analyzer import FeatureExtractor
+
+ replay = ReplayFetcher(root)
+ extractor = FeatureExtractor()
+ detector = Detector()
+ # Read-only, empty store: no cross-response accumulation, so scoring
+ # cannot depend on completion order.
+ fingerprints = FingerprintStore(tmp_path / "fp.json", read_only=True)
+ out = {}
+ for site, url, username, _ in order:
+ fetch = asyncio.run(replay(None, url, ScannerConfig(), None, _NullLogger()))
+ bundle = extractor.extract(
+ fetch.content, fetch.final_url, username, site,
+ fetch.status, fetch.response_time, fetch.headers, fetch.redirect_count,
+ )
+ fp = build_fingerprint(bundle.fingerprint)
+ scored = fingerprints.score(site, fp)
+ bundle.features["fingerprint_match_found"] = float(scored["match_found"])
+ bundle.features["fingerprint_match_not_found"] = float(scored["match_not_found"])
+ fingerprints.add(site, "found", fp)
+ detection = detector.predict(bundle.features)
+ out[url] = (detection.status, detection.confidence, detection.probability)
+ return out
+
+ first = score(jobs)
+ assert first == score(jobs), "same order must reproduce"
+ assert first == score(list(reversed(jobs))), "reversed order must reproduce"
+
+
+def test_readonly_fingerprint_store_does_not_accumulate(tmp_path, found_html):
+ store = FingerprintStore(tmp_path / "fp.json", read_only=True)
+ fp = {"title_hash": "a", "meta_hash": "b", "dom_signature": "c", "server": "d"}
+ store.add("github", "found", fp)
+ assert store.score("github", fp)["match_found"] == 0
+ store.save()
+ assert not (tmp_path / "fp.json").exists()
+
+
+def test_writable_fingerprint_store_still_accumulates(tmp_path):
+ store = FingerprintStore(tmp_path / "fp.json")
+ fp = {"title_hash": "a", "meta_hash": "b", "dom_signature": "c", "server": "d"}
+ store.add("github", "found", fp)
+ assert store.score("github", fp)["match_found"] > 0
+
+
+# --- storage --------------------------------------------------------------
+
+
+def test_identical_bodies_are_stored_once(tmp_path, monkeypatch, not_found_html):
+ """Error pages repeat across every negative; the corpus must not duplicate them."""
+ urls = {
+ f"https://github.com/ghost{i}": (404, not_found_html) for i in range(5)
+ }
+ jobs = [("github", url, f"ghost{i}", 0) for i, url in enumerate(urls)]
+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs)
+
+ bodies = list((root / "bodies").rglob("*.gz"))
+ assert len(bodies) == 1
+ assert ReplayFetcher(root).stats()["records"] == 5
+
+
+def test_body_digest_is_content_addressed():
+ assert body_digest("abc") == body_digest("abc")
+ assert body_digest("abc") != body_digest("abd")
+ assert body_digest("") == ""
+
+
+def test_records_are_written_in_stable_order(tmp_path, monkeypatch, found_html):
+ urls = {f"https://github.com/u{i}": (200, found_html + str(i)) for i in range(4)}
+ jobs = [("github", f"https://github.com/u{i}", f"u{i}", i % 2) for i in range(4)]
+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs)
+ lines = (root / "records.jsonl").read_text("utf-8").strip().splitlines()
+ keys = [(json.loads(x)["site"], json.loads(x)["label"], json.loads(x)["username"]) for x in lines]
+ assert keys == sorted(keys)
+
+
+# --- guards ---------------------------------------------------------------
+
+
+def test_strict_replay_raises_on_missing_url(tmp_path, monkeypatch, found_html):
+ url = "https://github.com/torvalds"
+ root, _ = make_corpus(
+ tmp_path, monkeypatch, {url: (200, found_html)},
+ [("github", url, "torvalds", 1)],
+ )
+ replay = ReplayFetcher(root, strict=True)
+ with pytest.raises(CorpusError, match="not in the corpus"):
+ asyncio.run(replay(None, "https://github.com/absent", ScannerConfig(), None, _NullLogger()))
+
+
+def test_lenient_replay_reports_miss_as_fetch_error(tmp_path, monkeypatch, found_html):
+ url = "https://github.com/torvalds"
+ root, _ = make_corpus(
+ tmp_path, monkeypatch, {url: (200, found_html)},
+ [("github", url, "torvalds", 1)],
+ )
+ replay = ReplayFetcher(root, strict=False)
+ result = asyncio.run(
+ replay(None, "https://github.com/absent", ScannerConfig(), None, _NullLogger())
+ )
+ assert result.error == MISSING_ERROR
+ assert replay.misses == ["https://github.com/absent"]
+
+
+def test_missing_corpus_raises(tmp_path):
+ with pytest.raises(CorpusError, match="No corpus manifest"):
+ ReplayFetcher(tmp_path / "nope")
+
+
+def test_version_mismatch_raises(tmp_path, monkeypatch, found_html):
+ url = "https://github.com/torvalds"
+ root, _ = make_corpus(
+ tmp_path, monkeypatch, {url: (200, found_html)},
+ [("github", url, "torvalds", 1)],
+ )
+ manifest = json.loads((root / "manifest.json").read_text("utf-8"))
+ manifest["corpus_version"] = 999
+ (root / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
+ with pytest.raises(CorpusError, match="re-record"):
+ ReplayFetcher(root)
+
+
+# --- job construction -----------------------------------------------------
+
+
+def test_build_jobs_covers_positives_and_negatives():
+ import random
+
+ ground_truth = {"github": ["torvalds", "mojombo"], "reddit": ["spez"]}
+ jobs = build_jobs(SITES, ground_truth, negatives_per_site=4, rng=random.Random(1))
+ positives = [j for j in jobs if j[3] == 1]
+ negatives = [j for j in jobs if j[3] == 0]
+ assert len(positives) == 3
+ assert len(negatives) == 8
+ assert {j[0] for j in jobs} == {"github", "reddit"}
+
+
+def test_build_jobs_skips_sites_absent_from_sites_json():
+ import random
+
+ jobs = build_jobs(SITES, {"nowhere": ["someone"]}, 2, random.Random(1))
+ assert jobs == []
+
+
+def test_build_jobs_is_seed_reproducible():
+ import random
+
+ gt = {"github": ["torvalds"]}
+ a = build_jobs(SITES, gt, 4, random.Random(7))
+ b = build_jobs(SITES, gt, 4, random.Random(7))
+ assert a == b
+
+
+def test_negative_pools_have_different_morphology():
+ """Random negatives are separable on shape alone; plausible ones are not.
+
+ A negative class of 16-char random strings is an artificially easy problem
+ and inflates every metric, which is why the corpus mixes in handles built
+ from ordinary word morphology.
+ """
+ import random
+
+ rng = random.Random(3)
+ randoms = [random_negative(rng) for _ in range(50)]
+ plausibles = [plausible_negative(rng) for _ in range(50)]
+ assert all(len(u) >= 14 for u in randoms)
+ assert min(len(u) for u in plausibles) < 14
+ # Random strings are ~uniform over the alphabet; word-built handles are not.
+ assert sum(c.isdigit() for u in randoms for c in u) > sum(
+ c.isdigit() for u in plausibles for c in u
+ )
+
+
+def test_plausible_negatives_vary():
+ import random
+
+ rng = random.Random(11)
+ assert len({plausible_negative(rng) for _ in range(40)}) > 20
+
+
+# --- corpus as the evaluation set -----------------------------------------
+
+
+def test_eval_jobs_returns_labelled_rows(tmp_path, monkeypatch, found_html, not_found_html):
+ urls = {
+ "https://github.com/torvalds": (200, found_html),
+ "https://github.com/ghost1": (404, not_found_html),
+ "https://www.reddit.com/user/spez": (200, found_html),
+ }
+ jobs_in = [
+ ("github", "https://github.com/torvalds", "torvalds", 1),
+ ("github", "https://github.com/ghost1", "ghost1", 0),
+ ("reddit", "https://www.reddit.com/user/spez", "spez", 1),
+ ]
+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs_in)
+
+ jobs = ReplayFetcher(root).eval_jobs()
+ assert sorted(jobs) == sorted(jobs_in)
+ assert jobs == sorted(jobs, key=lambda j: (j[0], j[3], j[2])), "must be stably ordered"
+
+
+def test_eval_jobs_filters_by_site(tmp_path, monkeypatch, found_html):
+ urls = {
+ "https://github.com/torvalds": (200, found_html),
+ "https://www.reddit.com/user/spez": (200, found_html),
+ }
+ jobs_in = [
+ ("github", "https://github.com/torvalds", "torvalds", 1),
+ ("reddit", "https://www.reddit.com/user/spez", "spez", 1),
+ ]
+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs_in)
+
+ jobs = ReplayFetcher(root).eval_jobs({"github"})
+ assert [j[0] for j in jobs] == ["github"]
+
+
+def test_eval_jobs_hit_the_corpus_exactly(tmp_path, monkeypatch, found_html, not_found_html):
+ """Regression: the eval set must come from the corpus, not be regenerated.
+
+ Regenerating negatives at eval time yields URLs the corpus never captured,
+ so every negative misses and the negative class is empty -- which silently
+ reports a false-positive rate computed over nothing.
+ """
+ urls = {
+ "https://github.com/torvalds": (200, found_html),
+ "https://github.com/ghost1": (404, not_found_html),
+ "https://github.com/ghost2": (404, not_found_html),
+ }
+ jobs_in = [
+ ("github", "https://github.com/torvalds", "torvalds", 1),
+ ("github", "https://github.com/ghost1", "ghost1", 0),
+ ("github", "https://github.com/ghost2", "ghost2", 0),
+ ]
+ root, _ = make_corpus(tmp_path, monkeypatch, urls, jobs_in)
+
+ replay = ReplayFetcher(root, strict=True)
+ jobs = replay.eval_jobs()
+
+ async def drive():
+ for _, url, _, _ in jobs:
+ await replay(None, url, ScannerConfig(), None, _NullLogger())
+
+ asyncio.run(drive())
+ assert replay.misses == []
+ assert replay.hits == len(jobs)
+ # The negative class is non-empty, so FPR is computed over something.
+ assert sum(1 for j in jobs if j[3] == 0) == 2
+
+
+def test_eval_jobs_skips_unlabelled_records(tmp_path, monkeypatch, found_html):
+ url = "https://github.com/torvalds"
+ root, _ = make_corpus(
+ tmp_path, monkeypatch, {url: (200, found_html)},
+ [("github", url, "torvalds", None)],
+ )
+ assert ReplayFetcher(root).eval_jobs() == []