Skip to content

AI powered OSINT engine to discover and map 840+ social media identities across the web

fix: split ground truth site-disjoint to remove train/eval leakage

arxhr007cf7194244a0c
diff --git a/src/aliens_eye/core/detector.py b/src/aliens_eye/core/detector.py
index f5a568c..67b515d 100644
--- a/src/aliens_eye/core/detector.py
+++ b/src/aliens_eye/core/detector.py
@@ -12,8 +12,12 @@ class DetectionResult:
probability: float | None
-# Blend weights and probability thresholds, calibrated by out-of-fold grid
-# search on the seed dataset and validated against the selfcheck set.
+# FALLBACK blend weight and probability thresholds, used only when no model is
+# loaded or when a model file omits these fields. A loaded model overrides all
+# three (see Detector.predict and MLModel.from_dict) -- the shipped
+# data/model.json carries its own out-of-fold grid-searched values, so these
+# constants do NOT describe the default runtime behaviour of the packaged tool.
+# tests/test_detector.py pins the shipped model's values against the docs.
ML_WEIGHT = 0.4
HEURISTIC_WEIGHT = 0.6
HEURISTIC_SIGMOID_SCALE = 6.0
@@ -45,13 +49,24 @@ class Detector:
self.model = None
logger.debug("ML model unavailable, using heuristics only: %s", exc)
- def predict(self, features: dict[str, float]) -> DetectionResult:
+ def judges(self, features: dict[str, float]) -> tuple[float, float, float | None]:
+ """Both judges, unblended: (heuristic_score, heuristic_prob, ml_prob).
+
+ ml_prob is None when no model is loaded. Exposed so an ablation can
+ weigh the two judges differently without reimplementing either -- see
+ aliens_eye.eval.ablate. Mutates features["heuristic_score"], which
+ the ML model consumes as an input feature.
+ """
heuristic_score = self._heuristic_score(features)
heuristic_prob = _sigmoid(heuristic_score / HEURISTIC_SIGMOID_SCALE)
features["heuristic_score"] = heuristic_score
+ ml_prob = self.model.predict_proba(features) if self.model is not None else None
+ return heuristic_score, heuristic_prob, ml_prob
+
+ def predict(self, features: dict[str, float]) -> DetectionResult:
+ heuristic_score, heuristic_prob, ml_prob = self.judges(features)
- if self.model is not None:
- ml_prob = self.model.predict_proba(features)
+ if ml_prob is not None:
ml_w = getattr(self.model, "ml_weight", ML_WEIGHT)
probability = ml_w * ml_prob + (1.0 - ml_w) * heuristic_prob
method = "ml+heuristic"
diff --git a/src/aliens_eye/data/eval_holdout.json b/src/aliens_eye/data/eval_holdout.json
new file mode 100644
index 0000000..8c9c0aa
--- /dev/null
+++ b/src/aliens_eye/data/eval_holdout.json
@@ -0,0 +1,43 @@
+{
+ "vimeo": [
+ "staff"
+ ],
+ "duolingo": [
+ "luis"
+ ],
+ "dribbble": [
+ "dan"
+ ],
+ "steamcommunity": [
+ "gabelogannewell"
+ ],
+ "hackernews": [
+ "pg",
+ "dang"
+ ],
+ "last.fm": [
+ "rj",
+ "lastfm"
+ ],
+ "tumblr": [
+ "staff"
+ ],
+ "linktree": [
+ "linktree"
+ ],
+ "unsplash": [
+ "nasa"
+ ],
+ "codechef": [
+ "tourist"
+ ],
+ "bitbucket": [
+ "atlassian"
+ ],
+ "archive.org": [
+ "brewster_kahle"
+ ],
+ "monkeytype": [
+ "miodec"
+ ]
+}
diff --git a/src/aliens_eye/data/selfcheck.json b/src/aliens_eye/data/selfcheck.json
index 4cee3e3..526338d 100644
--- a/src/aliens_eye/data/selfcheck.json
+++ b/src/aliens_eye/data/selfcheck.json
@@ -1,45 +1,107 @@
{
- "github": ["torvalds", "gvanrossum", "mojombo"],
- "reddit": ["spez", "kn0thing"],
- "gitlab": ["sytses"],
- "twitch": ["shroud", "ninja"],
- "medium": ["ev"],
- "dev.to": ["ben", "jess"],
- "npm": ["sindresorhus", "isaacs"],
- "pypi": ["dstufft"],
- "soundcloud": ["skrillex", "diplo"],
- "vimeo": ["staff"],
- "imgur": ["sarah"],
- "patreon": ["jacksepticeye"],
- "telegram": ["durov", "telegram"],
- "chess.com": ["hikaru", "magnuscarlsen"],
- "lichess": ["thibault", "drnykterstein"],
- "hackernews": ["pg", "dang"],
- "duolingo": ["luis"],
- "dribbble": ["dan"],
- "mastodon.social": ["Gargron"],
- "steamcommunity": ["gabelogannewell"],
- "pinterest": ["pinterest"],
- "devrant": ["dfox", "trogus"],
- "spotify": ["spotify"],
- "kaggle": ["kaggle", "abhishek"],
- "wikipedia": ["Jimbo_Wales"],
- "youtube user": ["pewdiepie"],
- "codepen": ["chriscoyier", "cassiecodes"],
- "last.fm": ["rj", "lastfm"],
- "wordpressorg": ["matt", "automattic"],
- "docker hub": ["docker"],
- "scratch": ["Scratch", "griffpatch"],
- "tumblr": ["staff"],
- "linktree": ["linktree"],
- "replit.com": ["replit"],
- "unsplash": ["nasa"],
- "producthunt": ["ryanhoover"],
- "codechef": ["tourist"],
- "artstation": ["artstation"],
- "bitbucket": ["atlassian"],
- "codewars": ["jhoffner"],
- "archive.org": ["brewster_kahle"],
- "freecodecamp": ["QuincyLarson"],
- "monkeytype": ["miodec"]
+ "github": [
+ "torvalds",
+ "gvanrossum",
+ "mojombo"
+ ],
+ "reddit": [
+ "spez",
+ "kn0thing"
+ ],
+ "gitlab": [
+ "sytses"
+ ],
+ "twitch": [
+ "shroud",
+ "ninja"
+ ],
+ "medium": [
+ "ev"
+ ],
+ "dev.to": [
+ "ben",
+ "jess"
+ ],
+ "npm": [
+ "sindresorhus",
+ "isaacs"
+ ],
+ "pypi": [
+ "dstufft"
+ ],
+ "soundcloud": [
+ "skrillex",
+ "diplo"
+ ],
+ "imgur": [
+ "sarah"
+ ],
+ "patreon": [
+ "jacksepticeye"
+ ],
+ "telegram": [
+ "durov",
+ "telegram"
+ ],
+ "chess.com": [
+ "hikaru",
+ "magnuscarlsen"
+ ],
+ "lichess": [
+ "thibault",
+ "drnykterstein"
+ ],
+ "mastodon.social": [
+ "Gargron"
+ ],
+ "pinterest": [
+ "pinterest"
+ ],
+ "devrant": [
+ "dfox",
+ "trogus"
+ ],
+ "spotify": [
+ "spotify"
+ ],
+ "kaggle": [
+ "kaggle",
+ "abhishek"
+ ],
+ "wikipedia": [
+ "Jimbo_Wales"
+ ],
+ "youtube user": [
+ "pewdiepie"
+ ],
+ "codepen": [
+ "chriscoyier",
+ "cassiecodes"
+ ],
+ "wordpressorg": [
+ "matt",
+ "automattic"
+ ],
+ "docker hub": [
+ "docker"
+ ],
+ "scratch": [
+ "Scratch",
+ "griffpatch"
+ ],
+ "replit.com": [
+ "replit"
+ ],
+ "producthunt": [
+ "ryanhoover"
+ ],
+ "artstation": [
+ "artstation"
+ ],
+ "codewars": [
+ "jhoffner"
+ ],
+ "freecodecamp": [
+ "QuincyLarson"
+ ]
}
diff --git a/src/aliens_eye/ml/collect.py b/src/aliens_eye/ml/collect.py
index 1fbbfd8..689d546 100644
--- a/src/aliens_eye/ml/collect.py
+++ b/src/aliens_eye/ml/collect.py
@@ -1,9 +1,17 @@
"""Build a labeled training dataset by scanning ground-truth accounts.
-Positives come from a curated map of sites to usernames known to exist
-(data/selfcheck.json). Negatives are randomized usernames that almost
-certainly do not exist. Each scan produces one row of FEATURE_SCHEMA values
-plus a label column.
+Positives come from a curated map of sites to usernames known to exist.
+Negatives are randomized usernames that almost certainly do not exist. Each
+scan produces one row of FEATURE_SCHEMA values plus a label column.
+
+The ground truth is split **site-disjoint** into two files:
+
+* ``data/selfcheck.json`` -- the train split; the only split training reads.
+* ``data/eval_holdout.json`` -- held out; never seen by training.
+
+The split is site-disjoint rather than account-disjoint because the detector
+learns per-site page structure: holding out only accounts from a site whose
+layout the model already trained on would overstate generalization.
"""
from __future__ import annotations
@@ -15,6 +23,7 @@ import random
import string
from importlib import resources
from pathlib import Path
+from typing import Any
import aiohttp
@@ -26,11 +35,42 @@ from aliens_eye.core.http import fetch_url
from aliens_eye.core.rate_limit import DomainRateLimiter
from aliens_eye.core.scanner import format_site_url
-
-def load_selfcheck_data() -> dict[str, list[str]]:
- """Map of site name to usernames known to exist there."""
- text = (resources.files("aliens_eye.data") / "selfcheck.json").read_text("utf-8")
- data = json.loads(text)
+TRAIN_SPLIT_RESOURCE = "selfcheck.json"
+HOLDOUT_SPLIT_RESOURCE = "eval_holdout.json"
+SPLIT_RESOURCES = {"train": TRAIN_SPLIT_RESOURCE, "holdout": HOLDOUT_SPLIT_RESOURCE}
+
+
+def _read_split(split: str, path: Path | None) -> dict[str, Any]:
+ if path is not None:
+ text = Path(path).read_text("utf-8")
+ else:
+ try:
+ resource = SPLIT_RESOURCES[split]
+ except KeyError:
+ raise ValueError(
+ f"Unknown ground-truth split {split!r}; expected one of "
+ f"{', '.join(sorted(SPLIT_RESOURCES))} or 'all'."
+ ) from None
+ text = (resources.files("aliens_eye.data") / resource).read_text("utf-8")
+ return json.loads(text)
+
+
+def load_selfcheck_data(
+ split: str = "train",
+ path: Path | None = None,
+) -> dict[str, list[str]]:
+ """Map of site name to usernames known to exist there.
+
+ ``split`` selects the ground-truth file: ``"train"`` (default, the only
+ split training may read), ``"holdout"``, or ``"all"`` to merge both. Pass
+ ``path`` to load an arbitrary ground-truth JSON instead.
+ """
+ if path is None and split == "all":
+ data: dict[str, Any] = {}
+ for name in SPLIT_RESOURCES:
+ data.update(_read_split(name, None))
+ else:
+ data = _read_split(split, path)
return {
site: [value] if isinstance(value, str) else list(value)
for site, value in data.items()
@@ -65,9 +105,10 @@ async def _collect_row(
username: str,
label: int,
logger,
+ fetch=fetch_url,
) -> list[float] | None:
url = format_site_url(site, url_template, username)
- fetch = await fetch_url(session, url, config, rate_limiter, logger)
+ fetch = await fetch(session, url, config, rate_limiter, logger)
if fetch.error:
return None
try:
@@ -94,9 +135,16 @@ async def collect_dataset(
negatives_per_site: int = 2,
concurrency: int = 20,
seed: int | None = None,
+ split: str = "train",
+ ground_truth_path: Path | None = None,
+ fetch=fetch_url,
) -> int:
- """Scan ground-truth positives and random negatives, write a labeled CSV."""
- selfcheck = load_selfcheck_data()
+ """Scan ground-truth positives and random negatives, write a labeled CSV.
+
+ Reads the train split by default. Training must never read the holdout:
+ doing so reintroduces the train/eval leak the split exists to remove.
+ """
+ selfcheck = load_selfcheck_data(split, ground_truth_path)
rng = random.Random(seed)
config = ScannerConfig(retries=1, timeout=10.0)
extractor = FeatureExtractor()
@@ -124,7 +172,7 @@ async def collect_dataset(
async with semaphore:
row = await _collect_row(
session, extractor, rate_limiter, config,
- site, template, username, label, logger,
+ site, template, username, label, logger, fetch,
)
if row is not None:
rows.append(row)
diff --git a/src/aliens_eye/ml/inference.py b/src/aliens_eye/ml/inference.py
index 5c31494..7b145b5 100644
--- a/src/aliens_eye/ml/inference.py
+++ b/src/aliens_eye/ml/inference.py
@@ -13,6 +13,7 @@ from dataclasses import dataclass
from importlib import resources
from pathlib import Path
+from aliens_eye.core.detector import FOUND_THRESHOLD, ML_WEIGHT, NOT_FOUND_THRESHOLD
from aliens_eye.core.features import FEATURE_SCHEMA, vectorize_features
MODEL_RESOURCE = "model.json"
@@ -30,9 +31,9 @@ class MLModel:
coef: list[float]
intercept: float
version: str = "unknown"
- found_threshold: float = 0.6
- not_found_threshold: float = 0.35
- ml_weight: float = 0.4
+ found_threshold: float = FOUND_THRESHOLD
+ not_found_threshold: float = NOT_FOUND_THRESHOLD
+ ml_weight: float = ML_WEIGHT
@classmethod
def from_dict(cls, data: dict) -> MLModel:
@@ -45,9 +46,9 @@ class MLModel:
coef=[float(x) for x in data["coef"]],
intercept=float(data["intercept"]),
version=str(data.get("version", "unknown")),
- found_threshold=float(thresholds.get("found", 0.6)),
- not_found_threshold=float(thresholds.get("not_found", 0.35)),
- ml_weight=float(data.get("ml_weight", 0.4)),
+ found_threshold=float(thresholds.get("found", FOUND_THRESHOLD)),
+ not_found_threshold=float(thresholds.get("not_found", NOT_FOUND_THRESHOLD)),
+ ml_weight=float(data.get("ml_weight", ML_WEIGHT)),
)
except (KeyError, TypeError, ValueError) as exc:
raise ModelError(f"Malformed model file: {exc}") from exc
diff --git a/src/aliens_eye/selfcheck.py b/src/aliens_eye/selfcheck.py
index 3598ed0..2ca17d6 100644
--- a/src/aliens_eye/selfcheck.py
+++ b/src/aliens_eye/selfcheck.py
@@ -1,9 +1,13 @@
"""Validate detection accuracy against accounts known to exist (and to not exist).
-Scans each known ``(site, username)`` positive from data/selfcheck.json plus a few
-random negatives per site, then reports precision / recall / F1 / false-positive
-rate overall and per site. Doubles as a rot detector for sites.json and a
-calibration baseline for the ML model.
+Scans each known ``(site, username)`` positive plus a few random negatives per
+site, then reports precision / recall / F1 / false-positive rate overall and per
+site. Doubles as a rot detector for sites.json and a calibration baseline for
+the ML model.
+
+Ground truth is split site-disjoint (see ``aliens_eye.ml.collect``). Scoring the
+``holdout`` split measures generalization to platforms the model never trained
+on; scoring ``train`` measures fit and will read optimistically high.
"""
from __future__ import annotations
@@ -12,7 +16,7 @@ import asyncio
import json
import random
import string
-from importlib import resources
+from pathlib import Path
from typing import Any
import aiohttp
@@ -29,14 +33,21 @@ from aliens_eye.core.scanner import format_site_url
from aliens_eye.utils.console import get_console
-def load_selfcheck_data() -> dict[str, str]:
- """Map of site name to one username known to exist there."""
- text = (resources.files("aliens_eye.data") / "selfcheck.json").read_text("utf-8")
- data = json.loads(text)
+def load_selfcheck_data(
+ split: str = "train",
+ path: Path | None = None,
+) -> dict[str, str]:
+ """Map of site name to one username known to exist there.
+
+ Thin wrapper over :func:`aliens_eye.ml.collect.load_selfcheck_data` that
+ keeps one username per site; see there for the ``split`` values.
+ """
+ from aliens_eye.ml.collect import load_selfcheck_data as load_ground_truth
+
return {
- site: value if isinstance(value, str) else value[0]
- for site, value in data.items()
- if value
+ site: usernames[0]
+ for site, usernames in load_ground_truth(split, path).items()
+ if usernames
}
@@ -67,24 +78,39 @@ async def run_selfcheck(
negatives: int = 1,
report_format: str | None = None,
seed: int | None = 1234,
+ split: str = "train",
+ ground_truth_path: Path | None = None,
+ fetch=fetch_url,
+ jobs: list[tuple[str, str, str, int]] | None = None,
) -> dict[str, Any]:
"""Run the self-check. Returns a metrics dict; renders a report unless report_format='json'."""
console = get_console()
- selfcheck = load_selfcheck_data()
+ jobs_supplied = jobs is not None
+ selfcheck = load_selfcheck_data(split, ground_truth_path)
extractor = FeatureExtractor()
rate_limiter = DomainRateLimiter()
rng = random.Random(seed)
- # Build (site, username, expected_label) jobs. label 1 = should be Found.
- jobs: list[tuple[str, str, str, int]] = []
- for site, username in selfcheck.items():
- if site not in sites_data:
- continue
- jobs.append((site, sites_data[site], username, 1))
- for _ in range(max(0, negatives)):
- jobs.append((site, sites_data[site], _random_username(rng), 0))
-
- skipped = [site for site in selfcheck if site not in sites_data]
+ # Build (site, url, username, expected_label) jobs. label 1 = should be Found.
+ # A caller replaying a frozen corpus passes `jobs` instead: the corpus is
+ # the evaluation set, and regenerating usernames here would produce rows the
+ # corpus never captured.
+ if jobs is None:
+ jobs = []
+ for site, username in selfcheck.items():
+ if site not in sites_data:
+ continue
+ url = format_site_url(site, sites_data[site], username)
+ jobs.append((site, url, username, 1))
+ for _ in range(max(0, negatives)):
+ negative = _random_username(rng)
+ jobs.append(
+ (site, format_site_url(site, sites_data[site], negative), negative, 0)
+ )
+ skipped = [site for site in selfcheck if site not in sites_data]
+ else:
+ jobs = list(jobs)
+ skipped = []
if skipped and report_format != "json":
console.print(f"[yellow]Skipping sites missing from sites.json: {', '.join(skipped)}[/yellow]")
if not jobs:
@@ -97,10 +123,11 @@ async def run_selfcheck(
connector = aiohttp.TCPConnector(limit=10)
async with aiohttp.ClientSession(headers=DEFAULT_HEADERS, connector=connector) as session:
- async def check(site: str, template: str, username: str, label: int) -> None:
- url = format_site_url(site, template, username)
+ fetch_result = fetch
+
+ async def check(site: str, url: str, username: str, label: int) -> None:
async with semaphore:
- fetch = await fetch_url(session, url, config, rate_limiter, logger)
+ fetch = await fetch_result(session, url, config, rate_limiter, logger)
if fetch.error:
rows.append({"site": site, "username": username, "label": label,
"status": "Error", "confidence": 0, "predicted": None,
@@ -144,6 +171,7 @@ async def run_selfcheck(
overall = _metrics(tp, fp, fn, tn)
errors = sum(1 for r in rows if r["status"] == "Error")
result = {
+ "split": "corpus" if jobs_supplied else ("custom" if ground_truth_path else split),
"overall": overall,
"per_site": {s: _metrics(c["tp"], c["fp"], c["fn"], c["tn"]) for s, c in per_site.items()},
"errors": errors,
diff --git a/tests/test_detector.py b/tests/test_detector.py
index 8117d8a..ed39879 100644
--- a/tests/test_detector.py
+++ b/tests/test_detector.py
@@ -1,10 +1,22 @@
+import json
+from importlib import resources
+
from aliens_eye.core.detector import (
FOUND_THRESHOLD,
+ ML_WEIGHT,
NOT_FOUND_THRESHOLD,
Detector,
_sigmoid,
)
+# The blend weight and thresholds the packaged model actually runs with. These
+# are documented in README.md and WORKING.md; if the shipped model changes,
+# update both docs and this table in the same commit. The fallback constants in
+# core/detector.py are deliberately different and must not be confused for these.
+SHIPPED_MODEL_ML_WEIGHT = 0.6
+SHIPPED_MODEL_FOUND_THRESHOLD = 0.5559
+SHIPPED_MODEL_NOT_FOUND_THRESHOLD = 0.3224
+
def make_found_features():
return {
@@ -114,3 +126,47 @@ def test_sigmoid_bounds():
assert _sigmoid(1000) == 1.0
assert _sigmoid(-1000) < 1e-20
assert abs(_sigmoid(0) - 0.5) < 1e-9
+
+
+def test_shipped_model_matches_documented_values():
+ """Guard the doc/model drift that had WORKING.md stating the wrong blend.
+
+ The shipped model overrides the module fallbacks at runtime, so the docs
+ must describe the model's values, not the constants.
+ """
+ text = (resources.files("aliens_eye.data") / "model.json").read_text("utf-8")
+ data = json.loads(text)
+ assert data["ml_weight"] == SHIPPED_MODEL_ML_WEIGHT
+ assert data["thresholds"]["found"] == SHIPPED_MODEL_FOUND_THRESHOLD
+ assert data["thresholds"]["not_found"] == SHIPPED_MODEL_NOT_FOUND_THRESHOLD
+
+
+def test_shipped_model_actually_overrides_fallbacks(logger):
+ """If these ever coincide, the override path stops being exercised."""
+ detector = Detector()
+ detector.load_model(logger)
+ assert detector.model.ml_weight == SHIPPED_MODEL_ML_WEIGHT != ML_WEIGHT
+ assert detector.model.found_threshold == SHIPPED_MODEL_FOUND_THRESHOLD != FOUND_THRESHOLD
+ assert (
+ detector.model.not_found_threshold
+ == SHIPPED_MODEL_NOT_FOUND_THRESHOLD
+ != NOT_FOUND_THRESHOLD
+ )
+
+
+def test_model_defaults_fall_back_to_detector_constants():
+ """A model file omitting the tuning fields must inherit the constants."""
+ from aliens_eye.core.features import FEATURE_SCHEMA
+ from aliens_eye.ml.inference import MLModel
+
+ n = len(FEATURE_SCHEMA)
+ model = MLModel.from_dict({
+ "feature_schema": list(FEATURE_SCHEMA),
+ "mean": [0.0] * n,
+ "scale": [1.0] * n,
+ "coef": [0.0] * n,
+ "intercept": 0.0,
+ })
+ assert model.ml_weight == ML_WEIGHT
+ assert model.found_threshold == FOUND_THRESHOLD
+ assert model.not_found_threshold == NOT_FOUND_THRESHOLD
diff --git a/tests/test_sites.py b/tests/test_sites.py
index 314354c..5944ee3 100644
--- a/tests/test_sites.py
+++ b/tests/test_sites.py
@@ -59,8 +59,35 @@ def test_nsfw_list_subset_of_sites():
def test_selfcheck_sites_exist():
sites = load_sites_data()
- selfcheck = load_selfcheck_data()
+ selfcheck = load_selfcheck_data("all")
missing = [site for site in selfcheck if site not in sites]
assert not missing, f"selfcheck entries missing from sites.json: {missing}"
for site, usernames in selfcheck.items():
assert usernames, f"{site} has no usernames"
+
+
+def test_ground_truth_splits_are_site_disjoint():
+ """The train split and the eval holdout must share no site.
+
+ This is the guard against the train/eval leak: training reads
+ data/selfcheck.json and evaluation reads data/eval_holdout.json, so any
+ overlap means the detector is scored on platforms it trained on.
+ """
+ train = set(load_selfcheck_data("train"))
+ holdout = set(load_selfcheck_data("holdout"))
+ overlap = train & holdout
+ assert not overlap, f"train/holdout splits overlap on: {sorted(overlap)}"
+ assert train and holdout
+
+
+def test_ground_truth_all_split_is_the_union():
+ train = set(load_selfcheck_data("train"))
+ holdout = set(load_selfcheck_data("holdout"))
+ assert set(load_selfcheck_data("all")) == train | holdout
+
+
+def test_unknown_split_rejected():
+ import pytest
+
+ with pytest.raises(ValueError, match="Unknown ground-truth split"):
+ load_selfcheck_data("nope")