AI powered OSINT engine to discover and map 840+ social media identities across the web
Python aiartificial-intelligencedebainhackhackerhackinghackingtoolsinformationinformation-gatheringlinuxlinux-desktoposintpythonpython-scriptpython3social-mediasocialmediahackingtermuxtermux-tool
fix: split ground truth site-disjoint to remove train/eval leakage
arxhr007cf7194244a0c
diff --git a/src/aliens_eye/core/detector.py b/src/aliens_eye/core/detector.pyindex f5a568c..67b515d 100644--- a/src/aliens_eye/core/detector.py+++ b/src/aliens_eye/core/detector.py@@ -12,8 +12,12 @@ class DetectionResult:probability: float | None-# Blend weights and probability thresholds, calibrated by out-of-fold grid-# search on the seed dataset and validated against the selfcheck set.+# FALLBACK blend weight and probability thresholds, used only when no model is+# loaded or when a model file omits these fields. A loaded model overrides all+# three (see Detector.predict and MLModel.from_dict) -- the shipped+# data/model.json carries its own out-of-fold grid-searched values, so these+# constants do NOT describe the default runtime behaviour of the packaged tool.+# tests/test_detector.py pins the shipped model's values against the docs.ML_WEIGHT = 0.4HEURISTIC_WEIGHT = 0.6HEURISTIC_SIGMOID_SCALE = 6.0@@ -45,13 +49,24 @@ class Detector:self.model = Nonelogger.debug("ML model unavailable, using heuristics only: %s", exc)- def predict(self, features: dict[str, float]) -> DetectionResult:+ def judges(self, features: dict[str, float]) -> tuple[float, float, float | None]:+ """Both judges, unblended: (heuristic_score, heuristic_prob, ml_prob).++ ml_prob is None when no model is loaded. Exposed so an ablation can+ weigh the two judges differently without reimplementing either -- see+ aliens_eye.eval.ablate. Mutates features["heuristic_score"], which+ the ML model consumes as an input feature.+ """heuristic_score = self._heuristic_score(features)heuristic_prob = _sigmoid(heuristic_score / HEURISTIC_SIGMOID_SCALE)features["heuristic_score"] = heuristic_score+ ml_prob = self.model.predict_proba(features) if self.model is not None else None+ return heuristic_score, heuristic_prob, ml_prob++ def predict(self, features: dict[str, float]) -> DetectionResult:+ heuristic_score, heuristic_prob, ml_prob = self.judges(features)- if self.model is not None:- ml_prob = self.model.predict_proba(features)+ if ml_prob is not None:ml_w = getattr(self.model, "ml_weight", ML_WEIGHT)probability = ml_w * ml_prob + (1.0 - ml_w) * heuristic_probmethod = "ml+heuristic"diff --git a/src/aliens_eye/data/eval_holdout.json b/src/aliens_eye/data/eval_holdout.jsonnew file mode 100644index 0000000..8c9c0aa--- /dev/null+++ b/src/aliens_eye/data/eval_holdout.json@@ -0,0 +1,43 @@+{+ "vimeo": [+ "staff"+ ],+ "duolingo": [+ "luis"+ ],+ "dribbble": [+ "dan"+ ],+ "steamcommunity": [+ "gabelogannewell"+ ],+ "hackernews": [+ "pg",+ "dang"+ ],+ "last.fm": [+ "rj",+ "lastfm"+ ],+ "tumblr": [+ "staff"+ ],+ "linktree": [+ "linktree"+ ],+ "unsplash": [+ "nasa"+ ],+ "codechef": [+ "tourist"+ ],+ "bitbucket": [+ "atlassian"+ ],+ "archive.org": [+ "brewster_kahle"+ ],+ "monkeytype": [+ "miodec"+ ]+}diff --git a/src/aliens_eye/data/selfcheck.json b/src/aliens_eye/data/selfcheck.jsonindex 4cee3e3..526338d 100644--- a/src/aliens_eye/data/selfcheck.json+++ b/src/aliens_eye/data/selfcheck.json@@ -1,45 +1,107 @@{- "github": ["torvalds", "gvanrossum", "mojombo"],- "reddit": ["spez", "kn0thing"],- "gitlab": ["sytses"],- "twitch": ["shroud", "ninja"],- "medium": ["ev"],- "dev.to": ["ben", "jess"],- "npm": ["sindresorhus", "isaacs"],- "pypi": ["dstufft"],- "soundcloud": ["skrillex", "diplo"],- "vimeo": ["staff"],- "imgur": ["sarah"],- "patreon": ["jacksepticeye"],- "telegram": ["durov", "telegram"],- "chess.com": ["hikaru", "magnuscarlsen"],- "lichess": ["thibault", "drnykterstein"],- "hackernews": ["pg", "dang"],- "duolingo": ["luis"],- "dribbble": ["dan"],- "mastodon.social": ["Gargron"],- "steamcommunity": ["gabelogannewell"],- "pinterest": ["pinterest"],- "devrant": ["dfox", "trogus"],- "spotify": ["spotify"],- "kaggle": ["kaggle", "abhishek"],- "wikipedia": ["Jimbo_Wales"],- "youtube user": ["pewdiepie"],- "codepen": ["chriscoyier", "cassiecodes"],- "last.fm": ["rj", "lastfm"],- "wordpressorg": ["matt", "automattic"],- "docker hub": ["docker"],- "scratch": ["Scratch", "griffpatch"],- "tumblr": ["staff"],- "linktree": ["linktree"],- "replit.com": ["replit"],- "unsplash": ["nasa"],- "producthunt": ["ryanhoover"],- "codechef": ["tourist"],- "artstation": ["artstation"],- "bitbucket": ["atlassian"],- "codewars": ["jhoffner"],- "archive.org": ["brewster_kahle"],- "freecodecamp": ["QuincyLarson"],- "monkeytype": ["miodec"]+ "github": [+ "torvalds",+ "gvanrossum",+ "mojombo"+ ],+ "reddit": [+ "spez",+ "kn0thing"+ ],+ "gitlab": [+ "sytses"+ ],+ "twitch": [+ "shroud",+ "ninja"+ ],+ "medium": [+ "ev"+ ],+ "dev.to": [+ "ben",+ "jess"+ ],+ "npm": [+ "sindresorhus",+ "isaacs"+ ],+ "pypi": [+ "dstufft"+ ],+ "soundcloud": [+ "skrillex",+ "diplo"+ ],+ "imgur": [+ "sarah"+ ],+ "patreon": [+ "jacksepticeye"+ ],+ "telegram": [+ "durov",+ "telegram"+ ],+ "chess.com": [+ "hikaru",+ "magnuscarlsen"+ ],+ "lichess": [+ "thibault",+ "drnykterstein"+ ],+ "mastodon.social": [+ "Gargron"+ ],+ "pinterest": [+ "pinterest"+ ],+ "devrant": [+ "dfox",+ "trogus"+ ],+ "spotify": [+ "spotify"+ ],+ "kaggle": [+ "kaggle",+ "abhishek"+ ],+ "wikipedia": [+ "Jimbo_Wales"+ ],+ "youtube user": [+ "pewdiepie"+ ],+ "codepen": [+ "chriscoyier",+ "cassiecodes"+ ],+ "wordpressorg": [+ "matt",+ "automattic"+ ],+ "docker hub": [+ "docker"+ ],+ "scratch": [+ "Scratch",+ "griffpatch"+ ],+ "replit.com": [+ "replit"+ ],+ "producthunt": [+ "ryanhoover"+ ],+ "artstation": [+ "artstation"+ ],+ "codewars": [+ "jhoffner"+ ],+ "freecodecamp": [+ "QuincyLarson"+ ]}diff --git a/src/aliens_eye/ml/collect.py b/src/aliens_eye/ml/collect.pyindex 1fbbfd8..689d546 100644--- a/src/aliens_eye/ml/collect.py+++ b/src/aliens_eye/ml/collect.py@@ -1,9 +1,17 @@"""Build a labeled training dataset by scanning ground-truth accounts.-Positives come from a curated map of sites to usernames known to exist-(data/selfcheck.json). Negatives are randomized usernames that almost-certainly do not exist. Each scan produces one row of FEATURE_SCHEMA values-plus a label column.+Positives come from a curated map of sites to usernames known to exist.+Negatives are randomized usernames that almost certainly do not exist. Each+scan produces one row of FEATURE_SCHEMA values plus a label column.++The ground truth is split **site-disjoint** into two files:++* ``data/selfcheck.json`` -- the train split; the only split training reads.+* ``data/eval_holdout.json`` -- held out; never seen by training.++The split is site-disjoint rather than account-disjoint because the detector+learns per-site page structure: holding out only accounts from a site whose+layout the model already trained on would overstate generalization."""from __future__ import annotations@@ -15,6 +23,7 @@ import randomimport stringfrom importlib import resourcesfrom pathlib import Path+from typing import Anyimport aiohttp@@ -26,11 +35,42 @@ from aliens_eye.core.http import fetch_urlfrom aliens_eye.core.rate_limit import DomainRateLimiterfrom aliens_eye.core.scanner import format_site_url--def load_selfcheck_data() -> dict[str, list[str]]:- """Map of site name to usernames known to exist there."""- text = (resources.files("aliens_eye.data") / "selfcheck.json").read_text("utf-8")- data = json.loads(text)+TRAIN_SPLIT_RESOURCE = "selfcheck.json"+HOLDOUT_SPLIT_RESOURCE = "eval_holdout.json"+SPLIT_RESOURCES = {"train": TRAIN_SPLIT_RESOURCE, "holdout": HOLDOUT_SPLIT_RESOURCE}+++def _read_split(split: str, path: Path | None) -> dict[str, Any]:+ if path is not None:+ text = Path(path).read_text("utf-8")+ else:+ try:+ resource = SPLIT_RESOURCES[split]+ except KeyError:+ raise ValueError(+ f"Unknown ground-truth split {split!r}; expected one of "+ f"{', '.join(sorted(SPLIT_RESOURCES))} or 'all'."+ ) from None+ text = (resources.files("aliens_eye.data") / resource).read_text("utf-8")+ return json.loads(text)+++def load_selfcheck_data(+ split: str = "train",+ path: Path | None = None,+) -> dict[str, list[str]]:+ """Map of site name to usernames known to exist there.++ ``split`` selects the ground-truth file: ``"train"`` (default, the only+ split training may read), ``"holdout"``, or ``"all"`` to merge both. Pass+ ``path`` to load an arbitrary ground-truth JSON instead.+ """+ if path is None and split == "all":+ data: dict[str, Any] = {}+ for name in SPLIT_RESOURCES:+ data.update(_read_split(name, None))+ else:+ data = _read_split(split, path)return {site: [value] if isinstance(value, str) else list(value)for site, value in data.items()@@ -65,9 +105,10 @@ async def _collect_row(username: str,label: int,logger,+ fetch=fetch_url,) -> list[float] | None:url = format_site_url(site, url_template, username)- fetch = await fetch_url(session, url, config, rate_limiter, logger)+ fetch = await fetch(session, url, config, rate_limiter, logger)if fetch.error:return Nonetry:@@ -94,9 +135,16 @@ async def collect_dataset(negatives_per_site: int = 2,concurrency: int = 20,seed: int | None = None,+ split: str = "train",+ ground_truth_path: Path | None = None,+ fetch=fetch_url,) -> int:- """Scan ground-truth positives and random negatives, write a labeled CSV."""- selfcheck = load_selfcheck_data()+ """Scan ground-truth positives and random negatives, write a labeled CSV.++ Reads the train split by default. Training must never read the holdout:+ doing so reintroduces the train/eval leak the split exists to remove.+ """+ selfcheck = load_selfcheck_data(split, ground_truth_path)rng = random.Random(seed)config = ScannerConfig(retries=1, timeout=10.0)extractor = FeatureExtractor()@@ -124,7 +172,7 @@ async def collect_dataset(async with semaphore:row = await _collect_row(session, extractor, rate_limiter, config,- site, template, username, label, logger,+ site, template, username, label, logger, fetch,)if row is not None:rows.append(row)diff --git a/src/aliens_eye/ml/inference.py b/src/aliens_eye/ml/inference.pyindex 5c31494..7b145b5 100644--- a/src/aliens_eye/ml/inference.py+++ b/src/aliens_eye/ml/inference.py@@ -13,6 +13,7 @@ from dataclasses import dataclassfrom importlib import resourcesfrom pathlib import Path+from aliens_eye.core.detector import FOUND_THRESHOLD, ML_WEIGHT, NOT_FOUND_THRESHOLDfrom aliens_eye.core.features import FEATURE_SCHEMA, vectorize_featuresMODEL_RESOURCE = "model.json"@@ -30,9 +31,9 @@ class MLModel:coef: list[float]intercept: floatversion: str = "unknown"- found_threshold: float = 0.6- not_found_threshold: float = 0.35- ml_weight: float = 0.4+ found_threshold: float = FOUND_THRESHOLD+ not_found_threshold: float = NOT_FOUND_THRESHOLD+ ml_weight: float = ML_WEIGHT@classmethoddef from_dict(cls, data: dict) -> MLModel:@@ -45,9 +46,9 @@ class MLModel:coef=[float(x) for x in data["coef"]],intercept=float(data["intercept"]),version=str(data.get("version", "unknown")),- found_threshold=float(thresholds.get("found", 0.6)),- not_found_threshold=float(thresholds.get("not_found", 0.35)),- ml_weight=float(data.get("ml_weight", 0.4)),+ found_threshold=float(thresholds.get("found", FOUND_THRESHOLD)),+ not_found_threshold=float(thresholds.get("not_found", NOT_FOUND_THRESHOLD)),+ ml_weight=float(data.get("ml_weight", ML_WEIGHT)),)except (KeyError, TypeError, ValueError) as exc:raise ModelError(f"Malformed model file: {exc}") from excdiff --git a/src/aliens_eye/selfcheck.py b/src/aliens_eye/selfcheck.pyindex 3598ed0..2ca17d6 100644--- a/src/aliens_eye/selfcheck.py+++ b/src/aliens_eye/selfcheck.py@@ -1,9 +1,13 @@"""Validate detection accuracy against accounts known to exist (and to not exist).-Scans each known ``(site, username)`` positive from data/selfcheck.json plus a few-random negatives per site, then reports precision / recall / F1 / false-positive-rate overall and per site. Doubles as a rot detector for sites.json and a-calibration baseline for the ML model.+Scans each known ``(site, username)`` positive plus a few random negatives per+site, then reports precision / recall / F1 / false-positive rate overall and per+site. Doubles as a rot detector for sites.json and a calibration baseline for+the ML model.++Ground truth is split site-disjoint (see ``aliens_eye.ml.collect``). Scoring the+``holdout`` split measures generalization to platforms the model never trained+on; scoring ``train`` measures fit and will read optimistically high."""from __future__ import annotations@@ -12,7 +16,7 @@ import asyncioimport jsonimport randomimport string-from importlib import resources+from pathlib import Pathfrom typing import Anyimport aiohttp@@ -29,14 +33,21 @@ from aliens_eye.core.scanner import format_site_urlfrom aliens_eye.utils.console import get_console-def load_selfcheck_data() -> dict[str, str]:- """Map of site name to one username known to exist there."""- text = (resources.files("aliens_eye.data") / "selfcheck.json").read_text("utf-8")- data = json.loads(text)+def load_selfcheck_data(+ split: str = "train",+ path: Path | None = None,+) -> dict[str, str]:+ """Map of site name to one username known to exist there.++ Thin wrapper over :func:`aliens_eye.ml.collect.load_selfcheck_data` that+ keeps one username per site; see there for the ``split`` values.+ """+ from aliens_eye.ml.collect import load_selfcheck_data as load_ground_truth+return {- site: value if isinstance(value, str) else value[0]- for site, value in data.items()- if value+ site: usernames[0]+ for site, usernames in load_ground_truth(split, path).items()+ if usernames}@@ -67,24 +78,39 @@ async def run_selfcheck(negatives: int = 1,report_format: str | None = None,seed: int | None = 1234,+ split: str = "train",+ ground_truth_path: Path | None = None,+ fetch=fetch_url,+ jobs: list[tuple[str, str, str, int]] | None = None,) -> dict[str, Any]:"""Run the self-check. Returns a metrics dict; renders a report unless report_format='json'."""console = get_console()- selfcheck = load_selfcheck_data()+ jobs_supplied = jobs is not None+ selfcheck = load_selfcheck_data(split, ground_truth_path)extractor = FeatureExtractor()rate_limiter = DomainRateLimiter()rng = random.Random(seed)- # Build (site, username, expected_label) jobs. label 1 = should be Found.- jobs: list[tuple[str, str, str, int]] = []- for site, username in selfcheck.items():- if site not in sites_data:- continue- jobs.append((site, sites_data[site], username, 1))- for _ in range(max(0, negatives)):- jobs.append((site, sites_data[site], _random_username(rng), 0))-- skipped = [site for site in selfcheck if site not in sites_data]+ # Build (site, url, username, expected_label) jobs. label 1 = should be Found.+ # A caller replaying a frozen corpus passes `jobs` instead: the corpus is+ # the evaluation set, and regenerating usernames here would produce rows the+ # corpus never captured.+ if jobs is None:+ jobs = []+ for site, username in selfcheck.items():+ if site not in sites_data:+ continue+ url = format_site_url(site, sites_data[site], username)+ jobs.append((site, url, username, 1))+ for _ in range(max(0, negatives)):+ negative = _random_username(rng)+ jobs.append(+ (site, format_site_url(site, sites_data[site], negative), negative, 0)+ )+ skipped = [site for site in selfcheck if site not in sites_data]+ else:+ jobs = list(jobs)+ skipped = []if skipped and report_format != "json":console.print(f"[yellow]Skipping sites missing from sites.json: {', '.join(skipped)}[/yellow]")if not jobs:@@ -97,10 +123,11 @@ async def run_selfcheck(connector = aiohttp.TCPConnector(limit=10)async with aiohttp.ClientSession(headers=DEFAULT_HEADERS, connector=connector) as session:- async def check(site: str, template: str, username: str, label: int) -> None:- url = format_site_url(site, template, username)+ fetch_result = fetch++ async def check(site: str, url: str, username: str, label: int) -> None:async with semaphore:- fetch = await fetch_url(session, url, config, rate_limiter, logger)+ fetch = await fetch_result(session, url, config, rate_limiter, logger)if fetch.error:rows.append({"site": site, "username": username, "label": label,"status": "Error", "confidence": 0, "predicted": None,@@ -144,6 +171,7 @@ async def run_selfcheck(overall = _metrics(tp, fp, fn, tn)errors = sum(1 for r in rows if r["status"] == "Error")result = {+ "split": "corpus" if jobs_supplied else ("custom" if ground_truth_path else split),"overall": overall,"per_site": {s: _metrics(c["tp"], c["fp"], c["fn"], c["tn"]) for s, c in per_site.items()},"errors": errors,diff --git a/tests/test_detector.py b/tests/test_detector.pyindex 8117d8a..ed39879 100644--- a/tests/test_detector.py+++ b/tests/test_detector.py@@ -1,10 +1,22 @@+import json+from importlib import resources+from aliens_eye.core.detector import (FOUND_THRESHOLD,+ ML_WEIGHT,NOT_FOUND_THRESHOLD,Detector,_sigmoid,)+# The blend weight and thresholds the packaged model actually runs with. These+# are documented in README.md and WORKING.md; if the shipped model changes,+# update both docs and this table in the same commit. The fallback constants in+# core/detector.py are deliberately different and must not be confused for these.+SHIPPED_MODEL_ML_WEIGHT = 0.6+SHIPPED_MODEL_FOUND_THRESHOLD = 0.5559+SHIPPED_MODEL_NOT_FOUND_THRESHOLD = 0.3224+def make_found_features():return {@@ -114,3 +126,47 @@ def test_sigmoid_bounds():assert _sigmoid(1000) == 1.0assert _sigmoid(-1000) < 1e-20assert abs(_sigmoid(0) - 0.5) < 1e-9+++def test_shipped_model_matches_documented_values():+ """Guard the doc/model drift that had WORKING.md stating the wrong blend.++ The shipped model overrides the module fallbacks at runtime, so the docs+ must describe the model's values, not the constants.+ """+ text = (resources.files("aliens_eye.data") / "model.json").read_text("utf-8")+ data = json.loads(text)+ assert data["ml_weight"] == SHIPPED_MODEL_ML_WEIGHT+ assert data["thresholds"]["found"] == SHIPPED_MODEL_FOUND_THRESHOLD+ assert data["thresholds"]["not_found"] == SHIPPED_MODEL_NOT_FOUND_THRESHOLD+++def test_shipped_model_actually_overrides_fallbacks(logger):+ """If these ever coincide, the override path stops being exercised."""+ detector = Detector()+ detector.load_model(logger)+ assert detector.model.ml_weight == SHIPPED_MODEL_ML_WEIGHT != ML_WEIGHT+ assert detector.model.found_threshold == SHIPPED_MODEL_FOUND_THRESHOLD != FOUND_THRESHOLD+ assert (+ detector.model.not_found_threshold+ == SHIPPED_MODEL_NOT_FOUND_THRESHOLD+ != NOT_FOUND_THRESHOLD+ )+++def test_model_defaults_fall_back_to_detector_constants():+ """A model file omitting the tuning fields must inherit the constants."""+ from aliens_eye.core.features import FEATURE_SCHEMA+ from aliens_eye.ml.inference import MLModel++ n = len(FEATURE_SCHEMA)+ model = MLModel.from_dict({+ "feature_schema": list(FEATURE_SCHEMA),+ "mean": [0.0] * n,+ "scale": [1.0] * n,+ "coef": [0.0] * n,+ "intercept": 0.0,+ })+ assert model.ml_weight == ML_WEIGHT+ assert model.found_threshold == FOUND_THRESHOLD+ assert model.not_found_threshold == NOT_FOUND_THRESHOLDdiff --git a/tests/test_sites.py b/tests/test_sites.pyindex 314354c..5944ee3 100644--- a/tests/test_sites.py+++ b/tests/test_sites.py@@ -59,8 +59,35 @@ def test_nsfw_list_subset_of_sites():def test_selfcheck_sites_exist():sites = load_sites_data()- selfcheck = load_selfcheck_data()+ selfcheck = load_selfcheck_data("all")missing = [site for site in selfcheck if site not in sites]assert not missing, f"selfcheck entries missing from sites.json: {missing}"for site, usernames in selfcheck.items():assert usernames, f"{site} has no usernames"+++def test_ground_truth_splits_are_site_disjoint():+ """The train split and the eval holdout must share no site.++ This is the guard against the train/eval leak: training reads+ data/selfcheck.json and evaluation reads data/eval_holdout.json, so any+ overlap means the detector is scored on platforms it trained on.+ """+ train = set(load_selfcheck_data("train"))+ holdout = set(load_selfcheck_data("holdout"))+ overlap = train & holdout+ assert not overlap, f"train/holdout splits overlap on: {sorted(overlap)}"+ assert train and holdout+++def test_ground_truth_all_split_is_the_union():+ train = set(load_selfcheck_data("train"))+ holdout = set(load_selfcheck_data("holdout"))+ assert set(load_selfcheck_data("all")) == train | holdout+++def test_unknown_split_rejected():+ import pytest++ with pytest.raises(ValueError, match="Unknown ground-truth split"):+ load_selfcheck_data("nope")