Skip to content

AI powered OSINT engine to discover and map 840+ social media identities across the web

11.1 KB
Raw Blame
"""Validate detection accuracy against accounts known to exist (and to not exist).

Scans each known ``(site, username)`` positive plus a few random negatives per
site, then reports precision / recall / F1 / false-positive rate overall and per
site. Doubles as a rot detector for sites.json and a calibration baseline for
the ML model.

Ground truth is split site-disjoint (see ``aliens_eye.ml.collect``). Scoring the
``holdout`` split measures generalization to platforms the model never trained
on; scoring ``train`` measures fit and will read optimistically high.
"""

from __future__ import annotations

import asyncio
import json
import random
import string
from pathlib import Path
from typing import Any

from rich.panel import Panel
from rich.table import Table
from rich.text import Text

from aliens_eye.core.analyzer import FeatureExtractor
from aliens_eye.core.config import ScannerConfig
from aliens_eye.core.control import PageShape, control_username, page_shape, same_page
from aliens_eye.core.detector import Detector
from aliens_eye.core.http import fetch_url
from aliens_eye.core.rate_limit import DomainRateLimiter
from aliens_eye.core.scanner import build_session, format_site_url, overrule_found
from aliens_eye.utils.console import get_console


def load_selfcheck_data(
    split: str = "train",
    path: Path | None = None,
) -> dict[str, str]:
    """Map of site name to one username known to exist there.

    Thin wrapper over :func:`aliens_eye.ml.collect.load_selfcheck_data` that
    keeps one username per site; see there for the ``split`` values.
    """
    from aliens_eye.ml.collect import load_selfcheck_data as load_ground_truth

    return {
        site: usernames[0]
        for site, usernames in load_ground_truth(split, path).items()
        if usernames
    }


def _random_username(rng: random.Random) -> str:
    length = rng.randint(14, 20)
    return "".join(rng.choices(string.ascii_lowercase + string.digits, k=length))


def _metrics(tp: int, fp: int, fn: int, tn: int) -> dict[str, float]:
    precision = tp / (tp + fp) if (tp + fp) else 0.0
    recall = tp / (tp + fn) if (tp + fn) else 0.0
    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
    fpr = fp / (fp + tn) if (fp + tn) else 0.0
    return {
        "precision": round(precision, 4),
        "recall": round(recall, 4),
        "f1": round(f1, 4),
        "false_positive_rate": round(fpr, 4),
        "tp": tp, "fp": fp, "fn": fn, "tn": tn,
    }


async def run_selfcheck(
    sites_data: dict[str, str],
    detector: Detector,
    config: ScannerConfig,
    logger,
    negatives: int = 1,
    report_format: str | None = None,
    seed: int | None = 1234,
    split: str = "train",
    ground_truth_path: Path | None = None,
    fetch=fetch_url,
    jobs: list[tuple[str, str, str, int]] | None = None,
    detector_only: bool = False,
) -> dict[str, Any]:
    """Run the self-check. Returns a metrics dict; renders a report unless report_format='json'.

    By default a row is scored the way a scan reports it: the detector's verdict,
    then the checks that can overrule a Found (an HTTP error, a challenge page,
    the same page as a user who doesn't exist). ``detector_only`` scores the
    detector's verdict alone, which is what model training compares.
    """
    console = get_console()
    jobs_supplied = jobs is not None
    selfcheck = load_selfcheck_data(split, ground_truth_path)
    extractor = FeatureExtractor()
    rate_limiter = DomainRateLimiter()
    rng = random.Random(seed)

    # Build (site, url, username, expected_label) jobs. label 1 = should be Found.
    # A caller replaying a frozen corpus passes `jobs` instead: the corpus is
    # the evaluation set, and regenerating usernames here would produce rows the
    # corpus never captured.
    if jobs is None:
        jobs = []
        for site, username in selfcheck.items():
            if site not in sites_data:
                continue
            url = format_site_url(site, sites_data[site], username)
            jobs.append((site, url, username, 1))
            for _ in range(max(0, negatives)):
                negative = _random_username(rng)
                jobs.append(
                    (site, format_site_url(site, sites_data[site], negative), negative, 0)
                )
        skipped = [site for site in selfcheck if site not in sites_data]
    else:
        jobs = list(jobs)
        skipped = []
    if skipped and report_format != "json":
        console.print(f"[yellow]Skipping sites missing from sites.json: {', '.join(skipped)}[/yellow]")
    if not jobs:
        if report_format != "json":
            console.print("[red]No self-check sites available.[/red]")
        return {"overall": _metrics(0, 0, 0, 0), "per_site": {}, "errors": 0}

    rows: list[dict] = []
    semaphore = asyncio.Semaphore(min(10, len(jobs)))
    controls: dict[str, asyncio.Task] = {}
    async with build_session(config, 10) as session:

        fetch_result = fetch

        async def fetch_control(site: str) -> PageShape | None:
            """What ``site`` shows for a username that does not exist."""
            recorded = getattr(fetch_result, "control_for", None)
            name, url = control_username(), None
            if recorded is not None:
                name, url = recorded(site) or (name, None)
            if url is None:
                if site not in sites_data:
                    return None
                url = format_site_url(site, sites_data[site], name)
            try:
                async with semaphore:
                    page = await fetch_result(session, url, config, rate_limiter, logger)
            except Exception:  # noqa: BLE001 - no control means no second opinion
                return None
            return None if page.error else page_shape(page.content, page.final_url, name)

        async def check(site: str, url: str, username: str, label: int) -> None:
            async with semaphore:
                fetch = await fetch_result(session, url, config, rate_limiter, logger)
            if fetch.error:
                rows.append({"site": site, "username": username, "label": label,
                             "status": "Error", "confidence": 0, "predicted": None,
                             "note": fetch.error[:60]})
                return
            bundle = extractor.extract(
                fetch.content, fetch.final_url, username, site,
                fetch.status, fetch.response_time, fetch.headers, fetch.redirect_count,
            )
            bundle.features["fingerprint_match_found"] = 0.0
            bundle.features["fingerprint_match_not_found"] = 0.0
            detection = detector.predict(bundle.features)
            status, note = detection.status, detection.method
            if status == "Found" and not detector_only:
                async def same_as_missing_user() -> bool:
                    if site not in controls:     # one fetch per site, shared by its rows
                        controls[site] = asyncio.ensure_future(fetch_control(site))
                    control = await controls[site]
                    return control is not None and same_page(
                        page_shape(fetch.content, fetch.final_url, username), control
                    )

                overruled = await overrule_found(
                    fetch.status, bundle.signals,
                    same_as_missing_user if config.control_check else None,
                )
                if overruled is not None:
                    status, note = overruled[0], f"{note}-{overruled[1]}"
            rows.append({"site": site, "username": username, "label": label,
                         "status": status, "confidence": detection.confidence,
                         "predicted": 1 if status == "Found" else 0,
                         "note": note})

        await asyncio.gather(*(check(*j) for j in jobs))

    # Aggregate confusion matrix overall + per site (ignoring network errors).
    per_site: dict[str, dict[str, int]] = {}
    tp = fp = fn = tn = 0
    for r in rows:
        if r["predicted"] is None:
            continue
        cell = per_site.setdefault(r["site"], {"tp": 0, "fp": 0, "fn": 0, "tn": 0})
        if r["label"] == 1 and r["predicted"] == 1:
            key = "tp"
        elif r["label"] == 1 and r["predicted"] == 0:
            key = "fn"
        elif r["label"] == 0 and r["predicted"] == 1:
            key = "fp"
        else:
            key = "tn"
        cell[key] += 1
        tp += key == "tp"
        fp += key == "fp"
        fn += key == "fn"
        tn += key == "tn"

    overall = _metrics(tp, fp, fn, tn)
    errors = sum(1 for r in rows if r["status"] == "Error")
    result = {
        "split": "corpus" if jobs_supplied else ("custom" if ground_truth_path else split),
        "overall": overall,
        "per_site": {s: _metrics(c["tp"], c["fp"], c["fn"], c["tn"]) for s, c in per_site.items()},
        "errors": errors,
        "samples": len([r for r in rows if r["predicted"] is not None]),
    }

    if report_format == "json":
        print(json.dumps(result, indent=2))
        return result

    _render(console, rows, result)
    return result


def _render(console, rows: list[dict], result: dict[str, Any]) -> None:
    rows.sort(key=lambda r: (r["label"] == r.get("predicted"), r["site"]))
    table = Table(title="Self-check: known accounts + random negatives", header_style="bold blue")
    table.add_column("Site", style="yellow")
    table.add_column("Username")
    table.add_column("Expect")
    table.add_column("Detected")
    table.add_column("Confidence", justify="right")
    table.add_column("Result")
    for r in rows:
        expect = "exists" if r["label"] == 1 else "absent"
        if r["predicted"] is None:
            outcome = Text("ERROR", style="dim red")
        else:
            correct = r["label"] == r["predicted"]
            outcome = Text("PASS" if correct else "FAIL", style="bold green" if correct else "bold red")
        table.add_row(
            r["site"], r["username"], expect,
            Text(r["status"], style="green" if r["status"] == "Found" else "red"),
            f"{r['confidence']}%", outcome,
        )
    console.print(table)

    # Per-site metrics table.
    site_table = Table(title="Per-site metrics", header_style="bold blue")
    site_table.add_column("Site", style="yellow")
    for col in ("Precision", "Recall", "F1", "FPR"):
        site_table.add_column(col, justify="right")
    for site, m in sorted(result["per_site"].items()):
        site_table.add_row(
            site, f"{m['precision']:.0%}", f"{m['recall']:.0%}",
            f"{m['f1']:.2f}", f"{m['false_positive_rate']:.0%}",
        )
    console.print(site_table)

    o = result["overall"]
    f1 = o["f1"]
    style = "green" if f1 >= 0.8 else "yellow" if f1 >= 0.6 else "red"
    console.print(
        Panel(
            f"[{style}]Precision {o['precision']:.0%}  Recall {o['recall']:.0%}  "
            f"F1 {o['f1']:.2f}  FPR {o['false_positive_rate']:.0%}[/{style}]  "
            f"[dim]({result['samples']} samples"
            + (f", {result['errors']} network errors" if result["errors"] else "")
            + ")[/dim]",
            border_style=style,
        )
    )