Skip to content

AI powered OSINT engine to discover and map 840+ social media identities across the web

10.9 KB
Raw Blame
"""Build an expanded ground-truth set from external projects, cross-sourced.

Sherlock and WhatsMyName each ship accounts asserted to exist -- Sherlock's
``username_claimed`` (one per site, MIT) and WhatsMyName's ``known`` (several per
site, CC BY-SA 4.0). Together they cover several hundred of this catalogue's
sites, which is the difference between a 43-site evaluation and a 400-site one.

**Those accounts are each project's own tuning set.** Sherlock's rules are
maintained so that ``username_claimed`` comes back Found; WhatsMyName's
``e_string`` markers are chosen against its ``known`` accounts. Scoring a tool on
the accounts its rules were tuned against measures memorisation, not detection.

So provenance is recorded per site and the comparison honours it: a site sourced
from Sherlock is excluded when scoring Sherlock, and likewise for WhatsMyName.
To keep both tools measurable, sites covered by both are alternated between the
two sources rather than all taken from one.

Accounts are **asserted, not verified**. Sherlock in particular uses generic
placeholder handles (``blue``, ``red``) on many sites. Every imported positive
carries ``verified: false`` until a capture pass confirms it resolves to a
profile; see ``verify_positives``.
"""

from __future__ import annotations

import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

from aliens_eye.eval.external import _netloc

CURATED = "curated"
SHERLOCK = "sherlock"
WHATSMYNAME = "whatsmyname"

#: Status codes that actually carry evidence about whether an account exists.
#: Anything else -- 401/403 (auth or bot wall), 429 (rate limited), 5xx, 0
#: (transport failure) -- says the request did not get a real answer. Treating
#: those as evidence of absence conflates a capture failure with a label error:
#: an early version of this check flagged Hacker News accounts that plainly
#: exist, purely because the capture was rate limited.
CONCLUSIVE_STATUSES = {200, 203, 204, 301, 302, 303, 307, 308, 404, 410}

#: Handles used as generic placeholders rather than real accounts. Present in a
#: source's data as test fixtures; they need verification before use as positives.
PLACEHOLDER_HANDLES = {
    "blue", "red", "green", "black", "white", "user", "username", "test",
    "admin", "adam", "john", "jsmith", "example", "demo", "none", "null",
}


@dataclass
class GroundTruthEntry:
    site: str
    usernames: list[str]
    source: str
    verified: bool = False
    notes: dict[str, Any] = field(default_factory=dict)

    def to_dict(self) -> dict[str, Any]:
        payload: dict[str, Any] = {"usernames": list(self.usernames), "source": self.source}
        if self.verified:
            payload["verified"] = True
        if self.notes:
            payload["notes"] = self.notes
        return payload


def _index_catalogue(sites_data: dict[str, str]) -> dict[str, str]:
    """Map host -> our site name, so external entries can be matched to ours."""
    index: dict[str, str] = {}
    for name, template in sites_data.items():
        host = _netloc(template)
        if host and host not in index:
            index[host] = name
    return index


def read_sherlock_accounts(path: Path, catalogue: dict[str, str]) -> dict[str, list[str]]:
    data = json.loads(Path(path).read_text(encoding="utf-8"))
    out: dict[str, list[str]] = {}
    for entry in data.values():
        if not isinstance(entry, dict):
            continue
        url, claimed = entry.get("url"), entry.get("username_claimed")
        if not url or not claimed:
            continue
        site = catalogue.get(_netloc(str(url)))
        if site and site not in out:
            out[site] = [str(claimed)]
    return out


def read_whatsmyname_accounts(path: Path, catalogue: dict[str, str]) -> dict[str, list[str]]:
    data = json.loads(Path(path).read_text(encoding="utf-8"))
    entries = data.get("sites", data if isinstance(data, list) else [])
    out: dict[str, list[str]] = {}
    for entry in entries:
        if not isinstance(entry, dict):
            continue
        uri, known = entry.get("uri_check"), entry.get("known")
        if not uri or not known:
            continue
        site = catalogue.get(_netloc(str(uri)))
        if site and site not in out:
            out[site] = [str(k) for k in known if k]
    return out


def cross_source(
    existing: dict[str, list[str]],
    sherlock: dict[str, list[str]],
    whatsmyname: dict[str, list[str]],
) -> dict[str, GroundTruthEntry]:
    """Assign each new site to exactly one source, alternating on overlap.

    Sites already curated here keep their own provenance and are never
    reassigned: they predate both imports and are usable for scoring either tool.
    """
    entries: dict[str, GroundTruthEntry] = {
        site: GroundTruthEntry(site, list(usernames), CURATED, verified=True)
        for site, usernames in existing.items()
    }

    both = sorted((set(sherlock) & set(whatsmyname)) - set(entries))
    only_sherlock = sorted(set(sherlock) - set(whatsmyname) - set(entries))
    only_wmn = sorted(set(whatsmyname) - set(sherlock) - set(entries))

    # Alternate the overlap so each tool has a comparable number of sites it can
    # be scored on. Deterministic: sorted order, fixed parity.
    for position, site in enumerate(both):
        source = WHATSMYNAME if position % 2 == 0 else SHERLOCK
        usernames = whatsmyname[site] if source == WHATSMYNAME else sherlock[site]
        entries[site] = GroundTruthEntry(site, list(usernames), source)

    for site in only_sherlock:
        entries[site] = GroundTruthEntry(site, list(sherlock[site]), SHERLOCK)
    for site in only_wmn:
        entries[site] = GroundTruthEntry(site, list(whatsmyname[site]), WHATSMYNAME)

    for entry in entries.values():
        placeholders = [u for u in entry.usernames if u.lower() in PLACEHOLDER_HANDLES]
        if placeholders:
            entry.notes["placeholder_handles"] = placeholders
    return entries


def assign_splits(
    entries: dict[str, GroundTruthEntry],
    existing_train: set[str],
    existing_holdout: set[str],
    holdout_fraction: float = 0.3,
) -> dict[str, str]:
    """Site-disjoint train/holdout assignment, preserving existing placements.

    New sites are interleaved by sorted position rather than shuffled, so the
    assignment is reproducible and the holdout is spread across sources instead
    of concentrating in whichever source sorts last.
    """
    splits: dict[str, str] = {}
    for site in entries:
        if site in existing_holdout:
            splits[site] = "holdout"
        elif site in existing_train:
            splits[site] = "train"

    step = max(2, round(1 / holdout_fraction)) if holdout_fraction > 0 else 0
    unassigned = sorted(set(entries) - set(splits))
    for position, site in enumerate(unassigned):
        splits[site] = "holdout" if step and position % step == 0 else "train"
    return splits


def write_splits(
    entries: dict[str, GroundTruthEntry],
    splits: dict[str, str],
    train_path: Path,
    holdout_path: Path,
) -> dict[str, Any]:
    """Write both split files in the provenance-carrying dict form."""
    buckets: dict[str, dict[str, Any]] = {"train": {}, "holdout": {}}
    for site, entry in sorted(entries.items()):
        buckets[splits[site]][site] = entry.to_dict()

    for name, path in (("train", train_path), ("holdout", holdout_path)):
        Path(path).write_text(
            json.dumps(buckets[name], indent=2, ensure_ascii=False, sort_keys=True) + "\n",
            encoding="utf-8",
        )

    def summarise(bucket: dict[str, Any]) -> dict[str, Any]:
        sources: dict[str, int] = {}
        for payload in bucket.values():
            sources[payload["source"]] = sources.get(payload["source"], 0) + 1
        return {
            "sites": len(bucket),
            "accounts": sum(len(p["usernames"]) for p in bucket.values()),
            "by_source": dict(sorted(sources.items())),
            "unverified_sites": sum(1 for p in bucket.values() if not p.get("verified")),
        }

    return {
        "train": summarise(buckets["train"]),
        "holdout": summarise(buckets["holdout"]),
        "total_sites": len(entries),
    }


def verify_positives(
    observations,
    engines: dict[str, Any],
    provenance: dict[str, str],
) -> dict[str, Any]:
    """Flag imported positives that both external rule sets call absent.

    Imported accounts are asserted by their source project, not verified here,
    and Sherlock in particular uses placeholder handles on many sites. A label
    error on a positive is worse than a missing site: it teaches the model that
    an absent-user page is a profile.

    The check is a consensus of signals *independent of the account's own
    source*: a positive is suspect when every other project's rules, applied to
    the captured response, say the account is absent. That is evidence of a bad
    label, not proof -- a rule can be stale -- so suspects are reported for
    review rather than deleted.

    Rows whose only ruling engine is the account's own source are skipped: that
    engine agreeing with its own tuning set proves nothing.
    """
    suspects: list[dict[str, Any]] = []
    confirmed = unrulable = inconclusive = 0

    for obs in observations:
        if obs.error or obs.label != 1:
            continue
        if obs.status_code not in CONCLUSIVE_STATUSES:
            # The capture never got a real answer, so it is not evidence either way.
            inconclusive += 1
            continue
        source = provenance.get(obs.site, CURATED)
        verdicts: dict[str, str] = {}
        for name, engine in engines.items():
            if name == source:
                continue  # circular: this engine was tuned on this account
            verdict = engine.decide(obs.url, obs.final_url or obs.url, obs.status_code, obs.body or "")
            if verdict is not None:
                verdicts[name] = verdict
        if not verdicts:
            unrulable += 1
            continue
        if all(v == "Not Found" for v in verdicts.values()):
            suspects.append({
                "site": obs.site,
                "username": obs.username,
                "url": obs.url,
                "status": obs.status_code,
                "source": source,
                "verdicts": verdicts,
                "placeholder": obs.username.lower() in PLACEHOLDER_HANDLES,
            })
        else:
            confirmed += 1

    by_source: dict[str, int] = {}
    for entry in suspects:
        by_source[entry["source"]] = by_source.get(entry["source"], 0) + 1
    return {
        "checked": confirmed + len(suspects),
        "corroborated": confirmed,
        "suspect": len(suspects),
        "no_independent_rule": unrulable,
        "inconclusive_capture": inconclusive,
        "suspect_by_source": dict(sorted(by_source.items())),
        "suspect_placeholder_handles": sum(1 for s in suspects if s["placeholder"]),
        "suspects": sorted(suspects, key=lambda s: (s["source"], s["site"])),
    }