AI powered OSINT engine to discover and map 840+ social media identities across the web
Python aiartificial-intelligencedebainhackhackerhackinghackingtoolsinformationinformation-gatheringlinuxlinux-desktoposintpythonpython-scriptpython3social-mediasocialmediahackingtermuxtermux-tool
"""Capture live responses into a frozen corpus.
The recorder wraps :func:`aliens_eye.core.http.fetch_url` rather than reimplementing
it, so captured responses go through the same retry, backoff, rate-limit and proxy
behaviour as a normal scan. Whatever the scanner would have seen is what lands on
disk.
"""
from __future__ import annotations
import asyncio
import random
import string
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import aiohttp
from aliens_eye import __version__
from aliens_eye.core.config import ScannerConfig
from aliens_eye.core.http import FetchResult, fetch_url
from aliens_eye.core.rate_limit import DomainRateLimiter
from aliens_eye.core.scanner import build_session, format_site_url
from .store import CorpusRecord, CorpusStore
# Ordinary-looking handle morphology, for negatives that are not trivially
# distinguishable from real usernames. A negative pool of random 16-character
# strings is an artificially easy class: a detector can separate it on length and
# character distribution alone, which inflates every metric.
_ADJECTIVES = [
"quiet", "amber", "rapid", "hollow", "silver", "brave", "narrow", "lunar",
"cobalt", "gentle", "rustic", "vivid", "solar", "hidden", "copper", "swift",
]
_NOUNS = [
"falcon", "harbor", "meadow", "lantern", "cipher", "willow", "canyon", "ember",
"pixel", "otter", "vector", "thicket", "beacon", "marlin", "quartz", "atlas",
]
def random_negative(rng: random.Random) -> str:
"""A high-entropy handle that is very unlikely to exist anywhere."""
length = rng.randint(14, 20)
return "".join(rng.choices(string.ascii_lowercase + string.digits, k=length))
def plausible_negative(rng: random.Random) -> str:
"""A handle with realistic morphology, which still probably does not exist.
Plausible negatives are *not* verified absent by construction -- on large
platforms one may collide with a real account. Records tag their negative
kind so a later labelling pass can check them; see WORKING.md.
"""
word = f"{rng.choice(_ADJECTIVES)}{rng.choice(_NOUNS)}"
style = rng.randint(0, 3)
if style == 0:
return word
if style == 1:
return f"{word}{rng.randint(1, 9999)}"
if style == 2:
return f"{word}_{rng.randint(10, 99)}"
return f"{word[: len(word) // 2]}_{word[len(word) // 2 :]}"
class RecordingFetcher:
"""A ``fetch_url``-compatible callable that persists what it fetches.
Context (site / username / label) cannot be recovered from the URL alone, so
the driver registers it up front with :meth:`register`. That keeps the call
signature identical to ``fetch_url``, which is what makes the recorder
droppable into the scanner without touching the scan path.
"""
def __init__(self, store: CorpusStore, tool_version: str = __version__) -> None:
self.store = store
self.tool_version = tool_version
self._context: dict[str, dict[str, Any]] = {}
self._records: list[CorpusRecord] = []
self._lock = asyncio.Lock()
self.flushed = 0
def register(
self,
url: str,
site: str,
username: str,
label: int | None = None,
notes: dict[str, Any] | None = None,
) -> None:
self._context[url] = {
"site": site,
"username": username,
"label": label,
"notes": notes or {},
}
@property
def records(self) -> list[CorpusRecord]:
return list(self._records)
async def __call__(
self,
session: aiohttp.ClientSession,
url: str,
config: ScannerConfig,
rate_limiter: DomainRateLimiter,
logger,
) -> FetchResult:
result = await fetch_url(session, url, config, rate_limiter, logger)
context = self._context.get(url, {})
async with self._lock:
digest = self.store.write_body(result.content)
self._records.append(
CorpusRecord(
url=url,
site=str(context.get("site", "")),
username=str(context.get("username", "")),
label=context.get("label"),
final_url=result.final_url,
status=result.status,
headers=dict(result.headers),
body_sha256=digest,
body_chars=len(result.content),
response_time=result.response_time,
redirect_count=result.redirect_count,
error=result.error,
captured_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
tool_version=self.tool_version,
notes=dict(context.get("notes") or {}),
)
)
return result
def flush(self) -> int:
"""Append buffered records to disk in a stable order. Returns the count."""
records = sorted(self._records, key=lambda r: (r.site, r.label or 0, r.username))
self.store.append_records(records)
count = len(records)
self._records.clear()
self.flushed += count
return count
async def maybe_flush(self, every: int) -> int:
"""Flush once the buffer reaches every records.
A long capture buffered entirely in memory loses everything if the run
dies partway; incremental flushing bounds that loss. Records are sorted
within each flush rather than globally, so the file is grouped rather
than fully ordered -- readers index by URL, and the corpus is a set, not
a sequence.
"""
if every <= 0 or len(self._records) < every:
return 0
async with self._lock:
return self.flush()
def build_jobs(
sites_data: dict[str, str],
ground_truth: dict[str, list[str]],
negatives_per_site: int,
rng: random.Random,
plausible_ratio: float = 0.5,
) -> list[tuple[str, str, str, int, dict[str, Any]]]:
"""Expand ground truth into ``(site, url, username, label, notes)`` jobs."""
jobs: list[tuple[str, str, str, int, dict[str, Any]]] = []
for site in sorted(ground_truth):
template = sites_data.get(site)
if not template:
continue
for username in ground_truth[site]:
jobs.append(
(site, format_site_url(site, template, username), username, 1,
{"positive_kind": "ground_truth"})
)
n_plausible = int(round(negatives_per_site * plausible_ratio))
for index in range(negatives_per_site):
if index < n_plausible:
username = plausible_negative(rng)
kind = "plausible"
else:
username = random_negative(rng)
kind = "random"
jobs.append(
(site, format_site_url(site, template, username), username, 0,
{"negative_kind": kind})
)
return jobs
async def record_corpus(
sites_data: dict[str, str],
ground_truth: dict[str, list[str]],
out_dir: Path,
logger,
negatives_per_site: int = 4,
concurrency: int = 20,
seed: int = 1234,
split: str = "all",
plausible_ratio: float = 0.5,
config: ScannerConfig | None = None,
flush_every: int = 100,
skip_urls: set[str] | None = None,
) -> dict[str, Any]:
"""Capture every ground-truth positive plus generated negatives to ``out_dir``."""
store = CorpusStore(Path(out_dir))
store.init()
rng = random.Random(seed)
config = config or ScannerConfig(retries=2, timeout=15.0)
rate_limiter = DomainRateLimiter()
recorder = RecordingFetcher(store)
jobs = build_jobs(sites_data, ground_truth, negatives_per_site, rng, plausible_ratio)
skipped = sorted(set(ground_truth) - set(sites_data))
if skip_urls:
# Resume: negatives are seed-derived, so regenerating the job list
# reproduces the same URLs and already-captured ones can be dropped.
jobs = [job for job in jobs if job[1] not in skip_urls]
for site, url, username, label, notes in jobs:
recorder.register(url, site, username, label, notes)
conn_limit = max(1, min(concurrency, len(jobs) or 1))
semaphore = asyncio.Semaphore(conn_limit)
async with build_session(config, conn_limit) as session:
async def run(url: str) -> None:
async with semaphore:
try:
await recorder(session, url, config, rate_limiter, logger)
except Exception as exc: # noqa: BLE001 - one bad host must not abort a capture
logger.debug("Capture failed for %s: %s", url, exc)
written = await recorder.maybe_flush(flush_every)
if written:
logger.info("Captured %d records so far", recorder.flushed)
await asyncio.gather(*(run(job[1]) for job in jobs))
written = recorder.flush()
total_written = recorder.flushed
# Read back from disk, not from the buffer: incremental flushing clears
# recorder.records as it goes, so summarising the buffer would describe only
# whatever happened to remain in the final batch.
persisted = list(store.iter_records())
errors = sum(1 for r in persisted if r.error)
manifest = {
"tool_version": __version__,
"created_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"split": split,
"seed": seed,
"negatives_per_site": negatives_per_site,
"plausible_ratio": plausible_ratio,
"requested": len(jobs),
"records": total_written,
"records_final_flush": written,
"errors": errors,
"sites": sorted({r.site for r in persisted if r.site}),
"skipped_sites": skipped,
}
store.write_manifest(manifest)
logger.info("Captured %d records (%d errors) to %s", total_written, errors, out_dir)
return manifest