Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

9.0 KB
Raw Blame
"""Reversible pseudonymization of evidence before it leaves the process.

Phase 1.4 of the world-class program. The README promises "no data
exfiltration", yet the default path sends raw evidence to a third-party LLM.
This module pseudonymizes the *customer's* identifying data — internal IPs,
emails, file paths, secrets, internal hostnames, usernames — through a per-run,
per-tenant, in-memory-only bidirectional map. The LLM reasons over ``USER_1``,
``HOST_2``, ``IP_3``; the ledger and console re-hydrate the real values locally
via :meth:`Pseudonymizer.rehydrate`.

Public threat indicators (external domains/IPs) are intentionally *not*
redacted by default: they are IOCs, not customer PII, and the agent needs them
to reason. This is a deliberate, documented trade-off (see docs/trust/).

Pure/synchronous, stdlib only, so it is unit-testable offline and gated: a
golden-corpus test asserts zero raw customer PII survives redaction.
"""

from __future__ import annotations

import ipaddress
import re
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any

__all__ = ["RedactionConfig", "Pseudonymizer", "default_pseudonymizer"]

# Field-name hints whose *values* are treated as usernames when redacting
# structured evidence.
_USER_FIELD_HINTS = frozenset({"user", "username", "user_name", "account", "actor", "principal", "samaccountname", "upn", "subject"})

_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")
# Windows (C:\...), UNC (\\host\share), and multi-segment unix paths.
_WIN_PATH_RE = re.compile(r"[A-Za-z]:\\[^\s\"']+")
_UNC_PATH_RE = re.compile(r"\\\\[^\s\"']+")
_UNIX_PATH_RE = re.compile(r"(?:/[A-Za-z0-9._\-]+){2,}/?")
# DOMAIN\user (down-level logon name).
_DOMAIN_USER_RE = re.compile(r"\b[A-Za-z0-9.\-]+\\[A-Za-z0-9._\-]+")

# A bare username in prose, found by the phrase that introduces it. The
# module docstring says usernames are pseudonymized, and until this they
# only were in the `DOMAIN\user` form or under a user-ish key: an alert
# narrative saying "running as priya.raghavan" sent the name to a hosted
# model in the clear, which a call-path test caught and a unit test of the
# redactor never could.
#
# Cue-anchored on purpose. A pattern broad enough to catch any bare
# identifier would swallow process names, rule names and ATT&CK ids, and a
# redactor that mangles the evidence gets switched off.
_USER_CUE_RE = re.compile(
    r"\b(?P<cue>user|username|account|actor|principal|logged in as|running as|owned by|"
    # The negative lookahead refuses a token this pass has already emitted.
    # Without it, `User ACME\\alice` becomes `User USER_1` on the DOMAIN\\user
    # pass and then `User USER_2` here, with USER_2 mapping to the string
    # "USER_1", so rehydrate returns a token instead of the name. The
    # round-trip test caught it.
    r"authenticated as|sign-?in by)\b(?P<sep>[:\s=]+)"
    r"(?!(?:USER|HOST|IP|EMAIL|PATH|SECRET)_\d+\b)"
    r"(?P<name>[A-Za-z][A-Za-z0-9._\-]{2,63})\b",
    re.IGNORECASE,
)
# Common secret shapes: AWS keys, OpenAI-style, bearer/JWT, private key headers.
_SECRET_RES: tuple[re.Pattern[str], ...] = (
    re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
    re.compile(r"\b(?:sk|rk)-[A-Za-z0-9]{20,}\b"),
    re.compile(r"\bghp_[A-Za-z0-9]{36}\b"),
    re.compile(r"\beyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\b"),  # JWT
    re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
)
_IPV4_RE = re.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}\b")
# FQDNs; only redacted when the suffix is in the internal set.
_FQDN_RE = re.compile(r"\b(?:[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,}\b")

_DEFAULT_INTERNAL_SUFFIXES = (".local", ".internal", ".corp", ".lan", ".intranet", ".home.arpa")


@dataclass(frozen=True)
class RedactionConfig:
    """What to pseudonymize. Defaults are on (fail-safe for the no-exfil claim)."""

    redact_internal_ips: bool = True
    redact_emails: bool = True
    redact_paths: bool = True
    redact_secrets: bool = True
    redact_internal_hostnames: bool = True
    redact_usernames: bool = True
    # Extra domain suffixes considered internal (customer-configurable).
    internal_domain_suffixes: tuple[str, ...] = _DEFAULT_INTERNAL_SUFFIXES


def _is_internal_ip(value: str) -> bool:
    try:
        ip = ipaddress.ip_address(value)
    except ValueError:
        return False
    return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved


class Pseudonymizer:
    """Per-run, per-tenant reversible entity map held only in memory."""

    def __init__(self, *, tenant_id: str = "", config: RedactionConfig | None = None) -> None:
        self.tenant_id = tenant_id
        self.config = config or RedactionConfig()
        self._to_token: dict[str, str] = {}
        self._to_original: dict[str, str] = {}
        self._counters: dict[str, int] = {}

    # -- public API ------------------------------------------------------------

    @property
    def mapping(self) -> dict[str, str]:
        """token -> original, for local (never-egress) re-hydration."""
        return dict(self._to_original)

    def redact(self, text: str) -> str:
        if not text or not isinstance(text, str):
            return text if isinstance(text, str) else ""
        out = text
        # Order matters: secrets/emails/paths/domain-user before bare IPs/hosts,
        # so a token like EMAIL_1 is never re-matched by a later pattern.
        if self.config.redact_secrets:
            for pat in _SECRET_RES:
                out = pat.sub(lambda m: self._token("SECRET", m.group(0)), out)
        if self.config.redact_emails:
            out = _EMAIL_RE.sub(lambda m: self._token("EMAIL", m.group(0)), out)
        # Paths before DOMAIN\user: a Windows path (C:\Users\alice\...) contains
        # backslash-separated segments that would otherwise be mis-matched as a
        # down-level logon name and fragment the path.
        if self.config.redact_paths:
            out = _WIN_PATH_RE.sub(lambda m: self._token("PATH", m.group(0)), out)
            out = _UNC_PATH_RE.sub(lambda m: self._token("PATH", m.group(0)), out)
            out = _UNIX_PATH_RE.sub(lambda m: self._token("PATH", m.group(0)), out)
        if self.config.redact_usernames:
            out = _DOMAIN_USER_RE.sub(lambda m: self._token("USER", m.group(0)), out)
            # Cue-anchored bare usernames. After the email and path passes,
            # so `running as [email protected]` is already EMAIL_1 and is not
            # re-matched here.
            out = _USER_CUE_RE.sub(
                lambda m: f"{m.group('cue')}{m.group('sep')}{self._token('USER', m.group('name'))}",
                out,
            )
        if self.config.redact_internal_hostnames:
            out = _FQDN_RE.sub(self._maybe_internal_host, out)
        if self.config.redact_internal_ips:
            out = _IPV4_RE.sub(self._maybe_internal_ip, out)
        return out

    def redact_value(self, value: Any, *, _key: str | None = None) -> Any:
        """Recursively redact a JSON-like value. Values under user-ish keys are
        pseudonymized as usernames even if they are plain identifiers."""
        if isinstance(value, str):
            if _key and self.config.redact_usernames and _key.lower() in _USER_FIELD_HINTS and "@" not in value:
                return self._token("USER", value)
            return self.redact(value)
        if isinstance(value, Mapping):
            return {k: self.redact_value(v, _key=str(k)) for k, v in value.items()}
        if isinstance(value, Sequence) and not isinstance(value, str | bytes):
            return [self.redact_value(v) for v in value]
        return value

    def rehydrate(self, text: str) -> str:
        """Reverse redaction locally (never send the result to the LLM)."""
        if not text:
            return text
        out = text
        # Replace longer tokens first to avoid USER_1 clobbering USER_10.
        for token in sorted(self._to_original, key=len, reverse=True):
            out = out.replace(token, self._to_original[token])
        return out

    # -- internals -------------------------------------------------------------

    def _token(self, kind: str, original: str) -> str:
        if original in self._to_token:
            return self._to_token[original]
        self._counters[kind] = self._counters.get(kind, 0) + 1
        token = f"{kind}_{self._counters[kind]}"
        self._to_token[original] = token
        self._to_original[token] = original
        return token

    def _maybe_internal_ip(self, m: re.Match[str]) -> str:
        val = m.group(0)
        return self._token("IP", val) if _is_internal_ip(val) else val

    def _maybe_internal_host(self, m: re.Match[str]) -> str:
        host = m.group(0)
        lowered = host.lower()
        if any(lowered.endswith(sfx) for sfx in self.config.internal_domain_suffixes):
            return self._token("HOST", host)
        return host


def default_pseudonymizer(tenant_id: str = "") -> Pseudonymizer:
    """A pseudonymizer with all redaction on (the safe default for no-exfil)."""
    return Pseudonymizer(tenant_id=tenant_id, config=RedactionConfig())