Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(api): reported-message intake adapters and campaign grouping
Beenu Arora323d04d09dcb
diff --git a/services/api/app/services/phishing/__init__.py b/services/api/app/services/phishing/__init__.pynew file mode 100644index 000000000..90eb3ea11--- /dev/null+++ b/services/api/app/services/phishing/__init__.py@@ -0,0 +1,13 @@+"""Reported-phish intake, campaign grouping and the governed purge.++Three modules, split by the question each answers:++* :mod:`app.services.phishing.intake` — what a reported message *is*, and how+ each of the three sources' shapes becomes one.+* :mod:`app.services.phishing.campaigns` — which reports are the same message,+ and who received it.+* :mod:`app.services.phishing.purge` — the governed fan-out across those+ recipients, one dispatch per mailbox.+"""++from __future__ import annotationsdiff --git a/services/api/app/services/phishing/campaigns.py b/services/api/app/services/phishing/campaigns.pynew file mode 100644index 000000000..9ec0ea813--- /dev/null+++ b/services/api/app/services/phishing/campaigns.py@@ -0,0 +1,328 @@+"""Which reports are the same message, and who received it.++Gap-closure 10.3: "group phishing submissions by the same sender, URL or+attachment hash, or a near-duplicate subject, and raise one governed purge+proposal across all recipients".++The grouping ladder+-------------------+Five attributes, tried in descending order of how much they prove:++1. ``internet_message_id`` — the same message, by definition. Added to the+ plan's four because it is both the strongest signal and the one the purge+ needs anyway, and grouping two copies of one message under "same sender"+ would be a weaker claim than the evidence supports.+2. ``attachment_hash`` — the same file reached several people.+3. ``url`` — the same landing page. The first URL in document order, so two+ reports of one message agree regardless of how many links it carried.+4. ``sender`` — the same envelope sender.+5. ``subject`` — a *judgement*, and the only one here that is. Normalised+ (reply/forward prefixes stripped, digits and long tokens collapsed) so+ "Invoice 10452 overdue" and "Invoice 98113 overdue" group, which is the+ whole point and is also exactly why it is last and why+ ``grouping_basis`` records it.++``grouping_basis`` is stored, not derived, because an analyst about to purge+across forty mailboxes is entitled to know whether the forty were joined by a+fact or by a similarity heuristic.++Why the counts are stored+-------------------------+``recipient_count`` feeds ``capability_contracts.scaled_impact``, which turns+it into MODERATE, HIGH or SEVERE and therefore into the approval tier. A+count recomputed on read could change between a proposal being graded and an+approver reading it, which would mean the two saw different actions. It is+recomputed from the recipient rows inside the same transaction that adds one.+"""++from __future__ import annotations++import hashlib+import re+import uuid+from dataclasses import dataclass+from typing import Any++import structlog+from sqlalchemy import text+from sqlalchemy.ext.asyncio import AsyncSession++from app.services.phishing.intake import ReportedMessage, utcnow++logger = structlog.get_logger(__name__)++#: Order matters: the first attribute a message carries decides the group.+GROUPING_ORDER: tuple[str, ...] = ("internet_message_id", "attachment_hash", "url", "sender", "subject")++#: Bases that are a fact about the message rather than a similarity+#: judgement. Published alongside a campaign so a reader does not have to+#: know the ladder to know how strong the claim is.+FACTUAL_BASES: frozenset[str] = frozenset({"internet_message_id", "attachment_hash", "url", "sender"})++_REPLY_PREFIX = re.compile(r"^(?:\s*(?:re|fwd?|fw|aw|tr|rv|enc|antwort)\s*:\s*)+", re.IGNORECASE)+_DIGITS = re.compile(r"\d+")+_WHITESPACE = re.compile(r"\s+")+#: Long opaque tokens — tracking ids, hex blobs, base64 fragments — differ per+#: recipient in exactly the campaigns worth grouping, so they collapse too.+_OPAQUE = re.compile(r"\b[0-9a-f]{8,}\b", re.IGNORECASE)+++def normalise_subject(subject: str) -> str:+ """Strip what varies between copies of one campaign's messages."""+ cleaned = _REPLY_PREFIX.sub("", str(subject or "")).strip().lower()+ cleaned = _OPAQUE.sub("#", cleaned)+ cleaned = _DIGITS.sub("#", cleaned)+ return _WHITESPACE.sub(" ", cleaned).strip()+++@dataclass(frozen=True)+class GroupingDecision:+ """Why two reports are the same campaign.++ ``value`` is kept beside the hash so the decision can be explained+ without reversing a digest — "grouped on the sender billing@…" is+ actionable, "grouped on 6f2a…" is not.+ """++ basis: str+ value: str+ key: str++ @property+ def is_factual(self) -> bool:+ return self.basis in FACTUAL_BASES+++def grouping_decision(message: ReportedMessage) -> GroupingDecision | None:+ """The strongest attribute this message carries, or ``None``.++ ``None`` is a real answer: a report with no message id, no attachment, no+ URL, no sender and no subject has nothing to group on, and inventing a+ group for it would put an unrelated message into somebody's purge list.+ """+ candidates: list[tuple[str, str]] = []+ if message.internet_message_id:+ candidates.append(("internet_message_id", message.internet_message_id))+ if message.attachment_hashes:+ candidates.append(("attachment_hash", sorted(message.attachment_hashes)[0]))+ if message.urls:+ candidates.append(("url", message.urls[0]))+ if message.sender:+ candidates.append(("sender", message.sender.lower()))+ normalised = normalise_subject(message.subject)+ if normalised:+ candidates.append(("subject", normalised))++ by_basis = dict(candidates)+ for basis in GROUPING_ORDER:+ value = by_basis.get(basis)+ if value:+ # The basis is inside the digest so two campaigns cannot collide+ # by having the same string mean different things — a sender+ # address and a URL are not interchangeable even when equal.+ digest = hashlib.sha256(f"{basis}\x00{value}".encode()).hexdigest()+ return GroupingDecision(basis=basis, value=value, key=digest)+ return None+++@dataclass+class CampaignRecord:+ id: uuid.UUID+ grouping_basis: str+ grouping_key: str+ sender: str+ subject: str+ report_count: int+ recipient_count: int+ status: str+ created: bool+++async def record_report(+ db: AsyncSession,+ *,+ tenant_id: uuid.UUID,+ message: ReportedMessage,+ submission_id: uuid.UUID | None = None,+) -> CampaignRecord | None:+ """Join a reported message to its campaign, creating one if needed.++ Returns ``None`` when the message carries nothing to group on, which the+ caller surfaces rather than hides: a submission that joined no campaign+ is still a submission, and saying so is how somebody notices an intake+ adapter that has stopped parsing recipients.++ The recipient rows are inserted with ``ON CONFLICT DO NOTHING`` on+ ``(campaign_id, recipient)``, so a second report of the same message by a+ second person does not duplicate the mailbox — and does not reset a+ ``purge_state`` the first report's purge already set.+ """+ decision = grouping_decision(message)+ if decision is None:+ logger.info("phishing_campaign.ungroupable", tenant_id=str(tenant_id), source=message.intake_source)+ return None++ now = utcnow()+ campaign_id = uuid.uuid4()+ row = (+ await db.execute(+ text("""+ INSERT INTO aisoc_phishing_campaigns+ (id, tenant_id, grouping_key, grouping_basis, sender, subject,+ report_count, recipient_count, first_reported_at, last_reported_at,+ created_at, updated_at)+ VALUES+ (:id, :tenant_id, :key, :basis, :sender, :subject,+ 1, 0, :now, :now, :now, :now)+ ON CONFLICT (tenant_id, grouping_key) DO UPDATE+ SET report_count = aisoc_phishing_campaigns.report_count + 1,+ last_reported_at = EXCLUDED.last_reported_at,+ updated_at = EXCLUDED.updated_at,+ -- Fill a field the first report did not carry, and+ -- never overwrite one it did: the earliest report is+ -- the closest to the message as delivered.+ sender = COALESCE(NULLIF(aisoc_phishing_campaigns.sender, ''), EXCLUDED.sender),+ subject = COALESCE(NULLIF(aisoc_phishing_campaigns.subject, ''), EXCLUDED.subject)+ RETURNING id, grouping_basis, grouping_key, sender, subject,+ report_count, recipient_count, status, (xmax = 0) AS inserted+ """).bindparams(+ id=campaign_id,+ tenant_id=tenant_id,+ key=decision.key,+ basis=decision.basis,+ sender=message.sender,+ subject=message.subject,+ now=now,+ )+ )+ ).fetchone()++ campaign_id = row.id+ for recipient in message.recipients:+ await db.execute(+ text("""+ INSERT INTO aisoc_phishing_campaign_recipients+ (id, campaign_id, tenant_id, recipient, internet_message_id, created_at)+ VALUES (:id, :campaign_id, :tenant_id, :recipient, :message_id, :now)+ ON CONFLICT (campaign_id, recipient) DO NOTHING+ """).bindparams(+ id=uuid.uuid4(),+ campaign_id=campaign_id,+ tenant_id=tenant_id,+ recipient=recipient,+ message_id=message.internet_message_id,+ now=now,+ )+ )++ # Recomputed from the rows rather than incremented, in the same+ # transaction. An incremented counter and a de-duplicating insert+ # disagree the first time two people report the same message, and the+ # counter is what decides the approval tier.+ recount = (+ await db.execute(+ text("""+ UPDATE aisoc_phishing_campaigns+ SET recipient_count = (+ SELECT count(*) FROM aisoc_phishing_campaign_recipients+ WHERE campaign_id = :id AND tenant_id = :tenant_id+ ),+ updated_at = :now+ WHERE id = :id AND tenant_id = :tenant_id+ RETURNING recipient_count+ """).bindparams(id=campaign_id, tenant_id=tenant_id, now=now)+ )+ ).fetchone()++ if submission_id is not None:+ await db.execute(+ text("UPDATE aisoc_phishing_submissions SET campaign_id = :cid WHERE id = :sid AND tenant_id = :tenant_id").bindparams(+ cid=campaign_id, sid=submission_id, tenant_id=tenant_id+ )+ )++ return CampaignRecord(+ id=campaign_id,+ grouping_basis=row.grouping_basis,+ grouping_key=row.grouping_key,+ sender=row.sender or message.sender,+ subject=row.subject or message.subject,+ report_count=row.report_count,+ recipient_count=int(recount.recipient_count if recount else 0),+ status=row.status,+ created=bool(row.inserted),+ )+++async def list_campaigns(db: AsyncSession, *, tenant_id: uuid.UUID, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:+ rows = (+ await db.execute(+ text("""+ SELECT id, grouping_basis, grouping_key, sender, subject, report_count,+ recipient_count, status, first_reported_at, last_reported_at+ FROM aisoc_phishing_campaigns+ WHERE tenant_id = :tenant_id+ ORDER BY last_reported_at DESC+ LIMIT :limit OFFSET :offset+ """).bindparams(tenant_id=tenant_id, limit=limit, offset=offset)+ )+ ).fetchall()+ return [_campaign_dict(r) for r in rows]+++async def get_campaign(db: AsyncSession, *, tenant_id: uuid.UUID, campaign_id: uuid.UUID) -> dict[str, Any] | None:+ row = (+ await db.execute(+ text("""+ SELECT id, grouping_basis, grouping_key, sender, subject, report_count,+ recipient_count, status, first_reported_at, last_reported_at+ FROM aisoc_phishing_campaigns+ WHERE tenant_id = :tenant_id AND id = :id+ """).bindparams(tenant_id=tenant_id, id=campaign_id)+ )+ ).fetchone()+ if row is None:+ return None+ payload = _campaign_dict(row)+ recipients = (+ await db.execute(+ text("""+ SELECT recipient, internet_message_id, purge_state, purge_detail, vendor, last_action_at+ FROM aisoc_phishing_campaign_recipients+ WHERE tenant_id = :tenant_id AND campaign_id = :id+ ORDER BY recipient+ """).bindparams(tenant_id=tenant_id, id=campaign_id)+ )+ ).fetchall()+ payload["recipients"] = [+ {+ "recipient": r.recipient,+ "internet_message_id": r.internet_message_id,+ "purge_state": r.purge_state,+ "purge_detail": r.purge_detail,+ "vendor": r.vendor,+ "last_action_at": r.last_action_at,+ }+ for r in recipients+ ]+ return payload+++def _campaign_dict(row: Any) -> dict[str, Any]:+ basis = row.grouping_basis+ return {+ "id": row.id,+ "grouping_basis": basis,+ "grouping_key": row.grouping_key,+ # Published so a console does not have to carry a second copy of+ # FACTUAL_BASES, and so the distinction cannot be dropped by a+ # renderer that does not know it exists.+ "grouping_is_factual": basis in FACTUAL_BASES,+ "sender": row.sender,+ "subject": row.subject,+ "report_count": row.report_count,+ "recipient_count": row.recipient_count,+ "status": row.status,+ "first_reported_at": row.first_reported_at,+ "last_reported_at": row.last_reported_at,+ }diff --git a/services/api/app/services/phishing/intake.py b/services/api/app/services/phishing/intake.pynew file mode 100644index 000000000..49401d013--- /dev/null+++ b/services/api/app/services/phishing/intake.py@@ -0,0 +1,373 @@+"""What a reported phishing message is, and how three sources become one.++Depth 7.1 names three intake doors: a Microsoft 365 report mailbox read+through Graph, Google Workspace user reports, and the existing+``email_inbox`` IMAP connector. They arrive in three unrelated shapes and the+rest of the phishing path must not know that, so this module is the one place+that does.++The canonical record is :class:`ReportedMessage`. Everything downstream —+grouping, the purge fan-out, header analysis — reads it and nothing reads a+vendor payload.++Two rules the adapters share+----------------------------+**A field that was not in the payload is empty, never invented.** A reported+message with no recipient list produces an empty tuple, and the campaign it+joins reports a recipient count of zero rather than guessing one. A purge+proposal over a guessed recipient list is an action over mailboxes nobody+named.++**The vendor's own shape is the input, not a convenient one.** Each adapter+parses what the vendor documents: a Graph ``message`` resource with+``internetMessageHeaders``, a Gmail ``Message`` with+``payload.headers[].name/value``, and the flat ``subject``/``from``/``to``+dict ``EmailInboxConnector._to_event`` emits. Accepting a tidied shape and+testing against it is how an intake path passes every test and fails on the+first real message.++Why ``internet_message_id`` is normalised here+-----------------------------------------------+RFC 5322 brackets a Message-ID, Graph stores it bracketed and Gmail's+``rfc822msgid:`` operator wants it unbracketed. A reported message arrives+both ways — a forwarded header keeps the brackets, a JSON field usually+strips them. The canonical form here is **bracketed**, matching what Graph+stores and what the header itself contains, and the Gmail client strips them+at the point of use. Picking one and converting at the edges is the+difference between a purge that finds the message and one that reports an+empty mailbox.+"""++from __future__ import annotations++import base64+import binascii+import re+from dataclasses import dataclass, field+from datetime import UTC, datetime+from email.utils import getaddresses, parsedate_to_datetime+from typing import Any++#: The three doors depth 7.1 names, plus ``manual`` for ``POST+#: /phishing/submit``. A closed vocabulary so "the M365 report mailbox has+#: gone quiet" is a diagnosis somebody can make.+INTAKE_SOURCES: frozenset[str] = frozenset({"manual", "m365_graph", "google_workspace", "email_inbox"})++_SHA256 = re.compile(r"\b[0-9a-fA-F]{64}\b")++#: Deliberately conservative. A URL extractor that also matches bare domains+#: turns every signature block into an indicator, and the resulting campaign+#: groups on "example.com" — joining messages that have nothing to do with+#: each other and proposing a purge across all of them.+_URL = re.compile(r"https?://[^\s<>\"'\)\]]+", re.IGNORECASE)+++def normalise_message_id(value: str) -> str:+ """Canonical bracketed form, or ``""``.++ Bracketed is the canonical direction because that is what the header+ contains and what Graph stores; Gmail's search operator wants it bare and+ its client strips at the point of use.+ """+ cleaned = "".join(ch for ch in str(value or "") if ch.isprintable()).strip()+ if not cleaned:+ return ""+ cleaned = cleaned.strip("<>").strip()+ return f"<{cleaned}>" if cleaned else ""+++def _addresses(raw: Any) -> tuple[str, ...]:+ """Every address in a header value or list, lowercased and de-duplicated.++ ``getaddresses`` handles the forms a real ``To`` header takes — display+ names, quoting, groups, folded lines — which a ``split(",")`` does not.+ """+ if raw is None:+ return ()+ values = [str(v) for v in raw] if isinstance(raw, (list, tuple, set)) else [str(raw)]+ out: list[str] = []+ for _name, addr in getaddresses(values):+ cleaned = addr.strip().lower()+ if cleaned and "@" in cleaned and cleaned not in out:+ out.append(cleaned)+ return tuple(out)+++def _parse_time(value: Any) -> datetime | None:+ if not value:+ return None+ text = str(value).strip()+ try:+ return datetime.fromisoformat(text.replace("Z", "+00:00"))+ except ValueError:+ pass+ try:+ return parsedate_to_datetime(text)+ except (TypeError, ValueError):+ return None+++@dataclass(frozen=True)+class ReportedMessage:+ """One reported message, in the shape the rest of the path reads.++ Frozen because an intake record is evidence: a later stage that wants a+ different value should derive one rather than edit what the reporter+ sent.+ """++ intake_source: str+ internet_message_id: str = ""+ sender: str = ""+ subject: str = ""+ recipients: tuple[str, ...] = ()+ reported_by: str = ""+ received_at: datetime | None = None+ urls: tuple[str, ...] = ()+ attachment_hashes: tuple[str, ...] = ()+ #: Attachment metadata as the vendor gave it: name, content type, size.+ #: Never content, and never a digest the vendor did not supply —+ #: downloading bytes to hash them would make the report-mailbox poller an+ #: unattended file-egress path out of the customer's tenant.+ attachments: tuple[dict[str, Any], ...] = ()+ #: Raw internet headers, lowercased name -> value. 7.2 reads these; 7.1+ #: carries them so a later analysis does not have to re-fetch the+ #: message from a mailbox that may by then have been purged.+ headers: dict[str, str] = field(default_factory=dict)+ body_text: str = ""+ #: The vendor's own id for the message in the *report* mailbox, so the+ #: poller can mark it processed without a second search.+ source_message_id: str = ""++ def __post_init__(self) -> None:+ if self.intake_source not in INTAKE_SOURCES:+ raise ValueError(f"unknown intake source {self.intake_source!r}; expected one of {sorted(INTAKE_SOURCES)}")++ @property+ def recipient_count(self) -> int:+ return len(self.recipients)+++def _headers_from_pairs(pairs: Any, name_key: str, value_key: str) -> dict[str, str]:+ """Fold a vendor's ``[{name, value}, ...]`` header list into a dict.++ Repeated headers are joined with a newline rather than overwritten.+ ``Received`` appears once per hop and the hop chain is the whole of its+ value; keeping the last one would describe a one-hop delivery for a+ message that crossed five.+ """+ out: dict[str, str] = {}+ if not isinstance(pairs, (list, tuple)):+ return out+ for entry in pairs:+ if not isinstance(entry, dict):+ continue+ name = str(entry.get(name_key) or "").strip().lower()+ value = str(entry.get(value_key) or "").strip()+ if not name:+ continue+ out[name] = f"{out[name]}\n{value}" if name in out else value+ return out+++def _indicators(body: str, headers: dict[str, str], attachments: tuple[dict[str, Any], ...]) -> tuple[tuple[str, ...], tuple[str, ...]]:+ """URLs and SHA-256 digests visible in the message, without fetching anything."""+ haystack = "\n".join([body, headers.get("subject", "")])+ urls = tuple(dict.fromkeys(_URL.findall(haystack)))[:50]+ digests: list[str] = []+ for attachment in attachments:+ for key in ("sha256", "hash", "fileHash", "contentSha256"):+ candidate = str(attachment.get(key) or "")+ if _SHA256.fullmatch(candidate):+ digests.append(candidate.lower())+ break+ return urls, tuple(dict.fromkeys(digests))+++# ── Microsoft 365, through Graph ────────────────────────────────────────────+++def from_graph_message(payload: dict[str, Any], *, reported_by: str = "") -> ReportedMessage:+ """A Microsoft Graph ``message`` resource from a report mailbox.++ The shape is the documented one: ``internetMessageId``,+ ``from.emailAddress.address``, ``toRecipients[].emailAddress.address``,+ ``receivedDateTime``, ``internetMessageHeaders[].name/value``,+ ``body.content`` and an ``attachments`` collection of ``name`` /+ ``contentType`` / ``size``.++ A user who reports phish by **forwarding** it makes themselves the sender+ of the outer message, so ``reported_by`` is taken from the envelope and+ the *reported* sender is read from the original's headers when the+ forward preserved them. Where it did not, the outer sender is recorded as+ the reporter and the sender field stays empty rather than naming the+ victim as the attacker.+ """+ headers = _headers_from_pairs(payload.get("internetMessageHeaders"), "name", "value")+ envelope_sender = str((payload.get("from") or {}).get("emailAddress", {}).get("address") or "").lower()++ # The original sender, preferred from the preserved headers.+ sender = ""+ for key in ("from", "x-ms-exchange-organization-originalfrom", "reply-to", "return-path"):+ candidates = _addresses(headers.get(key))+ if candidates and candidates[0] != envelope_sender:+ sender = candidates[0]+ break++ recipients = _addresses(+ [str((r or {}).get("emailAddress", {}).get("address") or "") for r in (payload.get("toRecipients") or [])]+ + [str((r or {}).get("emailAddress", {}).get("address") or "") for r in (payload.get("ccRecipients") or [])]+ )+ # A forward lands in the report mailbox addressed to the report mailbox,+ # which is not a recipient of the phish. The original recipients are in+ # the preserved `to`/`cc` headers when there are any.+ header_recipients = _addresses(headers.get("to")) + _addresses(headers.get("cc"))+ if header_recipients:+ recipients = tuple(dict.fromkeys(header_recipients))++ attachments = tuple(+ {+ "name": str(a.get("name") or ""),+ "content_type": str(a.get("contentType") or ""),+ "size": a.get("size"),+ }+ for a in (payload.get("attachments") or [])+ if isinstance(a, dict)+ )+ body = str((payload.get("body") or {}).get("content") or "")+ headers.setdefault("subject", str(payload.get("subject") or ""))+ urls, digests = _indicators(body, headers, attachments)++ return ReportedMessage(+ intake_source="m365_graph",+ internet_message_id=normalise_message_id(+ headers.get("message-id") or str(payload.get("internetMessageId") or "")+ ),+ sender=sender,+ subject=str(payload.get("subject") or ""),+ recipients=recipients,+ reported_by=reported_by or envelope_sender,+ received_at=_parse_time(payload.get("receivedDateTime")),+ urls=urls,+ attachment_hashes=digests,+ attachments=attachments,+ headers=headers,+ body_text=body,+ source_message_id=str(payload.get("id") or ""),+ )+++# ── Google Workspace, through Gmail ─────────────────────────────────────────+++def from_gmail_message(payload: dict[str, Any], *, reported_by: str = "") -> ReportedMessage:+ """A Gmail ``Message`` resource, as ``users.messages.get`` returns it.++ ``payload.headers`` is a list of ``{name, value}``; ``snippet`` is the+ only body text a ``format=metadata`` read returns, and that read is+ deliberate — see ``GmailClient.get_message``. Where a caller supplied a+ full ``format=full`` message the base64url body is decoded.+ """+ gmail_payload = payload.get("payload") or {}+ headers = _headers_from_pairs(gmail_payload.get("headers"), "name", "value")++ body = str(payload.get("snippet") or "")+ encoded = ((gmail_payload.get("body") or {}).get("data")) if isinstance(gmail_payload.get("body"), dict) else None+ if encoded:+ try:+ body = base64.urlsafe_b64decode(str(encoded) + "==").decode("utf-8", "replace")+ except (binascii.Error, ValueError):+ # A body that will not decode is not a reason to lose the+ # snippet, and it is certainly not a reason to fail the intake.+ pass++ attachments = tuple(+ {+ "name": str(part.get("filename") or ""),+ "content_type": str(part.get("mimeType") or ""),+ "size": (part.get("body") or {}).get("size"),+ }+ for part in (gmail_payload.get("parts") or [])+ if isinstance(part, dict) and part.get("filename")+ )+ urls, digests = _indicators(body, headers, attachments)+ subject = headers.get("subject", "")++ return ReportedMessage(+ intake_source="google_workspace",+ internet_message_id=normalise_message_id(headers.get("message-id", "")),+ sender=(_addresses(headers.get("from")) or ("",))[0],+ subject=subject,+ recipients=tuple(dict.fromkeys(_addresses(headers.get("to")) + _addresses(headers.get("cc")))),+ reported_by=reported_by or (_addresses(headers.get("delivered-to")) or ("",))[0],+ received_at=_parse_time(headers.get("date")),+ urls=urls,+ attachment_hashes=digests,+ attachments=attachments,+ headers=headers,+ body_text=body,+ source_message_id=str(payload.get("id") or ""),+ )+++# ── The existing email_inbox connector ──────────────────────────────────────+++def from_email_inbox_event(payload: dict[str, Any], *, reported_by: str = "") -> ReportedMessage:+ """The flat dict ``EmailInboxConnector._to_event`` emits.++ Its keys are ``subject`` / ``from`` / ``to`` / ``received_at`` /+ ``message_id`` / ``body``, which that connector's docstring pins to the+ ingest-side ``email-forwarded`` template. This adapter reads exactly+ those and tolerates a ``headers`` dict when a caller has one, because the+ webhook sibling (``POST /v1/inbox/email/{token}``) can carry more than+ IMAP's six fields.+ """+ headers = {str(k).lower(): str(v) for k, v in (payload.get("headers") or {}).items() if isinstance(payload.get("headers"), dict)}+ body = str(payload.get("body") or "")+ subject = str(payload.get("subject") or headers.get("subject", ""))+ headers.setdefault("subject", subject)+ urls, digests = _indicators(body, headers, ())++ return ReportedMessage(+ intake_source="email_inbox",+ internet_message_id=normalise_message_id(payload.get("message_id") or headers.get("message-id", "")),+ sender=(_addresses(payload.get("from") or headers.get("from")) or ("",))[0],+ subject=subject,+ recipients=_addresses(payload.get("to") or headers.get("to")),+ reported_by=reported_by,+ received_at=_parse_time(payload.get("received_at") or headers.get("date")),+ urls=urls,+ attachment_hashes=digests,+ attachments=(),+ headers=headers,+ body_text=body,+ )+++#: ``intake source -> adapter``. A dict rather than an ``if`` chain so the+#: route can validate the source against the same set it dispatches on, and+#: so adding a fourth door is one entry rather than two edits that can+#: disagree.+ADAPTERS = {+ "m365_graph": from_graph_message,+ "google_workspace": from_gmail_message,+ "email_inbox": from_email_inbox_event,+}+++def normalise(source: str, payload: dict[str, Any], *, reported_by: str = "") -> ReportedMessage:+ """Turn one vendor payload into a :class:`ReportedMessage`.++ Raises ``ValueError`` for an unknown source rather than falling back to a+ lenient parser. A lenient fallback is how the ingest normaliser came to+ stamp "Splunk" onto events from twenty-six other connectors.+ """+ adapter = ADAPTERS.get(source)+ if adapter is None:+ raise ValueError(f"unknown intake source {source!r}; expected one of {sorted(ADAPTERS)}")+ return adapter(payload, reported_by=reported_by)+++def utcnow() -> datetime:+ return datetime.now(UTC)