Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(api): reported-message intake adapters and campaign grouping

Beenu Arora323d04d09dcb
diff --git a/services/api/app/services/phishing/__init__.py b/services/api/app/services/phishing/__init__.py
new file mode 100644
index 000000000..90eb3ea11
--- /dev/null
+++ b/services/api/app/services/phishing/__init__.py
@@ -0,0 +1,13 @@
+"""Reported-phish intake, campaign grouping and the governed purge.
+
+Three modules, split by the question each answers:
+
+* :mod:`app.services.phishing.intake` — what a reported message *is*, and how
+ each of the three sources' shapes becomes one.
+* :mod:`app.services.phishing.campaigns` — which reports are the same message,
+ and who received it.
+* :mod:`app.services.phishing.purge` — the governed fan-out across those
+ recipients, one dispatch per mailbox.
+"""
+
+from __future__ import annotations
diff --git a/services/api/app/services/phishing/campaigns.py b/services/api/app/services/phishing/campaigns.py
new file mode 100644
index 000000000..9ec0ea813
--- /dev/null
+++ b/services/api/app/services/phishing/campaigns.py
@@ -0,0 +1,328 @@
+"""Which reports are the same message, and who received it.
+
+Gap-closure 10.3: "group phishing submissions by the same sender, URL or
+attachment hash, or a near-duplicate subject, and raise one governed purge
+proposal across all recipients".
+
+The grouping ladder
+-------------------
+Five attributes, tried in descending order of how much they prove:
+
+1. ``internet_message_id`` — the same message, by definition. Added to the
+ plan's four because it is both the strongest signal and the one the purge
+ needs anyway, and grouping two copies of one message under "same sender"
+ would be a weaker claim than the evidence supports.
+2. ``attachment_hash`` — the same file reached several people.
+3. ``url`` — the same landing page. The first URL in document order, so two
+ reports of one message agree regardless of how many links it carried.
+4. ``sender`` — the same envelope sender.
+5. ``subject`` — a *judgement*, and the only one here that is. Normalised
+ (reply/forward prefixes stripped, digits and long tokens collapsed) so
+ "Invoice 10452 overdue" and "Invoice 98113 overdue" group, which is the
+ whole point and is also exactly why it is last and why
+ ``grouping_basis`` records it.
+
+``grouping_basis`` is stored, not derived, because an analyst about to purge
+across forty mailboxes is entitled to know whether the forty were joined by a
+fact or by a similarity heuristic.
+
+Why the counts are stored
+-------------------------
+``recipient_count`` feeds ``capability_contracts.scaled_impact``, which turns
+it into MODERATE, HIGH or SEVERE and therefore into the approval tier. A
+count recomputed on read could change between a proposal being graded and an
+approver reading it, which would mean the two saw different actions. It is
+recomputed from the recipient rows inside the same transaction that adds one.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import uuid
+from dataclasses import dataclass
+from typing import Any
+
+import structlog
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.services.phishing.intake import ReportedMessage, utcnow
+
+logger = structlog.get_logger(__name__)
+
+#: Order matters: the first attribute a message carries decides the group.
+GROUPING_ORDER: tuple[str, ...] = ("internet_message_id", "attachment_hash", "url", "sender", "subject")
+
+#: Bases that are a fact about the message rather than a similarity
+#: judgement. Published alongside a campaign so a reader does not have to
+#: know the ladder to know how strong the claim is.
+FACTUAL_BASES: frozenset[str] = frozenset({"internet_message_id", "attachment_hash", "url", "sender"})
+
+_REPLY_PREFIX = re.compile(r"^(?:\s*(?:re|fwd?|fw|aw|tr|rv|enc|antwort)\s*:\s*)+", re.IGNORECASE)
+_DIGITS = re.compile(r"\d+")
+_WHITESPACE = re.compile(r"\s+")
+#: Long opaque tokens — tracking ids, hex blobs, base64 fragments — differ per
+#: recipient in exactly the campaigns worth grouping, so they collapse too.
+_OPAQUE = re.compile(r"\b[0-9a-f]{8,}\b", re.IGNORECASE)
+
+
+def normalise_subject(subject: str) -> str:
+ """Strip what varies between copies of one campaign's messages."""
+ cleaned = _REPLY_PREFIX.sub("", str(subject or "")).strip().lower()
+ cleaned = _OPAQUE.sub("#", cleaned)
+ cleaned = _DIGITS.sub("#", cleaned)
+ return _WHITESPACE.sub(" ", cleaned).strip()
+
+
+@dataclass(frozen=True)
+class GroupingDecision:
+ """Why two reports are the same campaign.
+
+ ``value`` is kept beside the hash so the decision can be explained
+ without reversing a digest — "grouped on the sender billing@…" is
+ actionable, "grouped on 6f2a…" is not.
+ """
+
+ basis: str
+ value: str
+ key: str
+
+ @property
+ def is_factual(self) -> bool:
+ return self.basis in FACTUAL_BASES
+
+
+def grouping_decision(message: ReportedMessage) -> GroupingDecision | None:
+ """The strongest attribute this message carries, or ``None``.
+
+ ``None`` is a real answer: a report with no message id, no attachment, no
+ URL, no sender and no subject has nothing to group on, and inventing a
+ group for it would put an unrelated message into somebody's purge list.
+ """
+ candidates: list[tuple[str, str]] = []
+ if message.internet_message_id:
+ candidates.append(("internet_message_id", message.internet_message_id))
+ if message.attachment_hashes:
+ candidates.append(("attachment_hash", sorted(message.attachment_hashes)[0]))
+ if message.urls:
+ candidates.append(("url", message.urls[0]))
+ if message.sender:
+ candidates.append(("sender", message.sender.lower()))
+ normalised = normalise_subject(message.subject)
+ if normalised:
+ candidates.append(("subject", normalised))
+
+ by_basis = dict(candidates)
+ for basis in GROUPING_ORDER:
+ value = by_basis.get(basis)
+ if value:
+ # The basis is inside the digest so two campaigns cannot collide
+ # by having the same string mean different things — a sender
+ # address and a URL are not interchangeable even when equal.
+ digest = hashlib.sha256(f"{basis}\x00{value}".encode()).hexdigest()
+ return GroupingDecision(basis=basis, value=value, key=digest)
+ return None
+
+
+@dataclass
+class CampaignRecord:
+ id: uuid.UUID
+ grouping_basis: str
+ grouping_key: str
+ sender: str
+ subject: str
+ report_count: int
+ recipient_count: int
+ status: str
+ created: bool
+
+
+async def record_report(
+ db: AsyncSession,
+ *,
+ tenant_id: uuid.UUID,
+ message: ReportedMessage,
+ submission_id: uuid.UUID | None = None,
+) -> CampaignRecord | None:
+ """Join a reported message to its campaign, creating one if needed.
+
+ Returns ``None`` when the message carries nothing to group on, which the
+ caller surfaces rather than hides: a submission that joined no campaign
+ is still a submission, and saying so is how somebody notices an intake
+ adapter that has stopped parsing recipients.
+
+ The recipient rows are inserted with ``ON CONFLICT DO NOTHING`` on
+ ``(campaign_id, recipient)``, so a second report of the same message by a
+ second person does not duplicate the mailbox — and does not reset a
+ ``purge_state`` the first report's purge already set.
+ """
+ decision = grouping_decision(message)
+ if decision is None:
+ logger.info("phishing_campaign.ungroupable", tenant_id=str(tenant_id), source=message.intake_source)
+ return None
+
+ now = utcnow()
+ campaign_id = uuid.uuid4()
+ row = (
+ await db.execute(
+ text("""
+ INSERT INTO aisoc_phishing_campaigns
+ (id, tenant_id, grouping_key, grouping_basis, sender, subject,
+ report_count, recipient_count, first_reported_at, last_reported_at,
+ created_at, updated_at)
+ VALUES
+ (:id, :tenant_id, :key, :basis, :sender, :subject,
+ 1, 0, :now, :now, :now, :now)
+ ON CONFLICT (tenant_id, grouping_key) DO UPDATE
+ SET report_count = aisoc_phishing_campaigns.report_count + 1,
+ last_reported_at = EXCLUDED.last_reported_at,
+ updated_at = EXCLUDED.updated_at,
+ -- Fill a field the first report did not carry, and
+ -- never overwrite one it did: the earliest report is
+ -- the closest to the message as delivered.
+ sender = COALESCE(NULLIF(aisoc_phishing_campaigns.sender, ''), EXCLUDED.sender),
+ subject = COALESCE(NULLIF(aisoc_phishing_campaigns.subject, ''), EXCLUDED.subject)
+ RETURNING id, grouping_basis, grouping_key, sender, subject,
+ report_count, recipient_count, status, (xmax = 0) AS inserted
+ """).bindparams(
+ id=campaign_id,
+ tenant_id=tenant_id,
+ key=decision.key,
+ basis=decision.basis,
+ sender=message.sender,
+ subject=message.subject,
+ now=now,
+ )
+ )
+ ).fetchone()
+
+ campaign_id = row.id
+ for recipient in message.recipients:
+ await db.execute(
+ text("""
+ INSERT INTO aisoc_phishing_campaign_recipients
+ (id, campaign_id, tenant_id, recipient, internet_message_id, created_at)
+ VALUES (:id, :campaign_id, :tenant_id, :recipient, :message_id, :now)
+ ON CONFLICT (campaign_id, recipient) DO NOTHING
+ """).bindparams(
+ id=uuid.uuid4(),
+ campaign_id=campaign_id,
+ tenant_id=tenant_id,
+ recipient=recipient,
+ message_id=message.internet_message_id,
+ now=now,
+ )
+ )
+
+ # Recomputed from the rows rather than incremented, in the same
+ # transaction. An incremented counter and a de-duplicating insert
+ # disagree the first time two people report the same message, and the
+ # counter is what decides the approval tier.
+ recount = (
+ await db.execute(
+ text("""
+ UPDATE aisoc_phishing_campaigns
+ SET recipient_count = (
+ SELECT count(*) FROM aisoc_phishing_campaign_recipients
+ WHERE campaign_id = :id AND tenant_id = :tenant_id
+ ),
+ updated_at = :now
+ WHERE id = :id AND tenant_id = :tenant_id
+ RETURNING recipient_count
+ """).bindparams(id=campaign_id, tenant_id=tenant_id, now=now)
+ )
+ ).fetchone()
+
+ if submission_id is not None:
+ await db.execute(
+ text("UPDATE aisoc_phishing_submissions SET campaign_id = :cid WHERE id = :sid AND tenant_id = :tenant_id").bindparams(
+ cid=campaign_id, sid=submission_id, tenant_id=tenant_id
+ )
+ )
+
+ return CampaignRecord(
+ id=campaign_id,
+ grouping_basis=row.grouping_basis,
+ grouping_key=row.grouping_key,
+ sender=row.sender or message.sender,
+ subject=row.subject or message.subject,
+ report_count=row.report_count,
+ recipient_count=int(recount.recipient_count if recount else 0),
+ status=row.status,
+ created=bool(row.inserted),
+ )
+
+
+async def list_campaigns(db: AsyncSession, *, tenant_id: uuid.UUID, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:
+ rows = (
+ await db.execute(
+ text("""
+ SELECT id, grouping_basis, grouping_key, sender, subject, report_count,
+ recipient_count, status, first_reported_at, last_reported_at
+ FROM aisoc_phishing_campaigns
+ WHERE tenant_id = :tenant_id
+ ORDER BY last_reported_at DESC
+ LIMIT :limit OFFSET :offset
+ """).bindparams(tenant_id=tenant_id, limit=limit, offset=offset)
+ )
+ ).fetchall()
+ return [_campaign_dict(r) for r in rows]
+
+
+async def get_campaign(db: AsyncSession, *, tenant_id: uuid.UUID, campaign_id: uuid.UUID) -> dict[str, Any] | None:
+ row = (
+ await db.execute(
+ text("""
+ SELECT id, grouping_basis, grouping_key, sender, subject, report_count,
+ recipient_count, status, first_reported_at, last_reported_at
+ FROM aisoc_phishing_campaigns
+ WHERE tenant_id = :tenant_id AND id = :id
+ """).bindparams(tenant_id=tenant_id, id=campaign_id)
+ )
+ ).fetchone()
+ if row is None:
+ return None
+ payload = _campaign_dict(row)
+ recipients = (
+ await db.execute(
+ text("""
+ SELECT recipient, internet_message_id, purge_state, purge_detail, vendor, last_action_at
+ FROM aisoc_phishing_campaign_recipients
+ WHERE tenant_id = :tenant_id AND campaign_id = :id
+ ORDER BY recipient
+ """).bindparams(tenant_id=tenant_id, id=campaign_id)
+ )
+ ).fetchall()
+ payload["recipients"] = [
+ {
+ "recipient": r.recipient,
+ "internet_message_id": r.internet_message_id,
+ "purge_state": r.purge_state,
+ "purge_detail": r.purge_detail,
+ "vendor": r.vendor,
+ "last_action_at": r.last_action_at,
+ }
+ for r in recipients
+ ]
+ return payload
+
+
+def _campaign_dict(row: Any) -> dict[str, Any]:
+ basis = row.grouping_basis
+ return {
+ "id": row.id,
+ "grouping_basis": basis,
+ "grouping_key": row.grouping_key,
+ # Published so a console does not have to carry a second copy of
+ # FACTUAL_BASES, and so the distinction cannot be dropped by a
+ # renderer that does not know it exists.
+ "grouping_is_factual": basis in FACTUAL_BASES,
+ "sender": row.sender,
+ "subject": row.subject,
+ "report_count": row.report_count,
+ "recipient_count": row.recipient_count,
+ "status": row.status,
+ "first_reported_at": row.first_reported_at,
+ "last_reported_at": row.last_reported_at,
+ }
diff --git a/services/api/app/services/phishing/intake.py b/services/api/app/services/phishing/intake.py
new file mode 100644
index 000000000..49401d013
--- /dev/null
+++ b/services/api/app/services/phishing/intake.py
@@ -0,0 +1,373 @@
+"""What a reported phishing message is, and how three sources become one.
+
+Depth 7.1 names three intake doors: a Microsoft 365 report mailbox read
+through Graph, Google Workspace user reports, and the existing
+``email_inbox`` IMAP connector. They arrive in three unrelated shapes and the
+rest of the phishing path must not know that, so this module is the one place
+that does.
+
+The canonical record is :class:`ReportedMessage`. Everything downstream —
+grouping, the purge fan-out, header analysis — reads it and nothing reads a
+vendor payload.
+
+Two rules the adapters share
+----------------------------
+**A field that was not in the payload is empty, never invented.** A reported
+message with no recipient list produces an empty tuple, and the campaign it
+joins reports a recipient count of zero rather than guessing one. A purge
+proposal over a guessed recipient list is an action over mailboxes nobody
+named.
+
+**The vendor's own shape is the input, not a convenient one.** Each adapter
+parses what the vendor documents: a Graph ``message`` resource with
+``internetMessageHeaders``, a Gmail ``Message`` with
+``payload.headers[].name/value``, and the flat ``subject``/``from``/``to``
+dict ``EmailInboxConnector._to_event`` emits. Accepting a tidied shape and
+testing against it is how an intake path passes every test and fails on the
+first real message.
+
+Why ``internet_message_id`` is normalised here
+-----------------------------------------------
+RFC 5322 brackets a Message-ID, Graph stores it bracketed and Gmail's
+``rfc822msgid:`` operator wants it unbracketed. A reported message arrives
+both ways — a forwarded header keeps the brackets, a JSON field usually
+strips them. The canonical form here is **bracketed**, matching what Graph
+stores and what the header itself contains, and the Gmail client strips them
+at the point of use. Picking one and converting at the edges is the
+difference between a purge that finds the message and one that reports an
+empty mailbox.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import re
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from email.utils import getaddresses, parsedate_to_datetime
+from typing import Any
+
+#: The three doors depth 7.1 names, plus ``manual`` for ``POST
+#: /phishing/submit``. A closed vocabulary so "the M365 report mailbox has
+#: gone quiet" is a diagnosis somebody can make.
+INTAKE_SOURCES: frozenset[str] = frozenset({"manual", "m365_graph", "google_workspace", "email_inbox"})
+
+_SHA256 = re.compile(r"\b[0-9a-fA-F]{64}\b")
+
+#: Deliberately conservative. A URL extractor that also matches bare domains
+#: turns every signature block into an indicator, and the resulting campaign
+#: groups on "example.com" — joining messages that have nothing to do with
+#: each other and proposing a purge across all of them.
+_URL = re.compile(r"https?://[^\s<>\"'\)\]]+", re.IGNORECASE)
+
+
+def normalise_message_id(value: str) -> str:
+ """Canonical bracketed form, or ``""``.
+
+ Bracketed is the canonical direction because that is what the header
+ contains and what Graph stores; Gmail's search operator wants it bare and
+ its client strips at the point of use.
+ """
+ cleaned = "".join(ch for ch in str(value or "") if ch.isprintable()).strip()
+ if not cleaned:
+ return ""
+ cleaned = cleaned.strip("<>").strip()
+ return f"<{cleaned}>" if cleaned else ""
+
+
+def _addresses(raw: Any) -> tuple[str, ...]:
+ """Every address in a header value or list, lowercased and de-duplicated.
+
+ ``getaddresses`` handles the forms a real ``To`` header takes — display
+ names, quoting, groups, folded lines — which a ``split(",")`` does not.
+ """
+ if raw is None:
+ return ()
+ values = [str(v) for v in raw] if isinstance(raw, (list, tuple, set)) else [str(raw)]
+ out: list[str] = []
+ for _name, addr in getaddresses(values):
+ cleaned = addr.strip().lower()
+ if cleaned and "@" in cleaned and cleaned not in out:
+ out.append(cleaned)
+ return tuple(out)
+
+
+def _parse_time(value: Any) -> datetime | None:
+ if not value:
+ return None
+ text = str(value).strip()
+ try:
+ return datetime.fromisoformat(text.replace("Z", "+00:00"))
+ except ValueError:
+ pass
+ try:
+ return parsedate_to_datetime(text)
+ except (TypeError, ValueError):
+ return None
+
+
+@dataclass(frozen=True)
+class ReportedMessage:
+ """One reported message, in the shape the rest of the path reads.
+
+ Frozen because an intake record is evidence: a later stage that wants a
+ different value should derive one rather than edit what the reporter
+ sent.
+ """
+
+ intake_source: str
+ internet_message_id: str = ""
+ sender: str = ""
+ subject: str = ""
+ recipients: tuple[str, ...] = ()
+ reported_by: str = ""
+ received_at: datetime | None = None
+ urls: tuple[str, ...] = ()
+ attachment_hashes: tuple[str, ...] = ()
+ #: Attachment metadata as the vendor gave it: name, content type, size.
+ #: Never content, and never a digest the vendor did not supply —
+ #: downloading bytes to hash them would make the report-mailbox poller an
+ #: unattended file-egress path out of the customer's tenant.
+ attachments: tuple[dict[str, Any], ...] = ()
+ #: Raw internet headers, lowercased name -> value. 7.2 reads these; 7.1
+ #: carries them so a later analysis does not have to re-fetch the
+ #: message from a mailbox that may by then have been purged.
+ headers: dict[str, str] = field(default_factory=dict)
+ body_text: str = ""
+ #: The vendor's own id for the message in the *report* mailbox, so the
+ #: poller can mark it processed without a second search.
+ source_message_id: str = ""
+
+ def __post_init__(self) -> None:
+ if self.intake_source not in INTAKE_SOURCES:
+ raise ValueError(f"unknown intake source {self.intake_source!r}; expected one of {sorted(INTAKE_SOURCES)}")
+
+ @property
+ def recipient_count(self) -> int:
+ return len(self.recipients)
+
+
+def _headers_from_pairs(pairs: Any, name_key: str, value_key: str) -> dict[str, str]:
+ """Fold a vendor's ``[{name, value}, ...]`` header list into a dict.
+
+ Repeated headers are joined with a newline rather than overwritten.
+ ``Received`` appears once per hop and the hop chain is the whole of its
+ value; keeping the last one would describe a one-hop delivery for a
+ message that crossed five.
+ """
+ out: dict[str, str] = {}
+ if not isinstance(pairs, (list, tuple)):
+ return out
+ for entry in pairs:
+ if not isinstance(entry, dict):
+ continue
+ name = str(entry.get(name_key) or "").strip().lower()
+ value = str(entry.get(value_key) or "").strip()
+ if not name:
+ continue
+ out[name] = f"{out[name]}\n{value}" if name in out else value
+ return out
+
+
+def _indicators(body: str, headers: dict[str, str], attachments: tuple[dict[str, Any], ...]) -> tuple[tuple[str, ...], tuple[str, ...]]:
+ """URLs and SHA-256 digests visible in the message, without fetching anything."""
+ haystack = "\n".join([body, headers.get("subject", "")])
+ urls = tuple(dict.fromkeys(_URL.findall(haystack)))[:50]
+ digests: list[str] = []
+ for attachment in attachments:
+ for key in ("sha256", "hash", "fileHash", "contentSha256"):
+ candidate = str(attachment.get(key) or "")
+ if _SHA256.fullmatch(candidate):
+ digests.append(candidate.lower())
+ break
+ return urls, tuple(dict.fromkeys(digests))
+
+
+# ── Microsoft 365, through Graph ────────────────────────────────────────────
+
+
+def from_graph_message(payload: dict[str, Any], *, reported_by: str = "") -> ReportedMessage:
+ """A Microsoft Graph ``message`` resource from a report mailbox.
+
+ The shape is the documented one: ``internetMessageId``,
+ ``from.emailAddress.address``, ``toRecipients[].emailAddress.address``,
+ ``receivedDateTime``, ``internetMessageHeaders[].name/value``,
+ ``body.content`` and an ``attachments`` collection of ``name`` /
+ ``contentType`` / ``size``.
+
+ A user who reports phish by **forwarding** it makes themselves the sender
+ of the outer message, so ``reported_by`` is taken from the envelope and
+ the *reported* sender is read from the original's headers when the
+ forward preserved them. Where it did not, the outer sender is recorded as
+ the reporter and the sender field stays empty rather than naming the
+ victim as the attacker.
+ """
+ headers = _headers_from_pairs(payload.get("internetMessageHeaders"), "name", "value")
+ envelope_sender = str((payload.get("from") or {}).get("emailAddress", {}).get("address") or "").lower()
+
+ # The original sender, preferred from the preserved headers.
+ sender = ""
+ for key in ("from", "x-ms-exchange-organization-originalfrom", "reply-to", "return-path"):
+ candidates = _addresses(headers.get(key))
+ if candidates and candidates[0] != envelope_sender:
+ sender = candidates[0]
+ break
+
+ recipients = _addresses(
+ [str((r or {}).get("emailAddress", {}).get("address") or "") for r in (payload.get("toRecipients") or [])]
+ + [str((r or {}).get("emailAddress", {}).get("address") or "") for r in (payload.get("ccRecipients") or [])]
+ )
+ # A forward lands in the report mailbox addressed to the report mailbox,
+ # which is not a recipient of the phish. The original recipients are in
+ # the preserved `to`/`cc` headers when there are any.
+ header_recipients = _addresses(headers.get("to")) + _addresses(headers.get("cc"))
+ if header_recipients:
+ recipients = tuple(dict.fromkeys(header_recipients))
+
+ attachments = tuple(
+ {
+ "name": str(a.get("name") or ""),
+ "content_type": str(a.get("contentType") or ""),
+ "size": a.get("size"),
+ }
+ for a in (payload.get("attachments") or [])
+ if isinstance(a, dict)
+ )
+ body = str((payload.get("body") or {}).get("content") or "")
+ headers.setdefault("subject", str(payload.get("subject") or ""))
+ urls, digests = _indicators(body, headers, attachments)
+
+ return ReportedMessage(
+ intake_source="m365_graph",
+ internet_message_id=normalise_message_id(
+ headers.get("message-id") or str(payload.get("internetMessageId") or "")
+ ),
+ sender=sender,
+ subject=str(payload.get("subject") or ""),
+ recipients=recipients,
+ reported_by=reported_by or envelope_sender,
+ received_at=_parse_time(payload.get("receivedDateTime")),
+ urls=urls,
+ attachment_hashes=digests,
+ attachments=attachments,
+ headers=headers,
+ body_text=body,
+ source_message_id=str(payload.get("id") or ""),
+ )
+
+
+# ── Google Workspace, through Gmail ─────────────────────────────────────────
+
+
+def from_gmail_message(payload: dict[str, Any], *, reported_by: str = "") -> ReportedMessage:
+ """A Gmail ``Message`` resource, as ``users.messages.get`` returns it.
+
+ ``payload.headers`` is a list of ``{name, value}``; ``snippet`` is the
+ only body text a ``format=metadata`` read returns, and that read is
+ deliberate — see ``GmailClient.get_message``. Where a caller supplied a
+ full ``format=full`` message the base64url body is decoded.
+ """
+ gmail_payload = payload.get("payload") or {}
+ headers = _headers_from_pairs(gmail_payload.get("headers"), "name", "value")
+
+ body = str(payload.get("snippet") or "")
+ encoded = ((gmail_payload.get("body") or {}).get("data")) if isinstance(gmail_payload.get("body"), dict) else None
+ if encoded:
+ try:
+ body = base64.urlsafe_b64decode(str(encoded) + "==").decode("utf-8", "replace")
+ except (binascii.Error, ValueError):
+ # A body that will not decode is not a reason to lose the
+ # snippet, and it is certainly not a reason to fail the intake.
+ pass
+
+ attachments = tuple(
+ {
+ "name": str(part.get("filename") or ""),
+ "content_type": str(part.get("mimeType") or ""),
+ "size": (part.get("body") or {}).get("size"),
+ }
+ for part in (gmail_payload.get("parts") or [])
+ if isinstance(part, dict) and part.get("filename")
+ )
+ urls, digests = _indicators(body, headers, attachments)
+ subject = headers.get("subject", "")
+
+ return ReportedMessage(
+ intake_source="google_workspace",
+ internet_message_id=normalise_message_id(headers.get("message-id", "")),
+ sender=(_addresses(headers.get("from")) or ("",))[0],
+ subject=subject,
+ recipients=tuple(dict.fromkeys(_addresses(headers.get("to")) + _addresses(headers.get("cc")))),
+ reported_by=reported_by or (_addresses(headers.get("delivered-to")) or ("",))[0],
+ received_at=_parse_time(headers.get("date")),
+ urls=urls,
+ attachment_hashes=digests,
+ attachments=attachments,
+ headers=headers,
+ body_text=body,
+ source_message_id=str(payload.get("id") or ""),
+ )
+
+
+# ── The existing email_inbox connector ──────────────────────────────────────
+
+
+def from_email_inbox_event(payload: dict[str, Any], *, reported_by: str = "") -> ReportedMessage:
+ """The flat dict ``EmailInboxConnector._to_event`` emits.
+
+ Its keys are ``subject`` / ``from`` / ``to`` / ``received_at`` /
+ ``message_id`` / ``body``, which that connector's docstring pins to the
+ ingest-side ``email-forwarded`` template. This adapter reads exactly
+ those and tolerates a ``headers`` dict when a caller has one, because the
+ webhook sibling (``POST /v1/inbox/email/{token}``) can carry more than
+ IMAP's six fields.
+ """
+ headers = {str(k).lower(): str(v) for k, v in (payload.get("headers") or {}).items() if isinstance(payload.get("headers"), dict)}
+ body = str(payload.get("body") or "")
+ subject = str(payload.get("subject") or headers.get("subject", ""))
+ headers.setdefault("subject", subject)
+ urls, digests = _indicators(body, headers, ())
+
+ return ReportedMessage(
+ intake_source="email_inbox",
+ internet_message_id=normalise_message_id(payload.get("message_id") or headers.get("message-id", "")),
+ sender=(_addresses(payload.get("from") or headers.get("from")) or ("",))[0],
+ subject=subject,
+ recipients=_addresses(payload.get("to") or headers.get("to")),
+ reported_by=reported_by,
+ received_at=_parse_time(payload.get("received_at") or headers.get("date")),
+ urls=urls,
+ attachment_hashes=digests,
+ attachments=(),
+ headers=headers,
+ body_text=body,
+ )
+
+
+#: ``intake source -> adapter``. A dict rather than an ``if`` chain so the
+#: route can validate the source against the same set it dispatches on, and
+#: so adding a fourth door is one entry rather than two edits that can
+#: disagree.
+ADAPTERS = {
+ "m365_graph": from_graph_message,
+ "google_workspace": from_gmail_message,
+ "email_inbox": from_email_inbox_event,
+}
+
+
+def normalise(source: str, payload: dict[str, Any], *, reported_by: str = "") -> ReportedMessage:
+ """Turn one vendor payload into a :class:`ReportedMessage`.
+
+ Raises ``ValueError`` for an unknown source rather than falling back to a
+ lenient parser. A lenient fallback is how the ingest normaliser came to
+ stamp "Splunk" onto events from twenty-six other connectors.
+ """
+ adapter = ADAPTERS.get(source)
+ if adapter is None:
+ raise ValueError(f"unknown intake source {source!r}; expected one of {sorted(ADAPTERS)}")
+ return adapter(payload, reported_by=reported_by)
+
+
+def utcnow() -> datetime:
+ return datetime.now(UTC)