Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

feat(api): report-mailbox intake worker, registered in the lifespan

Beenu Aroraf96b70a2e183
diff --git a/services/actions/app/live_actions/investigation_reads.py b/services/actions/app/live_actions/investigation_reads.py
index 10ae07df3..edbe38c7e 100644
--- a/services/actions/app/live_actions/investigation_reads.py
+++ b/services/actions/app/live_actions/investigation_reads.py
@@ -955,6 +955,14 @@ class GmailSearchMailboxes(LiveActionExecutor):
sender=str(params.get("sender") or ""),
limit=int(params.get("limit") or 25),
)
+ if params.get("hydrate", True):
+ # Gmail's list endpoint answers `{id, threadId}` and nothing
+ # else, while Graph's answers a full message resource. A
+ # verb that returns an opaque id on one vendor and a
+ # readable message on the other is one capability with two
+ # meanings, and the caller ends up writing the vendor check
+ # the registry exists to remove. Headers only, never bodies.
+ matches = [await client.get_message(target, str(m.get("id") or "")) for m in matches if m.get("id")]
except ValueError as exc:
# The Gmail query sanitiser refuses rather than escapes, and a
# refusal is a failure rather than an empty result: an escaped
diff --git a/services/api/app/core/config.py b/services/api/app/core/config.py
index 9b1c740f5..4b89f5f50 100644
--- a/services/api/app/core/config.py
+++ b/services/api/app/core/config.py
@@ -468,6 +468,19 @@ class Settings(BaseSettings):
)
RETENTION_WORKER_INTERVAL_SECONDS: int = 21600 # 6h
+ # Depth 7.1 — the reported-phish report-mailbox poller. Off by default
+ # for the same reason the mailbox arms are: reading a tenant's mailbox
+ # on a schedule is a capability an operator opts into, not one an
+ # upgrade acquires. Enabling it alone changes nothing either — the
+ # worker also needs `connector_config.phishing_report_mailbox` on an
+ # enabled Microsoft 365 or Google Workspace connector, and it never
+ # guesses an address.
+ PHISHING_INTAKE_WORKER_ENABLED: bool = Field(
+ default=False,
+ validation_alias=AliasChoices("PHISHING_INTAKE_WORKER_ENABLED", "AISOC_PHISHING_INTAKE_WORKER_ENABLED"),
+ )
+ PHISHING_INTAKE_WORKER_INTERVAL_SECONDS: int = 300 # 5m
+
# Approval-SLA expiry (deferral 9b). On by default, unlike the retention
# purge: that one deletes data, this one marks an already-abandoned
# request as abandoned and dispatches nothing. Leaving it off would keep
diff --git a/services/api/app/main.py b/services/api/app/main.py
index e6a2a04d3..30cd08490 100644
--- a/services/api/app/main.py
+++ b/services/api/app/main.py
@@ -41,6 +41,7 @@ from app.workers.autonomy_drift import run_forever as run_autonomy_drift
from app.workers.hunt_scheduler import run_forever as run_hunt_scheduler
from app.workers.oauth_refresh import run_forever as run_oauth_refresh
from app.workers.outbound_webhook_delivery import run_forever as run_outbound_webhook_delivery
+from app.workers.phishing_report_intake import run_forever as run_phishing_report_intake
from app.workers.report_scheduler import run_forever as run_report_scheduler
from app.workers.retention_purge import run_forever as run_retention_purge
from app.workers.retro_hunt_consumer import run_forever as run_retro_hunt_consumer
@@ -66,6 +67,10 @@ _HUNT_SCHEDULER_LOCK_TTL_SECONDS = 300
# ClickHouse mutation with mutations_sync=1 blocks until it lands. 30m keeps
# a second replica from starting a concurrent sweep mid-delete.
_RETENTION_PURGE_LOCK_TTL_SECONDS = 1800
+# A report-mailbox poll is one vendor read plus a bounded number of
+# inserts, so a lock held long after a crashed replica would stall intake
+# for much longer than the work takes. Sized to the poll interval.
+_PHISHING_INTAKE_LOCK_TTL_SECONDS = 300
# Shorter than the others: the sweep is a single bounded UPDATE, so a lock
# held for half an hour after a crashed replica would leave approvals
# un-expired for far longer than the work takes.
@@ -448,6 +453,26 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception as exc:
logger.warning("retention_purge worker failed to start", error=str(exc))
+ # Depth 7.1. Pulls reported phish out of the tenant's nominated report
+ # mailbox. Default off: reading a mailbox on a schedule is a capability
+ # an operator opts into. Enabling it alone still polls nothing until a
+ # connector nominates a mailbox, and the sweep says which half is
+ # missing rather than going quiet.
+ phishing_intake_task: asyncio.Task | None = None
+ if settings.PHISHING_INTAKE_WORKER_ENABLED:
+ try:
+ phishing_intake_task = asyncio.create_task(
+ _run_guarded_scheduler_worker(
+ job_name="phishing_report_intake",
+ ttl_seconds=_PHISHING_INTAKE_LOCK_TTL_SECONDS,
+ worker=run_phishing_report_intake,
+ ),
+ name="phishing_report_intake_worker",
+ )
+ logger.info("phishing_report_intake worker started")
+ except Exception as exc:
+ logger.warning("phishing_report_intake worker failed to start", error=str(exc))
+
hunt_scheduler_task: asyncio.Task | None = None
if settings.HUNT_SCHEDULER_ENABLED:
try:
@@ -679,6 +704,15 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception as exc:
logger.warning("retention_purge worker shutdown error", error=type(exc).__name__)
+ if phishing_intake_task is not None and not phishing_intake_task.done():
+ phishing_intake_task.cancel()
+ try:
+ await phishing_intake_task
+ except asyncio.CancelledError:
+ logger.debug("phishing_report_intake worker cancelled during shutdown")
+ except Exception as exc:
+ logger.warning("phishing_report_intake worker shutdown error", error=type(exc).__name__)
+
if shadow_reconcile_task is not None and not shadow_reconcile_task.done():
shadow_reconcile_task.cancel()
try:
diff --git a/services/api/app/workers/phishing_report_intake.py b/services/api/app/workers/phishing_report_intake.py
new file mode 100644
index 000000000..5389d03ba
--- /dev/null
+++ b/services/api/app/workers/phishing_report_intake.py
@@ -0,0 +1,316 @@
+"""Poll the tenant's phishing report mailbox and turn what is in it into reports.
+
+Depth 7.1 names three intake doors. ``POST /phishing/reports`` is the door a
+caller pushes through and the ``email_inbox`` connector is the door IMAP
+pushes through; this worker is the third — the one that *pulls*, from a
+Microsoft 365 report mailbox over Graph or a Google Workspace label.
+
+Without it the intake adapters would be a mechanism with no production
+caller: the exact "exists, is unit-tested, and nothing calls it on the path
+that matters" shape this repository keeps rediscovering. The module
+reachability gate would even have passed, because the route calls the
+adapters; what it cannot see is that no code path reads a report mailbox.
+
+How a tenant turns it on
+------------------------
+Two things, both deliberate:
+
+1. ``PHISHING_INTAKE_WORKER_ENABLED`` on the deployment. Default off. Reading
+ a mailbox on a schedule is a capability an upgrade must not acquire by
+ itself, the same reasoning the mailbox arms ship disabled under.
+2. ``connector_config.phishing_report_mailbox`` on an enabled ``m365_audit``,
+ ``azure_entra`` or ``google_workspace`` connector. No mailbox, no polling:
+ the worker never guesses an address, because a guess here reads somebody's
+ mail.
+
+A tenant that has done one and not the other is logged once per sweep at
+``info`` with which half is missing. "Nothing is happening" with no reason is
+the diagnostic failure that made a clean-room install take four attempts.
+
+What a poll does and does not do
+---------------------------------
+It asks ``search_mailboxes`` for messages received since the last watermark,
+normalises each through the same adapters the route uses, and writes a
+submission plus a campaign membership. It does **not** mark the source
+message read, move it, or delete it: this worker has a read verb and should
+not acquire a write one to keep its own bookkeeping. De-duplication is by
+``internet_message_id`` per tenant, which is the identity of the message
+rather than of the copy, so the same phish reported twice joins one campaign
+instead of creating two submissions.
+
+The watermark lives in the connector row
+-----------------------------------------
+``connector_config.phishing_report_watermark``, with a deliberate overlap of
+:data:`WATERMARK_OVERLAP_SECONDS` on each poll. Mail is indexed after it is
+delivered, so a watermark with no overlap drops anything that landed in the
+gap — the same lesson the Splunk connector's high-watermark carries, and the
+same reason the overlap is paid for by de-duplication rather than avoided.
+It is not held on an instance attribute: a worker object is rebuilt on
+restart and there are several replicas, so in-process state would silently
+re-read or skip.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import uuid
+from dataclasses import dataclass, field
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.config import settings
+from app.db.cross_tenant import assert_cross_tenant_session
+from app.db.database import AsyncSessionLocal
+from app.services.phishing import campaigns as phishing_campaigns
+from app.services.phishing import intake as phishing_intake
+from app.services.playbook_step_dispatch import dispatch_step
+from app.workers._tick_failures import TickFailures
+
+logger = logging.getLogger("aisoc.phishing_report_intake")
+
+#: Connector types that can hold a report mailbox, and the intake adapter
+#: each one's payloads are parsed by. A dict rather than an ``if`` chain so
+#: the two cannot disagree about which vendor a connector speaks.
+MAILBOX_CONNECTORS: dict[str, str] = {
+ "m365_audit": "m365_graph",
+ "azure_entra": "m365_graph",
+ "google_workspace": "google_workspace",
+}
+
+#: The key an operator sets on the connector to nominate a report mailbox.
+MAILBOX_CONFIG_KEY = "phishing_report_mailbox"
+WATERMARK_CONFIG_KEY = "phishing_report_watermark"
+
+#: How far back a poll reaches beyond the last watermark. Mail is indexed
+#: after it is delivered, so a watermark with no overlap drops whatever
+#: landed in the gap. Paid for by de-duplication on the message id.
+WATERMARK_OVERLAP_SECONDS = 120
+
+#: How far back a *first* poll reaches. Bounded so enabling the worker on a
+#: mailbox with four years of reports does not ingest four years of reports.
+FIRST_POLL_LOOKBACK_HOURS = 24
+
+#: Per poll, per mailbox. The vendor caps its own page at 100.
+MAX_MESSAGES_PER_POLL = 50
+
+
+@dataclass
+class SweepReport:
+ """What one sweep did, for the log line and for the tests."""
+
+ tenants_considered: int = 0
+ mailboxes_polled: int = 0
+ messages_seen: int = 0
+ reports_created: int = 0
+ duplicates_skipped: int = 0
+ campaigns_touched: set[uuid.UUID] = field(default_factory=set)
+ #: Tenants that configured half of what the worker needs. Named rather
+ #: than counted: "nothing is happening" with no reason is the diagnostic
+ #: failure worth avoiding.
+ incomplete: list[str] = field(default_factory=list)
+ errors: list[str] = field(default_factory=list)
+
+
+async def _mailbox_connectors(db: AsyncSession) -> list[Any]:
+ """Enabled connectors across every tenant that nominate a report mailbox."""
+ rows = (
+ await db.execute(
+ text("""
+ SELECT id, tenant_id, connector_type, connector_config
+ FROM connectors
+ WHERE is_enabled = TRUE
+ AND connector_type = ANY(CAST(:types AS text[]))
+ ORDER BY tenant_id, connector_type
+ """).bindparams(types=sorted(MAILBOX_CONNECTORS))
+ )
+ ).fetchall()
+ return list(rows)
+
+
+def _watermark(config: dict[str, Any]) -> datetime:
+ raw = str((config or {}).get(WATERMARK_CONFIG_KEY) or "")
+ if raw:
+ try:
+ parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+ return parsed - timedelta(seconds=WATERMARK_OVERLAP_SECONDS)
+ except ValueError:
+ logger.warning("phishing_report_intake: unparseable watermark %r, falling back to the first-poll window", raw[:64])
+ return datetime.now(UTC) - timedelta(hours=FIRST_POLL_LOOKBACK_HOURS)
+
+
+async def _already_seen(db: AsyncSession, *, tenant_id: uuid.UUID, message_id: str) -> bool:
+ if not message_id:
+ # A message with no id cannot be de-duplicated, and ingesting it
+ # anyway would create a new submission on every poll for as long as
+ # it sits in the mailbox. Skipping is the lesser error and it is
+ # counted, so a mailbox full of them is visible.
+ return True
+ found = (
+ await db.execute(
+ text("SELECT 1 FROM aisoc_phishing_submissions WHERE tenant_id = :tenant_id AND internet_message_id = :mid LIMIT 1").bindparams(
+ tenant_id=tenant_id, mid=message_id
+ )
+ )
+ ).fetchone()
+ return found is not None
+
+
+async def _record(db: AsyncSession, *, tenant_id: uuid.UUID, message: phishing_intake.ReportedMessage) -> uuid.UUID | None:
+ """Write one submission and join it to its campaign.
+
+ Triage is deliberately **not** run here. The worker ingests whatever is
+ in a report mailbox, which on a busy tenant is hundreds of messages a
+ day, and an LLM call per message is a cost nobody asked for on a path
+ nobody is watching. ``POST /phishing/{id}/retriage`` is the door for
+ that, and the verdict stays ``pending`` until somebody opens it — which
+ is honest, where a heuristic verdict applied silently would not be.
+ """
+ submission_id = uuid.uuid4()
+ now = datetime.now(UTC)
+ await db.execute(
+ text("""
+ INSERT INTO aisoc_phishing_submissions (
+ id, tenant_id, submitted_by, artifact_kind, raw_content, sender, subject,
+ urls, verdict, confidence, indicators, mitre_technique,
+ internet_message_id, recipients, reported_by, intake_source,
+ submitted_at, triaged_at, created_at
+ ) VALUES (
+ :id, :tenant_id, 'phishing-report-intake', 'email', :content, :sender, :subject,
+ CAST(:urls AS text[]), 'pending', NULL, CAST('[]' AS jsonb), NULL,
+ :message_id, CAST(:recipients AS text[]), :reported_by, :source,
+ :now, NULL, :now
+ )
+ """).bindparams(
+ id=submission_id,
+ tenant_id=tenant_id,
+ content=message.body_text,
+ sender=message.sender,
+ subject=message.subject,
+ urls=list(message.urls),
+ message_id=message.internet_message_id,
+ recipients=list(message.recipients),
+ reported_by=message.reported_by,
+ source=message.intake_source,
+ now=now,
+ )
+ )
+ campaign = await phishing_campaigns.record_report(db, tenant_id=tenant_id, message=message, submission_id=submission_id)
+ return campaign.id if campaign else None
+
+
+async def run_once() -> SweepReport:
+ """One sweep across every tenant with a nominated report mailbox."""
+ report = SweepReport()
+ async with AsyncSessionLocal() as db:
+ assert_cross_tenant_session(db, reason="phishing report intake sweeps every tenant's nominated report mailbox")
+ connectors = await _mailbox_connectors(db)
+ report.tenants_considered = len({str(c.tenant_id) for c in connectors})
+
+ for connector in connectors:
+ config = dict(connector.connector_config or {})
+ mailbox = str(config.get(MAILBOX_CONFIG_KEY) or "").strip()
+ if not mailbox:
+ report.incomplete.append(
+ f"tenant {connector.tenant_id} has an enabled {connector.connector_type} connector and no "
+ f"{MAILBOX_CONFIG_KEY} in its connector_config, so nothing is polled"
+ )
+ continue
+
+ source = MAILBOX_CONNECTORS[connector.connector_type]
+ since = _watermark(config)
+ polled_at = datetime.now(UTC)
+
+ step = await dispatch_step(
+ db,
+ tenant_id=connector.tenant_id,
+ capability="search_mailboxes",
+ target=mailbox,
+ params={
+ "received_after": since.isoformat().replace("+00:00", "Z"),
+ "limit": MAX_MESSAGES_PER_POLL,
+ },
+ vendor_id="m365" if source == "m365_graph" else "google_workspace",
+ dry_run=False,
+ requested_by="aisoc-phishing-report-intake",
+ )
+ if not step.executed:
+ # Reported with the step's own reason. "The report mailbox
+ # has gone quiet" and "the capability is disabled on this
+ # deployment" send an operator to different places, and the
+ # step vocabulary already distinguishes them.
+ report.errors.append(f"tenant {connector.tenant_id} {mailbox}: {step.status} — {step.summary}")
+ continue
+
+ report.mailboxes_polled += 1
+ matches = list((step.details or {}).get("matches") or [])
+ report.messages_seen += len(matches)
+
+ for payload in matches:
+ if not isinstance(payload, dict):
+ continue
+ try:
+ message = phishing_intake.normalise(source, payload)
+ except ValueError as exc:
+ report.errors.append(f"tenant {connector.tenant_id}: {exc}")
+ continue
+ if await _already_seen(db, tenant_id=connector.tenant_id, message_id=message.internet_message_id):
+ report.duplicates_skipped += 1
+ continue
+ campaign_id = await _record(db, tenant_id=connector.tenant_id, message=message)
+ report.reports_created += 1
+ if campaign_id:
+ report.campaigns_touched.add(campaign_id)
+
+ # The watermark advances only on a poll that reached the vendor,
+ # which is why it is set here and not before the dispatch: a
+ # failed poll that moved the watermark would skip whatever
+ # arrived during the outage.
+ config[WATERMARK_CONFIG_KEY] = polled_at.isoformat().replace("+00:00", "Z")
+ await db.execute(
+ text("UPDATE connectors SET connector_config = CAST(:cfg AS jsonb) WHERE id = :id").bindparams(
+ cfg=json.dumps(config), id=connector.id
+ )
+ )
+
+ await db.commit()
+
+ if report.incomplete:
+ logger.info("phishing_report_intake: %s", "; ".join(report.incomplete[:5]))
+ for problem in report.errors[:5]:
+ logger.warning("phishing_report_intake: %s", problem)
+ logger.info(
+ "phishing_report_intake swept tenants=%d mailboxes=%d seen=%d created=%d duplicates=%d campaigns=%d",
+ report.tenants_considered,
+ report.mailboxes_polled,
+ report.messages_seen,
+ report.reports_created,
+ report.duplicates_skipped,
+ len(report.campaigns_touched),
+ )
+ return report
+
+
+async def run_forever() -> None:
+ """Tick until cancelled. Owned by the API ``lifespan``, like the other workers."""
+ interval = max(int(getattr(settings, "PHISHING_INTAKE_WORKER_INTERVAL_SECONDS", 300)), 60)
+ logger.info("phishing_report_intake started interval=%ds", interval)
+ failures = TickFailures("phishing_report_intake", logger)
+ try:
+ while True:
+ try:
+ await run_once()
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc: # pragma: no cover - defensive
+ failures.record_failure(exc)
+ else:
+ failures.record_success()
+ await asyncio.sleep(interval)
+ except asyncio.CancelledError:
+ logger.info("phishing_report_intake stopped")
+ raise