Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(connectors): nine cloud and SaaS connectors
Beenu Arora2b1181fc0cfa
diff --git a/services/connectors/app/connectors/azure_activity.py b/services/connectors/app/connectors/azure_activity.pynew file mode 100644index 000000000..2f65ebdf6--- /dev/null+++ b/services/connectors/app/connectors/azure_activity.py@@ -0,0 +1,245 @@+"""+Azure Activity Logs connector.++Pulls subscription-level Azure Activity Log entries — the audit record of+every control-plane operation against Azure resources (creates, deletes,+RBAC changes, policy changes, etc.).++Auth model: Azure AD app registration with the **Reader** role assigned at+the subscription scope (or higher, e.g. management group). We use the+client-credentials flow against the ARM management endpoint.++API: ``/subscriptions/{sub}/providers/Microsoft.Insights/eventtypes/management/values``+which is the documented Activity Log read API. Resource Graph would be+faster for very large estates, but the Activity Log API is the canonical+source and avoids needing the Resource Graph SDK.+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta+from typing import Any++import httpx+import structlog++from app.connectors.base import BaseConnector, ConnectorSchema, Field, OAuthHints++logger = structlog.get_logger()++_TOKEN_URL = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"+_ARM_BASE = "https://management.azure.com"+_ACTIVITY_API_VERSION = "2015-04-01"++# Verbs that move the cluster — we boost severity for these so analysts+# actually see them in the queue. Everything else is informational unless+# the operation explicitly failed.+# Substrings (matched against the lower-cased operationName.value) that we+# treat as inherently high blast radius. Azure operation names follow the+# pattern ``Microsoft.<rp>/<resource>/<verb>``, e.g.+# ``Microsoft.Authorization/roleAssignments/write``, so each entry is the+# tail of the path we want to flag.+_HIGH_BLAST_RADIUS_VERBS = (+ "/delete",+ "roleassignments/write",+ "roleassignmentscheduleinstances/write",+ "policyassignments/write",+ "policydefinitions/write",+ "/deallocate",+ "/disable",+ "diagnosticsettings/delete",+)+++class AzureActivityConnector(BaseConnector):+ """Azure Resource Manager Activity Logs (control-plane audit)."""++ connector_id = "azure_activity"+ connector_name = "Azure Activity Logs"+ connector_category = "cloud"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description=(+ "Azure subscription Activity Log — every control-plane "+ "operation (resource CRUD, RBAC changes, policy changes). "+ "Requires Reader role on the subscription."+ ),+ docs_url="/docs/connectors/azure-activity",+ fields=[+ Field("tenant_id", "string", "Tenant ID"),+ Field("client_id", "string", "Application (Client) ID"),+ Field("client_secret", "secret", "Client Secret"),+ Field(+ "subscription_id",+ "string",+ "Subscription ID",+ placeholder="00000000-0000-0000-0000-000000000000",+ ),+ ],+ oauth=OAuthHints(+ supported_in_hosted=True,+ authorize_url="https://login.microsoftonline.com/common/oauth2/v2.0/authorize",+ token_url="https://login.microsoftonline.com/common/oauth2/v2.0/token",+ scopes=["https://management.azure.com/user_impersonation"],+ ),+ )++ def __init__(+ self,+ tenant_id: str,+ client_id: str,+ client_secret: str,+ subscription_id: str,+ ):+ self._tenant_id = tenant_id+ self._client_id = client_id+ self._client_secret = client_secret+ self._subscription_id = subscription_id+ self._access_token: str | None = None++ # --------------------------- auth ---------------------------++ async def _authenticate(self) -> str:+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.post(+ _TOKEN_URL.format(tenant_id=self._tenant_id),+ data={+ "client_id": self._client_id,+ "client_secret": self._client_secret,+ "scope": "https://management.azure.com/.default",+ "grant_type": "client_credentials",+ },+ )+ resp.raise_for_status()+ self._access_token = resp.json()["access_token"]+ return self._access_token++ def _headers(self) -> dict[str, str]:+ return {+ "Authorization": f"Bearer {self._access_token}",+ "Accept": "application/json",+ }++ # ------------------------- contract -------------------------++ async def test_connection(self) -> dict[str, Any]:+ try:+ await self._authenticate()+ # Confirm the subscription is actually reachable with this token+ # (catches "wrong tenant" and "no Reader role" mistakes early).+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.get(+ f"{_ARM_BASE}/subscriptions/{self._subscription_id}",+ headers=self._headers(),+ params={"api-version": "2020-01-01"},+ )+ resp.raise_for_status()+ sub = resp.json()+ return {+ "success": True,+ "connector": self.connector_id,+ "subscription_id": self._subscription_id,+ "subscription_name": sub.get("displayName"),+ }+ except httpx.HTTPStatusError as exc:+ return {+ "success": False,+ "connector": self.connector_id,+ "error": f"HTTP {exc.response.status_code}: {exc.response.text[:300]}",+ }+ except Exception as exc:+ return {"success": False, "connector": self.connector_id, "error": str(exc)}++ async def fetch_alerts(self, since_seconds: int = 300) -> list[dict[str, Any]]:+ if not self._access_token:+ await self._authenticate()++ # Activity log filter is OData; quote the literal datetime.+ since = (datetime.now(UTC) - timedelta(seconds=since_seconds)).strftime(+ "%Y-%m-%dT%H:%M:%SZ"+ )+ until = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")++ url = (+ f"{_ARM_BASE}/subscriptions/{self._subscription_id}"+ f"/providers/Microsoft.Insights/eventtypes/management/values"+ )+ params = {+ "api-version": _ACTIVITY_API_VERSION,+ "$filter": (+ f"eventTimestamp ge '{since}' and eventTimestamp le '{until}'"+ ),+ }++ events: list[dict[str, Any]] = []+ async with httpx.AsyncClient(timeout=30.0) as client:+ resp = await client.get(url, headers=self._headers(), params=params)+ if resp.status_code == 401:+ await self._authenticate()+ resp = await client.get(url, headers=self._headers(), params=params)+ if resp.status_code != 200:+ logger.warning(+ "azure_activity.fetch_failed",+ status=resp.status_code,+ body=resp.text[:300],+ )+ return []+ events = resp.json().get("value", [])++ # Activity log responses are paginated via ``nextLink``; we cap+ # at a few pages per poll so a single noisy subscription can't+ # starve the scheduler.+ for _ in range(4):+ next_link = resp.json().get("nextLink")+ if not next_link:+ break+ resp = await client.get(next_link, headers=self._headers())+ if resp.status_code != 200:+ break+ events.extend(resp.json().get("value", []))++ return [self.normalize(e) for e in events]++ # ----------------------- normalize --------------------------++ def normalize(self, raw: dict[str, Any]) -> dict[str, Any]:+ op_name = (raw.get("operationName", {}) or {}).get("value") or ""+ status = (raw.get("status", {}) or {}).get("value") or ""+ sub_status = (raw.get("subStatus", {}) or {}).get("value") or ""+ level = (raw.get("level") or "").lower()+ caller = raw.get("caller", "")++ op_lower = op_name.lower()+ severity = "info"+ if level in ("error", "critical"):+ severity = "high"+ elif level == "warning":+ severity = "medium"+ if status.lower() == "failed":+ severity = "medium" if severity == "info" else severity+ if any(verb in op_lower for verb in _HIGH_BLAST_RADIUS_VERBS):+ severity = "high"++ resource_id = raw.get("resourceId") or ""++ return {+ "source": self.connector_id,+ "external_id": raw.get("eventDataId") or raw.get("id", ""),+ "title": op_name or "Azure Activity Log Event",+ "description": (+ f"status={status}; subStatus={sub_status}; "+ f"resource={resource_id}"+ ),+ "severity": severity,+ "actor": caller,+ "actor_email": caller if "@" in caller else None,+ "event_type": f"azure.activity.{op_lower or 'unknown'}",+ "resource": resource_id,+ "raw_event": raw,+ "created_at": raw.get("eventTimestamp"),+ }diff --git a/services/connectors/app/connectors/azure_defender.py b/services/connectors/app/connectors/azure_defender.pynew file mode 100644index 000000000..ad2a9db96--- /dev/null+++ b/services/connectors/app/connectors/azure_defender.py@@ -0,0 +1,202 @@+"""+Microsoft Defender / Microsoft 365 Defender alerts connector.++Pulls alerts from the unified Microsoft Graph Security ``alerts_v2`` endpoint.+This single feed covers Defender for Cloud, Defender for Endpoint, Defender+for Identity, Defender for Office 365, and Defender for Cloud Apps — i.e.+the post-Sentinel "XDR" alert surface.++Auth model: Azure AD app registration with the following Microsoft Graph+*application* permissions granted with admin consent:++ SecurityAlert.Read.All++(``SecurityIncident.Read.All`` is needed if you also want incidents, but+we stick to alerts here so the contract matches the other connectors.)+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta+from typing import Any++import httpx+import structlog++from app.connectors.base import BaseConnector, ConnectorSchema, Field, OAuthHints++logger = structlog.get_logger()++_TOKEN_URL = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"+_GRAPH_BASE = "https://graph.microsoft.com/v1.0"+_PAGE_SIZE = 200+++class AzureDefenderConnector(BaseConnector):+ """Microsoft 365 Defender / Defender for Cloud alerts via Graph."""++ connector_id = "azure_defender"+ connector_name = "Microsoft Defender"+ connector_category = "edr"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description=(+ "Microsoft 365 Defender unified alerts feed (Defender for "+ "Cloud, Endpoint, Identity, Office 365, Cloud Apps) via "+ "Microsoft Graph. Requires SecurityAlert.Read.All."+ ),+ docs_url="/docs/connectors/azure-defender",+ fields=[+ Field("tenant_id", "string", "Tenant ID"),+ Field("client_id", "string", "Application (Client) ID"),+ Field("client_secret", "secret", "Client Secret"),+ ],+ oauth=OAuthHints(+ supported_in_hosted=True,+ authorize_url="https://login.microsoftonline.com/common/oauth2/v2.0/authorize",+ token_url="https://login.microsoftonline.com/common/oauth2/v2.0/token",+ scopes=["https://graph.microsoft.com/SecurityAlert.Read.All"],+ ),+ )++ def __init__(self, tenant_id: str, client_id: str, client_secret: str):+ self._tenant_id = tenant_id+ self._client_id = client_id+ self._client_secret = client_secret+ self._access_token: str | None = None++ # --------------------------- auth ---------------------------++ async def _authenticate(self) -> str:+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.post(+ _TOKEN_URL.format(tenant_id=self._tenant_id),+ data={+ "client_id": self._client_id,+ "client_secret": self._client_secret,+ "scope": "https://graph.microsoft.com/.default",+ "grant_type": "client_credentials",+ },+ )+ resp.raise_for_status()+ self._access_token = resp.json()["access_token"]+ return self._access_token++ def _headers(self) -> dict[str, str]:+ return {+ "Authorization": f"Bearer {self._access_token}",+ "Accept": "application/json",+ }++ # ------------------------- contract -------------------------++ async def test_connection(self) -> dict[str, Any]:+ try:+ await self._authenticate()+ # Pull a single alert just to confirm scope/token combo works.+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.get(+ f"{_GRAPH_BASE}/security/alerts_v2",+ headers=self._headers(),+ params={"$top": 1},+ )+ resp.raise_for_status()+ return {+ "success": True,+ "connector": self.connector_id,+ "tenant_id": self._tenant_id,+ }+ except httpx.HTTPStatusError as exc:+ return {+ "success": False,+ "connector": self.connector_id,+ "error": f"HTTP {exc.response.status_code}: {exc.response.text[:300]}",+ }+ except Exception as exc:+ return {"success": False, "connector": self.connector_id, "error": str(exc)}++ async def fetch_alerts(self, since_seconds: int = 300) -> list[dict[str, Any]]:+ if not self._access_token:+ await self._authenticate()++ since = (datetime.now(UTC) - timedelta(seconds=since_seconds)).strftime(+ "%Y-%m-%dT%H:%M:%S.000Z"+ )+ params = {+ "$filter": f"createdDateTime ge {since}",+ "$top": _PAGE_SIZE,+ "$orderby": "createdDateTime desc",+ }++ alerts: list[dict[str, Any]] = []+ async with httpx.AsyncClient(timeout=30.0) as client:+ resp = await client.get(+ f"{_GRAPH_BASE}/security/alerts_v2",+ headers=self._headers(),+ params=params,+ )+ if resp.status_code == 401:+ await self._authenticate()+ resp = await client.get(+ f"{_GRAPH_BASE}/security/alerts_v2",+ headers=self._headers(),+ params=params,+ )+ if resp.status_code != 200:+ logger.warning(+ "azure_defender.fetch_failed",+ status=resp.status_code,+ body=resp.text[:300],+ )+ return []+ alerts = resp.json().get("value", [])++ return [self.normalize(a) for a in alerts]++ # ----------------------- normalize --------------------------++ def normalize(self, raw: dict[str, Any]) -> dict[str, Any]:+ # Graph alerts_v2 uses {low, medium, high, informational}; map+ # informational → info to match the AiSOC scale.+ sev_in = (raw.get("severity") or "medium").lower()+ severity = {"informational": "info"}.get(sev_in, sev_in)++ # Pull the first user/host evidence so the analyst sees something+ # actionable in the queue without expanding raw_event.+ evidences = raw.get("evidence") or []+ actor = None+ actor_email = None+ host = None+ for ev in evidences:+ odata_type = ev.get("@odata.type", "")+ if "userEvidence" in odata_type and not actor:+ user_account = ev.get("userAccount", {}) or {}+ actor = user_account.get("displayName") or user_account.get("accountName")+ actor_email = user_account.get("userPrincipalName")+ elif "deviceEvidence" in odata_type and not host:+ host = ev.get("deviceDnsName") or ev.get("hostName")++ return {+ "source": self.connector_id,+ "external_id": raw.get("id", ""),+ "title": raw.get("title", "Defender Alert"),+ "description": raw.get("description", ""),+ "severity": severity,+ "status": raw.get("status"),+ "actor": actor,+ "actor_email": actor_email,+ "host": host,+ "service_source": raw.get("serviceSource"),+ "tactics": raw.get("mitreTechniques", []),+ "category": raw.get("category"),+ "event_type": (+ f"azure.defender.{(raw.get('serviceSource') or 'unknown').lower()}"+ ),+ "raw_event": raw,+ "created_at": raw.get("createdDateTime"),+ }diff --git a/services/connectors/app/connectors/azure_entra.py b/services/connectors/app/connectors/azure_entra.pynew file mode 100644index 000000000..51c56c981--- /dev/null+++ b/services/connectors/app/connectors/azure_entra.py@@ -0,0 +1,265 @@+"""+Azure Entra ID (formerly Azure AD) connector.++Pulls directory audit logs and risky sign-in events via Microsoft Graph.+This is the identity-plane counterpart to ``azure_activity`` (control plane)+and ``azure_defender`` (security plane).++Auth model: Azure AD app registration with the following Microsoft Graph+*application* permissions granted with admin consent:++ AuditLog.Read.All+ Directory.Read.All+ IdentityRiskyUser.Read.All (only if you want risky-user signals)++We use the OAuth2 client-credentials flow, scoped to ``.default`` so the+token reflects exactly the consented permissions — that way the connector+fails closed if an operator forgets to grant a scope.+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta+from typing import Any++import httpx+import structlog++from app.connectors.base import BaseConnector, ConnectorSchema, Field, OAuthHints++logger = structlog.get_logger()++_TOKEN_URL = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"+_GRAPH_BASE = "https://graph.microsoft.com/v1.0"++# Microsoft Graph caps `$top` at 1000 for these collections; 200 keeps each+# round-trip fast and rate-limit friendly while still covering normal volume.+_PAGE_SIZE = 200+++class AzureEntraConnector(BaseConnector):+ """Microsoft Entra ID directory audits + risky sign-ins."""++ connector_id = "azure_entra"+ connector_name = "Microsoft Entra ID"+ connector_category = "iam"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description=(+ "Microsoft Entra ID (Azure AD) directory audits and risky "+ "sign-ins via Microsoft Graph. Requires AuditLog.Read.All "+ "and Directory.Read.All application permissions."+ ),+ docs_url="/docs/connectors/azure-entra",+ fields=[+ Field("tenant_id", "string", "Tenant ID", placeholder="00000000-0000-0000-0000-000000000000"),+ Field("client_id", "string", "Application (Client) ID"),+ Field("client_secret", "secret", "Client Secret"),+ ],+ oauth=OAuthHints(+ supported_in_hosted=True,+ authorize_url="https://login.microsoftonline.com/common/oauth2/v2.0/authorize",+ token_url="https://login.microsoftonline.com/common/oauth2/v2.0/token",+ scopes=[+ "https://graph.microsoft.com/AuditLog.Read.All",+ "https://graph.microsoft.com/Directory.Read.All",+ ],+ ),+ )++ def __init__(self, tenant_id: str, client_id: str, client_secret: str):+ self._tenant_id = tenant_id+ self._client_id = client_id+ self._client_secret = client_secret+ self._access_token: str | None = None++ # --------------------------- auth ---------------------------++ async def _authenticate(self) -> str:+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.post(+ _TOKEN_URL.format(tenant_id=self._tenant_id),+ data={+ "client_id": self._client_id,+ "client_secret": self._client_secret,+ "scope": "https://graph.microsoft.com/.default",+ "grant_type": "client_credentials",+ },+ )+ resp.raise_for_status()+ self._access_token = resp.json()["access_token"]+ return self._access_token++ def _headers(self) -> dict[str, str]:+ return {+ "Authorization": f"Bearer {self._access_token}",+ "Accept": "application/json",+ }++ # ------------------------- contract -------------------------++ async def test_connection(self) -> dict[str, Any]:+ try:+ await self._authenticate()+ # Hit a cheap Graph endpoint to confirm the token can actually+ # reach the directory; the token call alone doesn't verify scopes.+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.get(+ f"{_GRAPH_BASE}/organization?$select=id,displayName",+ headers=self._headers(),+ )+ resp.raise_for_status()+ orgs = resp.json().get("value", [])+ return {+ "success": True,+ "connector": self.connector_id,+ "tenant_id": self._tenant_id,+ "organization": orgs[0].get("displayName") if orgs else None,+ }+ except httpx.HTTPStatusError as exc:+ return {+ "success": False,+ "connector": self.connector_id,+ "error": f"HTTP {exc.response.status_code}: {exc.response.text[:300]}",+ }+ except Exception as exc:+ return {"success": False, "connector": self.connector_id, "error": str(exc)}++ async def fetch_alerts(self, since_seconds: int = 300) -> list[dict[str, Any]]:+ if not self._access_token:+ await self._authenticate()++ # Graph wants ISO 8601 with milliseconds, e.g. ``2024-01-01T00:00:00.000Z``.+ since = (datetime.now(UTC) - timedelta(seconds=since_seconds)).strftime(+ "%Y-%m-%dT%H:%M:%S.000Z"+ )++ events: list[dict[str, Any]] = []+ async with httpx.AsyncClient(timeout=30.0) as client:+ # Directory audits — admin/role/policy/group changes etc.+ audit_params = {+ "$filter": f"activityDateTime ge {since}",+ "$top": _PAGE_SIZE,+ "$orderby": "activityDateTime desc",+ }+ audit_resp = await client.get(+ f"{_GRAPH_BASE}/auditLogs/directoryAudits",+ headers=self._headers(),+ params=audit_params,+ )+ if audit_resp.status_code == 401:+ await self._authenticate()+ audit_resp = await client.get(+ f"{_GRAPH_BASE}/auditLogs/directoryAudits",+ headers=self._headers(),+ params=audit_params,+ )+ if audit_resp.status_code == 200:+ for raw in audit_resp.json().get("value", []):+ raw["_aisoc_event_kind"] = "directoryAudit"+ events.append(raw)+ else:+ logger.warning(+ "azure_entra.directory_audits.fetch_failed",+ status=audit_resp.status_code,+ body=audit_resp.text[:300],+ )++ # Risky sign-ins — these are higher-signal than every signIn.+ signin_params = {+ "$filter": (+ f"createdDateTime ge {since} and "+ f"riskLevelAggregated ne 'none'"+ ),+ "$top": _PAGE_SIZE,+ "$orderby": "createdDateTime desc",+ }+ signin_resp = await client.get(+ f"{_GRAPH_BASE}/auditLogs/signIns",+ headers=self._headers(),+ params=signin_params,+ )+ if signin_resp.status_code == 200:+ for raw in signin_resp.json().get("value", []):+ raw["_aisoc_event_kind"] = "riskySignIn"+ events.append(raw)+ else:+ logger.warning(+ "azure_entra.risky_signins.fetch_failed",+ status=signin_resp.status_code,+ body=signin_resp.text[:300],+ )++ return [self.normalize(e) for e in events]++ # ----------------------- normalize --------------------------++ def normalize(self, raw: dict[str, Any]) -> dict[str, Any]:+ kind = raw.get("_aisoc_event_kind", "directoryAudit")++ if kind == "riskySignIn":+ risk = (raw.get("riskLevelAggregated") or "low").lower()+ severity = {+ "none": "info",+ "low": "low",+ "medium": "medium",+ "high": "high",+ }.get(risk, "medium")+ return {+ "source": self.connector_id,+ "external_id": raw.get("id", ""),+ "title": f"Risky sign-in: {raw.get('userPrincipalName', 'unknown user')}",+ "description": (+ f"riskLevel={risk}; "+ f"riskState={raw.get('riskState')}; "+ f"riskEventTypes={raw.get('riskEventTypes_v2', raw.get('riskEventTypes', []))}"+ ),+ "severity": severity,+ "src_ip": raw.get("ipAddress"),+ "actor": raw.get("userDisplayName"),+ "actor_email": raw.get("userPrincipalName"),+ "event_type": "azure.entra.signin.risky",+ "raw_event": raw,+ "created_at": raw.get("createdDateTime"),+ }++ # directoryAudit+ result = (raw.get("result") or "").lower()+ category = raw.get("category", "")+ # Failed admin actions or anything in the AuthenticationMethod /+ # RoleManagement category gets bumped to medium since those are+ # the high-blast-radius levers.+ severity = "info"+ if result == "failure":+ severity = "medium"+ if category in ("RoleManagement", "AuthenticationMethod", "ConditionalAccess"):+ severity = "medium" if severity == "info" else severity+ if category == "RoleManagement" and result == "success":+ # role assignments are always worth surfacing+ severity = "high"++ initiator = raw.get("initiatedBy", {}) or {}+ user_part = initiator.get("user", {}) or {}+ app_part = initiator.get("app", {}) or {}++ return {+ "source": self.connector_id,+ "external_id": raw.get("id", ""),+ "title": raw.get("activityDisplayName", "Entra Directory Audit"),+ "description": (+ f"category={category}; "+ f"result={raw.get('result')}; "+ f"resultReason={raw.get('resultReason', '')}"+ ),+ "severity": severity,+ "actor": user_part.get("displayName") or app_part.get("displayName"),+ "actor_email": user_part.get("userPrincipalName"),+ "event_type": f"azure.entra.audit.{category.lower() or 'unknown'}",+ "raw_event": raw,+ "created_at": raw.get("activityDateTime"),+ }diff --git a/services/connectors/app/connectors/cloudflare.py b/services/connectors/app/connectors/cloudflare.pynew file mode 100644index 000000000..637a83b36--- /dev/null+++ b/services/connectors/app/connectors/cloudflare.py@@ -0,0 +1,240 @@+"""+Cloudflare audit logs connector.++Pulls the account-level audit log from the Cloudflare API. Auth is the+modern API token model — recommend a token scoped to ``Account: Audit Logs:+Read`` (and optionally ``Account: Account Settings: Read`` if you want+metadata). Account-scoped tokens beat global API keys for blast radius.++Endpoint: ``GET /accounts/{account_id}/audit_logs`` returns user/admin+activity such as zone changes, API token grants, member adds/removes, WAF+rule edits, and so on. Volume is generally low (most accounts produce+single-digit events per minute even at scale) so we use a flat list call+without pagination on the hot path.+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta+from typing import Any++import httpx+import structlog++from app.connectors.base import BaseConnector, ConnectorSchema, Field, OAuthHints++logger = structlog.get_logger()++_BASE = "https://api.cloudflare.com/client/v4"+_PER_PAGE = 100+++class CloudflareConnector(BaseConnector):+ """Cloudflare account audit logs."""++ connector_id = "cloudflare"+ connector_name = "Cloudflare"+ # ``saas`` covers admin/config audit; the actual edge product is a+ # network service but our data here is operator activity, not packets.+ connector_category = "saas"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description=(+ "Cloudflare account audit logs (zone changes, API token "+ "grants, member adds, WAF/firewall edits) via the v4 API. "+ "Requires an API token scoped to Account: Audit Logs: Read."+ ),+ docs_url="/docs/connectors/cloudflare",+ fields=[+ Field(+ "account_id",+ "string",+ "Account ID",+ placeholder="abc123def456...",+ help_text=(+ "Found in the Cloudflare dashboard under any zone's "+ "Overview, lower-right pane (\"Account ID\")."+ ),+ ),+ Field(+ "api_token",+ "secret",+ "API Token",+ help_text=(+ "Account-scoped API token with the Audit Logs: Read "+ "permission. Avoid using the legacy Global API Key."+ ),+ ),+ ],+ # Cloudflare doesn't have a meaningful OAuth flow for this API;+ # the token model is the recommended path for service-to-service.+ oauth=OAuthHints(+ supported_in_hosted=False,+ authorize_url=None,+ token_url=None,+ scopes=[],+ ),+ )++ def __init__(self, account_id: str, api_token: str):+ self._account_id = account_id+ self._api_token = api_token++ # --------------------------- auth ---------------------------++ def _headers(self) -> dict[str, str]:+ # Cloudflare's recommended header for API tokens — distinct from+ # the legacy ``X-Auth-Email`` + ``X-Auth-Key`` global-key model.+ return {+ "Authorization": f"Bearer {self._api_token}",+ "Accept": "application/json",+ }++ # ------------------------- contract -------------------------++ async def test_connection(self) -> dict[str, Any]:+ try:+ async with httpx.AsyncClient(timeout=15.0) as client:+ # ``/user/tokens/verify`` is the canonical token-health+ # endpoint; it returns 200 + ``status: active`` for a good+ # token without needing any account scope.+ resp = await client.get(+ f"{_BASE}/user/tokens/verify",+ headers=self._headers(),+ )+ if resp.status_code != 200:+ return {+ "success": False,+ "connector": self.connector_id,+ "error": f"HTTP {resp.status_code}: {resp.text[:300]}",+ }+ token_status = resp.json().get("result", {}).get("status")+ if token_status != "active":+ return {+ "success": False,+ "connector": self.connector_id,+ "error": f"Token status: {token_status}",+ }++ # Now confirm the token has audit-log access for this account+ # by issuing a 1-record list. 403 here means "token is good+ # but missing the Audit Logs: Read permission" — that's a+ # different and more useful error than a generic auth fail.+ audit_resp = await client.get(+ f"{_BASE}/accounts/{self._account_id}/audit_logs",+ headers=self._headers(),+ params={"per_page": 1},+ )+ if audit_resp.status_code != 200:+ return {+ "success": False,+ "connector": self.connector_id,+ "error": (+ f"Token verified but audit_logs failed: "+ f"HTTP {audit_resp.status_code}: "+ f"{audit_resp.text[:200]}"+ ),+ }++ return {+ "success": True,+ "connector": self.connector_id,+ "account_id": self._account_id,+ }+ except Exception as exc:+ return {"success": False, "connector": self.connector_id, "error": str(exc)}++ async def fetch_alerts(self, since_seconds: int = 300) -> list[dict[str, Any]]:+ since = (datetime.now(UTC) - timedelta(seconds=since_seconds)).strftime(+ "%Y-%m-%dT%H:%M:%SZ"+ )+ params = {+ "since": since,+ "per_page": _PER_PAGE,+ "direction": "desc",+ }++ async with httpx.AsyncClient(timeout=30.0) as client:+ resp = await client.get(+ f"{_BASE}/accounts/{self._account_id}/audit_logs",+ headers=self._headers(),+ params=params,+ )+ if resp.status_code != 200:+ logger.warning(+ "cloudflare.fetch_failed",+ status=resp.status_code,+ body=resp.text[:300],+ )+ return []+ entries = resp.json().get("result", []) or []++ return [self.normalize(e) for e in entries]++ # ----------------------- normalize --------------------------++ # Cloudflare ``action.type`` values that we always escalate. The full+ # taxonomy is documented at+ # https://developers.cloudflare.com/api/operations/audit-logs-list-account-audit-logs+ # — these are the ones with material security impact.+ _HIGH_RISK_ACTIONS = (+ # API token / authentication changes+ "tokenCreate",+ "tokenDelete",+ "tokenUpdate",+ # account membership+ "memberAdd",+ "memberRemove",+ "memberRoleUpdate",+ "ownerChange",+ # firewall/WAF rule deletions are a classic ATT&CK T1562 pattern+ "firewallDelete",+ "wafPackageDelete",+ "wafRuleDelete",+ # zone / DNS deletions+ "zoneDelete",+ "dnsRecordDelete",+ # access (Zero Trust) policy changes+ "accessPolicyDelete",+ "accessApplicationDelete",+ "accessIdpDelete",+ )++ def normalize(self, raw: dict[str, Any]) -> dict[str, Any]:+ action = raw.get("action") or {}+ action_type = action.get("type", "")+ actor = raw.get("actor") or {}+ resource = raw.get("resource") or {}++ # Severity heuristic: high-risk actions → high; failed actions+ # (action.result == false) → low; ``info`` for everything else.+ severity = "info"+ if any(h == action_type for h in self._HIGH_RISK_ACTIONS):+ severity = "high"+ elif action.get("result") is False:+ severity = "low"++ actor_email = actor.get("email")++ return {+ "source": self.connector_id,+ "external_id": raw.get("id", ""),+ "title": action_type or "Cloudflare audit event",+ "description": (+ f"actor={actor_email or actor.get('id', 'unknown')}; "+ f"action={action_type}; "+ f"resource={resource.get('type', '')}/{resource.get('id', '')}"+ ),+ "severity": severity,+ "actor": actor.get("email") or actor.get("id"),+ "actor_email": actor_email,+ "src_ip": raw.get("actor", {}).get("ip"),+ "event_type": f"cloudflare.{action_type}" if action_type else "cloudflare.audit",+ "raw_event": raw,+ "created_at": raw.get("when"),+ }diff --git a/services/connectors/app/connectors/gcp_cloud_audit.py b/services/connectors/app/connectors/gcp_cloud_audit.pynew file mode 100644index 000000000..0e69bd5c2--- /dev/null+++ b/services/connectors/app/connectors/gcp_cloud_audit.py@@ -0,0 +1,334 @@+"""+GCP Cloud Audit Logs connector.++Pulls Admin Activity, Data Access, and System Event audit logs from Google+Cloud Logging via the ``entries.list`` API. These logs cover IAM changes,+resource mutations, key creates/deletes, and privileged data access — the+control-plane events most SOCs care about.++Auth model: a Google service-account JSON key with at least+``roles/logging.viewer`` (or ``roles/logging.privateLogViewer`` to read+Data Access logs). The full key blob is pasted into the secret field and+encrypted at rest by the credential vault.++The connector talks to two Google services:++ * ``oauth2.googleapis.com/token`` — exchange the signed JWT for+ a short-lived access token.+ * ``logging.googleapis.com/v2/entries:list`` — fetch log entries.++We construct the JWT manually (RS256) so we don't have to depend on the+heavy ``google-auth`` package. Cryptography is already installed for the+credential vault, so reusing it costs nothing.+"""++from __future__ import annotations++import base64+import json+import time+from datetime import UTC, datetime, timedelta+from typing import Any++import httpx+import structlog+from cryptography.hazmat.primitives import hashes, serialization+from cryptography.hazmat.primitives.asymmetric import padding++from app.connectors.base import BaseConnector, ConnectorSchema, Field, OAuthHints++logger = structlog.get_logger()++_TOKEN_URL = "https://oauth2.googleapis.com/token"+_LOGGING_URL = "https://logging.googleapis.com/v2/entries:list"+_SCOPE = "https://www.googleapis.com/auth/logging.read"++# The Logging API caps page size at 1000; 200 keeps each round-trip fast+# and stays well below per-minute quotas for a 5-minute polling cadence.+_PAGE_SIZE = 200+++def _b64url(data: bytes) -> str:+ """Standard JWT base64url with no padding."""+ return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")+++class GCPCloudAuditConnector(BaseConnector):+ """Google Cloud Audit Logs (Admin Activity + Data Access + System Event)."""++ connector_id = "gcp_cloud_audit"+ connector_name = "GCP Cloud Audit Logs"+ connector_category = "cloud"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description=(+ "Google Cloud Audit Logs (Admin Activity, Data Access, System "+ "Event) via Cloud Logging. Requires a service account with "+ "roles/logging.viewer (and roles/logging.privateLogViewer for "+ "Data Access logs)."+ ),+ docs_url="/docs/connectors/gcp-cloud-audit",+ fields=[+ Field(+ "project_id",+ "string",+ "Project ID",+ placeholder="my-prod-project",+ help_text=(+ "The GCP project to read audit logs from. Use a "+ "log-sink aggregator project if you've consolidated."+ ),+ ),+ Field(+ "service_account_json",+ "secret",+ "Service account JSON key",+ help_text=(+ "Paste the full JSON key file. It will be encrypted "+ "at rest by the credential vault."+ ),+ ),+ ],+ oauth=OAuthHints(+ supported_in_hosted=False,+ authorize_url=None,+ token_url=_TOKEN_URL,+ scopes=[_SCOPE],+ ),+ )++ def __init__(self, project_id: str, service_account_json: str):+ self._project_id = project_id+ self._sa_info = self._parse_sa(service_account_json)+ self._access_token: str | None = None+ self._token_expiry: float = 0.0++ # --------------------------- auth ---------------------------++ @staticmethod+ def _parse_sa(blob: str) -> dict[str, Any]:+ """Accept either a raw JSON string or an already-parsed dict-as-string."""+ try:+ sa = json.loads(blob)+ except json.JSONDecodeError as exc:+ raise ValueError(+ "service_account_json is not valid JSON. Paste the entire "+ "key file contents."+ ) from exc+ for required in ("client_email", "private_key", "token_uri"):+ if required not in sa:+ raise ValueError(+ f"service_account_json missing required field: {required}"+ )+ return sa++ def _build_jwt(self) -> str:+ """Build and sign a short-lived JWT for the Google token endpoint."""+ now = int(time.time())+ header = {"alg": "RS256", "typ": "JWT"}+ claims = {+ "iss": self._sa_info["client_email"],+ "scope": _SCOPE,+ "aud": self._sa_info.get("token_uri", _TOKEN_URL),+ "iat": now,+ # 1 hour is the maximum Google accepts for a JWT assertion.+ "exp": now + 3600,+ }+ signing_input = (+ _b64url(json.dumps(header, separators=(",", ":")).encode())+ + "."+ + _b64url(json.dumps(claims, separators=(",", ":")).encode())+ ).encode("ascii")++ private_key = serialization.load_pem_private_key(+ self._sa_info["private_key"].encode("utf-8"),+ password=None,+ )+ signature = private_key.sign( # type: ignore[union-attr]+ signing_input,+ padding.PKCS1v15(),+ hashes.SHA256(),+ )+ return signing_input.decode("ascii") + "." + _b64url(signature)++ async def _authenticate(self) -> str:+ # Cache tokens; Google issues 1-hour tokens, but we refresh at 55 min+ # to leave headroom for clock skew and slow polls.+ if self._access_token and time.time() < self._token_expiry - 60:+ return self._access_token++ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.post(+ self._sa_info.get("token_uri", _TOKEN_URL),+ data={+ "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",+ "assertion": self._build_jwt(),+ },+ )+ resp.raise_for_status()+ payload = resp.json()+ self._access_token = payload["access_token"]+ self._token_expiry = time.time() + int(payload.get("expires_in", 3600))+ return self._access_token++ def _headers(self) -> dict[str, str]:+ return {+ "Authorization": f"Bearer {self._access_token}",+ "Content-Type": "application/json",+ }++ # ------------------------- contract -------------------------++ async def test_connection(self) -> dict[str, Any]:+ try:+ await self._authenticate()+ # A zero-result list call confirms the token, project access, and+ # that the Logging API is enabled — without paying for log scan.+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.post(+ _LOGGING_URL,+ headers=self._headers(),+ json={+ "resourceNames": [f"projects/{self._project_id}"],+ "filter": 'logName=~"cloudaudit.googleapis.com"',+ "pageSize": 1,+ },+ )+ resp.raise_for_status()+ return {+ "success": True,+ "connector": self.connector_id,+ "project_id": self._project_id,+ "service_account": self._sa_info["client_email"],+ }+ except httpx.HTTPStatusError as exc:+ return {+ "success": False,+ "connector": self.connector_id,+ "error": f"HTTP {exc.response.status_code}: {exc.response.text[:300]}",+ }+ except Exception as exc:+ return {"success": False, "connector": self.connector_id, "error": str(exc)}++ async def fetch_alerts(self, since_seconds: int = 300) -> list[dict[str, Any]]:+ await self._authenticate()+ since = (datetime.now(UTC) - timedelta(seconds=since_seconds)).strftime(+ "%Y-%m-%dT%H:%M:%SZ"+ )++ # Filter to the three Cloud Audit log streams. We exclude DataAccess+ # by default because most projects emit it at huge volume; operators+ # can set ``connector_config.include_data_access=true`` to opt in.+ # For now we keep the filter conservative.+ log_filter = (+ 'logName=~"cloudaudit.googleapis.com%2Factivity" '+ 'OR logName=~"cloudaudit.googleapis.com%2Fsystem_event" '+ f'AND timestamp >= "{since}"'+ )++ events: list[dict[str, Any]] = []+ async with httpx.AsyncClient(timeout=30.0) as client:+ resp = await client.post(+ _LOGGING_URL,+ headers=self._headers(),+ json={+ "resourceNames": [f"projects/{self._project_id}"],+ "filter": log_filter,+ "orderBy": "timestamp desc",+ "pageSize": _PAGE_SIZE,+ },+ )+ if resp.status_code == 401:+ # Token may have raced past expiry; refresh and retry once.+ self._access_token = None+ await self._authenticate()+ resp = await client.post(+ _LOGGING_URL,+ headers=self._headers(),+ json={+ "resourceNames": [f"projects/{self._project_id}"],+ "filter": log_filter,+ "orderBy": "timestamp desc",+ "pageSize": _PAGE_SIZE,+ },+ )+ if resp.status_code != 200:+ logger.warning(+ "gcp_cloud_audit.fetch_failed",+ status=resp.status_code,+ body=resp.text[:300],+ )+ return []++ for raw in resp.json().get("entries", []):+ events.append(raw)++ return [self.normalize(e) for e in events]++ # ----------------------- normalize --------------------------++ # GCP audit method names for the common high-blast-radius operations.+ # These are matched as substrings against ``protoPayload.methodName``.+ _HIGH_BLAST_METHODS = (+ "SetIamPolicy",+ ".delete",+ ".disable",+ "DeleteServiceAccount",+ "DeleteServiceAccountKey",+ "CreateServiceAccountKey",+ "DisableServiceAccount",+ "google.iam.admin.v1.DeleteRole",+ "google.cloud.kms.v1.DestroyCryptoKeyVersion",+ )++ def normalize(self, raw: dict[str, Any]) -> dict[str, Any]:+ proto = raw.get("protoPayload", {}) or {}+ method = proto.get("methodName", "")+ service = proto.get("serviceName", "")+ resource = raw.get("resource", {}) or {}+ auth_info = proto.get("authenticationInfo", {}) or {}+ request_meta = proto.get("requestMetadata", {}) or {}++ status = proto.get("status", {}) or {}+ succeeded = "code" not in status or status.get("code", 0) == 0++ # Severity heuristic: high-blast-radius IAM/KMS operations land at+ # ``high``; failed control-plane writes at ``medium``; everything+ # else at ``info``. SOC analysts will tune this with detections.+ severity = "info"+ if any(m in method for m in self._HIGH_BLAST_METHODS):+ severity = "high"+ elif not succeeded:+ severity = "medium"+ # Cloud Logging severity is also a useful hint when present.+ log_severity = (raw.get("severity") or "").upper()+ if log_severity in ("ERROR", "CRITICAL", "ALERT", "EMERGENCY"):+ severity = "high"+ elif log_severity == "WARNING" and severity == "info":+ severity = "low"++ actor = auth_info.get("principalEmail") or "unknown"++ return {+ "source": self.connector_id,+ "external_id": raw.get("insertId", ""),+ "title": method or "GCP Cloud Audit",+ "description": (+ f"service={service}; "+ f"method={method}; "+ f"resource={resource.get('type', '')}; "+ f"status={'success' if succeeded else status.get('message', 'failed')}"+ ),+ "severity": severity,+ "src_ip": request_meta.get("callerIp"),+ "actor": actor,+ "actor_email": actor if "@" in actor else None,+ "event_type": f"gcp.audit.{service.split('.')[0] or 'unknown'}",+ "raw_event": raw,+ "created_at": raw.get("timestamp"),+ }diff --git a/services/connectors/app/connectors/gcp_scc.py b/services/connectors/app/connectors/gcp_scc.pynew file mode 100644index 000000000..af2315b0d--- /dev/null+++ b/services/connectors/app/connectors/gcp_scc.py@@ -0,0 +1,278 @@+"""+GCP Security Command Center (SCC) connector.++Pulls SCC findings — the curated security signals Google synthesizes from+Cloud Audit Logs, VPC Flow Logs, Container Threat Detection, Web Security+Scanner, and partner integrations. SCC is essentially Google's equivalent+of AWS Security Hub or Microsoft Defender for Cloud, and is the highest-+signal feed available for GCP estates.++Auth model: a Google service-account JSON key with+``roles/securitycenter.findingsViewer`` (org or folder-scoped). The full+key blob is pasted into the secret field and encrypted at rest.++We share JWT signing with ``gcp_cloud_audit`` via duplicate logic rather+than a shared module — connectors are intentionally standalone so they+can be lifted into the marketplace independently. If we ever extract a+``gcp_auth`` helper, both should switch in lock-step.+"""++from __future__ import annotations++import base64+import json+import time+from datetime import UTC, datetime, timedelta+from typing import Any++import httpx+import structlog+from cryptography.hazmat.primitives import hashes, serialization+from cryptography.hazmat.primitives.asymmetric import padding++from app.connectors.base import BaseConnector, ConnectorSchema, Field, OAuthHints++logger = structlog.get_logger()++_TOKEN_URL = "https://oauth2.googleapis.com/token"+_SCC_BASE = "https://securitycenter.googleapis.com/v1"+_SCOPE = "https://www.googleapis.com/auth/cloud-platform"++_PAGE_SIZE = 200+++def _b64url(data: bytes) -> str:+ return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")+++class GCPSCCConnector(BaseConnector):+ """Google Cloud Security Command Center findings."""++ connector_id = "gcp_scc"+ connector_name = "GCP Security Command Center"+ connector_category = "cloud"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description=(+ "Google Cloud Security Command Center findings. Requires a "+ "service account with roles/securitycenter.findingsViewer at "+ "the organization or folder level."+ ),+ docs_url="/docs/connectors/gcp-scc",+ fields=[+ Field(+ "organization_id",+ "string",+ "Organization ID",+ placeholder="1234567890",+ help_text=(+ "Numeric GCP organization ID. SCC is org-scoped; the "+ "service account must have findingsViewer at this "+ "level (or on a parent folder/source)."+ ),+ ),+ Field(+ "service_account_json",+ "secret",+ "Service account JSON key",+ help_text=(+ "Paste the full JSON key file. It will be encrypted "+ "at rest by the credential vault."+ ),+ ),+ ],+ oauth=OAuthHints(+ supported_in_hosted=False,+ authorize_url=None,+ token_url=_TOKEN_URL,+ scopes=[_SCOPE],+ ),+ )++ def __init__(self, organization_id: str, service_account_json: str):+ self._organization_id = organization_id+ self._sa_info = self._parse_sa(service_account_json)+ self._access_token: str | None = None+ self._token_expiry: float = 0.0++ # --------------------------- auth ---------------------------++ @staticmethod+ def _parse_sa(blob: str) -> dict[str, Any]:+ try:+ sa = json.loads(blob)+ except json.JSONDecodeError as exc:+ raise ValueError(+ "service_account_json is not valid JSON. Paste the entire "+ "key file contents."+ ) from exc+ for required in ("client_email", "private_key", "token_uri"):+ if required not in sa:+ raise ValueError(+ f"service_account_json missing required field: {required}"+ )+ return sa++ def _build_jwt(self) -> str:+ now = int(time.time())+ header = {"alg": "RS256", "typ": "JWT"}+ claims = {+ "iss": self._sa_info["client_email"],+ "scope": _SCOPE,+ "aud": self._sa_info.get("token_uri", _TOKEN_URL),+ "iat": now,+ "exp": now + 3600,+ }+ signing_input = (+ _b64url(json.dumps(header, separators=(",", ":")).encode())+ + "."+ + _b64url(json.dumps(claims, separators=(",", ":")).encode())+ ).encode("ascii")++ private_key = serialization.load_pem_private_key(+ self._sa_info["private_key"].encode("utf-8"),+ password=None,+ )+ signature = private_key.sign( # type: ignore[union-attr]+ signing_input,+ padding.PKCS1v15(),+ hashes.SHA256(),+ )+ return signing_input.decode("ascii") + "." + _b64url(signature)++ async def _authenticate(self) -> str:+ if self._access_token and time.time() < self._token_expiry - 60:+ return self._access_token+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.post(+ self._sa_info.get("token_uri", _TOKEN_URL),+ data={+ "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",+ "assertion": self._build_jwt(),+ },+ )+ resp.raise_for_status()+ payload = resp.json()+ self._access_token = payload["access_token"]+ self._token_expiry = time.time() + int(payload.get("expires_in", 3600))+ return self._access_token++ def _headers(self) -> dict[str, str]:+ return {+ "Authorization": f"Bearer {self._access_token}",+ "Content-Type": "application/json",+ }++ # ------------------------- contract -------------------------++ async def test_connection(self) -> dict[str, Any]:+ try:+ await self._authenticate()+ # Listing sources is the cheapest verification: it confirms the+ # token, that the org ID is correct, and that SCC is enabled.+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.get(+ f"{_SCC_BASE}/organizations/{self._organization_id}/sources",+ headers=self._headers(),+ params={"pageSize": 1},+ )+ resp.raise_for_status()+ return {+ "success": True,+ "connector": self.connector_id,+ "organization_id": self._organization_id,+ "service_account": self._sa_info["client_email"],+ }+ except httpx.HTTPStatusError as exc:+ return {+ "success": False,+ "connector": self.connector_id,+ "error": f"HTTP {exc.response.status_code}: {exc.response.text[:300]}",+ }+ except Exception as exc:+ return {"success": False, "connector": self.connector_id, "error": str(exc)}++ async def fetch_alerts(self, since_seconds: int = 300) -> list[dict[str, Any]]:+ await self._authenticate()+ since = (datetime.now(UTC) - timedelta(seconds=since_seconds)).strftime(+ "%Y-%m-%dT%H:%M:%SZ"+ )++ # ``-`` as the source ID means "across all sources" — the standard+ # way to enumerate findings in an org without iterating sources.+ # We filter on ``eventTime`` so we get net-new findings since the+ # last poll, and exclude already-resolved findings.+ scc_filter = (+ f'state="ACTIVE" '+ f'AND eventTime >= "{since}"'+ )++ async with httpx.AsyncClient(timeout=30.0) as client:+ resp = await client.get(+ f"{_SCC_BASE}/organizations/{self._organization_id}/sources/-/findings",+ headers=self._headers(),+ params={"filter": scc_filter, "pageSize": _PAGE_SIZE},+ )+ if resp.status_code == 401:+ self._access_token = None+ await self._authenticate()+ resp = await client.get(+ f"{_SCC_BASE}/organizations/{self._organization_id}/sources/-/findings",+ headers=self._headers(),+ params={"filter": scc_filter, "pageSize": _PAGE_SIZE},+ )+ if resp.status_code != 200:+ logger.warning(+ "gcp_scc.fetch_failed",+ status=resp.status_code,+ body=resp.text[:300],+ )+ return []++ entries = resp.json().get("listFindingsResults", [])++ return [self.normalize(item) for item in entries]++ # ----------------------- normalize --------------------------++ def normalize(self, raw: dict[str, Any]) -> dict[str, Any]:+ # SCC wraps findings in {"finding": {...}, "resource": {...}, ...}+ finding = raw.get("finding", {}) or raw # tolerate either shape+ resource = raw.get("resource", {}) or {}++ # SCC severity comes through as one of CRITICAL / HIGH / MEDIUM /+ # LOW / SEVERITY_UNSPECIFIED. Map directly to AiSOC's lowercase+ # severities — Critical collapses to ``high`` since downstream+ # severity-mapping pipelines expect that vocabulary.+ scc_sev = (finding.get("severity") or "").upper()+ severity = {+ "CRITICAL": "high",+ "HIGH": "high",+ "MEDIUM": "medium",+ "LOW": "low",+ }.get(scc_sev, "info")++ return {+ "source": self.connector_id,+ "external_id": finding.get("name", ""),+ "title": finding.get("category", "GCP SCC Finding"),+ "description": (+ finding.get("description")+ or finding.get("category", "")+ or "GCP Security Command Center finding"+ ),+ "severity": severity,+ "actor": (finding.get("access", {}) or {}).get("principalEmail"),+ "actor_email": (finding.get("access", {}) or {}).get("principalEmail"),+ "event_type": f"gcp.scc.{(finding.get('category') or 'finding').lower()}",+ "raw_event": raw,+ "created_at": finding.get("eventTime") or finding.get("createTime"),+ # Resource context is high-signal for triage; surface it explicitly.+ "resource_name": resource.get("name") or finding.get("resourceName"),+ "resource_type": resource.get("type"),+ }diff --git a/services/connectors/app/connectors/github.py b/services/connectors/app/connectors/github.pynew file mode 100644index 000000000..2a490dccb--- /dev/null+++ b/services/connectors/app/connectors/github.py@@ -0,0 +1,356 @@+"""+GitHub organization audit log + code scanning alerts connector.++Pulls two distinct streams in a single connector:++1. **Audit log** — ``GET /orgs/{org}/audit-log`` covers every admin/security+ action: member adds, role changes, repo creation/deletion, PAT grants,+ secret-scanning policy changes, branch protection edits, etc. Requires+ the ``read:audit_log`` scope. Org must be on a plan that exposes the+ audit log API (Team and above).++2. **Code Scanning alerts** — ``GET /orgs/{org}/code-scanning/alerts``+ surfaces CodeQL / third-party scanner findings across every repo in the+ org. Requires ``security_events`` scope.++Auth: fine-grained personal access token OR a GitHub App installation+token. We accept the token directly; for App-based auth, the operator+generates an installation token externally and pastes it. Hosted OAuth+flow is deferred — the schema advertises that explicitly.+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta+from typing import Any++import httpx+import structlog++from app.connectors.base import BaseConnector, ConnectorSchema, Field, OAuthHints++logger = structlog.get_logger()++_BASE = "https://api.github.com"+_PER_PAGE = 100+++class GitHubConnector(BaseConnector):+ """GitHub org audit log + code-scanning alerts."""++ connector_id = "github"+ connector_name = "GitHub"+ connector_category = "vcs"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description=(+ "GitHub organization audit log and Code Scanning alerts. "+ "Requires a token with read:audit_log and security_events "+ "scopes (fine-grained PAT or GitHub App installation token)."+ ),+ docs_url="/docs/connectors/github",+ fields=[+ Field(+ "organization",+ "string",+ "Organization slug",+ placeholder="my-org",+ ),+ Field(+ "token",+ "secret",+ "Personal Access Token or App Installation Token",+ help_text=(+ "Fine-grained PAT recommended. Required scopes: "+ "read:audit_log, security_events. For GitHub App "+ "auth, paste a fresh installation token."+ ),+ ),+ ],+ oauth=OAuthHints(+ supported_in_hosted=False,+ authorize_url="https://github.com/login/oauth/authorize",+ token_url="https://github.com/login/oauth/access_token",+ scopes=["read:audit_log", "security_events"],+ ),+ )++ def __init__(self, organization: str, token: str):+ self._org = organization+ self._token = token++ # --------------------------- auth ---------------------------++ def _headers(self) -> dict[str, str]:+ return {+ "Authorization": f"Bearer {self._token}",+ "Accept": "application/vnd.github+json",+ # Pin the API version to avoid surprise breakages when GitHub+ # rolls out v2-style changes. ``2022-11-28`` is the long-stable+ # current version.+ "X-GitHub-Api-Version": "2022-11-28",+ }++ # ------------------------- contract -------------------------++ async def test_connection(self) -> dict[str, Any]:+ try:+ async with httpx.AsyncClient(timeout=15.0) as client:+ # ``/orgs/{org}`` is cheap and returns 200 for any token+ # that can read the org. 401/403 here is the canonical+ # "your token is bad or doesn't have access" signal.+ resp = await client.get(+ f"{_BASE}/orgs/{self._org}",+ headers=self._headers(),+ )+ if resp.status_code != 200:+ return {+ "success": False,+ "connector": self.connector_id,+ "error": f"HTTP {resp.status_code}: {resp.text[:300]}",+ }++ # Audit log probe — confirms the token actually has+ # ``read:audit_log``. The endpoint returns 200 + [] on a+ # quiet org, 403 if the scope is missing, 404 if the org+ # plan doesn't include audit logs.+ audit_resp = await client.get(+ f"{_BASE}/orgs/{self._org}/audit-log",+ headers=self._headers(),+ params={"per_page": 1},+ )+ if audit_resp.status_code not in (200, 404):+ return {+ "success": False,+ "connector": self.connector_id,+ "error": (+ f"Org reachable but audit-log probe failed: "+ f"HTTP {audit_resp.status_code}: "+ f"{audit_resp.text[:200]}"+ ),+ }+ audit_available = audit_resp.status_code == 200++ return {+ "success": True,+ "connector": self.connector_id,+ "organization": self._org,+ "audit_log_available": audit_available,+ }+ except Exception as exc:+ return {"success": False, "connector": self.connector_id, "error": str(exc)}++ async def fetch_alerts(self, since_seconds: int = 300) -> list[dict[str, Any]]:+ # GitHub timestamps audit-log events in milliseconds-since-epoch;+ # the ``phrase`` query param is the supported way to filter by+ # time window (``created:>=YYYY-MM-DDTHH:MM:SSZ``).+ start = datetime.now(UTC) - timedelta(seconds=since_seconds)+ start_iso = start.strftime("%Y-%m-%dT%H:%M:%SZ")+ start_ms = int(start.timestamp() * 1000)++ events: list[dict[str, Any]] = []+ async with httpx.AsyncClient(timeout=30.0) as client:+ # 1) audit log+ audit_resp = await client.get(+ f"{_BASE}/orgs/{self._org}/audit-log",+ headers=self._headers(),+ params={+ "phrase": f"created:>={start_iso}",+ "per_page": _PER_PAGE,+ "order": "desc",+ },+ )+ if audit_resp.status_code == 200:+ for entry in audit_resp.json() or []:+ entry["_aisoc_stream"] = "audit_log"+ events.append(entry)+ elif audit_resp.status_code == 404:+ # Plan doesn't expose the audit log — log once and continue+ # with code-scanning. Don't error out the whole poll.+ logger.info(+ "github.audit_log_unavailable",+ organization=self._org,+ )+ else:+ logger.warning(+ "github.audit_log_failed",+ status=audit_resp.status_code,+ body=audit_resp.text[:300],+ )++ # 2) code scanning alerts. Filtering by ``created`` is cleaner+ # than ``updated`` for our poll model — we want net-new+ # findings, not state churn (which would re-emit the same+ # alert every time a triager moves it through review).+ cs_resp = await client.get(+ f"{_BASE}/orgs/{self._org}/code-scanning/alerts",+ headers=self._headers(),+ params={+ "state": "open",+ "per_page": _PER_PAGE,+ "sort": "created",+ "direction": "desc",+ },+ )+ if cs_resp.status_code == 200:+ for alert in cs_resp.json() or []:+ # Client-side filter on creation time. GitHub doesn't+ # accept ``since`` on this endpoint, and pulling >100+ # results just to filter would be wasteful at scale.+ created_at = alert.get("created_at")+ if created_at:+ try:+ created_dt = datetime.fromisoformat(+ created_at.replace("Z", "+00:00")+ )+ if created_dt.timestamp() * 1000 < start_ms:+ # Sorted desc → first too-old result means+ # all subsequent ones are also too old.+ break+ except ValueError:+ pass+ alert["_aisoc_stream"] = "code_scanning"+ events.append(alert)+ elif cs_resp.status_code in (403, 404):+ # 403 = scope missing or feature disabled; 404 = no+ # advanced security on this plan. Both are configuration+ # issues, not transient failures.+ logger.info(+ "github.code_scanning_unavailable",+ status=cs_resp.status_code,+ )+ else:+ logger.warning(+ "github.code_scanning_failed",+ status=cs_resp.status_code,+ body=cs_resp.text[:300],+ )++ return [self.normalize(e) for e in events]++ # ----------------------- normalize --------------------------++ # GitHub ``action`` strings that we always treat as high. These all+ # have the shape ``<category>.<verb>`` — list maintained from+ # https://docs.github.com/en/organizations/keeping-your-organization-secure/managing-security-settings-for-your-organization/audit-log-events-for-your-organization+ _HIGH_RISK_ACTIONS = (+ # org membership / ownership+ "org.add_member",+ "org.remove_member",+ "org.update_member", # role changes (member → owner is the canary)+ "org.transfer",+ # access tokens / SSH / SSO+ "personal_access_token.create",+ "personal_access_token.access_granted",+ "oauth_authorization.create",+ "public_key.create",+ "deploy_key.create",+ "two_factor_authentication.disabled",+ # branch protection bypass / deletion+ "protected_branch.destroy",+ "protected_branch.update_admin_enforced",+ # repository deletions+ "repo.destroy",+ "repo.transfer",+ "repo.access", # visibility changes+ # secret scanning / dependabot tampering+ "secret_scanning.disable",+ "secret_scanning_alert.resolve",+ "dependabot_alerts.disable",+ # GitHub Apps with admin scopes+ "integration_installation.create",+ "integration_installation.repositories_added",+ )++ def _normalize_audit(self, raw: dict[str, Any]) -> dict[str, Any]:+ action = raw.get("action", "")+ actor = raw.get("actor", "unknown")++ if any(h == action for h in self._HIGH_RISK_ACTIONS):+ severity = "high"+ elif action.endswith(".destroy") or action.endswith(".delete"):+ # Catch-all for delete-style actions we haven't enumerated+ # above. Better to over-flag deletions than miss them.+ severity = "medium"+ else:+ severity = "info"++ # ``created_at`` is a ms-since-epoch integer in audit log entries.+ created_ms = raw.get("created_at") or raw.get("@timestamp")+ created_iso: str | None = None+ if isinstance(created_ms, int | float):+ created_iso = datetime.fromtimestamp(created_ms / 1000, tz=UTC).isoformat()+ elif isinstance(created_ms, str):+ created_iso = created_ms++ return {+ "source": self.connector_id,+ "external_id": str(raw.get("_document_id") or raw.get("@timestamp") or ""),+ "title": action or "GitHub audit event",+ "description": (+ f"actor={actor}; "+ f"action={action}; "+ f"repo={raw.get('repo', '')}; "+ f"org={raw.get('org', self._org)}"+ ),+ "severity": severity,+ "actor": actor,+ "actor_email": raw.get("user_email"),+ "src_ip": raw.get("actor_ip"),+ "event_type": f"github.{action}" if action else "github.audit",+ "raw_event": raw,+ "created_at": created_iso,+ }++ # Code scanning severity values come from the rule, not the alert.+ # GitHub uses ``security-severity`` (CVSS-style numeric) and a+ # categorical ``severity`` (note/warning/error/critical/high/medium/low).+ # We honor the categorical first since it's set on every result.+ _CODE_SEVERITY_MAP = {+ "critical": "high", # collapse into AiSOC's 4-tier ladder+ "high": "high",+ "error": "high",+ "medium": "medium",+ "warning": "medium",+ "low": "low",+ "note": "info",+ }++ def _normalize_code_scanning(self, raw: dict[str, Any]) -> dict[str, Any]:+ rule = raw.get("rule", {}) or {}+ gh_severity = (rule.get("security_severity_level") or rule.get("severity") or "").lower()+ severity = self._CODE_SEVERITY_MAP.get(gh_severity, "info")++ repo = (raw.get("repository") or {}).get("full_name", "")+ rule_id = rule.get("id", "")+ rule_desc = rule.get("description", "")++ return {+ "source": self.connector_id,+ "external_id": f"code-scanning-{raw.get('number', '')}",+ "title": f"Code Scanning: {rule_id or rule_desc or 'finding'}",+ "description": (+ f"repo={repo}; "+ f"rule={rule_id}; "+ f"tool={(raw.get('tool') or {}).get('name', '')}; "+ f"path={(raw.get('most_recent_instance') or {}).get('location', {}).get('path', '')}"+ ),+ "severity": severity,+ "actor": "code-scanning",+ "actor_email": None,+ "src_ip": None,+ "event_type": f"github.code_scanning.{rule_id}" if rule_id else "github.code_scanning",+ "raw_event": raw,+ "created_at": raw.get("created_at"),+ }++ def normalize(self, raw: dict[str, Any]) -> dict[str, Any]:+ stream = raw.get("_aisoc_stream", "audit_log")+ if stream == "code_scanning":+ return self._normalize_code_scanning(raw)+ return self._normalize_audit(raw)diff --git a/services/connectors/app/connectors/google_workspace.py b/services/connectors/app/connectors/google_workspace.pynew file mode 100644index 000000000..3d4ddb0be--- /dev/null+++ b/services/connectors/app/connectors/google_workspace.py@@ -0,0 +1,345 @@+"""+Google Workspace audit logs connector.++Pulls activity events for the high-signal Workspace apps (login, admin,+drive, token, mobile) via the Reports API. Auth uses a service account JSON+key with domain-wide delegation, plus an admin email to impersonate — that's+the model Google requires for org-wide audit log access.++Reuses the manual JWT signing pattern from the GCP connectors. The crucial+difference is that Workspace requires the JWT ``sub`` claim to be set to the+admin user being impersonated; without that, the token endpoint returns+``invalid_grant`` even when the service account is otherwise correct.++ApplicationName values monitored by default:+ login - all login activity (success + failure + suspicious)+ admin - super-admin / delegated-admin operations (high blast radius)+ drive - file shares, ACL changes, external shares+ token - OAuth token grants (consent screen activity, app authorization)+ mobile - device events (lost device, wipe, jailbreak detection)+"""++from __future__ import annotations++import base64+import json+import time+from datetime import UTC, datetime, timedelta+from typing import Any++import httpx+import structlog+from cryptography.hazmat.primitives import hashes, serialization+from cryptography.hazmat.primitives.asymmetric import padding++from app.connectors.base import BaseConnector, ConnectorSchema, Field, OAuthHints++logger = structlog.get_logger()++_TOKEN_URL = "https://oauth2.googleapis.com/token"+_REPORTS_BASE = "https://admin.googleapis.com/admin/reports/v1/activity"+_SCOPE = "https://www.googleapis.com/auth/admin.reports.audit.readonly"++_DEFAULT_APPS = ("login", "admin", "drive", "token", "mobile")+_PAGE_SIZE = 500+++def _b64url(data: bytes) -> str:+ return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")+++class GoogleWorkspaceConnector(BaseConnector):+ """Google Workspace (Admin SDK Reports API) audit log connector."""++ connector_id = "google_workspace"+ connector_name = "Google Workspace"+ connector_category = "saas"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description=(+ "Google Workspace audit logs (login, admin, drive, token, "+ "mobile) via the Admin SDK Reports API. Requires a service "+ "account with domain-wide delegation and an admin email to "+ "impersonate."+ ),+ docs_url="/docs/connectors/google-workspace",+ fields=[+ Field(+ "admin_email",+ "string",+ "Super-admin email",+ placeholder="[email protected]",+ help_text=(+ "The Google Workspace super-admin to impersonate via "+ "domain-wide delegation. The service account's "+ "client_id must be authorized in the Admin Console "+ "with the admin.reports.audit.readonly scope."+ ),+ ),+ Field(+ "service_account_json",+ "secret",+ "Service account JSON key",+ help_text=(+ "Paste the full JSON key file. Encrypted at rest by "+ "the credential vault."+ ),+ ),+ ],+ oauth=OAuthHints(+ supported_in_hosted=False,+ authorize_url=None,+ token_url=_TOKEN_URL,+ scopes=[_SCOPE],+ ),+ )++ def __init__(self, admin_email: str, service_account_json: str):+ self._admin_email = admin_email+ self._sa_info = self._parse_sa(service_account_json)+ self._access_token: str | None = None+ self._token_expiry: float = 0.0++ # --------------------------- auth ---------------------------++ @staticmethod+ def _parse_sa(blob: str) -> dict[str, Any]:+ try:+ sa = json.loads(blob)+ except json.JSONDecodeError as exc:+ raise ValueError(+ "service_account_json is not valid JSON. Paste the entire "+ "key file contents."+ ) from exc+ for required in ("client_email", "private_key", "token_uri"):+ if required not in sa:+ raise ValueError(+ f"service_account_json missing required field: {required}"+ )+ return sa++ def _build_jwt(self) -> str:+ """Build a JWT with the ``sub`` claim set to the impersonated admin —+ this is what makes domain-wide delegation actually work."""+ now = int(time.time())+ header = {"alg": "RS256", "typ": "JWT"}+ claims = {+ "iss": self._sa_info["client_email"],+ "scope": _SCOPE,+ "aud": self._sa_info.get("token_uri", _TOKEN_URL),+ "iat": now,+ "exp": now + 3600,+ # The "act on behalf of" admin user. Without this, Google+ # rejects the token even when DWD is correctly configured.+ "sub": self._admin_email,+ }+ signing_input = (+ _b64url(json.dumps(header, separators=(",", ":")).encode())+ + "."+ + _b64url(json.dumps(claims, separators=(",", ":")).encode())+ ).encode("ascii")+ private_key = serialization.load_pem_private_key(+ self._sa_info["private_key"].encode("utf-8"),+ password=None,+ )+ signature = private_key.sign( # type: ignore[union-attr]+ signing_input,+ padding.PKCS1v15(),+ hashes.SHA256(),+ )+ return signing_input.decode("ascii") + "." + _b64url(signature)++ async def _authenticate(self) -> str:+ if self._access_token and time.time() < self._token_expiry - 60:+ return self._access_token+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.post(+ self._sa_info.get("token_uri", _TOKEN_URL),+ data={+ "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",+ "assertion": self._build_jwt(),+ },+ )+ resp.raise_for_status()+ payload = resp.json()+ self._access_token = payload["access_token"]+ self._token_expiry = time.time() + int(payload.get("expires_in", 3600))+ return self._access_token++ def _headers(self) -> dict[str, str]:+ return {+ "Authorization": f"Bearer {self._access_token}",+ "Accept": "application/json",+ }++ # ------------------------- contract -------------------------++ async def test_connection(self) -> dict[str, Any]:+ try:+ await self._authenticate()+ # Pull a single login event as the cheapest verification: it+ # confirms (a) the token, (b) DWD is configured for our SA,+ # and (c) the admin email exists and has permission.+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.get(+ f"{_REPORTS_BASE}/users/all/applications/login",+ headers=self._headers(),+ params={"maxResults": 1},+ )+ resp.raise_for_status()+ return {+ "success": True,+ "connector": self.connector_id,+ "admin_email": self._admin_email,+ "service_account": self._sa_info["client_email"],+ }+ except httpx.HTTPStatusError as exc:+ return {+ "success": False,+ "connector": self.connector_id,+ "error": f"HTTP {exc.response.status_code}: {exc.response.text[:300]}",+ }+ except Exception as exc:+ return {"success": False, "connector": self.connector_id, "error": str(exc)}++ async def fetch_alerts(self, since_seconds: int = 300) -> list[dict[str, Any]]:+ await self._authenticate()+ start_time = (datetime.now(UTC) - timedelta(seconds=since_seconds)).strftime(+ "%Y-%m-%dT%H:%M:%S.000Z"+ )++ events: list[dict[str, Any]] = []+ async with httpx.AsyncClient(timeout=30.0) as client:+ for app_name in _DEFAULT_APPS:+ resp = await client.get(+ f"{_REPORTS_BASE}/users/all/applications/{app_name}",+ headers=self._headers(),+ params={"startTime": start_time, "maxResults": _PAGE_SIZE},+ )+ if resp.status_code == 401:+ self._access_token = None+ await self._authenticate()+ resp = await client.get(+ f"{_REPORTS_BASE}/users/all/applications/{app_name}",+ headers=self._headers(),+ params={"startTime": start_time, "maxResults": _PAGE_SIZE},+ )+ if resp.status_code != 200:+ logger.warning(+ "google_workspace.fetch_failed",+ application=app_name,+ status=resp.status_code,+ body=resp.text[:300],+ )+ continue+ items = resp.json().get("items", [])+ # Tag each item with the app name so normalize() doesn't+ # have to crack open ``id.applicationName`` (which is+ # present, but explicit beats implicit).+ for item in items:+ item["_aisoc_application"] = app_name+ events.append(item)++ return [self.normalize(e) for e in events]++ # ----------------------- normalize --------------------------++ # Workspace event names that we always treat as high-severity. These are+ # all admin-console actions or auth events that materially change the+ # security posture of the org.+ _HIGH_RISK_EVENTS = (+ # admin app+ "GRANT_ADMIN_PRIVILEGE",+ "REVOKE_ADMIN_PRIVILEGE",+ "CREATE_ROLE",+ "DELETE_ROLE",+ "ASSIGN_ROLE",+ "DELEGATE_ADMIN_PRIVILEGES",+ # 2sv / mfa weakening+ "ENFORCE_STRONG_AUTHENTICATION",+ "UNENROLL_USER_FROM_STRONG_AUTH",+ "ALLOW_STRONG_AUTHENTICATION",+ "DISABLE_USER_2SV",+ # password resets and account changes+ "RESET_PASSWORD",+ "CHANGE_PASSWORD",+ "SUSPEND_USER",+ "UNSUSPEND_USER",+ # OAuth grants are the textbook M365 / Workspace BEC vector+ "authorize",+ "activity",+ # mobile+ "DEVICE_COMPROMISED_EVENT",+ "FAILED_PASSWORD_ATTEMPTS_EVENT",+ )++ # login app event names that mean "this auth attempt was suspicious"+ _SUSPICIOUS_LOGIN_EVENTS = (+ "login_failure",+ "suspicious_login",+ "suspicious_login_less_secure_app",+ "suspicious_programmatic_login",+ "gov_attack_warning",+ "login_challenge",+ )++ def normalize(self, raw: dict[str, Any]) -> dict[str, Any]:+ application = raw.get("_aisoc_application") or raw.get("id", {}).get(+ "applicationName"+ )+ actor = (raw.get("actor") or {}).get("email", "unknown")+ # The Reports API encodes individual events as a list under each+ # activity. Most activities have a single event; a few bundle+ # multiple. We pick the first to drive severity / title and+ # surface the full list in raw_event.+ events_list = raw.get("events") or []+ primary_event = events_list[0] if events_list else {}+ event_name = primary_event.get("name", "")++ # Severity heuristic mirrors the M365 connector: high-risk admin+ # ops and OAuth grants → high; suspicious login → medium; failed+ # logins → low; everything else → info.+ severity = "info"+ if any(h.lower() in event_name.lower() for h in self._HIGH_RISK_EVENTS):+ severity = "high"+ elif application == "login" and event_name in self._SUSPICIOUS_LOGIN_EVENTS:+ severity = "medium" if event_name != "login_failure" else "low"+ elif application == "drive" and event_name in (+ "change_acl_editors",+ "change_user_access",+ "create_link_sharing",+ "shared_drive_settings_change",+ ):+ severity = "medium"++ # Pull the IP address out of the activity envelope; not all events+ # have it, but login/admin/drive almost always do.+ ip_address = raw.get("ipAddress")++ return {+ "source": self.connector_id,+ "external_id": (raw.get("id") or {}).get("uniqueQualifier", ""),+ "title": event_name or f"Workspace {application} event",+ "description": (+ f"application={application}; "+ f"event={event_name}; "+ f"actor={actor}"+ ),+ "severity": severity,+ "actor": actor,+ "actor_email": actor if "@" in actor else None,+ "src_ip": ip_address,+ "application": application,+ "event_type": (+ f"workspace.{application}.{event_name}".lower()+ if application and event_name+ else "workspace.audit"+ ),+ "raw_event": raw,+ "created_at": (raw.get("id") or {}).get("time"),+ }diff --git a/services/connectors/app/connectors/m365_audit.py b/services/connectors/app/connectors/m365_audit.pynew file mode 100644index 000000000..267d0b23a--- /dev/null+++ b/services/connectors/app/connectors/m365_audit.py@@ -0,0 +1,332 @@+"""+Microsoft 365 / Office 365 Management Activity API connector.++Pulls unified audit events covering Azure AD, Exchange, SharePoint/OneDrive,+DLP, and other "general" Microsoft 365 audit content. Reuses the same Azure+AD app registration pattern as the ``azure_*`` connectors, but talks to the+Management Activity API instead of Graph.++Auth model: Azure AD app registration with the Office 365 Management API+permission ``ActivityFeed.Read`` granted with admin consent.++Why this is more involved than Graph alerts: the Activity API doesn't have+a simple ``/alerts`` endpoint. Instead Microsoft enforces a publish/poll+pattern where you (1) ensure a subscription is started for each content type+you care about, (2) list content blobs available since a window, and (3)+fetch each blob URL to materialise the events. We cover (1) and (2) in+``test_connection`` / ``fetch_alerts``, and inline the blob fetch so the+connector contract still returns flat events.++Scope: we deliberately limit to Audit.* content types and skip DLP.All by+default. DLP can be re-enabled per-instance later via ``connector_config``+once the analyst opt-in path lands.+"""++from __future__ import annotations++from datetime import UTC, datetime, timedelta+from typing import Any++import httpx+import structlog++from app.connectors.base import BaseConnector, ConnectorSchema, Field, OAuthHints++logger = structlog.get_logger()++_TOKEN_URL = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"+_BASE = "https://manage.office.com/api/v1.0/{tenant_id}/activity/feed"++# Content types we pull by default. Audit.General captures Power BI, Teams,+# Dynamics, etc.; the noisy DLP.All stream is left off to keep volumes sane.+_DEFAULT_CONTENT_TYPES = (+ "Audit.AzureActiveDirectory",+ "Audit.Exchange",+ "Audit.SharePoint",+ "Audit.General",+)+++class M365AuditConnector(BaseConnector):+ """Microsoft 365 unified audit log via Office 365 Management Activity API."""++ connector_id = "m365_audit"+ connector_name = "Microsoft 365 Audit"+ connector_category = "saas"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description=(+ "Microsoft 365 unified audit log (Azure AD, Exchange, "+ "SharePoint/OneDrive, Teams, Power BI) via the Office 365 "+ "Management Activity API. Requires ActivityFeed.Read on an "+ "Azure AD app registration."+ ),+ docs_url="/docs/connectors/m365-audit",+ fields=[+ Field(+ "tenant_id",+ "string",+ "Tenant ID",+ placeholder="00000000-0000-0000-0000-000000000000",+ ),+ Field("client_id", "string", "Application (Client) ID"),+ Field("client_secret", "secret", "Client Secret"),+ ],+ oauth=OAuthHints(+ supported_in_hosted=True,+ authorize_url="https://login.microsoftonline.com/common/oauth2/v2.0/authorize",+ token_url="https://login.microsoftonline.com/common/oauth2/v2.0/token",+ scopes=["https://manage.office.com/ActivityFeed.Read"],+ ),+ )++ def __init__(self, tenant_id: str, client_id: str, client_secret: str):+ self._tenant_id = tenant_id+ self._client_id = client_id+ self._client_secret = client_secret+ self._access_token: str | None = None++ # --------------------------- auth ---------------------------++ async def _authenticate(self) -> str:+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.post(+ _TOKEN_URL.format(tenant_id=self._tenant_id),+ data={+ "client_id": self._client_id,+ "client_secret": self._client_secret,+ # Note: Management API uses a *resource* parameter rather+ # than a Graph-style scope. Both ``/.default`` and the+ # explicit ``resource`` form are accepted by v2.0/token,+ # but ``resource`` is what Microsoft documents for this+ # API and is the safer choice across tenants.+ "scope": "https://manage.office.com/.default",+ "grant_type": "client_credentials",+ },+ )+ resp.raise_for_status()+ self._access_token = resp.json()["access_token"]+ return self._access_token++ def _headers(self) -> dict[str, str]:+ return {+ "Authorization": f"Bearer {self._access_token}",+ "Accept": "application/json",+ }++ def _base(self) -> str:+ return _BASE.format(tenant_id=self._tenant_id)++ # ------------------------- contract -------------------------++ async def test_connection(self) -> dict[str, Any]:+ try:+ await self._authenticate()+ # Listing current subscriptions is the canonical "is the auth+ # wired up correctly" probe — it returns 200 + JSON list even+ # when zero subscriptions exist, and 403 if ActivityFeed.Read+ # admin consent is missing.+ async with httpx.AsyncClient(timeout=15.0) as client:+ resp = await client.get(+ f"{self._base()}/subscriptions/list",+ headers=self._headers(),+ )+ resp.raise_for_status()+ return {+ "success": True,+ "connector": self.connector_id,+ "tenant_id": self._tenant_id,+ "active_subscriptions": [+ s.get("contentType") for s in resp.json() if s.get("status") == "enabled"+ ],+ }+ except httpx.HTTPStatusError as exc:+ return {+ "success": False,+ "connector": self.connector_id,+ "error": f"HTTP {exc.response.status_code}: {exc.response.text[:300]}",+ }+ except Exception as exc:+ return {"success": False, "connector": self.connector_id, "error": str(exc)}++ async def _ensure_subscriptions(self, client: httpx.AsyncClient) -> None:+ """Idempotently start any of our default content-type subscriptions+ that aren't already running. ``/subscriptions/start`` is a POST that+ returns 200 if the subscription was created, 400 if it was already+ active. We treat both as success.+ """+ for content_type in _DEFAULT_CONTENT_TYPES:+ try:+ resp = await client.post(+ f"{self._base()}/subscriptions/start",+ headers=self._headers(),+ params={"contentType": content_type},+ )+ if resp.status_code not in (200, 400):+ logger.warning(+ "m365_audit.subscribe_unexpected_status",+ content_type=content_type,+ status=resp.status_code,+ body=resp.text[:200],+ )+ except httpx.RequestError as exc:+ # Don't let a transient network blip wedge fetch_alerts;+ # we'll retry on the next poll.+ logger.warning(+ "m365_audit.subscribe_request_failed",+ content_type=content_type,+ error=str(exc),+ )++ async def fetch_alerts(self, since_seconds: int = 300) -> list[dict[str, Any]]:+ if not self._access_token:+ await self._authenticate()++ # Management API requires both startTime and endTime, and the spec+ # forbids more than 24h between them. Our default poll window is+ # well under that, but we still clamp to a sane upper bound to+ # protect against operator misconfig.+ end = datetime.now(UTC)+ start = end - timedelta(seconds=min(since_seconds, 24 * 3600))+ fmt = "%Y-%m-%dT%H:%M:%S"+ common_params = {+ "startTime": start.strftime(fmt),+ "endTime": end.strftime(fmt),+ }++ events: list[dict[str, Any]] = []+ async with httpx.AsyncClient(timeout=30.0) as client:+ await self._ensure_subscriptions(client)++ # For each content type, list available content blobs in the+ # window then fetch each blob URI. The blob URIs are short-lived+ # (~24h) signed URLs hosted on the same management API.+ for content_type in _DEFAULT_CONTENT_TYPES:+ resp = await client.get(+ f"{self._base()}/subscriptions/content",+ headers=self._headers(),+ params={**common_params, "contentType": content_type},+ )+ if resp.status_code == 401:+ await self._authenticate()+ resp = await client.get(+ f"{self._base()}/subscriptions/content",+ headers=self._headers(),+ params={**common_params, "contentType": content_type},+ )+ if resp.status_code != 200:+ logger.warning(+ "m365_audit.list_failed",+ content_type=content_type,+ status=resp.status_code,+ body=resp.text[:300],+ )+ continue++ blobs = resp.json()+ for blob in blobs:+ blob_uri = blob.get("contentUri")+ if not blob_uri:+ continue+ blob_resp = await client.get(blob_uri, headers=self._headers())+ if blob_resp.status_code != 200:+ logger.warning(+ "m365_audit.blob_fetch_failed",+ blob_id=blob.get("contentId"),+ status=blob_resp.status_code,+ )+ continue+ body = blob_resp.json()+ if isinstance(body, list):+ events.extend(body)+ elif isinstance(body, dict):+ # Some content types wrap the array; tolerate both.+ events.extend(body.get("value", []))++ return [self.normalize(e) for e in events]++ # ----------------------- normalize --------------------------++ # M365 ``Operation`` names that are inherently sensitive. Matched as a+ # case-insensitive substring.+ _HIGH_RISK_OPERATIONS = (+ "AddRoleScopeMember", # privileged role grants+ "Add member to role",+ "Add app role assignment",+ "Add service principal",+ "Set company information",+ "Update user", # specifically catches admin-on-admin elevation+ "Update application",+ "Add owner to",+ "Disable Strong Authentication",+ "Disable account",+ "Reset user password",+ "ResetPassword",+ # SharePoint sharing escalations+ "AnonymousLinkCreated",+ "SharingInvitationCreated",+ # Mailbox forwarding rules are a textbook BEC IOC+ "New-InboxRule",+ "Set-InboxRule",+ "UpdateInboxRules",+ )++ def normalize(self, raw: dict[str, Any]) -> dict[str, Any]:+ operation = raw.get("Operation", "")+ result_status = (raw.get("ResultStatus") or "").lower()+ workload = raw.get("Workload", "")++ # Severity heuristic: high-risk ops → high; failed sensitive ops →+ # medium (failed brute force / failed MFA / failed admin op are+ # interesting); failed normal ops → low; everything else → info.+ op_lower = operation.lower()+ is_high_risk = any(h.lower() in op_lower for h in self._HIGH_RISK_OPERATIONS)++ if is_high_risk:+ severity = "high"+ elif result_status in ("failed", "failure") and "login" in op_lower:+ # Brute-force-like patterns warrant low rather than info because+ # downstream detections will multiply low-severity auth failures+ # to flag spray attacks.+ severity = "low"+ elif result_status in ("failed", "failure"):+ severity = "low"+ else:+ severity = "info"++ actor = raw.get("UserId") or raw.get("UserKey") or "unknown"+ user_type_int = raw.get("UserType")+ # UserType: 0=Regular, 1=Reserved, 2=Admin, 3=DcAdmin, 4=System,+ # 5=Application, 6=ServicePrincipal. Admin/DcAdmin actions get a+ # severity bump because privileged operator activity is always+ # higher-stakes than end-user noise.+ if user_type_int in (2, 3) and severity == "info":+ severity = "low"++ return {+ "source": self.connector_id,+ "external_id": raw.get("Id", ""),+ "title": operation or f"M365 {workload} event",+ "description": (+ f"workload={workload}; "+ f"operation={operation}; "+ f"result={result_status or 'success'}"+ ),+ "severity": severity,+ "actor": actor,+ "actor_email": actor if "@" in actor else None,+ "src_ip": raw.get("ClientIP"),+ "workload": workload,+ "event_type": (+ f"m365.{workload.lower()}.{operation.replace(' ', '').lower()}"+ if workload and operation+ else "m365.audit"+ ),+ "raw_event": raw,+ "created_at": raw.get("CreationTime"),+ }diff --git a/services/connectors/tests/test_azure_connectors.py b/services/connectors/tests/test_azure_connectors.pynew file mode 100644index 000000000..c0c9f789b--- /dev/null+++ b/services/connectors/tests/test_azure_connectors.py@@ -0,0 +1,352 @@+"""+Unit tests for the Azure family of connectors.++We exercise three concerns per connector:++1. ``schema()`` is shaped correctly and registry-wired (smoke tests above+ in ``test_schemas.py`` cover the contract bits, here we assert the+ *Azure-specific* expectations like field names).+2. ``normalize()`` produces an event dict that downstream code can rely on.+3. ``test_connection()`` and ``fetch_alerts()`` route through ``httpx`` the+ way we expect — driven by ``respx`` so we never hit real Azure APIs.+"""++from __future__ import annotations++import httpx+import pytest+import respx++from app.connectors.azure_activity import AzureActivityConnector+from app.connectors.azure_defender import AzureDefenderConnector+from app.connectors.azure_entra import AzureEntraConnector++_TENANT = "00000000-0000-0000-0000-000000000001"+_CLIENT = "00000000-0000-0000-0000-000000000002"+_SECRET = "super-secret-value"+_SUB = "00000000-0000-0000-0000-000000000003"+++# ---------------------------------------------------------------------------+# Schema sanity (Azure-specific; generic contract checks live in test_schemas)+# ---------------------------------------------------------------------------+++def test_azure_entra_schema_has_required_fields():+ schema = AzureEntraConnector.schema()+ field_names = {f.name for f in schema.fields}+ assert {"tenant_id", "client_id", "client_secret"} <= field_names+ assert schema.category == "iam"+ assert schema.oauth is not None+ assert schema.oauth.supported_in_hosted is True+++def test_azure_activity_schema_includes_subscription_field():+ schema = AzureActivityConnector.schema()+ field_names = {f.name for f in schema.fields}+ assert "subscription_id" in field_names, (+ "azure_activity must collect a subscription_id; without it the polling "+ "loop has no scope to query"+ )+ assert {"tenant_id", "client_id", "client_secret"} <= field_names+ assert schema.category == "cloud"+++def test_azure_defender_schema_marks_secret_field():+ schema = AzureDefenderConnector.schema()+ secret_field = next(f for f in schema.fields if f.name == "client_secret")+ assert secret_field.type == "secret"+ assert schema.category == "edr"+++# ---------------------------------------------------------------------------+# Normalize: shape + severity rules+# ---------------------------------------------------------------------------+++def test_azure_entra_normalize_directory_audit_role_change_is_high():+ connector = AzureEntraConnector(_TENANT, _CLIENT, _SECRET)+ raw = {+ "id": "audit-1",+ "category": "RoleManagement",+ "result": "success",+ "activityDisplayName": "Add member to role",+ "activityDateTime": "2026-01-01T00:00:00Z",+ "initiatedBy": {+ "user": {+ "displayName": "Alice Admin",+ "userPrincipalName": "[email protected]",+ }+ },+ "_aisoc_event_kind": "directoryAudit",+ }+ out = connector.normalize(raw)+ assert out["source"] == "azure_entra"+ assert out["external_id"] == "audit-1"+ assert out["severity"] == "high", "successful RoleManagement audits must be high"+ assert out["actor"] == "Alice Admin"+ assert out["actor_email"] == "[email protected]"+ assert out["event_type"].startswith("azure.entra.audit.")+++def test_azure_entra_normalize_risky_signin_maps_severity():+ connector = AzureEntraConnector(_TENANT, _CLIENT, _SECRET)+ raw = {+ "id": "signin-1",+ "userPrincipalName": "[email protected]",+ "userDisplayName": "Bob User",+ "ipAddress": "1.2.3.4",+ "createdDateTime": "2026-01-01T00:00:00Z",+ "riskLevelAggregated": "high",+ "riskState": "atRisk",+ "riskEventTypes_v2": ["unfamiliarFeatures"],+ "_aisoc_event_kind": "riskySignIn",+ }+ out = connector.normalize(raw)+ assert out["severity"] == "high"+ assert out["src_ip"] == "1.2.3.4"+ assert out["actor_email"] == "[email protected]"+ assert out["event_type"] == "azure.entra.signin.risky"+++def test_azure_activity_normalize_role_assignment_write_is_high():+ connector = AzureActivityConnector(_TENANT, _CLIENT, _SECRET, _SUB)+ raw = {+ "eventDataId": "act-1",+ "operationName": {+ "value": "Microsoft.Authorization/roleAssignments/write",+ },+ "status": {"value": "Succeeded"},+ "subStatus": {"value": "OK"},+ "level": "Informational",+ "caller": "[email protected]",+ "resourceId": "/subscriptions/sub/resourceGroups/rg",+ "eventTimestamp": "2026-01-01T00:00:00Z",+ }+ out = connector.normalize(raw)+ assert out["severity"] == "high", (+ "writing role assignments is a high-blast-radius operation"+ )+ assert out["external_id"] == "act-1"+ assert out["actor"] == "[email protected]"+ assert out["actor_email"] == "[email protected]"+ assert out["event_type"].startswith("azure.activity.")+++def test_azure_activity_normalize_routine_read_is_info():+ connector = AzureActivityConnector(_TENANT, _CLIENT, _SECRET, _SUB)+ raw = {+ "eventDataId": "act-2",+ "operationName": {+ "value": "Microsoft.Storage/storageAccounts/read",+ },+ "status": {"value": "Succeeded"},+ "level": "Informational",+ "caller": "[email protected]",+ "resourceId": "/subscriptions/sub",+ "eventTimestamp": "2026-01-01T00:00:00Z",+ }+ out = connector.normalize(raw)+ assert out["severity"] == "info"+++def test_azure_defender_normalize_maps_informational_to_info():+ connector = AzureDefenderConnector(_TENANT, _CLIENT, _SECRET)+ raw = {+ "id": "def-1",+ "title": "Suspicious sign-in",+ "description": "Unusual location",+ "severity": "informational",+ "status": "newAlert",+ "createdDateTime": "2026-01-01T00:00:00Z",+ "serviceSource": "microsoftDefenderForEndpoint",+ "category": "InitialAccess",+ "evidence": [+ {+ "@odata.type": "#microsoft.graph.security.userEvidence",+ "userAccount": {+ "displayName": "Eve User",+ "userPrincipalName": "[email protected]",+ "accountName": "eve",+ },+ },+ {+ "@odata.type": "#microsoft.graph.security.deviceEvidence",+ "deviceDnsName": "host01.example.com",+ },+ ],+ }+ out = connector.normalize(raw)+ assert out["severity"] == "info"+ assert out["actor"] == "Eve User"+ assert out["actor_email"] == "[email protected]"+ assert out["host"] == "host01.example.com"+ assert out["service_source"] == "microsoftDefenderForEndpoint"+ assert out["event_type"] == "azure.defender.microsoftdefenderforendpoint"+++def test_azure_defender_normalize_preserves_high_severity():+ connector = AzureDefenderConnector(_TENANT, _CLIENT, _SECRET)+ raw = {+ "id": "def-2",+ "title": "Ransomware execution",+ "severity": "high",+ "createdDateTime": "2026-01-01T00:00:00Z",+ "serviceSource": "microsoftDefenderForEndpoint",+ }+ out = connector.normalize(raw)+ assert out["severity"] == "high"+++# ---------------------------------------------------------------------------+# Live HTTP routing via respx+# ---------------------------------------------------------------------------+++async def test_azure_entra_test_connection_success():+ respx.post(f"https://login.microsoftonline.com/{_TENANT}/oauth2/v2.0/token").mock(+ return_value=httpx.Response(200, json={"access_token": "abc"})+ )+ respx.get("https://graph.microsoft.com/v1.0/organization").mock(+ return_value=httpx.Response(+ 200, json={"value": [{"id": "org-1", "displayName": "Acme Corp"}]}+ )+ )++ connector = AzureEntraConnector(_TENANT, _CLIENT, _SECRET)+ result = await connector.test_connection()+ assert result["success"] is True+ assert result["organization"] == "Acme Corp"+++async def test_azure_entra_test_connection_bubbles_http_error():+ respx.post(f"https://login.microsoftonline.com/{_TENANT}/oauth2/v2.0/token").mock(+ return_value=httpx.Response(401, text="invalid_client")+ )++ connector = AzureEntraConnector(_TENANT, _CLIENT, _SECRET)+ result = await connector.test_connection()+ assert result["success"] is False+ assert "401" in result["error"]+++async def test_azure_entra_fetch_alerts_returns_normalized_events():+ respx.post(f"https://login.microsoftonline.com/{_TENANT}/oauth2/v2.0/token").mock(+ return_value=httpx.Response(200, json={"access_token": "abc"})+ )+ respx.get("https://graph.microsoft.com/v1.0/auditLogs/directoryAudits").mock(+ return_value=httpx.Response(+ 200,+ json={+ "value": [+ {+ "id": "a-1",+ "category": "UserManagement",+ "result": "success",+ "activityDisplayName": "Add user",+ "activityDateTime": "2026-01-01T00:00:00Z",+ "initiatedBy": {"user": {"displayName": "Alice"}},+ }+ ]+ },+ )+ )+ respx.get("https://graph.microsoft.com/v1.0/auditLogs/signIns").mock(+ return_value=httpx.Response(200, json={"value": []})+ )++ connector = AzureEntraConnector(_TENANT, _CLIENT, _SECRET)+ events = await connector.fetch_alerts(since_seconds=300)+ assert len(events) == 1+ assert events[0]["source"] == "azure_entra"+ assert events[0]["external_id"] == "a-1"+++async def test_azure_activity_test_connection_success():+ respx.post(f"https://login.microsoftonline.com/{_TENANT}/oauth2/v2.0/token").mock(+ return_value=httpx.Response(200, json={"access_token": "abc"})+ )+ respx.get(f"https://management.azure.com/subscriptions/{_SUB}").mock(+ return_value=httpx.Response(+ 200, json={"id": f"/subscriptions/{_SUB}", "displayName": "Production"}+ )+ )++ connector = AzureActivityConnector(_TENANT, _CLIENT, _SECRET, _SUB)+ result = await connector.test_connection()+ assert result["success"] is True+ assert result["subscription_name"] == "Production"+++async def test_azure_activity_fetch_alerts_handles_empty_response():+ respx.post(f"https://login.microsoftonline.com/{_TENANT}/oauth2/v2.0/token").mock(+ return_value=httpx.Response(200, json={"access_token": "abc"})+ )+ activity_url = (+ f"https://management.azure.com/subscriptions/{_SUB}"+ "/providers/Microsoft.Insights/eventtypes/management/values"+ )+ respx.get(activity_url).mock(+ return_value=httpx.Response(200, json={"value": []})+ )++ connector = AzureActivityConnector(_TENANT, _CLIENT, _SECRET, _SUB)+ events = await connector.fetch_alerts(since_seconds=600)+ assert events == []+++async def test_azure_defender_test_connection_handles_failure():+ respx.post(f"https://login.microsoftonline.com/{_TENANT}/oauth2/v2.0/token").mock(+ return_value=httpx.Response(200, json={"access_token": "abc"})+ )+ respx.get("https://graph.microsoft.com/v1.0/security/alerts_v2").mock(+ return_value=httpx.Response(403, text="insufficient_privileges")+ )++ connector = AzureDefenderConnector(_TENANT, _CLIENT, _SECRET)+ result = await connector.test_connection()+ assert result["success"] is False+ assert "403" in result["error"]+++async def test_azure_defender_fetch_alerts_returns_normalized_events():+ respx.post(f"https://login.microsoftonline.com/{_TENANT}/oauth2/v2.0/token").mock(+ return_value=httpx.Response(200, json={"access_token": "abc"})+ )+ respx.get("https://graph.microsoft.com/v1.0/security/alerts_v2").mock(+ return_value=httpx.Response(+ 200,+ json={+ "value": [+ {+ "id": "alert-1",+ "title": "Malware detected",+ "description": "Trojan found on host",+ "severity": "high",+ "createdDateTime": "2026-01-01T00:00:00Z",+ "serviceSource": "microsoftDefenderForEndpoint",+ }+ ]+ },+ )+ )++ connector = AzureDefenderConnector(_TENANT, _CLIENT, _SECRET)+ events = await connector.fetch_alerts(since_seconds=300)+ assert len(events) == 1+ assert events[0]["severity"] == "high"+ assert events[0]["source"] == "azure_defender"diff --git a/services/connectors/tests/test_gcp_connectors.py b/services/connectors/tests/test_gcp_connectors.pynew file mode 100644index 000000000..4a3fbdbc5--- /dev/null+++ b/services/connectors/tests/test_gcp_connectors.py@@ -0,0 +1,404 @@+"""+Unit tests for the GCP family of connectors.++These exercise the same three concerns as the Azure suite: schema sanity,+``normalize()`` severity rules, and HTTP routing through ``respx``. The+extra wrinkle is that GCP service accounts authenticate by signing a JWT+with an RSA private key, so we generate a throwaway 2048-bit key once per+test module and forge a realistic service-account JSON blob from it. That+way ``_build_jwt`` exercises real RSA signing — we just intercept the+token endpoint to return a canned access token.+"""++from __future__ import annotations++import json++import httpx+import pytest+import respx+from cryptography.hazmat.primitives import serialization+from cryptography.hazmat.primitives.asymmetric import rsa++from app.connectors.gcp_cloud_audit import GCPCloudAuditConnector+from app.connectors.gcp_scc import GCPSCCConnector++_PROJECT = "aisoc-test-project"+_ORG = "1234567890"+++# ---------------------------------------------------------------------------+# One throwaway RSA key per test module — generating per-test would slow+# the suite (~150ms per 2048-bit key). Module-scope keeps tests honest+# (real signing path) without paying the cost on every test.+# ---------------------------------------------------------------------------++[email protected](scope="module")+def fake_sa_json() -> str:+ key = rsa.generate_private_key(public_exponent=65537, key_size=2048)+ pem = key.private_bytes(+ encoding=serialization.Encoding.PEM,+ format=serialization.PrivateFormat.PKCS8,+ encryption_algorithm=serialization.NoEncryption(),+ ).decode("ascii")+ return json.dumps(+ {+ "type": "service_account",+ "project_id": _PROJECT,+ "private_key_id": "abc123",+ "private_key": pem,+ "client_email": "[email protected]",+ "client_id": "111111111111111111111",+ "token_uri": "https://oauth2.googleapis.com/token",+ }+ )+++# ---------------------------------------------------------------------------+# Schema sanity+# ---------------------------------------------------------------------------+++def test_gcp_cloud_audit_schema_has_required_fields():+ schema = GCPCloudAuditConnector.schema()+ field_names = {f.name for f in schema.fields}+ assert {"project_id", "service_account_json"} <= field_names+ assert schema.category == "cloud"+ sa_field = next(f for f in schema.fields if f.name == "service_account_json")+ assert sa_field.type == "secret", "SA key must be a secret field"+++def test_gcp_scc_schema_has_required_fields():+ schema = GCPSCCConnector.schema()+ field_names = {f.name for f in schema.fields}+ assert {"organization_id", "service_account_json"} <= field_names+ assert schema.category == "cloud"+ sa_field = next(f for f in schema.fields if f.name == "service_account_json")+ assert sa_field.type == "secret"+++def test_gcp_cloud_audit_schema_oauth_hosted_is_false():+ # Service-account auth doesn't fit the hosted-OAuth flow at all; the+ # frontend should not advertise "hosted OAuth" for these connectors.+ schema = GCPCloudAuditConnector.schema()+ assert schema.oauth is not None+ assert schema.oauth.supported_in_hosted is False+++# ---------------------------------------------------------------------------+# Constructor input validation+# ---------------------------------------------------------------------------+++def test_gcp_connector_rejects_non_json_service_account(fake_sa_json):+ with pytest.raises(ValueError, match="not valid JSON"):+ GCPCloudAuditConnector(_PROJECT, "this-is-not-json")+++def test_gcp_connector_rejects_sa_missing_required_fields():+ bad = json.dumps({"client_email": "[email protected]"})+ with pytest.raises(ValueError, match="missing required field"):+ GCPCloudAuditConnector(_PROJECT, bad)+++# ---------------------------------------------------------------------------+# Normalize: cloud audit severity rules+# ---------------------------------------------------------------------------+++def test_cloud_audit_normalize_setiampolicy_is_high(fake_sa_json):+ connector = GCPCloudAuditConnector(_PROJECT, fake_sa_json)+ raw = {+ "insertId": "audit-1",+ "timestamp": "2026-01-01T00:00:00Z",+ "severity": "NOTICE",+ "protoPayload": {+ "methodName": "SetIamPolicy",+ "serviceName": "cloudresourcemanager.googleapis.com",+ "authenticationInfo": {"principalEmail": "[email protected]"},+ "requestMetadata": {"callerIp": "1.2.3.4"},+ "status": {},+ },+ "resource": {"type": "project"},+ }+ out = connector.normalize(raw)+ assert out["source"] == "gcp_cloud_audit"+ assert out["severity"] == "high", "SetIamPolicy is high blast radius"+ assert out["actor"] == "[email protected]"+ assert out["actor_email"] == "[email protected]"+ assert out["src_ip"] == "1.2.3.4"+ assert out["event_type"].startswith("gcp.audit.")+++def test_cloud_audit_normalize_failed_op_bumps_to_medium(fake_sa_json):+ connector = GCPCloudAuditConnector(_PROJECT, fake_sa_json)+ raw = {+ "insertId": "audit-2",+ "timestamp": "2026-01-01T00:00:00Z",+ "severity": "INFO",+ "protoPayload": {+ "methodName": "google.compute.v1.instances.start",+ "serviceName": "compute.googleapis.com",+ "authenticationInfo": {"principalEmail": "[email protected]"},+ "status": {"code": 7, "message": "PERMISSION_DENIED"},+ },+ "resource": {"type": "gce_instance"},+ }+ out = connector.normalize(raw)+ assert out["severity"] == "medium", (+ "failed control-plane operations should escalate above info"+ )+++def test_cloud_audit_normalize_routine_read_is_info(fake_sa_json):+ connector = GCPCloudAuditConnector(_PROJECT, fake_sa_json)+ raw = {+ "insertId": "audit-3",+ "timestamp": "2026-01-01T00:00:00Z",+ "severity": "INFO",+ "protoPayload": {+ "methodName": "google.storage.buckets.list",+ "serviceName": "storage.googleapis.com",+ "authenticationInfo": {"principalEmail": "[email protected]"},+ "status": {},+ },+ "resource": {"type": "gcs_bucket"},+ }+ out = connector.normalize(raw)+ assert out["severity"] == "info"+++def test_cloud_audit_normalize_critical_log_severity_is_high(fake_sa_json):+ connector = GCPCloudAuditConnector(_PROJECT, fake_sa_json)+ raw = {+ "insertId": "audit-4",+ "timestamp": "2026-01-01T00:00:00Z",+ "severity": "CRITICAL",+ "protoPayload": {+ "methodName": "google.compute.v1.instances.detachDisk",+ "serviceName": "compute.googleapis.com",+ "authenticationInfo": {"principalEmail": "[email protected]"},+ "status": {},+ },+ "resource": {"type": "gce_instance"},+ }+ out = connector.normalize(raw)+ assert out["severity"] == "high"+++# ---------------------------------------------------------------------------+# Normalize: SCC severity mapping+# ---------------------------------------------------------------------------+++def test_scc_normalize_critical_collapses_to_high(fake_sa_json):+ connector = GCPSCCConnector(_ORG, fake_sa_json)+ raw = {+ "finding": {+ "name": "organizations/123/sources/456/findings/abc",+ "category": "MALWARE_BAD_DOMAIN",+ "description": "Connection to known-bad domain",+ "severity": "CRITICAL",+ "eventTime": "2026-01-01T00:00:00Z",+ "access": {"principalEmail": "[email protected]"},+ },+ "resource": {+ "name": "//compute.googleapis.com/projects/p/zones/z/instances/i",+ "type": "google.compute.Instance",+ },+ }+ out = connector.normalize(raw)+ assert out["source"] == "gcp_scc"+ assert out["severity"] == "high", (+ "SCC CRITICAL collapses to AiSOC 'high' since downstream pipelines "+ "expect that vocabulary"+ )+ assert out["title"] == "MALWARE_BAD_DOMAIN"+ assert out["actor_email"] == "[email protected]"+ assert out["resource_type"] == "google.compute.Instance"+ assert out["event_type"] == "gcp.scc.malware_bad_domain"+++def test_scc_normalize_low_severity_maps_to_low(fake_sa_json):+ connector = GCPSCCConnector(_ORG, fake_sa_json)+ raw = {+ "finding": {+ "name": "organizations/123/sources/456/findings/xyz",+ "category": "OPEN_FIREWALL",+ "severity": "LOW",+ "eventTime": "2026-01-01T00:00:00Z",+ },+ "resource": {+ "name": "//compute.googleapis.com/projects/p/global/firewalls/f",+ "type": "google.compute.Firewall",+ },+ }+ out = connector.normalize(raw)+ assert out["severity"] == "low"+++def test_scc_normalize_unknown_severity_falls_back_to_info(fake_sa_json):+ connector = GCPSCCConnector(_ORG, fake_sa_json)+ raw = {"finding": {"name": "x", "category": "UNKNOWN_THING"}}+ out = connector.normalize(raw)+ assert out["severity"] == "info"+++# ---------------------------------------------------------------------------+# HTTP routing via respx+# ---------------------------------------------------------------------------+++async def test_cloud_audit_test_connection_success(fake_sa_json):+ respx.post("https://oauth2.googleapis.com/token").mock(+ return_value=httpx.Response(+ 200, json={"access_token": "tok-abc", "expires_in": 3600}+ )+ )+ respx.post("https://logging.googleapis.com/v2/entries:list").mock(+ return_value=httpx.Response(200, json={"entries": []})+ )++ connector = GCPCloudAuditConnector(_PROJECT, fake_sa_json)+ result = await connector.test_connection()+ assert result["success"] is True+ assert result["project_id"] == _PROJECT+ assert "service_account" in result+++async def test_cloud_audit_test_connection_surfaces_403(fake_sa_json):+ respx.post("https://oauth2.googleapis.com/token").mock(+ return_value=httpx.Response(+ 200, json={"access_token": "tok-abc", "expires_in": 3600}+ )+ )+ respx.post("https://logging.googleapis.com/v2/entries:list").mock(+ return_value=httpx.Response(403, text="caller does not have permission")+ )++ connector = GCPCloudAuditConnector(_PROJECT, fake_sa_json)+ result = await connector.test_connection()+ assert result["success"] is False+ assert "403" in result["error"]+++async def test_cloud_audit_fetch_alerts_returns_normalized_events(fake_sa_json):+ respx.post("https://oauth2.googleapis.com/token").mock(+ return_value=httpx.Response(+ 200, json={"access_token": "tok-abc", "expires_in": 3600}+ )+ )+ respx.post("https://logging.googleapis.com/v2/entries:list").mock(+ return_value=httpx.Response(+ 200,+ json={+ "entries": [+ {+ "insertId": "log-1",+ "timestamp": "2026-01-01T00:00:00Z",+ "severity": "NOTICE",+ "protoPayload": {+ "methodName": "SetIamPolicy",+ "serviceName": "cloudresourcemanager.googleapis.com",+ "authenticationInfo": {+ "principalEmail": "[email protected]"+ },+ "status": {},+ },+ "resource": {"type": "project"},+ }+ ]+ },+ )+ )++ connector = GCPCloudAuditConnector(_PROJECT, fake_sa_json)+ events = await connector.fetch_alerts(since_seconds=300)+ assert len(events) == 1+ assert events[0]["source"] == "gcp_cloud_audit"+ assert events[0]["severity"] == "high"+ assert events[0]["actor_email"] == "[email protected]"+++async def test_cloud_audit_fetch_alerts_returns_empty_on_500(fake_sa_json):+ respx.post("https://oauth2.googleapis.com/token").mock(+ return_value=httpx.Response(+ 200, json={"access_token": "tok-abc", "expires_in": 3600}+ )+ )+ respx.post("https://logging.googleapis.com/v2/entries:list").mock(+ return_value=httpx.Response(500, text="internal")+ )++ connector = GCPCloudAuditConnector(_PROJECT, fake_sa_json)+ events = await connector.fetch_alerts(since_seconds=300)+ assert events == []+++async def test_scc_test_connection_success(fake_sa_json):+ respx.post("https://oauth2.googleapis.com/token").mock(+ return_value=httpx.Response(+ 200, json={"access_token": "tok-abc", "expires_in": 3600}+ )+ )+ respx.get(+ f"https://securitycenter.googleapis.com/v1/organizations/{_ORG}/sources"+ ).mock(return_value=httpx.Response(200, json={"sources": []}))++ connector = GCPSCCConnector(_ORG, fake_sa_json)+ result = await connector.test_connection()+ assert result["success"] is True+ assert result["organization_id"] == _ORG+++async def test_scc_fetch_alerts_returns_normalized_findings(fake_sa_json):+ respx.post("https://oauth2.googleapis.com/token").mock(+ return_value=httpx.Response(+ 200, json={"access_token": "tok-abc", "expires_in": 3600}+ )+ )+ respx.get(+ f"https://securitycenter.googleapis.com/v1/organizations/{_ORG}/sources/-/findings"+ ).mock(+ return_value=httpx.Response(+ 200,+ json={+ "listFindingsResults": [+ {+ "finding": {+ "name": "organizations/x/sources/y/findings/z",+ "category": "PUBLIC_BUCKET_ACL",+ "severity": "HIGH",+ "eventTime": "2026-01-01T00:00:00Z",+ "description": "GCS bucket has allUsers in ACL",+ },+ "resource": {+ "name": "//storage.googleapis.com/projects/p/buckets/b",+ "type": "google.cloud.storage.Bucket",+ },+ }+ ]+ },+ )+ )++ connector = GCPSCCConnector(_ORG, fake_sa_json)+ events = await connector.fetch_alerts(since_seconds=300)+ assert len(events) == 1+ assert events[0]["source"] == "gcp_scc"+ assert events[0]["severity"] == "high"+ assert events[0]["title"] == "PUBLIC_BUCKET_ACL"+ assert events[0]["resource_type"] == "google.cloud.storage.Bucket"diff --git a/services/connectors/tests/test_saas_connectors.py b/services/connectors/tests/test_saas_connectors.pynew file mode 100644index 000000000..87ad7f951--- /dev/null+++ b/services/connectors/tests/test_saas_connectors.py@@ -0,0 +1,866 @@+"""+Unit tests for the four SaaS connectors landed in the connectors PR:++- ``m365_audit`` — Microsoft 365 Management Activity API+- ``google_workspace`` — Admin SDK Reports API+- ``cloudflare`` — Account audit logs+- ``github`` — Org audit log + Code Scanning alerts++The shape mirrors ``test_azure_connectors.py`` and ``test_gcp_connectors.py``:+schema sanity, normalize() severity rules, then HTTP routing through respx.++For ``google_workspace`` we reuse the GCP-style fake-service-account JSON+fixture (real RS256 signing, mocked token endpoint) so the JWT path is+actually exercised — the ``sub`` claim for domain-wide delegation is the+single most-broken piece of Workspace integrations and needs to round-trip+through real signing to be useful.+"""++from __future__ import annotations++import json++import httpx+import pytest+import respx+from cryptography.hazmat.primitives import serialization+from cryptography.hazmat.primitives.asymmetric import rsa++from app.connectors.cloudflare import CloudflareConnector+from app.connectors.github import GitHubConnector+from app.connectors.google_workspace import GoogleWorkspaceConnector+from app.connectors.m365_audit import M365AuditConnector++_TENANT = "00000000-0000-0000-0000-000000000000"+_CLIENT_ID = "11111111-1111-1111-1111-111111111111"+_CLIENT_SECRET = "super-secret"+_ACCOUNT_ID = "abc123def456"+_TOKEN = "ghp_fakeFakeFakeFakeFakeFakeFakeFake"+_ORG = "aisoc-test-org"+_ADMIN_EMAIL = "[email protected]"+++# ---------------------------------------------------------------------------+# Fake Workspace service-account JSON — module-scope to avoid paying the+# 2048-bit RSA generation cost on every test.+# ---------------------------------------------------------------------------++[email protected](scope="module")+def fake_workspace_sa_json() -> str:+ key = rsa.generate_private_key(public_exponent=65537, key_size=2048)+ pem = key.private_bytes(+ encoding=serialization.Encoding.PEM,+ format=serialization.PrivateFormat.PKCS8,+ encryption_algorithm=serialization.NoEncryption(),+ ).decode("ascii")+ return json.dumps(+ {+ "type": "service_account",+ "project_id": "aisoc-workspace",+ "private_key_id": "abc123",+ "private_key": pem,+ "client_email": "[email protected]",+ "client_id": "111111111111111111111",+ "token_uri": "https://oauth2.googleapis.com/token",+ }+ )+++# ===========================================================================+# M365 Audit+# ===========================================================================+++def test_m365_audit_schema_has_required_fields():+ schema = M365AuditConnector.schema()+ field_names = {f.name for f in schema.fields}+ assert {"tenant_id", "client_id", "client_secret"} <= field_names+ assert schema.category == "saas"+ secret = next(f for f in schema.fields if f.name == "client_secret")+ assert secret.type == "secret"+++def test_m365_audit_schema_advertises_hosted_oauth():+ # M365 has a real Azure AD OAuth flow — distinct from the GCP/Workspace+ # service-account model — so the frontend should be told this is a+ # candidate for hosted OAuth in a follow-up.+ schema = M365AuditConnector.schema()+ assert schema.oauth is not None+ assert schema.oauth.supported_in_hosted is True+ assert "ActivityFeed.Read" in (schema.oauth.scopes or [""])[0]+++def test_m365_normalize_high_risk_op_is_high():+ connector = M365AuditConnector(_TENANT, _CLIENT_ID, _CLIENT_SECRET)+ raw = {+ "Id": "evt-1",+ "Operation": "Add member to role",+ "Workload": "AzureActiveDirectory",+ "ResultStatus": "Success",+ "UserId": "[email protected]",+ "ClientIP": "1.2.3.4",+ "CreationTime": "2026-01-01T00:00:00Z",+ }+ out = connector.normalize(raw)+ assert out["source"] == "m365_audit"+ assert out["severity"] == "high"+ assert out["actor_email"] == "[email protected]"+ assert out["src_ip"] == "1.2.3.4"+++def test_m365_normalize_inboxrule_is_high_bec_indicator():+ # BEC playbook: attacker compromises mailbox, sets a forwarding rule.+ # New-InboxRule must be high regardless of result_status.+ connector = M365AuditConnector(_TENANT, _CLIENT_ID, _CLIENT_SECRET)+ raw = {+ "Id": "evt-2",+ "Operation": "New-InboxRule",+ "Workload": "Exchange",+ "ResultStatus": "Success",+ "UserId": "[email protected]",+ }+ out = connector.normalize(raw)+ assert out["severity"] == "high"+++def test_m365_normalize_failed_login_is_low():+ connector = M365AuditConnector(_TENANT, _CLIENT_ID, _CLIENT_SECRET)+ raw = {+ "Id": "evt-3",+ "Operation": "UserLoggedIn",+ "Workload": "AzureActiveDirectory",+ "ResultStatus": "Failed",+ "UserId": "[email protected]",+ }+ out = connector.normalize(raw)+ # Failed logins land at low so spray detections can multiply them up.+ assert out["severity"] == "low"+++def test_m365_normalize_admin_routine_op_bumps_to_low():+ # Plain "info" admin actions should escalate one notch when UserType is+ # admin (2) or DcAdmin (3). Operator activity is always higher-stakes.+ connector = M365AuditConnector(_TENANT, _CLIENT_ID, _CLIENT_SECRET)+ raw = {+ "Id": "evt-4",+ "Operation": "FilePreviewed",+ "Workload": "SharePoint",+ "ResultStatus": "Success",+ "UserId": "[email protected]",+ "UserType": 2,+ }+ out = connector.normalize(raw)+ assert out["severity"] == "low"+++def test_m365_normalize_routine_user_op_is_info():+ connector = M365AuditConnector(_TENANT, _CLIENT_ID, _CLIENT_SECRET)+ raw = {+ "Id": "evt-5",+ "Operation": "FileAccessed",+ "Workload": "SharePoint",+ "ResultStatus": "Success",+ "UserId": "[email protected]",+ "UserType": 0,+ }+ out = connector.normalize(raw)+ assert out["severity"] == "info"+ assert out["event_type"].startswith("m365.sharepoint.")+++async def test_m365_test_connection_success():+ respx.post(+ f"https://login.microsoftonline.com/{_TENANT}/oauth2/v2.0/token"+ ).mock(return_value=httpx.Response(200, json={"access_token": "tok-abc"}))+ respx.get(+ f"https://manage.office.com/api/v1.0/{_TENANT}/activity/feed/subscriptions/list"+ ).mock(+ return_value=httpx.Response(+ 200,+ json=[+ {"contentType": "Audit.AzureActiveDirectory", "status": "enabled"},+ {"contentType": "Audit.Exchange", "status": "disabled"},+ ],+ )+ )++ connector = M365AuditConnector(_TENANT, _CLIENT_ID, _CLIENT_SECRET)+ result = await connector.test_connection()+ assert result["success"] is True+ assert result["tenant_id"] == _TENANT+ # Only enabled subscriptions surface; disabled ones are filtered out.+ assert result["active_subscriptions"] == ["Audit.AzureActiveDirectory"]+++async def test_m365_test_connection_surfaces_403():+ respx.post(+ f"https://login.microsoftonline.com/{_TENANT}/oauth2/v2.0/token"+ ).mock(return_value=httpx.Response(200, json={"access_token": "tok-abc"}))+ respx.get(+ f"https://manage.office.com/api/v1.0/{_TENANT}/activity/feed/subscriptions/list"+ ).mock(+ return_value=httpx.Response(403, text="ActivityFeed.Read not granted")+ )++ connector = M365AuditConnector(_TENANT, _CLIENT_ID, _CLIENT_SECRET)+ result = await connector.test_connection()+ assert result["success"] is False+ assert "403" in result["error"]+++async def test_m365_fetch_alerts_pulls_blob_content():+ # The Activity API is a two-step flow: list content blobs, then GET each+ # blob URI. We mock both layers and assert the connector materializes+ # the inner events through normalize().+ respx.post(+ f"https://login.microsoftonline.com/{_TENANT}/oauth2/v2.0/token"+ ).mock(return_value=httpx.Response(200, json={"access_token": "tok-abc"}))++ # Idempotent subscribe on every content type — return 200 OK.+ respx.post(+ f"https://manage.office.com/api/v1.0/{_TENANT}/activity/feed/subscriptions/start"+ ).mock(return_value=httpx.Response(200, json={"contentType": "ok"}))++ blob_uri = "https://manage.office.com/api/v1.0/blob/abc"+ # Only the AzureActiveDirectory listing returns a blob; the rest are+ # empty so we can assert filtering plus de-duplication of empty content+ # types in a single test.+ aad_url = (+ f"https://manage.office.com/api/v1.0/{_TENANT}/activity/feed/subscriptions/content"+ )++ def list_handler(request: httpx.Request) -> httpx.Response:+ if request.url.params.get("contentType") == "Audit.AzureActiveDirectory":+ return httpx.Response(200, json=[{"contentUri": blob_uri, "contentId": "b1"}])+ return httpx.Response(200, json=[])++ respx.get(aad_url).mock(side_effect=list_handler)+ respx.get(blob_uri).mock(+ return_value=httpx.Response(+ 200,+ json=[+ {+ "Id": "evt-1",+ "Operation": "Add member to role",+ "Workload": "AzureActiveDirectory",+ "ResultStatus": "Success",+ "UserId": "[email protected]",+ }+ ],+ )+ )++ connector = M365AuditConnector(_TENANT, _CLIENT_ID, _CLIENT_SECRET)+ events = await connector.fetch_alerts(since_seconds=300)+ assert len(events) == 1+ assert events[0]["source"] == "m365_audit"+ assert events[0]["severity"] == "high"+++# ===========================================================================+# Google Workspace+# ===========================================================================+++def test_workspace_schema_has_required_fields():+ schema = GoogleWorkspaceConnector.schema()+ field_names = {f.name for f in schema.fields}+ assert {"admin_email", "service_account_json"} <= field_names+ assert schema.category == "saas"+ sa = next(f for f in schema.fields if f.name == "service_account_json")+ assert sa.type == "secret"+++def test_workspace_schema_oauth_hosted_is_false():+ schema = GoogleWorkspaceConnector.schema()+ assert schema.oauth is not None+ assert schema.oauth.supported_in_hosted is False+++def test_workspace_rejects_non_json_service_account():+ with pytest.raises(ValueError, match="not valid JSON"):+ GoogleWorkspaceConnector(_ADMIN_EMAIL, "not-json-at-all")+++def test_workspace_rejects_sa_missing_required_fields():+ bad = json.dumps({"client_email": "[email protected]"})+ with pytest.raises(ValueError, match="missing required field"):+ GoogleWorkspaceConnector(_ADMIN_EMAIL, bad)+++def test_workspace_normalize_grant_admin_is_high(fake_workspace_sa_json):+ connector = GoogleWorkspaceConnector(_ADMIN_EMAIL, fake_workspace_sa_json)+ raw = {+ "_aisoc_application": "admin",+ "id": {"uniqueQualifier": "evt-1", "time": "2026-01-01T00:00:00Z"},+ "actor": {"email": "[email protected]"},+ "ipAddress": "1.2.3.4",+ "events": [{"name": "GRANT_ADMIN_PRIVILEGE", "type": "DELEGATED_ADMIN_SETTINGS"}],+ }+ out = connector.normalize(raw)+ assert out["source"] == "google_workspace"+ assert out["severity"] == "high"+ assert out["actor_email"] == "[email protected]"+ assert out["src_ip"] == "1.2.3.4"+ assert out["event_type"] == "workspace.admin.grant_admin_privilege"+++def test_workspace_normalize_suspicious_login_is_medium(fake_workspace_sa_json):+ connector = GoogleWorkspaceConnector(_ADMIN_EMAIL, fake_workspace_sa_json)+ raw = {+ "_aisoc_application": "login",+ "id": {"uniqueQualifier": "evt-2"},+ "actor": {"email": "[email protected]"},+ "events": [{"name": "suspicious_login", "type": "login"}],+ }+ out = connector.normalize(raw)+ assert out["severity"] == "medium"+++def test_workspace_normalize_failed_login_is_low(fake_workspace_sa_json):+ connector = GoogleWorkspaceConnector(_ADMIN_EMAIL, fake_workspace_sa_json)+ raw = {+ "_aisoc_application": "login",+ "id": {"uniqueQualifier": "evt-3"},+ "actor": {"email": "[email protected]"},+ "events": [{"name": "login_failure", "type": "login"}],+ }+ out = connector.normalize(raw)+ assert out["severity"] == "low"+++def test_workspace_normalize_drive_acl_change_is_medium(fake_workspace_sa_json):+ connector = GoogleWorkspaceConnector(_ADMIN_EMAIL, fake_workspace_sa_json)+ raw = {+ "_aisoc_application": "drive",+ "id": {"uniqueQualifier": "evt-4"},+ "actor": {"email": "[email protected]"},+ "events": [{"name": "change_acl_editors", "type": "acl_change"}],+ }+ out = connector.normalize(raw)+ assert out["severity"] == "medium"+++def test_workspace_normalize_routine_event_is_info(fake_workspace_sa_json):+ connector = GoogleWorkspaceConnector(_ADMIN_EMAIL, fake_workspace_sa_json)+ raw = {+ "_aisoc_application": "drive",+ "id": {"uniqueQualifier": "evt-5"},+ "actor": {"email": "[email protected]"},+ "events": [{"name": "view", "type": "access"}],+ }+ out = connector.normalize(raw)+ assert out["severity"] == "info"+++async def test_workspace_test_connection_success(fake_workspace_sa_json):+ respx.post("https://oauth2.googleapis.com/token").mock(+ return_value=httpx.Response(+ 200, json={"access_token": "tok-abc", "expires_in": 3600}+ )+ )+ respx.get(+ "https://admin.googleapis.com/admin/reports/v1/activity/users/all/applications/login"+ ).mock(return_value=httpx.Response(200, json={"items": []}))++ connector = GoogleWorkspaceConnector(_ADMIN_EMAIL, fake_workspace_sa_json)+ result = await connector.test_connection()+ assert result["success"] is True+ assert result["admin_email"] == _ADMIN_EMAIL+ assert "service_account" in result+++async def test_workspace_test_connection_surfaces_403(fake_workspace_sa_json):+ respx.post("https://oauth2.googleapis.com/token").mock(+ return_value=httpx.Response(+ 200, json={"access_token": "tok-abc", "expires_in": 3600}+ )+ )+ respx.get(+ "https://admin.googleapis.com/admin/reports/v1/activity/users/all/applications/login"+ ).mock(+ return_value=httpx.Response(+ 403, text="The user is not authorized to access this resource"+ )+ )++ connector = GoogleWorkspaceConnector(_ADMIN_EMAIL, fake_workspace_sa_json)+ result = await connector.test_connection()+ assert result["success"] is False+ assert "403" in result["error"]+++async def test_workspace_fetch_alerts_iterates_apps(fake_workspace_sa_json):+ respx.post("https://oauth2.googleapis.com/token").mock(+ return_value=httpx.Response(+ 200, json={"access_token": "tok-abc", "expires_in": 3600}+ )+ )++ # Each ApplicationName has its own URL. We return one event for+ # "admin" and empty for the rest, then assert the connector tagged+ # the event with the right application.+ apps_with_data = {+ "admin": [+ {+ "id": {"uniqueQualifier": "evt-1", "time": "2026-01-01T00:00:00Z"},+ "actor": {"email": "[email protected]"},+ "events": [{"name": "GRANT_ADMIN_PRIVILEGE"}],+ }+ ],+ }++ def app_handler(request: httpx.Request) -> httpx.Response:+ # URL path ends with .../applications/<app>+ app = request.url.path.rsplit("/", 1)[-1]+ return httpx.Response(200, json={"items": apps_with_data.get(app, [])})++ respx.get(+ url__regex=r"^https://admin\.googleapis\.com/admin/reports/v1/activity/users/all/applications/.*"+ ).mock(side_effect=app_handler)++ connector = GoogleWorkspaceConnector(_ADMIN_EMAIL, fake_workspace_sa_json)+ events = await connector.fetch_alerts(since_seconds=300)+ assert len(events) == 1+ assert events[0]["severity"] == "high"+ assert events[0]["application"] == "admin"+++# ===========================================================================+# Cloudflare+# ===========================================================================+++def test_cloudflare_schema_has_required_fields():+ schema = CloudflareConnector.schema()+ field_names = {f.name for f in schema.fields}+ assert {"account_id", "api_token"} <= field_names+ assert schema.category == "saas"+ token_field = next(f for f in schema.fields if f.name == "api_token")+ assert token_field.type == "secret"+++def test_cloudflare_schema_oauth_hosted_is_false():+ # API-token model only — no hosted OAuth flow.+ schema = CloudflareConnector.schema()+ assert schema.oauth is not None+ assert schema.oauth.supported_in_hosted is False+++def test_cloudflare_normalize_token_create_is_high():+ connector = CloudflareConnector(_ACCOUNT_ID, "cf-token")+ raw = {+ "id": "evt-1",+ "when": "2026-01-01T00:00:00Z",+ "action": {"type": "tokenCreate", "result": True},+ "actor": {"email": "[email protected]", "id": "actor-1", "ip": "1.2.3.4"},+ "resource": {"type": "user", "id": "u-1"},+ }+ out = connector.normalize(raw)+ assert out["source"] == "cloudflare"+ assert out["severity"] == "high"+ assert out["actor_email"] == "[email protected]"+ assert out["src_ip"] == "1.2.3.4"+++def test_cloudflare_normalize_waf_delete_is_high():+ # Disabling/deleting WAF rules is the textbook ATT&CK T1562 pattern.+ connector = CloudflareConnector(_ACCOUNT_ID, "cf-token")+ raw = {+ "id": "evt-2",+ "when": "2026-01-01T00:00:00Z",+ "action": {"type": "wafRuleDelete", "result": True},+ "actor": {"email": "[email protected]"},+ "resource": {"type": "zone", "id": "z-1"},+ }+ out = connector.normalize(raw)+ assert out["severity"] == "high"+++def test_cloudflare_normalize_failed_action_is_low():+ connector = CloudflareConnector(_ACCOUNT_ID, "cf-token")+ raw = {+ "id": "evt-3",+ "when": "2026-01-01T00:00:00Z",+ "action": {"type": "zoneUpdate", "result": False},+ "actor": {"email": "[email protected]"},+ "resource": {"type": "zone", "id": "z-1"},+ }+ out = connector.normalize(raw)+ assert out["severity"] == "low"+++def test_cloudflare_normalize_routine_action_is_info():+ connector = CloudflareConnector(_ACCOUNT_ID, "cf-token")+ raw = {+ "id": "evt-4",+ "when": "2026-01-01T00:00:00Z",+ "action": {"type": "pageRuleUpdate", "result": True},+ "actor": {"email": "[email protected]"},+ "resource": {"type": "zone", "id": "z-1"},+ }+ out = connector.normalize(raw)+ assert out["severity"] == "info"+++async def test_cloudflare_test_connection_success():+ respx.get("https://api.cloudflare.com/client/v4/user/tokens/verify").mock(+ return_value=httpx.Response(200, json={"result": {"status": "active"}})+ )+ respx.get(+ f"https://api.cloudflare.com/client/v4/accounts/{_ACCOUNT_ID}/audit_logs"+ ).mock(return_value=httpx.Response(200, json={"result": []}))++ connector = CloudflareConnector(_ACCOUNT_ID, "cf-token")+ result = await connector.test_connection()+ assert result["success"] is True+ assert result["account_id"] == _ACCOUNT_ID+++async def test_cloudflare_test_connection_inactive_token_fails():+ respx.get("https://api.cloudflare.com/client/v4/user/tokens/verify").mock(+ return_value=httpx.Response(200, json={"result": {"status": "expired"}})+ )++ connector = CloudflareConnector(_ACCOUNT_ID, "cf-token")+ result = await connector.test_connection()+ assert result["success"] is False+ assert "expired" in result["error"]+++async def test_cloudflare_test_connection_missing_audit_perm_fails():+ respx.get("https://api.cloudflare.com/client/v4/user/tokens/verify").mock(+ return_value=httpx.Response(200, json={"result": {"status": "active"}})+ )+ respx.get(+ f"https://api.cloudflare.com/client/v4/accounts/{_ACCOUNT_ID}/audit_logs"+ ).mock(return_value=httpx.Response(403, text="missing scope"))++ connector = CloudflareConnector(_ACCOUNT_ID, "cf-token")+ result = await connector.test_connection()+ assert result["success"] is False+ assert "audit_logs failed" in result["error"]+++async def test_cloudflare_fetch_alerts_returns_normalized_events():+ respx.get(+ f"https://api.cloudflare.com/client/v4/accounts/{_ACCOUNT_ID}/audit_logs"+ ).mock(+ return_value=httpx.Response(+ 200,+ json={+ "result": [+ {+ "id": "evt-1",+ "when": "2026-01-01T00:00:00Z",+ "action": {"type": "tokenCreate", "result": True},+ "actor": {+ "email": "[email protected]",+ "id": "actor-1",+ "ip": "1.2.3.4",+ },+ "resource": {"type": "user", "id": "u-1"},+ }+ ]+ },+ )+ )++ connector = CloudflareConnector(_ACCOUNT_ID, "cf-token")+ events = await connector.fetch_alerts(since_seconds=300)+ assert len(events) == 1+ assert events[0]["source"] == "cloudflare"+ assert events[0]["severity"] == "high"+++async def test_cloudflare_fetch_alerts_returns_empty_on_500():+ respx.get(+ f"https://api.cloudflare.com/client/v4/accounts/{_ACCOUNT_ID}/audit_logs"+ ).mock(return_value=httpx.Response(500, text="bad gateway"))++ connector = CloudflareConnector(_ACCOUNT_ID, "cf-token")+ events = await connector.fetch_alerts(since_seconds=300)+ assert events == []+++# ===========================================================================+# GitHub+# ===========================================================================+++def test_github_schema_has_required_fields():+ schema = GitHubConnector.schema()+ field_names = {f.name for f in schema.fields}+ assert {"organization", "token"} <= field_names+ # vcs is the more precise category — GitHub is a version-control SaaS,+ # not a generic SaaS platform — and the schema test allows it.+ assert schema.category == "vcs"+ token_field = next(f for f in schema.fields if f.name == "token")+ assert token_field.type == "secret"+++def test_github_schema_oauth_hosted_is_false():+ # We accept tokens directly. The hosted device-flow OAuth is a follow-up.+ schema = GitHubConnector.schema()+ assert schema.oauth is not None+ assert schema.oauth.supported_in_hosted is False+ assert "read:audit_log" in (schema.oauth.scopes or [])+++def test_github_normalize_audit_high_risk_action_is_high():+ connector = GitHubConnector(_ORG, _TOKEN)+ raw = {+ "_aisoc_stream": "audit_log",+ "_document_id": "doc-1",+ "@timestamp": 1735689600000,+ "action": "org.update_member",+ "actor": "alice",+ "user_email": "[email protected]",+ "actor_ip": "1.2.3.4",+ "repo": "aisoc/aisoc",+ "org": _ORG,+ }+ out = connector.normalize(raw)+ assert out["source"] == "github"+ assert out["severity"] == "high"+ assert out["actor_email"] == "[email protected]"+ assert out["src_ip"] == "1.2.3.4"+ assert out["event_type"] == "github.org.update_member"+++def test_github_normalize_secret_scanning_disable_is_high():+ connector = GitHubConnector(_ORG, _TOKEN)+ raw = {+ "_aisoc_stream": "audit_log",+ "@timestamp": 1735689600000,+ "action": "secret_scanning.disable",+ "actor": "attacker",+ }+ out = connector.normalize(raw)+ assert out["severity"] == "high"+++def test_github_normalize_unknown_destroy_is_medium():+ # We don't enumerate every ``.destroy`` action in HIGH_RISK_ACTIONS,+ # so the catch-all suffix rule should land them at medium.+ connector = GitHubConnector(_ORG, _TOKEN)+ raw = {+ "_aisoc_stream": "audit_log",+ "@timestamp": 1735689600000,+ "action": "team.destroy",+ "actor": "alice",+ }+ out = connector.normalize(raw)+ assert out["severity"] == "medium"+++def test_github_normalize_routine_action_is_info():+ connector = GitHubConnector(_ORG, _TOKEN)+ raw = {+ "_aisoc_stream": "audit_log",+ "@timestamp": 1735689600000,+ "action": "issues.opened",+ "actor": "alice",+ }+ out = connector.normalize(raw)+ assert out["severity"] == "info"+++def test_github_normalize_code_scanning_critical_collapses_to_high():+ connector = GitHubConnector(_ORG, _TOKEN)+ raw = {+ "_aisoc_stream": "code_scanning",+ "number": 42,+ "created_at": "2026-01-01T00:00:00Z",+ "rule": {+ "id": "py/sql-injection",+ "description": "SQL injection",+ "security_severity_level": "critical",+ "severity": "error",+ },+ "repository": {"full_name": "aisoc/aisoc"},+ "tool": {"name": "CodeQL"},+ "most_recent_instance": {"location": {"path": "src/db.py"}},+ }+ out = connector.normalize(raw)+ assert out["severity"] == "high"+ assert out["external_id"] == "code-scanning-42"+ assert out["title"].startswith("Code Scanning:")+ assert out["event_type"] == "github.code_scanning.py/sql-injection"+++def test_github_normalize_code_scanning_low_is_low():+ connector = GitHubConnector(_ORG, _TOKEN)+ raw = {+ "_aisoc_stream": "code_scanning",+ "number": 7,+ "created_at": "2026-01-01T00:00:00Z",+ "rule": {+ "id": "py/style",+ "security_severity_level": "low",+ "severity": "warning",+ },+ "repository": {"full_name": "aisoc/aisoc"},+ "tool": {"name": "CodeQL"},+ "most_recent_instance": {"location": {"path": "src/x.py"}},+ }+ out = connector.normalize(raw)+ assert out["severity"] == "low"+++def test_github_normalize_code_scanning_unknown_severity_falls_back_to_info():+ connector = GitHubConnector(_ORG, _TOKEN)+ raw = {+ "_aisoc_stream": "code_scanning",+ "number": 99,+ "created_at": "2026-01-01T00:00:00Z",+ "rule": {"id": "rule-x"},+ "repository": {"full_name": "aisoc/aisoc"},+ }+ out = connector.normalize(raw)+ assert out["severity"] == "info"+++async def test_github_test_connection_success_with_audit_log():+ respx.get(f"https://api.github.com/orgs/{_ORG}").mock(+ return_value=httpx.Response(200, json={"login": _ORG})+ )+ respx.get(f"https://api.github.com/orgs/{_ORG}/audit-log").mock(+ return_value=httpx.Response(200, json=[])+ )++ connector = GitHubConnector(_ORG, _TOKEN)+ result = await connector.test_connection()+ assert result["success"] is True+ assert result["organization"] == _ORG+ assert result["audit_log_available"] is True+++async def test_github_test_connection_succeeds_when_audit_log_404():+ # An org on a Free/Team plan returns 404 for the audit log endpoint.+ # That's a legitimate operational state — we still consider the+ # connector "connected" but flag audit_log_available = False so the+ # frontend can warn the operator.+ respx.get(f"https://api.github.com/orgs/{_ORG}").mock(+ return_value=httpx.Response(200, json={"login": _ORG})+ )+ respx.get(f"https://api.github.com/orgs/{_ORG}/audit-log").mock(+ return_value=httpx.Response(404, text="not found")+ )++ connector = GitHubConnector(_ORG, _TOKEN)+ result = await connector.test_connection()+ assert result["success"] is True+ assert result["audit_log_available"] is False+++async def test_github_test_connection_403_org_fails():+ respx.get(f"https://api.github.com/orgs/{_ORG}").mock(+ return_value=httpx.Response(403, text="bad token")+ )++ connector = GitHubConnector(_ORG, _TOKEN)+ result = await connector.test_connection()+ assert result["success"] is False+ assert "403" in result["error"]+++async def test_github_fetch_alerts_merges_audit_and_code_scanning():+ respx.get(f"https://api.github.com/orgs/{_ORG}/audit-log").mock(+ return_value=httpx.Response(+ 200,+ json=[+ {+ "_document_id": "doc-1",+ "@timestamp": 1735689600000,+ "action": "org.update_member",+ "actor": "alice",+ }+ ],+ )+ )+ respx.get(f"https://api.github.com/orgs/{_ORG}/code-scanning/alerts").mock(+ return_value=httpx.Response(+ 200,+ json=[+ {+ "number": 1,+ "created_at": "2099-01-01T00:00:00Z",+ "rule": {+ "id": "py/sql-injection",+ "security_severity_level": "high",+ },+ "repository": {"full_name": "aisoc/aisoc"},+ }+ ],+ )+ )++ connector = GitHubConnector(_ORG, _TOKEN)+ events = await connector.fetch_alerts(since_seconds=300)+ sources = {e["event_type"] for e in events}+ # Confirm both streams flowed through to normalize().+ assert any(s == "github.org.update_member" for s in sources)+ assert any(s.startswith("github.code_scanning.") for s in sources)+ assert all(e["source"] == "github" for e in events)+++async def test_github_fetch_alerts_handles_audit_log_unavailable():+ # 404 on audit log shouldn't stop us from delivering code scanning.+ respx.get(f"https://api.github.com/orgs/{_ORG}/audit-log").mock(+ return_value=httpx.Response(404, text="not found")+ )+ respx.get(f"https://api.github.com/orgs/{_ORG}/code-scanning/alerts").mock(+ return_value=httpx.Response(+ 200,+ json=[+ {+ "number": 1,+ "created_at": "2099-01-01T00:00:00Z",+ "rule": {+ "id": "py/sql-injection",+ "security_severity_level": "high",+ },+ "repository": {"full_name": "aisoc/aisoc"},+ }+ ],+ )+ )++ connector = GitHubConnector(_ORG, _TOKEN)+ events = await connector.fetch_alerts(since_seconds=300)+ assert len(events) == 1+ assert events[0]["event_type"].startswith("github.code_scanning.")