Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
feat(connectors): credential vault + schema-driven registry + CRUD API + scheduler
Beenu Arora8dcf7080e5bf
diff --git a/.env.example b/.env.exampleindex c5440372f..801796c3b 100644--- a/.env.example+++ b/.env.example@@ -6,6 +6,24 @@ ENVIRONMENT=developmentLOG_LEVEL=infoSECRET_KEY=change-this-to-a-random-secret-key-at-least-32-chars+# Application-layer encryption key for connector credentials and other+# secret payloads stored in Postgres. Generate with:+# python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"+# Required outside development. Leave blank in development to auto-generate+# an ephemeral key (warning: connectors won't survive a process restart).+AISOC_CREDENTIAL_KEY=+# Comma-separated list of previous credential keys to support zero-downtime+# rotation. Leave blank unless you are mid-rotation.+AISOC_CREDENTIAL_KEY_ROTATION_FROM=++# Internal URL the API uses to reach the connectors microservice for+# catalog lookups and "Test connection" actions in the click-and-connect+# wizard. ``http://connectors:8003`` is correct for docker-compose;+# use ``http://localhost:8088`` when running the API on the host while+# connectors is exposed via the docker-compose port forward.+CONNECTORS_SERVICE_URL=http://connectors:8003+CONNECTORS_SERVICE_TIMEOUT_SECONDS=15+# ─── Database ─────────────────────────────────────────────────────────────────DATABASE_URL=postgresql+asyncpg://aisoc:aisoc_dev_secret@localhost:5432/aisocREDIS_URL=redis://:redis_dev_secret@localhost:6379/0diff --git a/services/api/app/api/v1/endpoints/connectors.py b/services/api/app/api/v1/endpoints/connectors.pyindex f57b9c94c..23eac195d 100644--- a/services/api/app/api/v1/endpoints/connectors.py+++ b/services/api/app/api/v1/endpoints/connectors.py@@ -1,26 +1,82 @@-"""Connector management endpoints."""+"""Connector management endpoints.+This module is the **operator-facing** half of the click-and-connect+connector platform. It speaks to two parties:++* The frontend AddConnector wizard, which needs the catalog of available+ connectors, their configuration schemas, full CRUD over connector+ instances scoped to the current tenant, and a way to "Test connection"+ before saving.+* The stateless ``services/connectors`` microservice, which owns the+ catalog, schemas, and the actual ``test_connection()`` runtimes for+ every concrete connector class.++Why this lives in the API service rather than connectors microservice:++* The API service owns Postgres and tenant scoping. Storing connector+ instance rows next to other tenant resources is the obvious place.+* The API service owns the credential vault. Cleartext credentials must+ *never* leave this service except inbound to the connectors microservice+ for an active "test connection" call, and only over the internal+ Docker / VPC network.+* The connectors microservice should stay stateless so it can scale+ horizontally and be redeployed without touching the database.++Tenant scoping invariants enforced here:++* Every read query filters on ``Connector.tenant_id == current_user.tenant_id``.+* Every write goes through ``require_permission("connectors:write")``.+* The catalog endpoint is tenant-agnostic — it advertises the build's+ capabilities, not any specific tenant's instances — so it lives behind+ ``connectors:read`` so unauthenticated discovery is impossible.+"""++from __future__ import annotations++import loggingimport uuidfrom datetime import UTC, datetime-from typing import Annotated+from typing import Annotated, Any+import httpxfrom fastapi import APIRouter, Depends, HTTPException, status-from pydantic import BaseModel+from pydantic import BaseModel, Fieldfrom sqlalchemy import select, updatefrom app.api.v1.deps import AuthUser, DBSession, require_permission+from app.core.config import settingsfrom app.models.connector import Connector+from app.security.credential_vault import CredentialVaultError, get_vault++logger = logging.getLogger(__name__)router = APIRouter(prefix="/connectors", tags=["connectors"])+# --------------------------------------------------------------- Pydantic schemas++class ConnectorResponse(BaseModel):+ """Public projection of a ``Connector`` row.++ ``auth_config`` is intentionally **omitted**. Even decrypted credentials+ should never round-trip back to the wizard — the user typed them once+ and they belong inside the vault from then on. Wizard-driven re-edit+ flows reset specific secret fields rather than reading the existing+ ones.++ ``connector_config`` *is* included because it holds non-secret runtime+ knobs (poll interval, region, log filters) that the wizard surfaces+ so an operator can tweak them without re-pasting credentials.+ """+id: uuid.UUIDtenant_id: uuid.UUIDname: strconnector_type: strcategory: stris_enabled: bool+ connector_config: dict[str, Any]health_status: strlast_health_check: datetime | Nonelast_sync: datetime | None@@ -34,29 +90,225 @@ class ConnectorResponse(BaseModel):class CreateConnectorRequest(BaseModel):- name: str- connector_type: str- category: str- auth_config: dict = {}- connector_config: dict = {}- tags: list[str] = []+ name: str = Field(min_length=1, max_length=255)+ connector_type: str = Field(min_length=1, max_length=100)+ category: str | None = Field(+ default=None,+ max_length=50,+ description="Optional override; defaults to the catalog entry's category.",+ )+ auth_config: dict[str, Any] = Field(default_factory=dict)+ connector_config: dict[str, Any] = Field(default_factory=dict)+ tags: list[str] = Field(default_factory=list)class UpdateConnectorRequest(BaseModel):name: str | None = Noneis_enabled: bool | None = None- auth_config: dict | None = None- connector_config: dict | None = None+ auth_config: dict[str, Any] | None = None+ connector_config: dict[str, Any] | None = Nonetags: list[str] | None = None+class TestConnectionRequest(BaseModel):+ """Inline credential-test payload used by the wizard *before* saving.++ The wizard collects credentials in-browser, calls this endpoint, and+ only persists the connector if the test succeeds. We intentionally+ do *not* let the wizard test against an existing instance — that's+ a separate path (:func:`test_existing_connector`) that decrypts the+ stored creds first.+ """++ connector_type: str = Field(min_length=1, max_length=100)+ auth_config: dict[str, Any] = Field(default_factory=dict)+ connector_config: dict[str, Any] = Field(default_factory=dict)+++# ------------------------------------------------------------- internal helpers+++_CATALOG_TIMEOUT = httpx.Timeout(+ settings.CONNECTORS_SERVICE_TIMEOUT_SECONDS,+ connect=min(5.0, settings.CONNECTORS_SERVICE_TIMEOUT_SECONDS),+)+++def _connectors_service_url(path: str) -> str:+ """Build a URL into the stateless connectors microservice."""+ base = settings.CONNECTORS_SERVICE_URL.rstrip("/")+ suffix = path if path.startswith("/") else f"/{path}"+ return f"{base}/api/v1{suffix}"+++async def _fetch_catalog() -> list[dict[str, Any]]:+ """Pull the connector catalog from the connectors microservice.++ Raises a clear 503 if the microservice is unreachable. We don't cache+ here on purpose: catalog lookups are infrequent (a wizard open per+ operator), and a stale cache would make the "I just added a connector+ class and redeployed" feedback loop confusing.+ """+ url = _connectors_service_url("/connectors/schemas")+ try:+ async with httpx.AsyncClient(timeout=_CATALOG_TIMEOUT) as client:+ resp = await client.get(url)+ resp.raise_for_status()+ except httpx.HTTPError as exc:+ logger.warning("connectors_service.catalog.unreachable url=%s err=%s", url, exc)+ raise HTTPException(+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,+ detail="connectors service is unavailable; cannot list connector catalog",+ ) from exc++ body = resp.json()+ schemas = body.get("schemas") or []+ if not isinstance(schemas, list):+ raise HTTPException(+ status_code=status.HTTP_502_BAD_GATEWAY,+ detail="connectors service returned a malformed catalog",+ )+ return schemas+++async def _validate_connector_type(connector_type: str) -> dict[str, Any]:+ """Return the catalog entry for ``connector_type`` or raise 422.++ Validating against the live catalog (rather than a hand-maintained+ enum) means a freshly-added connector is automatically usable as soon+ as the connectors microservice picks it up — no API-side allowlist to+ keep in sync.+ """+ catalog = await _fetch_catalog()+ for entry in catalog:+ if entry.get("connector_id") == connector_type:+ return entry+ known = sorted(e.get("connector_id", "?") for e in catalog)+ raise HTTPException(+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,+ detail=f"unknown connector_type '{connector_type}'. Known types: {', '.join(known) or '(none)'}",+ )+++async def _proxy_test_connection(+ connector_type: str,+ auth_config: dict[str, Any],+ connector_config: dict[str, Any],+) -> dict[str, Any]:+ """POST plaintext credentials at the stateless test endpoint.++ The credentials cross the trust boundary between the API service and+ the connectors microservice **only here**. We expect that boundary to+ be a Docker / k8s internal network; if you're putting the connectors+ service on the public internet, terminate TLS in front of it and+ add a shared-secret header (mirroring how the realtime push proxy in+ this codebase does it).+ """+ url = _connectors_service_url(f"/connectors/{connector_type}/test")+ payload = {+ "auth_config": auth_config,+ "connector_config": connector_config,+ }+ try:+ async with httpx.AsyncClient(timeout=_CATALOG_TIMEOUT) as client:+ resp = await client.post(url, json=payload)+ except httpx.HTTPError as exc:+ logger.warning("connectors_service.test.unreachable url=%s err=%s", url, exc)+ raise HTTPException(+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,+ detail="connectors service is unavailable; cannot test connection",+ ) from exc++ # 404 from the microservice = unknown connector_type. We've already+ # validated against the catalog by the time we get here, so a 404 now+ # means a race (someone redeployed the connectors service mid-request);+ # surface it as a 503 since it's transient.+ if resp.status_code == status.HTTP_404_NOT_FOUND:+ raise HTTPException(+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,+ detail=f"connector '{connector_type}' is no longer available in the connectors service",+ )+ if resp.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY:+ # Forward the schema-mismatch detail so the wizard can highlight+ # the offending field.+ raise HTTPException(+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,+ detail=resp.json().get("detail", "connector config does not match schema"),+ )+ if resp.status_code >= 500:+ raise HTTPException(+ status_code=status.HTTP_502_BAD_GATEWAY,+ detail="connectors service failed while testing connection",+ )++ body: dict[str, Any]+ try:+ body = resp.json()+ except ValueError as exc:+ raise HTTPException(+ status_code=status.HTTP_502_BAD_GATEWAY,+ detail="connectors service returned a non-JSON response",+ ) from exc+ if not isinstance(body, dict):+ body = {"success": False, "error": "malformed response"}+ return body+++# ---------------------------------------------------------------------- catalog++[email protected]("/catalog")+async def list_catalog(+ current_user: Annotated[AuthUser, Depends(require_permission("connectors:read"))],+) -> dict[str, Any]:+ """Return the catalog of available connectors with their schemas.++ The wizard calls this once on open to populate the connector picker+ and to render the schema-driven configuration form. It is tenant-agnostic+ — every tenant in this build sees the same catalog — but still gated+ behind authentication.+ """+ schemas = await _fetch_catalog()+ return {"connectors": schemas}+++# ------------------------------------------------------------------ pre-save test++[email protected]("/test")+async def test_connection(+ request: TestConnectionRequest,+ current_user: Annotated[AuthUser, Depends(require_permission("connectors:write"))],+) -> dict[str, Any]:+ """Wizard-side "Test connection" before saving.++ The wizard sends plaintext credentials directly here. We validate the+ connector type against the catalog, then forward the payload to the+ connectors microservice's stateless test endpoint. **Nothing is+ persisted** — these credentials never touch Postgres or the vault.+ """+ await _validate_connector_type(request.connector_type)+ return await _proxy_test_connection(+ request.connector_type,+ request.auth_config,+ request.connector_config,+ )+++# ------------------------------------------------------------------ instance CRUD++@router.get("", response_model=list[ConnectorResponse])async def list_connectors(current_user: Annotated[AuthUser, Depends(require_permission("connectors:read"))],db: DBSession,) -> list[ConnectorResponse]:- """List all connectors for the tenant."""- result = await db.execute(select(Connector).where(Connector.tenant_id == current_user.tenant_id).order_by(Connector.created_at))+ """List all connector instances for the caller's tenant."""+ result = await db.execute(+ select(Connector)+ .where(Connector.tenant_id == current_user.tenant_id)+ .order_by(Connector.created_at)+ )connectors = result.scalars().all()return [ConnectorResponse.model_validate(c) for c in connectors]@@ -67,13 +319,31 @@ async def create_connector(current_user: Annotated[AuthUser, Depends(require_permission("connectors:write"))],db: DBSession,) -> ConnectorResponse:- """Create a new connector."""+ """Create a new connector instance.++ ``connector_type`` is validated against the live catalog. ``category``+ defaults to the catalog entry's category if the caller doesn't override+ it — that's the common case and keeps the wizard payload small.+ ``auth_config`` is encrypted with the credential vault before it+ touches Postgres.+ """+ catalog_entry = await _validate_connector_type(request.connector_type)+ category = request.category or catalog_entry.get("category") or "uncategorized"++ try:+ encrypted_auth = get_vault().encrypt_dict(request.auth_config or {})+ except CredentialVaultError as exc:+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail=f"credential vault unavailable: {exc}",+ ) from exc+connector = Connector(tenant_id=current_user.tenant_id,name=request.name,connector_type=request.connector_type,- category=request.category,- auth_config=request.auth_config,+ category=category,+ auth_config=encrypted_auth,connector_config=request.connector_config,tags=request.tags,)@@ -89,7 +359,7 @@ async def get_connector(current_user: Annotated[AuthUser, Depends(require_permission("connectors:read"))],db: DBSession,) -> ConnectorResponse:- """Get a connector by ID."""+ """Get a connector instance by ID, scoped to the caller's tenant."""result = await db.execute(select(Connector).where(Connector.id == connector_id,@@ -109,17 +379,40 @@ async def update_connector(current_user: Annotated[AuthUser, Depends(require_permission("connectors:write"))],db: DBSession,) -> ConnectorResponse:- """Update a connector's configuration or state."""- result = await db.execute(select(Connector).where(Connector.id == connector_id, Connector.tenant_id == current_user.tenant_id))+ """Update a connector's configuration or state.++ The PATCH semantics here are deliberately conservative: only fields+ the caller actually supplies are touched. Notably, sending an empty+ ``auth_config`` dict will overwrite all secrets — the wizard must+ therefore omit the field entirely when the operator hasn't re-typed+ credentials.+ """+ result = await db.execute(+ select(Connector).where(+ Connector.id == connector_id,+ Connector.tenant_id == current_user.tenant_id,+ )+ )connector = result.scalar_one_or_none()if connector is None:raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Connector not found")- updates: dict = {}- for field in ["name", "is_enabled", "auth_config", "connector_config", "tags"]:+ updates: dict[str, Any] = {}+ for field in ("name", "is_enabled", "auth_config", "connector_config", "tags"):val = getattr(request, field, None)- if val is not None:- updates[field] = val+ if val is None:+ continue+ if field == "auth_config":+ # Re-encrypt on every write so a partial PATCH still leaves+ # all secret leaves under the current primary key.+ try:+ val = get_vault().encrypt_dict(val)+ except CredentialVaultError as exc:+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail=f"credential vault unavailable: {exc}",+ ) from exc+ updates[field] = valif updates:updates["updated_at"] = datetime.now(UTC)@@ -136,8 +429,13 @@ async def delete_connector(current_user: Annotated[AuthUser, Depends(require_permission("connectors:delete"))],db: DBSession,) -> None:- """Delete a connector."""- result = await db.execute(select(Connector).where(Connector.id == connector_id, Connector.tenant_id == current_user.tenant_id))+ """Delete a connector instance."""+ result = await db.execute(+ select(Connector).where(+ Connector.id == connector_id,+ Connector.tenant_id == current_user.tenant_id,+ )+ )connector = result.scalar_one_or_none()if connector is None:raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Connector not found")@@ -145,27 +443,62 @@ async def delete_connector(await db.commit()+# ----------------------------------------------------------- existing-instance test++@router.post("/{connector_id}/test", status_code=status.HTTP_200_OK)-async def test_connector(+async def test_existing_connector(connector_id: uuid.UUID,current_user: Annotated[AuthUser, Depends(require_permission("connectors:write"))],db: DBSession,-) -> dict:- """Test connectivity for a connector (stub - to be implemented per connector type)."""- result = await db.execute(select(Connector).where(Connector.id == connector_id, Connector.tenant_id == current_user.tenant_id))+) -> dict[str, Any]:+ """Run a live ``test_connection`` against a saved connector instance.++ Flow:++ 1. Fetch the row, scoped to the caller's tenant.+ 2. Decrypt ``auth_config`` via the vault.+ 3. Forward to the connectors microservice with both the decrypted+ credentials and the non-secret ``connector_config``.+ 4. Update the row's ``health_status`` / ``last_health_check`` based on+ the verdict so the dashboard reflects reality.+ """+ result = await db.execute(+ select(Connector).where(+ Connector.id == connector_id,+ Connector.tenant_id == current_user.tenant_id,+ )+ )connector = result.scalar_one_or_none()if connector is None:raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Connector not found")- # Update health check timestamp+ try:+ decrypted_auth = get_vault().decrypt_dict(connector.auth_config or {})+ except CredentialVaultError as exc:+ # A vault failure on *decryption* almost always means the stored+ # ciphertext was written under a key that's no longer in the keyring+ # (key rotation gone wrong). Surface it as 500 rather than pretending+ # the connector is unhealthy.+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail=f"could not decrypt stored credentials: {exc}",+ ) from exc++ verdict = await _proxy_test_connection(+ connector.connector_type,+ decrypted_auth,+ connector.connector_config or {},+ )++ health_status = "healthy" if verdict.get("success") else "unhealthy"await db.execute(update(Connector).where(Connector.id == connector_id).values(last_health_check=datetime.now(UTC),- health_status="healthy",+ health_status=health_status,))await db.commit()-- return {"status": "ok", "message": f"Connector {connector.name} test initiated"}+ return verdictdiff --git a/services/api/app/core/config.py b/services/api/app/core/config.pyindex 3258d451a..47b22eb7b 100644--- a/services/api/app/core/config.py+++ b/services/api/app/core/config.py@@ -66,6 +66,30 @@ class Settings(BaseSettings):# a kubernetes secret / fly.io secret in production.METRICS_TOKEN: str = ""+ # Application-layer encryption key for connector credentials and other+ # sensitive ``auth_config`` payloads stored in Postgres. Must be a 32-byte+ # url-safe base64 key (the Fernet format). Generate one with+ # ``python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"``+ # and load it via Fly secrets / k8s secret. When empty in a development+ # environment the API auto-generates a process-local key on first use and+ # logs a warning; outside development the API refuses to encrypt or+ # decrypt anything until it's set, so credentials cannot be silently+ # written in plaintext. Rotate by re-encrypting via+ # ``AISOC_CREDENTIAL_KEY_ROTATION_FROM`` (comma-separated previous keys).+ AISOC_CREDENTIAL_KEY: str = ""+ AISOC_CREDENTIAL_KEY_ROTATION_FROM: str = ""++ # Internal URL for the connectors microservice. The API service proxies+ # catalog lookups (``GET /connectors``) and stateless connection tests+ # (``POST /connectors/{type}/test``) to this URL so the wizard UI can+ # render schema-driven forms and verify credentials before saving.+ # Default targets the docker-compose hostname; override locally with+ # e.g. ``http://localhost:8088`` when the connectors service is+ # exposed on the host. Empty disables proxying entirely (catalog and+ # test endpoints will return 503).+ CONNECTORS_SERVICE_URL: str = "http://connectors:8003"+ CONNECTORS_SERVICE_TIMEOUT_SECONDS: float = 15.0+# DatabaseDATABASE_URL: PostgresDsn = "postgresql+asyncpg://aisoc:aisoc@localhost:5432/aisoc"DATABASE_POOL_SIZE: int = 20@@ -217,6 +241,12 @@ def warn_if_insecure_defaults(s: Settings | None = None) -> list[str]:if not _is_dev_env(s.ENVIRONMENT) and s.PLUGIN_TRUST_MODE.lower() == "disabled":msgs.append("PLUGIN_TRUST_MODE=disabled outside development — plugins will load without signature verification.")+ # Connector credential vault: refuse to silently boot without an encryption+ # key outside development. Without this, ``Connector.auth_config`` would+ # round-trip in plaintext.+ if not _is_dev_env(s.ENVIRONMENT) and not s.AISOC_CREDENTIAL_KEY:+ msgs.append("AISOC_CREDENTIAL_KEY is empty in a non-development environment — connector credentials cannot be encrypted at rest.")+logger = logging.getLogger("aisoc.config")for msg in msgs:# ``stacklevel=2`` so the warning points at the caller ofdiff --git a/services/api/app/security/__init__.py b/services/api/app/security/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/services/api/app/security/credential_vault.py b/services/api/app/security/credential_vault.pynew file mode 100644index 000000000..916068afe--- /dev/null+++ b/services/api/app/security/credential_vault.py@@ -0,0 +1,232 @@+"""Application-layer encryption for connector credentials.++This module wraps :mod:`cryptography.fernet` to give the rest of the API a+narrow, opinionated interface for storing connector ``auth_config`` payloads+encrypted at rest. The model column itself stays a regular JSONB blob so+operators can query for which tenant owns which connector without having to+decrypt anything; only the field values inside the JSON are ciphertext.++Why Fernet (AES-128-CBC + HMAC-SHA256, plus a versioned token format) rather+than rolling our own AES-GCM:++* Authenticated encryption out of the box — tampering raises+ :class:`InvalidToken`, never a silent partial decrypt.+* Built-in support for **multi-key rotation** via :class:`MultiFernet`. We use+ it so operators can rotate ``AISOC_CREDENTIAL_KEY`` without a downtime+ migration: new writes use the primary key, old reads still resolve through+ any key listed in ``AISOC_CREDENTIAL_KEY_ROTATION_FROM``.+* Battle-tested and audited; we'd rather lean on the upstream than ship a+ hand-rolled crypto module in an open-source SOC product.++What the vault deliberately does **not** do:++* It does not derive a key from the user's password. We expect operators to+ generate a key with :func:`cryptography.fernet.Fernet.generate_key` and+ mount it via Fly.io secret / k8s secret / vault. Plaintext keys never+ appear in logs.+* It does not encrypt the *whole* ``auth_config`` blob as a single+ ciphertext. Each leaf string value is encrypted individually so:+ 1. structural fields (e.g. region, base URL) stay queryable for ops, and+ 2. partial decryption failures don't lose the rest of the row.++The vault is intentionally pure — it does not touch the DB, doesn't take a+``Settings`` instance, and is therefore trivially unit-testable without a+live Postgres or Redis.+"""++from __future__ import annotations++import logging+from collections.abc import Mapping+from threading import Lock+from typing import Any, Final++from cryptography.fernet import Fernet, InvalidToken, MultiFernet++from app.core.config import settings++logger = logging.getLogger("aisoc.credential_vault")++# Sentinel prefix that marks a string value as already-encrypted ciphertext.+# We tag every value we write so the decrypt path can distinguish "this is+# encrypted, run it through Fernet" from "this is a plaintext we just got+# back from a never-encrypted legacy row, leave it alone". The prefix is+# deliberately not a valid Fernet token start byte (``gAAAAA...``) so we+# can never accidentally interpret a real ciphertext as plaintext.+_CIPHER_PREFIX: Final[str] = "vault:v1:"+++class CredentialVaultError(RuntimeError):+ """Raised when the vault cannot encrypt/decrypt safely.++ The caller should treat this as fatal for the current request rather than+ fall back to plaintext — that's the whole point of the vault.+ """+++def _split_keys(raw: str) -> list[bytes]:+ """Parse a comma-separated list of base64 Fernet keys."""+ return [k.strip().encode("ascii") for k in raw.split(",") if k.strip()]+++class CredentialVault:+ """Encrypt and decrypt connector credential payloads.++ Constructed from a primary key and an optional list of historical keys.+ The historical keys are only used for *decryption* via :class:`MultiFernet`+ so that a rotation is a two-step process:++ 1. Add the old key to ``AISOC_CREDENTIAL_KEY_ROTATION_FROM`` and the new+ key to ``AISOC_CREDENTIAL_KEY``. Restart. All new writes use the new+ key, all old rows still decrypt.+ 2. Run a maintenance job that re-saves every connector instance, which+ re-encrypts under the new primary. Once that's done, drop the old key+ from the rotation list.++ The class is *not* thread-locked at the per-call level (Fernet is+ thread-safe), but the lazy-init path uses a lock so concurrent requests+ don't race to generate a development-only ephemeral key.+ """++ def __init__(self, primary_key: bytes, historical_keys: list[bytes] | None = None) -> None:+ if not primary_key:+ raise CredentialVaultError("CredentialVault requires a non-empty primary key")+ try:+ primary = Fernet(primary_key)+ except (TypeError, ValueError) as exc:+ raise CredentialVaultError(f"AISOC_CREDENTIAL_KEY is not a valid Fernet key: {exc}") from exc++ keyring: list[Fernet] = [primary]+ for k in historical_keys or []:+ try:+ keyring.append(Fernet(k))+ except (TypeError, ValueError) as exc:+ # We log but don't fail — a bad rotation key shouldn't take the+ # whole API down. The operator will see the warning and the+ # affected ciphertexts will simply fail to decrypt.+ logger.warning("ignoring invalid rotation key: %s", exc)++ self._fernet = MultiFernet(keyring) if len(keyring) > 1 else primary++ # --------------------------------------------------------------------- core++ def encrypt(self, value: str) -> str:+ """Encrypt a single string and return ``vault:v1:<token>``.++ Returns the input unchanged if it's already a vault token, so the+ caller can re-encrypt an entire dict idempotently.+ """+ if not isinstance(value, str):+ raise CredentialVaultError(f"vault.encrypt expects str, got {type(value).__name__}")+ if value.startswith(_CIPHER_PREFIX):+ return value+ token = self._fernet.encrypt(value.encode("utf-8")).decode("ascii")+ return f"{_CIPHER_PREFIX}{token}"++ def decrypt(self, value: str) -> str:+ """Decrypt a vault-tagged string. Plaintext passes through.++ Plaintext pass-through is required so the vault stays compatible with+ rows written by older builds that didn't encrypt anything yet. We log+ on the plaintext path so operators have a clear migration signal.+ """+ if not isinstance(value, str):+ raise CredentialVaultError(f"vault.decrypt expects str, got {type(value).__name__}")+ if not value.startswith(_CIPHER_PREFIX):+ return value+ token = value[len(_CIPHER_PREFIX):].encode("ascii")+ try:+ return self._fernet.decrypt(token).decode("utf-8")+ except InvalidToken as exc:+ raise CredentialVaultError("ciphertext failed integrity check (key rotated without re-encrypt?)") from exc++ # ------------------------------------------------------------- dict helpers++ def encrypt_dict(self, payload: Mapping[str, Any], *, secret_keys: set[str] | None = None) -> dict[str, Any]:+ """Encrypt the *secret* string fields inside an ``auth_config`` blob.++ We default to encrypting **every** string value, which is the safe+ behaviour for connector secrets where almost everything (tokens,+ keys, JSON service-account blobs, even tenant IDs) leaks information+ an attacker could weaponise. Pass ``secret_keys`` if you want to+ restrict encryption to a specific subset of keys; everything else+ round-trips as-is.++ Lists and nested dicts are recursed into so a JSON service-account+ key (a deeply nested object) ends up fully encrypted at the leaves.+ """+ return self._walk(payload, encrypt=True, secret_keys=secret_keys)++ def decrypt_dict(self, payload: Mapping[str, Any], *, secret_keys: set[str] | None = None) -> dict[str, Any]:+ """Inverse of :meth:`encrypt_dict`. Safe to call on partially-encrypted dicts."""+ return self._walk(payload, encrypt=False, secret_keys=secret_keys)++ def _walk(self, payload: Any, *, encrypt: bool, secret_keys: set[str] | None, _key: str | None = None) -> Any:+ op = self.encrypt if encrypt else self.decrypt+ if isinstance(payload, Mapping):+ out: dict[str, Any] = {}+ for k, v in payload.items():+ # If the caller restricted to specific keys, leave everything+ # else alone. Otherwise fall through and recurse.+ if secret_keys is not None and k not in secret_keys and not isinstance(v, (Mapping, list)):+ out[k] = v+ else:+ out[k] = self._walk(v, encrypt=encrypt, secret_keys=secret_keys, _key=k)+ return out+ if isinstance(payload, list):+ return [self._walk(item, encrypt=encrypt, secret_keys=secret_keys, _key=_key) for item in payload]+ if isinstance(payload, str):+ return op(payload)+ # Booleans, numbers, None — no encryption to apply.+ return payload+++# --------------------------------------------------------------- module helper++_vault_singleton: CredentialVault | None = None+_vault_lock = Lock()+++def _is_dev_env() -> bool:+ return (settings.ENVIRONMENT or "").strip().lower() in {"development", "dev", "local", "demo", "test"}+++def get_vault() -> CredentialVault:+ """Return the process-wide :class:`CredentialVault`.++ On the first call we resolve the primary key from+ ``settings.AISOC_CREDENTIAL_KEY``. If unset:++ * In development we auto-generate an **ephemeral** key (logged as a+ warning) so localhost demos work without ceremony. The key is *not*+ persisted, so credentials encrypted in one process won't decrypt in+ a fresh one — that's fine for development and forces operators to+ configure a real key the moment they leave it.+ * Outside development we raise — silently encrypting under a throwaway+ key would be worse than refusing to run.+ """+ global _vault_singleton+ if _vault_singleton is not None:+ return _vault_singleton+ with _vault_lock:+ if _vault_singleton is not None: # pragma: no cover - racing init+ return _vault_singleton+ primary = (settings.AISOC_CREDENTIAL_KEY or "").strip().encode("ascii")+ if not primary:+ if not _is_dev_env():+ raise CredentialVaultError("AISOC_CREDENTIAL_KEY is required outside development; refusing to start credential vault.")+ primary = Fernet.generate_key()+ logger.warning(+ "AISOC_CREDENTIAL_KEY is not set; using ephemeral process-local key. Connector secrets will not survive a restart. "+ "Generate a key with `python -c \"from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())\"` and set AISOC_CREDENTIAL_KEY."+ )+ rotation = _split_keys(settings.AISOC_CREDENTIAL_KEY_ROTATION_FROM or "")+ _vault_singleton = CredentialVault(primary, historical_keys=rotation)+ return _vault_singleton+++def reset_vault_for_tests() -> None:+ """Drop the cached singleton. Test helper only."""+ global _vault_singleton+ with _vault_lock:+ _vault_singleton = Nonediff --git a/services/api/pyproject.toml b/services/api/pyproject.tomlindex 5f014c152..c6a0c3427 100644--- a/services/api/pyproject.toml+++ b/services/api/pyproject.toml@@ -18,6 +18,7 @@ alembic = "^1.13.1"redis = { version = "^5.0.4", extras = ["hiredis"] }httpx = "^0.27.0"python-jose = { version = "^3.3.0", extras = ["cryptography"] }+cryptography = "^42.0.0" # Direct dep for credential vault Fernetpasslib = { version = "^1.7.4", extras = ["bcrypt"] }python-multipart = "^0.0.9"tenacity = "^8.3.0"diff --git a/services/api/tests/test_connectors_endpoint.py b/services/api/tests/test_connectors_endpoint.pynew file mode 100644index 000000000..1bd58b63c--- /dev/null+++ b/services/api/tests/test_connectors_endpoint.py@@ -0,0 +1,217 @@+"""Unit tests for the connectors API proxy helpers.++Full end-to-end CRUD tests would need a Postgres fixture and the auth+stack wired up, which the rest of this test suite generally skips. The+parts that *are* worth covering in isolation are the inter-service+proxy helpers — they hold the catalog validation and error mapping+logic that the wizard depends on, and they fail in subtle ways when the+connectors microservice misbehaves.++We mock ``httpx.AsyncClient`` directly because:++* It's the seam between this service and the connectors microservice;+ pinning it in tests is enough to characterise our behaviour without+ pulling up a second FastAPI app.+* The error-mapping logic (503 / 502 / 422 passthrough) only has+ meaningful assertions when we can synthesise specific upstream+ status codes.+"""++from __future__ import annotations++from typing import Any+from unittest.mock import AsyncMock, MagicMock, patch++import httpx+import pytest+from app.api.v1.endpoints.connectors import (+ _fetch_catalog,+ _proxy_test_connection,+ _validate_connector_type,+)+from fastapi import HTTPException+++def _mock_response(status_code: int, json_body: Any) -> MagicMock:+ """Build a MagicMock that quacks like an httpx.Response."""+ resp = MagicMock(spec=httpx.Response)+ resp.status_code = status_code+ resp.json = MagicMock(return_value=json_body)+ if status_code >= 400:+ resp.raise_for_status = MagicMock(+ side_effect=httpx.HTTPStatusError(+ f"upstream {status_code}", request=MagicMock(), response=resp+ )+ )+ else:+ resp.raise_for_status = MagicMock()+ return resp+++def _patched_client(get_resp: MagicMock | None = None, post_resp: MagicMock | None = None):+ """Patch httpx.AsyncClient with a context-managed mock client."""+ client_instance = MagicMock()+ client_instance.get = AsyncMock(return_value=get_resp) if get_resp else AsyncMock()+ client_instance.post = AsyncMock(return_value=post_resp) if post_resp else AsyncMock()+ client_cm = MagicMock()+ client_cm.__aenter__ = AsyncMock(return_value=client_instance)+ client_cm.__aexit__ = AsyncMock(return_value=None)+ return patch(+ "app.api.v1.endpoints.connectors.httpx.AsyncClient",+ return_value=client_cm,+ )+++# ---------------------------------------------------------------- catalog+++async def test_fetch_catalog_returns_schemas() -> None:+ fake_schemas = [+ {"connector_id": "splunk", "name": "Splunk", "category": "siem", "fields": []},+ ]+ resp = _mock_response(200, {"schemas": fake_schemas})+ with _patched_client(get_resp=resp):+ result = await _fetch_catalog()+ assert result == fake_schemas+++async def test_fetch_catalog_raises_503_when_unreachable() -> None:+ """If the connectors service is down, surface 503 — not 500."""+ client_instance = MagicMock()+ client_instance.get = AsyncMock(side_effect=httpx.ConnectError("connection refused"))+ client_cm = MagicMock()+ client_cm.__aenter__ = AsyncMock(return_value=client_instance)+ client_cm.__aexit__ = AsyncMock(return_value=None)+ with patch(+ "app.api.v1.endpoints.connectors.httpx.AsyncClient",+ return_value=client_cm,+ ):+ with pytest.raises(HTTPException) as exc_info:+ await _fetch_catalog()+ assert exc_info.value.status_code == 503+++async def test_fetch_catalog_raises_502_for_malformed_body() -> None:+ """A 200 with the wrong shape is the connectors service's fault."""+ resp = _mock_response(200, {"schemas": "not-a-list"})+ with _patched_client(get_resp=resp):+ with pytest.raises(HTTPException) as exc_info:+ await _fetch_catalog()+ assert exc_info.value.status_code == 502+++# -------------------------------------------------------- type validation+++async def test_validate_connector_type_accepts_known() -> None:+ fake_schemas = [{"connector_id": "splunk", "category": "siem", "name": "Splunk"}]+ resp = _mock_response(200, {"schemas": fake_schemas})+ with _patched_client(get_resp=resp):+ entry = await _validate_connector_type("splunk")+ assert entry["connector_id"] == "splunk"+ assert entry["category"] == "siem"+++async def test_validate_connector_type_rejects_unknown() -> None:+ """Unknown connector_type → 422 with the list of known ids."""+ fake_schemas = [{"connector_id": "splunk", "category": "siem"}]+ resp = _mock_response(200, {"schemas": fake_schemas})+ with _patched_client(get_resp=resp):+ with pytest.raises(HTTPException) as exc_info:+ await _validate_connector_type("not_a_real_connector")+ assert exc_info.value.status_code == 422+ assert "splunk" in exc_info.value.detail+++# ------------------------------------------------------------ test proxy+++async def test_proxy_test_connection_passthrough_success() -> None:+ """A 200 from upstream is returned as-is."""+ upstream_body = {"success": True, "connector": "splunk", "version": "9.0"}+ resp = _mock_response(200, upstream_body)+ with _patched_client(post_resp=resp):+ result = await _proxy_test_connection("splunk", {"token": "x"}, {"host": "y"})+ assert result == upstream_body+++async def test_proxy_test_connection_passthrough_connector_failure() -> None:+ """A 200 with success=False is *also* a normal return — connector said no."""+ upstream_body = {"success": False, "connector": "splunk", "error": "401 Unauthorized"}+ resp = _mock_response(200, upstream_body)+ with _patched_client(post_resp=resp):+ result = await _proxy_test_connection("splunk", {"token": "bad"}, {})+ # Critically, this is NOT raised — the wizard wants to render the+ # connector's own error message.+ assert result["success"] is False+ assert result["error"] == "401 Unauthorized"+++async def test_proxy_test_connection_422_passes_detail() -> None:+ """422 from upstream means schema mismatch; surface the detail."""+ resp = _mock_response(422, {"detail": "missing field 'tenant_id'"})+ with _patched_client(post_resp=resp):+ with pytest.raises(HTTPException) as exc_info:+ await _proxy_test_connection("azure_entra", {}, {})+ assert exc_info.value.status_code == 422+ assert "tenant_id" in exc_info.value.detail+++async def test_proxy_test_connection_404_becomes_503() -> None:+ """A 404 mid-request means the connector class vanished — transient.++ We treat it as 503 (try again) rather than 404 (the user did something+ wrong) because the catalog check happens *before* this proxy call,+ so by the time we get a 404 the catalog and the runtime have drifted.+ """+ resp = _mock_response(404, {"detail": "not found"})+ with _patched_client(post_resp=resp):+ with pytest.raises(HTTPException) as exc_info:+ await _proxy_test_connection("splunk", {}, {})+ assert exc_info.value.status_code == 503+++async def test_proxy_test_connection_5xx_becomes_502() -> None:+ """Upstream 500s are bad-gateway from our perspective."""+ resp = _mock_response(500, {"detail": "boom"})+ with _patched_client(post_resp=resp):+ with pytest.raises(HTTPException) as exc_info:+ await _proxy_test_connection("splunk", {}, {})+ assert exc_info.value.status_code == 502+++async def test_proxy_test_connection_unreachable_becomes_503() -> None:+ """Network errors are 503 — connectors service is down."""+ client_instance = MagicMock()+ client_instance.post = AsyncMock(side_effect=httpx.ConnectError("conn refused"))+ client_cm = MagicMock()+ client_cm.__aenter__ = AsyncMock(return_value=client_instance)+ client_cm.__aexit__ = AsyncMock(return_value=None)+ with patch(+ "app.api.v1.endpoints.connectors.httpx.AsyncClient",+ return_value=client_cm,+ ):+ with pytest.raises(HTTPException) as exc_info:+ await _proxy_test_connection("splunk", {}, {})+ assert exc_info.value.status_code == 503+++async def test_proxy_test_connection_non_dict_body_normalised() -> None:+ """Defend against connectors microservice returning a non-dict 200."""+ resp = _mock_response(200, "ok")+ with _patched_client(post_resp=resp):+ result = await _proxy_test_connection("splunk", {}, {})+ assert isinstance(result, dict)+ assert result["success"] is Falsediff --git a/services/api/tests/test_credential_vault.py b/services/api/tests/test_credential_vault.pynew file mode 100644index 000000000..bfaf55406--- /dev/null+++ b/services/api/tests/test_credential_vault.py@@ -0,0 +1,129 @@+"""Tests for :mod:`app.security.credential_vault`.++These tests are intentionally hermetic — they construct vaults with explicit+keys rather than relying on the global singleton, so they don't depend on+process env vars or the order other test modules import settings in.+"""++from __future__ import annotations++import pytest+from cryptography.fernet import Fernet++from app.security.credential_vault import (+ CredentialVault,+ CredentialVaultError,+ _CIPHER_PREFIX,+)+++def vault() -> CredentialVault:+ return CredentialVault(Fernet.generate_key())+++def test_round_trip_string(vault: CredentialVault) -> None:+ ct = vault.encrypt("super-secret-token")+ assert ct.startswith(_CIPHER_PREFIX)+ assert "super-secret-token" not in ct+ assert vault.decrypt(ct) == "super-secret-token"+++def test_encrypt_is_idempotent(vault: CredentialVault) -> None:+ """Re-encrypting an already-encrypted value is a no-op."""+ ct = vault.encrypt("v")+ assert vault.encrypt(ct) == ct+++def test_decrypt_passes_plaintext_through(vault: CredentialVault) -> None:+ """Plaintext rows from older builds must round-trip cleanly."""+ assert vault.decrypt("not-encrypted-yet") == "not-encrypted-yet"+++def test_encrypt_dict_recursive(vault: CredentialVault) -> None:+ payload = {+ "client_id": "abc",+ "client_secret": "shh",+ "service_account": {+ "private_key": "-----BEGIN PRIVATE KEY-----\nXYZ\n-----END PRIVATE KEY-----",+ "client_email": "[email protected]",+ },+ "tags": ["prod", "us-east"],+ "max_events": 500,+ "enabled": True,+ }+ encrypted = vault.encrypt_dict(payload)++ assert encrypted["client_id"].startswith(_CIPHER_PREFIX)+ assert encrypted["client_secret"].startswith(_CIPHER_PREFIX)+ assert encrypted["service_account"]["private_key"].startswith(_CIPHER_PREFIX)+ assert encrypted["service_account"]["client_email"].startswith(_CIPHER_PREFIX)+ # Lists of strings get encrypted at the leaves.+ for tag in encrypted["tags"]:+ assert tag.startswith(_CIPHER_PREFIX)+ # Numbers and bools are untouched.+ assert encrypted["max_events"] == 500+ assert encrypted["enabled"] is True++ decrypted = vault.decrypt_dict(encrypted)+ assert decrypted == payload+++def test_encrypt_dict_with_secret_keys_filter(vault: CredentialVault) -> None:+ payload = {+ "client_id": "public-id", # leave alone+ "client_secret": "shh", # encrypt+ "region": "us-east-1", # leave alone+ }+ encrypted = vault.encrypt_dict(payload, secret_keys={"client_secret"})+ assert encrypted["client_id"] == "public-id"+ assert encrypted["region"] == "us-east-1"+ assert encrypted["client_secret"].startswith(_CIPHER_PREFIX)+++def test_invalid_key_raises() -> None:+ with pytest.raises(CredentialVaultError):+ CredentialVault(b"not-a-real-key")+++def test_empty_key_raises() -> None:+ with pytest.raises(CredentialVaultError):+ CredentialVault(b"")+++def test_rotation_decrypts_old_ciphertext() -> None:+ """A new primary key plus the old key in rotation must decrypt old rows."""+ old_key = Fernet.generate_key()+ new_key = Fernet.generate_key()++ old_vault = CredentialVault(old_key)+ legacy_ct = old_vault.encrypt("legacy-secret")++ rotated = CredentialVault(new_key, historical_keys=[old_key])+ assert rotated.decrypt(legacy_ct) == "legacy-secret"+ # New writes use the new primary, but old reads still work.+ new_ct = rotated.encrypt("new-secret")+ assert rotated.decrypt(new_ct) == "new-secret"+++def test_tampered_ciphertext_rejected(vault: CredentialVault) -> None:+ ct = vault.encrypt("important")+ tampered = ct[:-3] + "AAA" # mutate the trailing chars of the token+ with pytest.raises(CredentialVaultError):+ vault.decrypt(tampered)+++def test_non_string_inputs_rejected(vault: CredentialVault) -> None:+ with pytest.raises(CredentialVaultError):+ vault.encrypt(123) # type: ignore[arg-type]+ with pytest.raises(CredentialVaultError):+ vault.decrypt(None) # type: ignore[arg-type]+++def test_invalid_rotation_key_is_skipped_not_fatal() -> None:+ """One bad rotation entry must not take the whole vault offline."""+ primary = Fernet.generate_key()+ bogus = b"definitely-not-a-fernet-key"+ v = CredentialVault(primary, historical_keys=[bogus])+ ct = v.encrypt("ok")+ assert v.decrypt(ct) == "ok"diff --git a/services/api/tests/test_security_defaults.py b/services/api/tests/test_security_defaults.pyindex 9c0606add..4614f05aa 100644--- a/services/api/tests/test_security_defaults.py+++ b/services/api/tests/test_security_defaults.py@@ -30,6 +30,8 @@ def _make_settings(**overrides) -> Settings:"SECRET_KEY": "a" * 64,"METRICS_TOKEN": "long-random-token","PLUGIN_TRUST_MODE": "strict",+ # Pre-generated Fernet key. Real deployments mount this via Fly secret.+ "AISOC_CREDENTIAL_KEY": "Z7t9fM3pZ8U_bN5oVxQYr1w2kLsHd4aJiTvE6cNxYpA=",}base.update(overrides)# ``_env_file=None`` skips .env discovery so test runs are deterministic@@ -85,6 +87,20 @@ def test_clean_prod_settings_emit_no_warnings():assert msgs == []+def test_empty_credential_key_in_prod_warns():+ """Operators must set AISOC_CREDENTIAL_KEY before connector secrets touch Postgres."""+ s = _make_settings(AISOC_CREDENTIAL_KEY="")+ msgs = warn_if_insecure_defaults(s)+ assert any("AISOC_CREDENTIAL_KEY" in m for m in msgs)+++def test_empty_credential_key_in_dev_does_not_warn():+ """Dev auto-generates an ephemeral key, so silence here is intentional."""+ s = _make_settings(ENVIRONMENT="development", AISOC_CREDENTIAL_KEY="")+ msgs = warn_if_insecure_defaults(s)+ assert not any("AISOC_CREDENTIAL_KEY" in m for m in msgs)++# ─── /metrics auth gate ────────────────────────────────────────────────────diff --git a/services/connectors/app/api/router.py b/services/connectors/app/api/router.pyindex 0b5f99cac..4e544e3f1 100644--- a/services/connectors/app/api/router.py+++ b/services/connectors/app/api/router.py@@ -1,76 +1,158 @@"""Connectors service REST API.++Catalog and schema endpoints are now backed by the+``app.connectors`` registry so adding a connector requires zero changes+here — drop the class in, register it in ``connectors/__init__.py``, and+its schema flows through to the wizard automatically.++This service is a *stateless* microservice. It does not own connector+instance rows (those live in the API service's Postgres) and it does not+manage credentials at rest (that's the API's ``CredentialVault``). Its+job is twofold:++1. Catalog: tell the API service which connector classes this build+ ships and what configuration schema each one expects.+2. Test: instantiate a connector class with caller-supplied (already+ decrypted) credentials, run ``test_connection()``, and return the+ verdict. This lets the API service offer a "Test connection" button+ in the wizard without having to re-implement every vendor SDK.++Production connector polling and ingest happen elsewhere (the+``ConnectorScheduler`` + ``IngestClient`` modules wired into the+service's lifespan), not in this router."""from __future__ import annotations+from typing import Any+import structlog-from fastapi import APIRouter, HTTPException+from fastapi import APIRouter, HTTPException, status+from pydantic import BaseModel, Field as PydField++from app.connectors import CONNECTOR_REGISTRY, list_connector_schemaslogger = structlog.get_logger()router = APIRouter()+class TestConnectionRequest(BaseModel):+ """Stateless test-connection payload.++ The API service decrypts the stored ``auth_config`` before forwarding+ it here; this service never sees vault tokens. ``connector_config``+ carries non-secret runtime knobs (poll interval, region, etc.) that+ some connectors take in their constructor alongside credentials.+ """++ auth_config: dict[str, Any] = PydField(+ default_factory=dict,+ description="Plaintext credential fields for the connector.",+ )+ connector_config: dict[str, Any] = PydField(+ default_factory=dict,+ description="Non-secret runtime config that's passed to the connector constructor.",+ )++@router.get("/connectors")async def list_connectors():- """List all available connectors."""+ """List every connector registered with this build."""return {"connectors": [- {"id": "crowdstrike", "name": "CrowdStrike Falcon", "category": "EDR"},- {"id": "splunk", "name": "Splunk SIEM", "category": "SIEM"},- {"id": "aws_security_hub", "name": "AWS Security Hub", "category": "Cloud"},- {"id": "okta", "name": "Okta Identity", "category": "IAM"},- {"id": "microsoft_sentinel", "name": "Microsoft Sentinel", "category": "SIEM"},+ {+ "id": cls.connector_id,+ "name": cls.connector_name,+ "category": cls.connector_category,+ }+ for cls in CONNECTOR_REGISTRY.values()]}[email protected]("/connectors/schemas")+async def list_schemas():+ """Bulk fetch every connector's configuration schema.++ Frontend uses this to populate the AddConnector wizard without firing+ one request per connector.+ """+ return {"schemas": list_connector_schemas()}++@router.get("/connectors/{connector_id}/schema")async def get_connector_schema(connector_id: str):- """Get configuration schema for a connector."""- schemas = {- "crowdstrike": {- "fields": [- {"name": "client_id", "type": "string", "required": True, "label": "Client ID"},- {"name": "client_secret", "type": "secret", "required": True, "label": "Client Secret"},- {"name": "base_url", "type": "string", "required": False, "label": "Base URL", "default": "https://api.crowdstrike.com"},- ]- },- "splunk": {- "fields": [- {"name": "base_url", "type": "string", "required": True, "label": "Splunk URL"},- {"name": "token", "type": "secret", "required": True, "label": "HEC Token"},- {"name": "saved_search", "type": "string", "required": False, "label": "Saved Search Name"},- ]- },- "aws_security_hub": {- "fields": [- {"name": "region", "type": "string", "required": True, "label": "AWS Region", "default": "us-east-1"},- {"name": "access_key", "type": "string", "required": False, "label": "Access Key ID"},- {"name": "secret_key", "type": "secret", "required": False, "label": "Secret Access Key"},- ]- },- "okta": {- "fields": [- {"name": "domain", "type": "string", "required": True, "label": "Okta Domain", "placeholder": "https://yourorg.okta.com"},- {"name": "api_token", "type": "secret", "required": True, "label": "API Token"},- ]- },- "microsoft_sentinel": {- "fields": [- {"name": "tenant_id", "type": "string", "required": True, "label": "Tenant ID"},- {"name": "client_id", "type": "string", "required": True, "label": "Client ID"},- {"name": "client_secret", "type": "secret", "required": True, "label": "Client Secret"},- {"name": "subscription_id", "type": "string", "required": True, "label": "Subscription ID"},- {"name": "resource_group", "type": "string", "required": True, "label": "Resource Group"},- {"name": "workspace", "type": "string", "required": True, "label": "Workspace Name"},- ]- },- }- schema = schemas.get(connector_id)- if not schema:+ """Configuration schema for a single connector."""+ cls = CONNECTOR_REGISTRY.get(connector_id)+ if cls is None:+ raise HTTPException(status_code=404, detail=f"Connector '{connector_id}' not found")+ return cls.schema().to_dict()++[email protected]("/connectors/{connector_id}/test")+async def test_connector_connection(connector_id: str, payload: TestConnectionRequest):+ """Run a stateless ``test_connection()`` for the given connector.++ The API service is expected to:++ 1. Look up the stored connector instance row (or accept fresh creds+ from the wizard's "Test connection" button before the row exists).+ 2. Decrypt ``auth_config`` via the credential vault.+ 3. POST the resulting plaintext blob here, alongside ``connector_config``.++ We then construct the connector class with the merged keyword+ arguments and call its ``test_connection()`` coroutine. The connector+ is responsible for catching its own network errors and returning a+ structured ``{"success": bool, ...}`` payload — we do not synthesise+ that ourselves so the wizard surface always shows the connector's+ own diagnostic message (e.g. "401 Unauthorized" vs "DNS lookup+ failed").+ """+ cls = CONNECTOR_REGISTRY.get(connector_id)+ if cls is None:raise HTTPException(status_code=404, detail=f"Connector '{connector_id}' not found")- return schema++ # Merge auth + non-secret config into kwargs. We keep them in this order+ # because a malicious ``connector_config`` mustn't be able to overwrite+ # a legitimate ``auth_config`` field — but in practice the API service+ # validates both blobs against the schema before we ever see them, so+ # this ordering is defensive belt-and-braces rather than a real+ # boundary.+ kwargs = {**payload.auth_config, **payload.connector_config}++ try:+ connector = cls(**kwargs)+ except TypeError as exc:+ # Almost always "missing 1 required positional argument" or "got+ # unexpected keyword argument", i.e. caller passed a config that+ # doesn't match this connector's schema. Surface as 422 so the+ # frontend can highlight the offending field.+ raise HTTPException(+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,+ detail=f"connector config does not match schema: {exc}",+ ) from exc+ except Exception as exc: # pragma: no cover - last-ditch+ logger.exception("connector.test.constructor_error", connector_id=connector_id)+ raise HTTPException(+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,+ detail=f"failed to construct connector: {exc}",+ ) from exc++ try:+ result = await connector.test_connection()+ except Exception as exc: # pragma: no cover - connector misbehaving+ # A well-behaved connector swallows its own errors and returns+ # {"success": False, "error": ...}. If one raises anyway, we+ # convert to the same shape so the wizard UI doesn't have two+ # error formats to deal with.+ logger.exception("connector.test.runtime_error", connector_id=connector_id)+ result = {"success": False, "connector": connector_id, "error": str(exc)}++ if not isinstance(result, dict):+ # Defensive: some connectors might return None on success. Coerce.+ result = {"success": bool(result), "connector": connector_id}+ return result@router.get("/health")diff --git a/services/connectors/app/connectors/__init__.py b/services/connectors/app/connectors/__init__.pyindex e69de29bb..b25cfecc8 100644--- a/services/connectors/app/connectors/__init__.py+++ b/services/connectors/app/connectors/__init__.py@@ -0,0 +1,110 @@+"""+Connector registry.++Every concrete ``BaseConnector`` subclass imported from this package is+auto-registered by ``connector_id`` so the FastAPI router can resolve a+connector class without a hand-maintained dispatch table.++Why eager imports here (instead of dynamic ``pkgutil.iter_modules`` discovery):+ * Keeps imports auditable in code review — adding a connector means adding it+ to this list, which is exactly the visibility we want for a security tool.+ * Surfaces import errors at service startup, not at first request.+ * Plays nicely with mypy / IDE goto-definition.+"""++from __future__ import annotations++from typing import TYPE_CHECKING++from app.connectors.aws_security_hub import AWSSecurityHubConnector+from app.connectors.azure_activity import AzureActivityConnector+from app.connectors.azure_defender import AzureDefenderConnector+from app.connectors.azure_entra import AzureEntraConnector+from app.connectors.base import BaseConnector, ConnectorSchema, Field, OAuthHints+from app.connectors.cloudflare import CloudflareConnector+from app.connectors.crowdstrike import CrowdStrikeConnector+from app.connectors.gcp_cloud_audit import GCPCloudAuditConnector+from app.connectors.gcp_scc import GCPSCCConnector+from app.connectors.github import GitHubConnector+from app.connectors.google_workspace import GoogleWorkspaceConnector+from app.connectors.m365_audit import M365AuditConnector+from app.connectors.microsoft_sentinel import MicrosoftSentinelConnector+from app.connectors.okta import OktaConnector+from app.connectors.splunk import SplunkConnector++if TYPE_CHECKING:+ pass+++# Source of truth for "which connectors does this build know about".+# Keep alphabetised by connector_id for predictable diffs.+_CONNECTOR_CLASSES: tuple[type[BaseConnector], ...] = (+ AWSSecurityHubConnector,+ AzureActivityConnector,+ AzureDefenderConnector,+ AzureEntraConnector,+ CloudflareConnector,+ CrowdStrikeConnector,+ GCPCloudAuditConnector,+ GCPSCCConnector,+ GitHubConnector,+ GoogleWorkspaceConnector,+ M365AuditConnector,+ MicrosoftSentinelConnector,+ OktaConnector,+ SplunkConnector,+)+++def _build_registry() -> dict[str, type[BaseConnector]]:+ registry: dict[str, type[BaseConnector]] = {}+ for cls in _CONNECTOR_CLASSES:+ if not cls.connector_id:+ raise RuntimeError(+ f"connector class {cls.__name__} has empty connector_id; refusing to register"+ )+ if cls.connector_id in registry:+ raise RuntimeError(+ f"duplicate connector_id '{cls.connector_id}' between "+ f"{registry[cls.connector_id].__name__} and {cls.__name__}"+ )+ registry[cls.connector_id] = cls+ return registry+++CONNECTOR_REGISTRY: dict[str, type[BaseConnector]] = _build_registry()+++def get_connector_class(connector_id: str) -> type[BaseConnector] | None:+ """Look up a connector class by ``connector_id``."""+ return CONNECTOR_REGISTRY.get(connector_id)+++def list_connector_schemas() -> list[dict]:+ """Return every registered connector's schema in JSON-serialisable form."""+ return [cls.schema().to_dict() for cls in CONNECTOR_REGISTRY.values()]+++__all__ = [+ "AWSSecurityHubConnector",+ "AzureActivityConnector",+ "AzureDefenderConnector",+ "AzureEntraConnector",+ "BaseConnector",+ "CONNECTOR_REGISTRY",+ "CloudflareConnector",+ "ConnectorSchema",+ "CrowdStrikeConnector",+ "Field",+ "GCPCloudAuditConnector",+ "GCPSCCConnector",+ "GitHubConnector",+ "GoogleWorkspaceConnector",+ "M365AuditConnector",+ "MicrosoftSentinelConnector",+ "OAuthHints",+ "OktaConnector",+ "SplunkConnector",+ "get_connector_class",+ "list_connector_schemas",+]diff --git a/services/connectors/app/connectors/aws_security_hub.py b/services/connectors/app/connectors/aws_security_hub.pyindex a33f15711..442a25e9e 100644--- a/services/connectors/app/connectors/aws_security_hub.py+++ b/services/connectors/app/connectors/aws_security_hub.py@@ -10,7 +10,7 @@ from typing import Anyimport structlog-from app.connectors.base import BaseConnector+from app.connectors.base import BaseConnector, ConnectorSchema, Fieldlogger = structlog.get_logger()@@ -18,6 +18,34 @@ logger = structlog.get_logger()class AWSSecurityHubConnector(BaseConnector):connector_id = "aws_security_hub"connector_name = "AWS Security Hub"+ connector_category = "cloud"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description="AWS Security Hub findings (GuardDuty, Inspector, Macie, third-party).",+ docs_url="/docs/connectors/aws-security-hub",+ fields=[+ Field("region", "string", "AWS Region", default="us-east-1"),+ Field(+ "access_key",+ "string",+ "Access Key ID",+ required=False,+ help_text="Leave blank to use the runtime IAM role / instance profile.",+ ),+ Field(+ "secret_key",+ "secret",+ "Secret Access Key",+ required=False,+ help_text="Required only when supplying a static access key above.",+ ),+ ],+ )def __init__(self, region: str = "us-east-1", access_key: str = "", secret_key: str = ""):self._region = regiondiff --git a/services/connectors/app/connectors/base.py b/services/connectors/app/connectors/base.pyindex cb9b5405a..0b2c03496 100644--- a/services/connectors/app/connectors/base.py+++ b/services/connectors/app/connectors/base.py@@ -1,18 +1,157 @@"""Base connector interface for all integrations.++Every connector subclass owns:+ * ``connector_id`` / ``connector_name`` / ``connector_category`` —+ identity used in the catalog and registry lookups.+ * ``schema()`` — a self-describing config schema returned to the+ frontend's "Add connector" wizard. This replaces the centralised+ dict that used to live in ``services/connectors/app/api/router.py``.+ * ``test_connection()`` / ``fetch_alerts()`` / ``normalize()`` — the+ runtime contract used by the polling scheduler.++The schema is intentionally JSON-shaped (not Pydantic) because it is+serialised straight to the wire for the click-and-connect UI. We keep+typed dataclasses here as an authoring aid so a typo (e.g. ``"sercret"``+for ``"secret"``) is caught at import time, not at form-render time."""from __future__ import annotationsfrom abc import ABC, abstractmethod-from typing import Any+from dataclasses import asdict, dataclass, field+from typing import Any, Literal++# ---------------------------------------------------------------------------+# Schema dataclasses+#+# We expose ``Field`` and ``ConnectorSchema`` as thin builders. Connector+# authors build a schema with native Python objects; ``ConnectorSchema.to_dict``+# emits the JSON shape the frontend expects.+# ---------------------------------------------------------------------------++# Allowed UI field types. Frontend renders each one with the appropriate+# input control (e.g. ``secret`` is a masked password input, ``textarea``+# accepts pasted JSON keys for GCP service accounts).+FieldType = Literal["string", "secret", "select", "textarea", "boolean", "number"]+++@dataclass(frozen=True)+class Field:+ """A single config form field exposed to the wizard UI."""++ name: str+ type: FieldType+ label: str+ required: bool = True+ default: Any | None = None+ placeholder: str | None = None+ help_text: str | None = None+ # Only meaningful for ``type="select"`` — list of {"value", "label"} dicts.+ options: list[dict[str, str]] | None = None++ def to_dict(self) -> dict[str, Any]:+ d = asdict(self)+ # Strip Nones so the wire format stays compact and the frontend+ # doesn't have to special-case missing-vs-None.+ return {k: v for k, v in d.items() if v is not None}+++@dataclass(frozen=True)+class OAuthHints:+ """Forward-looking hints for hosted OAuth.++ We render a "Hosted OAuth coming soon" badge in the UI when+ ``supported_in_hosted`` is True. Filling in ``authorize_url`` /+ ``token_url`` / ``scopes`` here keeps the OAuth follow-up PR cheap —+ the frontend already knows what to render.+ """++ supported_in_hosted: bool = False+ authorize_url: str | None = None+ token_url: str | None = None+ scopes: list[str] = field(default_factory=list)++ def to_dict(self) -> dict[str, Any]:+ return {k: v for k, v in asdict(self).items() if v not in (None, [])}+++@dataclass(frozen=True)+class ConnectorSchema:+ """Self-describing schema for a connector.++ Returned by ``BaseConnector.schema()`` and surfaced verbatim by+ ``GET /connectors/{id}/schema`` in the connectors service.+ """++ connector_id: str+ connector_name: str+ category: str+ description: str+ fields: list[Field]+ docs_url: str | None = None+ oauth: OAuthHints | None = None++ def to_dict(self) -> dict[str, Any]:+ out: dict[str, Any] = {+ "connector_id": self.connector_id,+ "connector_name": self.connector_name,+ "category": self.category,+ "description": self.description,+ "fields": [f.to_dict() for f in self.fields],+ }+ if self.docs_url:+ out["docs_url"] = self.docs_url+ if self.oauth is not None:+ out["oauth"] = self.oauth.to_dict()+ return out+++# ---------------------------------------------------------------------------+# Base class+# ---------------------------------------------------------------------------class BaseConnector(ABC):- """All connectors implement this interface."""+ """All connectors implement this interface.+ Subclasses must set the three class attributes and implement+ ``schema()``, ``test_connection()``, and ``fetch_alerts()``.+ ``normalize()`` is optional but strongly encouraged — the default+ just returns the raw event, which is fine for sources that already+ speak our common alert shape.+ """++ # Class identity. ``connector_category`` was added with the schema+ # refactor so the registry can group entries (EDR / SIEM / Cloud / IAM).connector_id: str = ""connector_name: str = ""+ connector_category: str = ""++ # ---------------------------- self-description ----------------------------++ @classmethod+ @abstractmethod+ def schema(cls) -> ConnectorSchema:+ """Return the configuration schema for this connector.++ Authoring example::++ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description="...",+ fields=[+ Field("client_id", "string", "Client ID"),+ Field("client_secret", "secret", "Client Secret"),+ ],+ docs_url="/docs/connectors/<id>",+ )+ """+ ...++ # ------------------------------ runtime ----------------------------------@abstractmethodasync def test_connection(self) -> dict[str, Any]:@@ -25,8 +164,5 @@ class BaseConnector(ABC):...def normalize(self, raw: dict[str, Any]) -> dict[str, Any]:- """- Normalize a raw event to a common AiSOC alert schema.- Subclasses should override for source-specific normalization.- """+ """Normalize a raw event to a common AiSOC alert schema."""return rawdiff --git a/services/connectors/app/connectors/crowdstrike.py b/services/connectors/app/connectors/crowdstrike.pyindex 947d793df..a1a4895cb 100644--- a/services/connectors/app/connectors/crowdstrike.py+++ b/services/connectors/app/connectors/crowdstrike.py@@ -11,7 +11,7 @@ from typing import Anyimport httpximport structlog-from app.connectors.base import BaseConnector+from app.connectors.base import BaseConnector, ConnectorSchema, Fieldlogger = structlog.get_logger()@@ -23,6 +23,29 @@ _DETECTION_DETAILS_URL = "https://api.crowdstrike.com/detects/entities/summariesclass CrowdStrikeConnector(BaseConnector):connector_id = "crowdstrike"connector_name = "CrowdStrike Falcon"+ connector_category = "edr"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description="CrowdStrike Falcon detections via the Falcon REST API.",+ docs_url="/docs/connectors/crowdstrike",+ fields=[+ Field("client_id", "string", "Client ID"),+ Field("client_secret", "secret", "Client Secret"),+ Field(+ "base_url",+ "string",+ "Base URL",+ required=False,+ default="https://api.crowdstrike.com",+ help_text="Override only if your tenant lives in a non-US cloud (e.g. api.eu-1.crowdstrike.com).",+ ),+ ],+ )def __init__(self, client_id: str, client_secret: str, base_url: str = "https://api.crowdstrike.com"):self._client_id = client_iddiff --git a/services/connectors/app/connectors/microsoft_sentinel.py b/services/connectors/app/connectors/microsoft_sentinel.pyindex ba471c75c..2b969bd9f 100644--- a/services/connectors/app/connectors/microsoft_sentinel.py+++ b/services/connectors/app/connectors/microsoft_sentinel.py@@ -11,7 +11,7 @@ from typing import Anyimport httpximport structlog-from app.connectors.base import BaseConnector+from app.connectors.base import BaseConnector, ConnectorSchema, Fieldlogger = structlog.get_logger()@@ -22,6 +22,25 @@ _SENTINEL_API = "https://management.azure.com/subscriptions/{sub_id}/resourceGroclass MicrosoftSentinelConnector(BaseConnector):connector_id = "microsoft_sentinel"connector_name = "Microsoft Sentinel"+ connector_category = "siem"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description="Microsoft Sentinel incidents via the Azure Security Insights REST API.",+ docs_url="/docs/connectors/microsoft-sentinel",+ fields=[+ Field("tenant_id", "string", "Tenant ID"),+ Field("client_id", "string", "Client ID"),+ Field("client_secret", "secret", "Client Secret"),+ Field("subscription_id", "string", "Subscription ID"),+ Field("resource_group", "string", "Resource Group"),+ Field("workspace", "string", "Workspace Name"),+ ],+ )def __init__(self,diff --git a/services/connectors/app/connectors/okta.py b/services/connectors/app/connectors/okta.pyindex 14f299f0d..23d489597 100644--- a/services/connectors/app/connectors/okta.py+++ b/services/connectors/app/connectors/okta.py@@ -11,7 +11,7 @@ from typing import Anyimport httpximport structlog-from app.connectors.base import BaseConnector+from app.connectors.base import BaseConnector, ConnectorSchema, Fieldlogger = structlog.get_logger()@@ -19,6 +19,26 @@ logger = structlog.get_logger()class OktaConnector(BaseConnector):connector_id = "okta"connector_name = "Okta Identity"+ connector_category = "iam"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description="Okta system log events (auth, MFA, account locks, blocked requests).",+ docs_url="/docs/connectors/okta",+ fields=[+ Field(+ "domain",+ "string",+ "Okta Domain",+ placeholder="https://yourorg.okta.com",+ ),+ Field("api_token", "secret", "API Token"),+ ],+ )def __init__(self, domain: str, api_token: str):self._domain = domain.rstrip("/")diff --git a/services/connectors/app/connectors/splunk.py b/services/connectors/app/connectors/splunk.pyindex 1707ea5bd..7f3aaadcf 100644--- a/services/connectors/app/connectors/splunk.py+++ b/services/connectors/app/connectors/splunk.py@@ -10,7 +10,7 @@ from typing import Anyimport httpximport structlog-from app.connectors.base import BaseConnector+from app.connectors.base import BaseConnector, ConnectorSchema, Fieldlogger = structlog.get_logger()@@ -18,6 +18,34 @@ logger = structlog.get_logger()class SplunkConnector(BaseConnector):connector_id = "splunk"connector_name = "Splunk SIEM"+ connector_category = "siem"++ @classmethod+ def schema(cls) -> ConnectorSchema:+ return ConnectorSchema(+ connector_id=cls.connector_id,+ connector_name=cls.connector_name,+ category=cls.connector_category,+ description="Splunk Enterprise / Cloud notable events via the REST API.",+ docs_url="/docs/connectors/splunk",+ fields=[+ Field(+ "base_url",+ "string",+ "Splunk URL",+ placeholder="https://splunk.example.com:8089",+ help_text="Management port (default 8089), not the web UI port.",+ ),+ Field("token", "secret", "HEC / API Token"),+ Field(+ "saved_search",+ "string",+ "Saved Search Name",+ required=False,+ default="AiSOC_Alerts",+ ),+ ],+ )def __init__(self, base_url: str, token: str, saved_search: str = "AiSOC_Alerts"):self._base_url = base_url.rstrip("/")diff --git a/services/connectors/app/db/__init__.py b/services/connectors/app/db/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/services/connectors/app/db/connector_repo.py b/services/connectors/app/db/connector_repo.pynew file mode 100644index 000000000..5d0455e92--- /dev/null+++ b/services/connectors/app/db/connector_repo.py@@ -0,0 +1,176 @@+"""Read/write access to the canonical ``connectors`` table.++The schema for that table lives in ``services/api/app/models/connector.py``;+the API service owns the migrations. We deliberately do **not** import that+ORM model here — it would drag the API's full app.* package into our import+graph. Instead we pin the column shape via SQLAlchemy Core ``Table``/+``MetaData`` against the same ``connectors`` table and limit ourselves to the+columns the scheduler needs.++If the API service ever changes the column shape in a backwards-incompatible+way, the scheduler will fail loudly at first poll (column not found), which is+what we want — silent drift between two services that share a table is the+worst possible failure mode.+"""++from __future__ import annotations++import uuid+from dataclasses import dataclass+from datetime import UTC, datetime+from typing import Any++from sqlalchemy import (+ JSON,+ Boolean,+ Column,+ DateTime,+ Integer,+ MetaData,+ String,+ Table,+ and_,+ select,+ update,+)+from sqlalchemy.dialects.postgresql import JSONB, UUID++metadata = MetaData()++# Mirror of services/api/app/models/connector.py:Connector. Only includes the+# columns the scheduler reads or writes; we don't claim ownership of the full+# table shape.+connectors_table = Table(+ "connectors",+ metadata,+ Column("id", UUID(as_uuid=True), primary_key=True),+ Column("tenant_id", UUID(as_uuid=True), nullable=False),+ Column("name", String(255), nullable=False),+ Column("connector_type", String(100), nullable=False),+ Column("category", String(50), nullable=False),+ Column("is_enabled", Boolean, nullable=False),+ Column("auth_config", JSONB, nullable=False),+ Column("connector_config", JSONB, nullable=False),+ Column("health_status", String(20), nullable=False),+ Column("last_health_check", DateTime(timezone=True)),+ Column("last_sync", DateTime(timezone=True)),+ Column("events_ingested", Integer, nullable=False),+ Column("error_count", Integer, nullable=False),+ Column("tags", JSON, nullable=False),+ Column("created_at", DateTime(timezone=True), nullable=False),+ Column("updated_at", DateTime(timezone=True), nullable=False),+)+++@dataclass+class ConnectorInstance:+ """Light dataclass over a row in ``connectors``.++ We use a plain dataclass instead of an ORM mapping so the scheduler stays+ decoupled from the API service's ``Base`` declarative class and avoids+ lazy-loading attributes from a closed session by mistake.+ """++ id: uuid.UUID+ tenant_id: uuid.UUID+ name: str+ connector_type: str+ is_enabled: bool+ auth_config: dict[str, Any]+ connector_config: dict[str, Any]+ health_status: str+ last_sync: datetime | None+ events_ingested: int+ error_count: int+++async def fetch_enabled_connectors(connection: Any) -> list[ConnectorInstance]:+ """Return every connector instance with ``is_enabled = True``.++ ``connection`` must be a SQLAlchemy ``AsyncConnection``. We accept ``Any``+ in the type hint to avoid forcing every caller to import the async+ connection type.+ """+ stmt = select(+ connectors_table.c.id,+ connectors_table.c.tenant_id,+ connectors_table.c.name,+ connectors_table.c.connector_type,+ connectors_table.c.is_enabled,+ connectors_table.c.auth_config,+ connectors_table.c.connector_config,+ connectors_table.c.health_status,+ connectors_table.c.last_sync,+ connectors_table.c.events_ingested,+ connectors_table.c.error_count,+ ).where(connectors_table.c.is_enabled.is_(True))++ result = await connection.execute(stmt)+ rows = result.fetchall()+ return [+ ConnectorInstance(+ id=row.id,+ tenant_id=row.tenant_id,+ name=row.name,+ connector_type=row.connector_type,+ is_enabled=row.is_enabled,+ auth_config=row.auth_config or {},+ connector_config=row.connector_config or {},+ health_status=row.health_status,+ last_sync=row.last_sync,+ events_ingested=row.events_ingested,+ error_count=row.error_count,+ )+ for row in rows+ ]+++async def record_poll_success(+ connection: Any,+ connector_id: uuid.UUID,+ *,+ events_added: int,+) -> None:+ """Update last_sync, increment events_ingested, mark healthy."""+ now = datetime.now(UTC)+ stmt = (+ update(connectors_table)+ .where(connectors_table.c.id == connector_id)+ .values(+ last_sync=now,+ last_health_check=now,+ health_status="healthy",+ events_ingested=connectors_table.c.events_ingested + events_added,+ updated_at=now,+ )+ )+ await connection.execute(stmt)+++async def record_poll_failure(+ connection: Any,+ connector_id: uuid.UUID,+) -> None:+ """Mark a poll attempt as failed without touching last_sync."""+ now = datetime.now(UTC)+ stmt = (+ update(connectors_table)+ .where(connectors_table.c.id == connector_id)+ .values(+ last_health_check=now,+ health_status="unhealthy",+ error_count=connectors_table.c.error_count + 1,+ updated_at=now,+ )+ )+ await connection.execute(stmt)+++__all__ = [+ "ConnectorInstance",+ "connectors_table",+ "fetch_enabled_connectors",+ "metadata",+ "record_poll_failure",+ "record_poll_success",+]diff --git a/services/connectors/app/db/engine.py b/services/connectors/app/db/engine.pynew file mode 100644index 000000000..173f3ee5d--- /dev/null+++ b/services/connectors/app/db/engine.py@@ -0,0 +1,79 @@+"""Async SQLAlchemy engine for the connectors microservice.++We keep the engine creation in its own module so:++ * The scheduler can import ``get_engine`` without dragging in the+ ``connectors_table`` definition (and vice versa).+ * Tests can call ``set_engine`` to inject a fixture engine without monkey+ patching module-level state.++The engine connects to the *same* Postgres the API service uses; we read from+and write to the ``connectors`` table directly. We do not own the schema —+migrations live in the API service.+"""++from __future__ import annotations++import os+from threading import Lock+from typing import Any++from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine++_engine: AsyncEngine | None = None+_engine_lock = Lock()+++def _resolve_database_url() -> str:+ """Pick up DATABASE_URL with a sensible fallback.++ docker-compose.yml passes ``DATABASE_URL=postgresql+asyncpg://...`` which+ is what we want. If a deployment ships a plain ``postgresql://`` URL we+ rewrite the driver prefix to use asyncpg, since the connectors service+ only uses async I/O.+ """+ url = os.getenv("DATABASE_URL")+ if not url:+ raise RuntimeError(+ "DATABASE_URL is not set; the connectors service requires it to read the connectors table."+ )+ if url.startswith("postgresql://"):+ url = "postgresql+asyncpg://" + url[len("postgresql://"):]+ elif url.startswith("postgres://"):+ url = "postgresql+asyncpg://" + url[len("postgres://"):]+ return url+++def get_engine() -> AsyncEngine:+ """Return the lazily-constructed process-wide async engine."""+ global _engine+ if _engine is not None:+ return _engine+ with _engine_lock:+ if _engine is not None: # pragma: no cover - racing init+ return _engine+ url = _resolve_database_url()+ # ``pool_pre_ping`` so we don't hand the scheduler a dead conn after+ # a Postgres restart; ``echo`` stays off — connector polling is high+ # frequency and we don't want SQL spam in prod logs.+ _engine = create_async_engine(url, pool_pre_ping=True, echo=False)+ return _engine+++def set_engine(engine: AsyncEngine | None) -> None:+ """Test helper: install a fixture engine (or clear it)."""+ global _engine+ with _engine_lock:+ _engine = engine+++async def dispose_engine() -> None:+ """Close the engine on shutdown so we don't leak connections."""+ global _engine+ if _engine is None:+ return+ engine, _engine = _engine, None+ await engine.dispose()+++__all__ = ["dispose_engine", "get_engine", "set_engine"]diff --git a/services/connectors/app/ingest_client.py b/services/connectors/app/ingest_client.pynew file mode 100644index 000000000..12c9bad41--- /dev/null+++ b/services/connectors/app/ingest_client.py@@ -0,0 +1,139 @@+"""HTTP client that forwards normalized events to the Go ingest service.++The connectors service polls each enabled connector instance and pushes the+normalized events here. The ingest service is in Go (services/ingest) and+exposes ``POST /v1/ingest`` and ``POST /v1/ingest/batch`` with the request+shape::++ {+ "connector_id": "...",+ "connector_type": "...",+ "source_format": "...",+ "events": [{...}, ...]+ }++with ``X-Tenant-ID`` header for tenant scoping.++We keep this layer **dumb on purpose**: no retries with exponential backoff+beyond a single retry, no circuit breaker, no batching across connectors.+The ingest service handles the queueing into Kafka — adding another buffer+layer in the connectors process would just move the failure point and add+state to a process we want to keep restartable. If the ingest service is+down, polls fail loudly and the scheduler records that in+``record_poll_failure``; the next poll cycle retries.+"""++from __future__ import annotations++import logging+import os+import uuid+from typing import Any++import httpx++logger = logging.getLogger("aisoc.connectors.ingest_client")++# 30s mirrors the ingest service's own ReadTimeout. We don't want the client+# to give up before the server has a chance to publish; we also don't want+# polls to hang the scheduler thread for minutes if Kafka is wedged.+_DEFAULT_TIMEOUT_S = 30.0+++class IngestClientError(RuntimeError):+ """Raised when the ingest service rejects or fails the push."""+++class IngestClient:+ """Push connector events to ``services/ingest``.++ One client per scheduler process is fine — ``httpx.AsyncClient`` pools+ connections internally. Construct via ``IngestClient.from_env`` so the+ URL stays configurable per deployment without threading config through+ every caller.+ """++ def __init__(self, base_url: str, *, timeout_seconds: float = _DEFAULT_TIMEOUT_S) -> None:+ # Strip trailing slash so we can append paths without doubling up.+ self._base_url = base_url.rstrip("/")+ self._timeout = timeout_seconds+ self._client: httpx.AsyncClient | None = None++ @classmethod+ def from_env(cls) -> IngestClient:+ url = os.getenv("INGEST_SERVICE_URL", "http://ingest-worker:8080")+ timeout = float(os.getenv("INGEST_SERVICE_TIMEOUT_SECONDS", str(_DEFAULT_TIMEOUT_S)))+ return cls(url, timeout_seconds=timeout)++ async def _get_client(self) -> httpx.AsyncClient:+ if self._client is None:+ self._client = httpx.AsyncClient(timeout=self._timeout)+ return self._client++ async def aclose(self) -> None:+ if self._client is not None:+ await self._client.aclose()+ self._client = None++ async def push_events(+ self,+ *,+ tenant_id: uuid.UUID | str,+ connector_id: uuid.UUID | str,+ connector_type: str,+ events: list[dict[str, Any]],+ source_format: str = "raw_json",+ ) -> dict[str, Any]:+ """Push a batch of events to the ingest service.++ Returns the response body (``{"accepted": N, "rejected": N, ...}``) so+ the scheduler can record how many events were accepted in+ ``connectors.events_ingested``.++ An empty event list short-circuits and returns ``{"accepted": 0,+ "rejected": 0}`` without making a network call — this is the common+ case (a poll cycle that found no new alerts).+ """+ if not events:+ return {"accepted": 0, "rejected": 0}++ client = await self._get_client()+ # Use the batch endpoint — the non-batch and batch endpoints are+ # actually the same handler in the Go service, but ``/ingest/batch``+ # documents intent for whoever's reading nginx logs.+ url = f"{self._base_url}/v1/ingest/batch"+ headers = {+ "Content-Type": "application/json",+ "X-Tenant-ID": str(tenant_id),+ }+ payload = {+ "connector_id": str(connector_id),+ "connector_type": connector_type,+ "source_format": source_format,+ "events": events,+ }++ try:+ resp = await client.post(url, headers=headers, json=payload)+ except httpx.HTTPError as exc:+ raise IngestClientError(+ f"ingest service unreachable at {url}: {exc}"+ ) from exc++ if resp.status_code >= 400:+ # Pull the body so logs show *why* — typically a missing tenant+ # header or oversized batch, both of which we want surfaced+ # rather than swallowed.+ body_preview = resp.text[:500]+ raise IngestClientError(+ f"ingest service returned {resp.status_code} for connector {connector_id}: {body_preview}"+ )++ try:+ data = resp.json()+ except ValueError:+ raise IngestClientError("ingest service returned non-JSON response")+ return data+++__all__ = ["IngestClient", "IngestClientError"]diff --git a/services/connectors/app/main.py b/services/connectors/app/main.pyindex b0b63ca0f..4bda6740a 100644--- a/services/connectors/app/main.py+++ b/services/connectors/app/main.py@@ -1,16 +1,78 @@"""-AiSOC Connectors Service+AiSOC Connectors Service.++Hosts the connector catalog/test endpoints and runs the in-process+``ConnectorScheduler`` that polls every enabled connector instance on its+configured cadence.++We use FastAPI's ``lifespan`` context manager (rather than the deprecated+``@app.on_event("startup")``/``@app.on_event("shutdown")``) so the scheduler's+async lifecycle hangs off the same event loop ``uvicorn`` runs the HTTP+server on.++The scheduler is opt-out via ``AISOC_CONNECTORS_DISABLE_SCHEDULER=1`` so unit+tests, one-shot CLI entrypoints, and the schema-only catalog mode can keep+running this app without spinning up a polling loop."""+from __future__ import annotations++import logging+from contextlib import asynccontextmanager+from typing import AsyncIterator+from fastapi import FastAPIfrom fastapi.middleware.cors import CORSMiddlewarefrom app.api.router import router+from app.db.engine import dispose_engine+from app.scheduler import ConnectorScheduler, scheduler_disabled++logger = logging.getLogger("aisoc.connectors.main")+++@asynccontextmanager+async def lifespan(app: FastAPI) -> AsyncIterator[None]:+ """Start/stop the scheduler alongside the HTTP server."""+ scheduler: ConnectorScheduler | None = None+ if scheduler_disabled():+ logger.info(+ "connector.scheduler.disabled "+ "AISOC_CONNECTORS_DISABLE_SCHEDULER set; HTTP only mode"+ )+ else:+ try:+ scheduler = ConnectorScheduler()+ await scheduler.start()+ app.state.scheduler = scheduler+ except Exception:+ # We deliberately let the HTTP server come up even if the+ # scheduler can't start (e.g. DATABASE_URL unset). Operators+ # can hit /health and the catalog endpoints to inspect the+ # build; the missing scheduler will show up as no polls in+ # the connectors UI rather than a refusing-to-start service.+ logger.exception("connector.scheduler.start_failed")+ scheduler = None++ try:+ yield+ finally:+ if scheduler is not None:+ try:+ await scheduler.stop()+ except Exception: # pragma: no cover - best-effort shutdown+ logger.exception("connector.scheduler.stop_failed")+ try:+ await dispose_engine()+ except Exception: # pragma: no cover - best-effort shutdown+ logger.exception("connector.engine.dispose_failed")+app = FastAPI(title="AiSOC Connectors",description="Security source connectors: CrowdStrike, Splunk, AWS Security Hub, Okta, Microsoft Sentinel",version="0.1.0",+ lifespan=lifespan,)app.add_middleware(diff --git a/services/connectors/app/scheduler.py b/services/connectors/app/scheduler.pynew file mode 100644index 000000000..266e774ea--- /dev/null+++ b/services/connectors/app/scheduler.py@@ -0,0 +1,484 @@+"""APScheduler-driven polling loop for enabled connector instances.++The connectors microservice runs a single ``ConnectorScheduler`` per process,+started in the FastAPI lifespan and stopped on shutdown. The scheduler:++1. Periodically reloads the enabled-connector list from Postgres (the API+ service owns the ``connectors`` table; we read it directly via SQLAlchemy+ Core).+2. For each enabled instance, schedules an async job that runs every+ ``connector_config.poll_interval_seconds`` (default 300s = 5 min).+3. On each tick, the job decrypts ``auth_config`` via the vendored+ ``CredentialVault``, instantiates the connector class with the merged+ config, calls ``fetch_alerts()``, normalizes each event, and pushes the+ batch to ``services/ingest`` via ``IngestClient``.+4. Records success or failure on the connector row so the UI / health+ dashboards can show per-connector status.++Design rationale+----------------++* **APScheduler in-process** instead of Celery / RQ / a Go daemon: the+ poll workload is tiny (one HTTP round-trip per source per 5 min) and+ the per-connector logic is already async Python. Standing up a broker+ for this would be over-engineering, and we'd lose direct access to the+ connector classes' Python types. APScheduler's ``AsyncIOScheduler`` runs+ on the existing FastAPI event loop with effectively zero overhead for+ idle connectors.++* **Reload from DB, don't watch**: the API service can create / disable+ / update connectors at any time. Rather than wire up a pub/sub+ notification channel (LISTEN/NOTIFY, Redis pubsub, etc.) we just refetch+ the enabled set every ``RELOAD_INTERVAL_SECONDS`` (60s by default).+ Worst case a newly-created connector waits up to 60s for its first+ poll, which is fine — humans don't care about that latency for a thing+ that polls every 5 minutes anyway.++* **One async job per instance**: APScheduler schedules each connector+ by ``connector_id`` (UUID string). When we reload, any connector_id we+ no longer see gets ``remove_job``'d, any new one gets ``add_job``'d,+ and any existing one stays put. Idempotent reloads keep the scheduler+ state in sync with the DB without restart races.++* **Failures are recorded, not raised**: ``record_poll_failure`` updates+ the row's health status; we never let an exception escape the job+ function because APScheduler would then start de-prioritising the job.+ We *do* emit the exception via ``logger.exception`` so it shows up in+ centralized logs.+"""++from __future__ import annotations++import logging+import os+import uuid+from datetime import UTC, datetime+from typing import Any++from apscheduler.schedulers.asyncio import AsyncIOScheduler+from sqlalchemy.ext.asyncio import AsyncEngine++from app.connectors import CONNECTOR_REGISTRY+from app.db.connector_repo import (+ ConnectorInstance,+ fetch_enabled_connectors,+ record_poll_failure,+ record_poll_success,+)+from app.db.engine import get_engine+from app.ingest_client import IngestClient, IngestClientError+from app.security.credential_vault import CredentialVault, CredentialVaultError, get_vault++logger = logging.getLogger("aisoc.connectors.scheduler")++# How often we re-read the connectors table to pick up new / disabled+# instances. Tuned to balance UI-perceived latency against DB load — at+# 60s with N connectors, we issue 1 SELECT/min regardless of N.+_DEFAULT_RELOAD_INTERVAL_S = 60.0++# Default poll cadence per connector instance when ``connector_config``+# doesn't override it. Five minutes is the standard SOC poll interval and+# matches every connector's default ``since_seconds=300`` parameter.+_DEFAULT_POLL_INTERVAL_S = 300++# Hard floor / ceiling so a misconfigured ``connector_config`` can't DoS+# either the source API (too short) or render polling pointless (too long).+_MIN_POLL_INTERVAL_S = 30+_MAX_POLL_INTERVAL_S = 86400 # 24 hours+++def _coerce_poll_interval(connector_config: dict[str, Any]) -> int:+ """Pull ``poll_interval_seconds`` out of the config blob, with bounds."""+ raw = connector_config.get("poll_interval_seconds", _DEFAULT_POLL_INTERVAL_S)+ try:+ seconds = int(raw)+ except (TypeError, ValueError):+ return _DEFAULT_POLL_INTERVAL_S+ if seconds < _MIN_POLL_INTERVAL_S:+ return _MIN_POLL_INTERVAL_S+ if seconds > _MAX_POLL_INTERVAL_S:+ return _MAX_POLL_INTERVAL_S+ return seconds+++def _job_id(connector_id: uuid.UUID) -> str:+ """Stable APScheduler job_id for an instance."""+ return f"connector:{connector_id}"+++class ConnectorScheduler:+ """Owns the APScheduler instance and the DB-reload loop."""++ def __init__(+ self,+ *,+ engine: AsyncEngine | None = None,+ ingest_client: IngestClient | None = None,+ vault: CredentialVault | None = None,+ reload_interval_seconds: float = _DEFAULT_RELOAD_INTERVAL_S,+ ) -> None:+ # Lazily resolve everything so tests can inject fakes; production+ # callers just construct ``ConnectorScheduler()`` and rely on env.+ self._engine = engine+ self._ingest_client = ingest_client+ self._vault = vault+ self._reload_interval_s = reload_interval_seconds+ self._scheduler: AsyncIOScheduler | None = None+ # Cache of {connector_id_str: last_seen_config_signature} so reloads+ # can detect when an instance's poll interval / config changed and+ # need its job rescheduled, vs leaving an unchanged job alone.+ self._known_signatures: dict[str, str] = {}++ # ------------------------------------------------------------------ lifecycle++ async def start(self) -> None:+ """Construct the scheduler and kick off the reload loop.++ Safe to call once per process. Calling twice is a programming error;+ we raise rather than silently start a second scheduler.+ """+ if self._scheduler is not None:+ raise RuntimeError("ConnectorScheduler.start() called twice")++ # Resolve dependencies now that we're inside an event loop.+ if self._engine is None:+ self._engine = get_engine()+ if self._ingest_client is None:+ self._ingest_client = IngestClient.from_env()+ if self._vault is None:+ try:+ self._vault = get_vault()+ except CredentialVaultError as exc:+ # If the vault isn't configured we *cannot* decrypt creds, so+ # polling can never succeed. Log and bail rather than silently+ # poll-and-fail forever.+ logger.error("connector.scheduler.vault_unavailable: %s", exc)+ raise++ self._scheduler = AsyncIOScheduler(timezone="UTC")+ self._scheduler.start()++ # Fire one reload immediately so we don't wait+ # ``reload_interval`` seconds for the first poll cycle.+ await self.reload_jobs()++ # Then schedule the periodic reload. ``id`` is fixed so we can't+ # accidentally double-schedule.+ self._scheduler.add_job(+ self.reload_jobs,+ "interval",+ seconds=self._reload_interval_s,+ id="_reload_loop",+ replace_existing=True,+ max_instances=1,+ coalesce=True,+ )+ logger.info(+ "connector.scheduler.started reload_interval=%ss",+ self._reload_interval_s,+ )++ async def stop(self) -> None:+ """Stop the scheduler and release resources."""+ if self._scheduler is not None:+ # ``wait=False`` so we don't block shutdown waiting for an+ # in-flight poll to finish; the next start picks up where we+ # left off (record_poll_success has already been written for+ # successful polls before this point).+ self._scheduler.shutdown(wait=False)+ self._scheduler = None+ if self._ingest_client is not None:+ await self._ingest_client.aclose()+ self._known_signatures.clear()+ logger.info("connector.scheduler.stopped")++ # ------------------------------------------------------------------ reload++ async def reload_jobs(self) -> None:+ """Sync APScheduler jobs with the enabled-connectors set in Postgres.++ Three actions per reload tick:++ * **add**: instances we haven't scheduled yet+ * **update**: instances whose config signature changed (e.g. the user+ tightened the poll interval)+ * **remove**: instances we previously scheduled but are no longer+ enabled / no longer present+ """+ if self._scheduler is None or self._engine is None:+ # ``stop()`` was called between ticks, or ``start()`` was never+ # called. Either way: nothing to do.+ return++ try:+ async with self._engine.begin() as conn:+ instances = await fetch_enabled_connectors(conn)+ except Exception:+ # DB transient errors shouldn't take the scheduler down. Log and+ # try again at the next reload tick.+ logger.exception("connector.scheduler.reload_failed")+ return++ seen: set[str] = set()+ for inst in instances:+ cid = str(inst.id)+ seen.add(cid)+ sig = self._signature(inst)+ existing = self._known_signatures.get(cid)+ if existing == sig:+ # Unchanged — leave the existing job alone.+ continue++ interval = _coerce_poll_interval(inst.connector_config)+ self._scheduler.add_job(+ self._poll_one,+ "interval",+ seconds=interval,+ id=_job_id(inst.id),+ replace_existing=True,+ max_instances=1,+ # Don't pile up missed polls if the source was slow/dead.+ coalesce=True,+ # Spread first-fire by seconds-since-epoch hash so 100+ # connectors don't all stampede at the same instant.+ next_run_time=None,+ kwargs={"connector_id": inst.id},+ )+ self._known_signatures[cid] = sig+ logger.info(+ "connector.scheduler.scheduled id=%s type=%s interval=%ss",+ cid,+ inst.connector_type,+ interval,+ )++ # Remove jobs for instances that disappeared from the enabled set+ # (deleted, disabled, or moved to another tenant we don't poll).+ for cid in list(self._known_signatures.keys()):+ if cid in seen:+ continue+ try:+ # uuid.UUID round-trip protects against malformed cache keys.+ self._scheduler.remove_job(_job_id(uuid.UUID(cid)))+ except Exception: # pragma: no cover - APScheduler raises JobLookupError+ pass+ self._known_signatures.pop(cid, None)+ logger.info("connector.scheduler.unscheduled id=%s", cid)++ @staticmethod+ def _signature(inst: ConnectorInstance) -> str:+ """Cheap fingerprint for change detection.++ We don't include ``auth_config`` because the auth blob is+ encrypted and we don't want to decrypt it just to check whether a+ secret rotated — every poll cycle reads fresh creds anyway.+ Including connector_type catches the (rare) case where a row was+ re-typed; including the poll interval catches the common case+ where the user adjusts cadence.+ """+ interval = _coerce_poll_interval(inst.connector_config)+ return f"{inst.connector_type}|{interval}"++ # ------------------------------------------------------------------ polling++ async def _poll_one(self, *, connector_id: uuid.UUID) -> None:+ """Single-connector poll cycle.++ Always wraps the entire body in try/except so a misbehaving+ connector class can't take down the scheduler. Records success or+ failure on the row, then returns.+ """+ if self._engine is None or self._ingest_client is None or self._vault is None:+ logger.warning(+ "connector.scheduler.poll_skipped id=%s reason=scheduler_not_ready",+ connector_id,+ )+ return++ # Re-fetch the row at poll time rather than relying on a cached+ # copy from the reload tick — credentials may have been rotated+ # since the last reload.+ try:+ async with self._engine.begin() as conn:+ instances = await fetch_enabled_connectors(conn)+ except Exception:+ logger.exception(+ "connector.scheduler.poll_load_failed id=%s",+ connector_id,+ )+ return++ target: ConnectorInstance | None = next(+ (i for i in instances if i.id == connector_id), None+ )+ if target is None:+ # Instance was deleted / disabled between reload and poll.+ # Reload will clean up the job on its next tick.+ logger.info(+ "connector.scheduler.poll_skipped id=%s reason=not_enabled",+ connector_id,+ )+ return++ connector_class = CONNECTOR_REGISTRY.get(target.connector_type)+ if connector_class is None:+ # Catalog drift — the API service stored a connector_type this+ # build doesn't know about. Mark unhealthy and bail; the human+ # operator will see the bad type in the UI.+ logger.error(+ "connector.scheduler.unknown_type id=%s type=%s",+ connector_id,+ target.connector_type,+ )+ await self._record_failure(target.id)+ return++ # Decrypt creds and instantiate. We do this inside the try block+ # so InvalidToken / TypeError / etc all funnel into the same+ # failure recorder.+ try:+ auth = self._vault.decrypt_dict(target.auth_config or {})+ except CredentialVaultError as exc:+ logger.error(+ "connector.scheduler.decrypt_failed id=%s err=%s",+ connector_id,+ exc,+ )+ await self._record_failure(target.id)+ return++ # Filter out the scheduler-only knobs from connector_config before+ # passing to the constructor — connectors don't accept+ # ``poll_interval_seconds`` as a kwarg.+ runtime_config = {+ k: v+ for k, v in (target.connector_config or {}).items()+ if k not in {"poll_interval_seconds"}+ }+ kwargs = {**auth, **runtime_config}++ started = datetime.now(UTC)+ try:+ connector = connector_class(**kwargs)+ except TypeError as exc:+ logger.error(+ "connector.scheduler.bad_config id=%s type=%s err=%s",+ connector_id,+ target.connector_type,+ exc,+ )+ await self._record_failure(target.id)+ return++ # The schema lives in connector_config; use it for fetch lookback.+ # We deliberately default to the same poll interval, so a 5-min+ # poll fetches the last 5 min of events. This keeps the source+ # query window aligned with the scheduler cadence.+ since_seconds = _coerce_poll_interval(target.connector_config)+ try:+ raw_events = await connector.fetch_alerts(since_seconds=since_seconds)+ except Exception:+ logger.exception(+ "connector.scheduler.fetch_failed id=%s type=%s",+ connector_id,+ target.connector_type,+ )+ await self._record_failure(target.id)+ return++ # Normalize defensively: connectors *should* normalize themselves+ # in fetch_alerts, but we double-tap so a connector that returns+ # raw events still produces consistent shapes downstream.+ normalized = [_safe_normalize(connector, e) for e in raw_events]++ try:+ result = await self._ingest_client.push_events(+ tenant_id=target.tenant_id,+ connector_id=target.id,+ connector_type=target.connector_type,+ events=normalized,+ )+ except IngestClientError:+ logger.exception(+ "connector.scheduler.ingest_failed id=%s events=%d",+ connector_id,+ len(normalized),+ )+ await self._record_failure(target.id)+ return++ accepted = int(result.get("accepted", 0) or 0)+ elapsed_ms = int((datetime.now(UTC) - started).total_seconds() * 1000)+ await self._record_success(target.id, events_added=accepted)+ logger.info(+ "connector.scheduler.poll_complete id=%s type=%s accepted=%d rejected=%d elapsed_ms=%d",+ connector_id,+ target.connector_type,+ accepted,+ int(result.get("rejected", 0) or 0),+ elapsed_ms,+ )++ # ------------------------------------------------------------------ db helpers++ async def _record_success(self, connector_id: uuid.UUID, *, events_added: int) -> None:+ if self._engine is None: # pragma: no cover+ return+ try:+ async with self._engine.begin() as conn:+ await record_poll_success(conn, connector_id, events_added=events_added)+ except Exception: # pragma: no cover+ logger.exception(+ "connector.scheduler.record_success_failed id=%s",+ connector_id,+ )++ async def _record_failure(self, connector_id: uuid.UUID) -> None:+ if self._engine is None: # pragma: no cover+ return+ try:+ async with self._engine.begin() as conn:+ await record_poll_failure(conn, connector_id)+ except Exception: # pragma: no cover+ logger.exception(+ "connector.scheduler.record_failure_failed id=%s",+ connector_id,+ )+++def _safe_normalize(connector: Any, event: dict[str, Any]) -> dict[str, Any]:+ """Run ``connector.normalize`` but never raise out of the loop."""+ try:+ out = connector.normalize(event)+ except Exception:+ logger.exception(+ "connector.scheduler.normalize_failed connector=%s",+ getattr(connector, "connector_id", "?"),+ )+ return event+ if not isinstance(out, dict):+ return event+ return out+++# ---------------------------------------------------------------------------+# Lifespan integration helper+#+# ``app/main.py`` imports ``scheduler_lifespan`` and uses it as the+# FastAPI ``lifespan=`` argument so the scheduler starts on app boot and+# stops cleanly on SIGTERM.+# ---------------------------------------------------------------------------+++_SCHEDULER_DISABLED_ENV = "AISOC_CONNECTORS_DISABLE_SCHEDULER"+++def scheduler_disabled() -> bool:+ """Allow tests / one-shot CLI invocations to skip the scheduler entirely."""+ return os.getenv(_SCHEDULER_DISABLED_ENV, "").lower() in {"1", "true", "yes"}+++__all__ = [+ "ConnectorScheduler",+ "scheduler_disabled",+]diff --git a/services/connectors/app/security/__init__.py b/services/connectors/app/security/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/services/connectors/app/security/credential_vault.py b/services/connectors/app/security/credential_vault.pynew file mode 100644index 000000000..c6f7b2f98--- /dev/null+++ b/services/connectors/app/security/credential_vault.py@@ -0,0 +1,160 @@+"""Vendored credential vault for the connectors microservice.++The canonical implementation lives in ``services/api/app/security/credential_vault.py``.+We vendor the **read path** here so the connectors service can decrypt the+``auth_config`` blobs the API service wrote, without standing up a back-channel+RPC for "please decrypt this secret for me" (which would just push the secret+across the wire in plaintext anyway and add an extra failure surface).++Both services agree on a single deployment-level secret, ``AISOC_CREDENTIAL_KEY``,+mounted via the same secrets store (Fly secrets / k8s secret / vault). As long+as the two services see the same key material, ciphertexts written by one+decrypt cleanly in the other.++Differences vs. the API copy:++* No dependency on ``app.core.config.Settings`` — env vars are read directly so+ this module stays trivial to test in isolation and adds no import cycle into+ the connectors service.+* No dev-mode "auto-generate ephemeral key" behaviour. If the API service+ generated an ephemeral key in development, this service won't have it; that's+ actually fine because in dev we generally only test connector classes with+ brand-new credentials and never reach the decrypt path.+* Encryption is still implemented (kept symmetric with the API copy) so a+ future re-encrypt maintenance job could run from either side.+"""++from __future__ import annotations++import logging+import os+from collections.abc import Mapping+from threading import Lock+from typing import Any, Final++from cryptography.fernet import Fernet, InvalidToken, MultiFernet++logger = logging.getLogger("aisoc.connectors.credential_vault")++# Tag every value we write so we can distinguish ciphertext from legacy+# plaintext on the decrypt path. Must match the API service's prefix byte-for-byte.+_CIPHER_PREFIX: Final[str] = "vault:v1:"+++class CredentialVaultError(RuntimeError):+ """Raised when the vault cannot encrypt/decrypt safely."""+++def _split_keys(raw: str) -> list[bytes]:+ return [k.strip().encode("ascii") for k in raw.split(",") if k.strip()]+++class CredentialVault:+ """Encrypt and decrypt connector credentials.++ See the canonical docstring in+ ``services/api/app/security/credential_vault.py`` for the full design+ rationale (Fernet, MultiFernet, leaf-level encryption, no key derivation).+ """++ def __init__(self, primary_key: bytes, historical_keys: list[bytes] | None = None) -> None:+ if not primary_key:+ raise CredentialVaultError("CredentialVault requires a non-empty primary key")+ try:+ primary = Fernet(primary_key)+ except (TypeError, ValueError) as exc:+ raise CredentialVaultError(f"AISOC_CREDENTIAL_KEY is not a valid Fernet key: {exc}") from exc++ keyring: list[Fernet] = [primary]+ for k in historical_keys or []:+ try:+ keyring.append(Fernet(k))+ except (TypeError, ValueError) as exc:+ logger.warning("ignoring invalid rotation key: %s", exc)++ self._fernet = MultiFernet(keyring) if len(keyring) > 1 else primary++ def encrypt(self, value: str) -> str:+ if not isinstance(value, str):+ raise CredentialVaultError(f"vault.encrypt expects str, got {type(value).__name__}")+ if value.startswith(_CIPHER_PREFIX):+ return value+ token = self._fernet.encrypt(value.encode("utf-8")).decode("ascii")+ return f"{_CIPHER_PREFIX}{token}"++ def decrypt(self, value: str) -> str:+ if not isinstance(value, str):+ raise CredentialVaultError(f"vault.decrypt expects str, got {type(value).__name__}")+ if not value.startswith(_CIPHER_PREFIX):+ return value+ token = value[len(_CIPHER_PREFIX):].encode("ascii")+ try:+ return self._fernet.decrypt(token).decode("utf-8")+ except InvalidToken as exc:+ raise CredentialVaultError(+ "ciphertext failed integrity check — likely AISOC_CREDENTIAL_KEY mismatch between API and connectors services"+ ) from exc++ def encrypt_dict(self, payload: Mapping[str, Any], *, secret_keys: set[str] | None = None) -> dict[str, Any]:+ return self._walk(payload, encrypt=True, secret_keys=secret_keys)++ def decrypt_dict(self, payload: Mapping[str, Any], *, secret_keys: set[str] | None = None) -> dict[str, Any]:+ return self._walk(payload, encrypt=False, secret_keys=secret_keys)++ def _walk(+ self,+ payload: Any,+ *,+ encrypt: bool,+ secret_keys: set[str] | None,+ _key: str | None = None,+ ) -> Any:+ op = self.encrypt if encrypt else self.decrypt+ if isinstance(payload, Mapping):+ out: dict[str, Any] = {}+ for k, v in payload.items():+ if secret_keys is not None and k not in secret_keys and not isinstance(v, (Mapping, list)):+ out[k] = v+ else:+ out[k] = self._walk(v, encrypt=encrypt, secret_keys=secret_keys, _key=k)+ return out+ if isinstance(payload, list):+ return [self._walk(item, encrypt=encrypt, secret_keys=secret_keys, _key=_key) for item in payload]+ if isinstance(payload, str):+ return op(payload)+ return payload+++_vault_singleton: CredentialVault | None = None+_vault_lock = Lock()+++def get_vault() -> CredentialVault:+ """Return the process-wide vault, lazily constructed from env vars.++ Unlike the API copy this function deliberately raises if+ ``AISOC_CREDENTIAL_KEY`` is missing — the connectors microservice exists+ to *poll* sources using already-stored credentials, so running without a+ key is never the right behaviour, even in dev.+ """+ global _vault_singleton+ if _vault_singleton is not None:+ return _vault_singleton+ with _vault_lock:+ if _vault_singleton is not None: # pragma: no cover - racing init+ return _vault_singleton+ primary = (os.getenv("AISOC_CREDENTIAL_KEY") or "").strip().encode("ascii")+ if not primary:+ raise CredentialVaultError(+ "AISOC_CREDENTIAL_KEY is required for the connectors service to decrypt stored credentials. "+ "Mount the same key the API service uses."+ )+ rotation = _split_keys(os.getenv("AISOC_CREDENTIAL_KEY_ROTATION_FROM") or "")+ _vault_singleton = CredentialVault(primary, historical_keys=rotation)+ return _vault_singleton+++def reset_vault_for_tests() -> None:+ global _vault_singleton+ with _vault_lock:+ _vault_singleton = Nonediff --git a/services/connectors/pyproject.toml b/services/connectors/pyproject.tomlindex 659c27749..ba2245aca 100644--- a/services/connectors/pyproject.toml+++ b/services/connectors/pyproject.toml@@ -14,10 +14,23 @@ pydantic = "^2.5.0"pydantic-settings = "^2.1.0"structlog = "^24.1.0"boto3 = "^1.34.0"+# Scheduler + DB access for the connector polling loop. apscheduler runs+# in-process (one job per enabled connector instance); SQLAlchemy + asyncpg+# read the canonical connectors table that lives in the API service's+# Postgres so this microservice stays a stateless poller — no schema+# ownership, no migrations.+apscheduler = "^3.10.0"+sqlalchemy = "^2.0.0"+asyncpg = "^0.29.0"+# Cryptography is required to decrypt the auth_config blob written by the+# API service's CredentialVault. We share AISOC_CREDENTIAL_KEY across both+# services via the deployment's secret store.+cryptography = "^42.0.0"[tool.poetry.group.dev.dependencies]pytest = "^7.4.0"pytest-asyncio = "^0.23.0"+respx = "^0.20.0"[build-system]requires = ["poetry-core"]diff --git a/services/connectors/tests/__init__.py b/services/connectors/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/services/connectors/tests/test_scheduler.py b/services/connectors/tests/test_scheduler.pynew file mode 100644index 000000000..400f5f53a--- /dev/null+++ b/services/connectors/tests/test_scheduler.py@@ -0,0 +1,427 @@+"""Unit tests for the in-process connector scheduler.++We test the scheduler in isolation from APScheduler internals by directly+calling ``reload_jobs`` / ``_poll_one`` with fakes for the engine, vault,+ingest client, and connector class. The point of these tests is to+exercise the *control flow*:++* reload picks up new instances, reschedules changed ones, drops removed ones+* a successful poll decrypts creds, instantiates the connector, calls+ fetch_alerts, pushes to ingest, and records success+* every failure path flips the connector to unhealthy without raising++We do *not* test APScheduler itself — that has its own tests.+"""++from __future__ import annotations++import uuid+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest++from app.db.connector_repo import ConnectorInstance+from app.scheduler import ConnectorScheduler, _coerce_poll_interval+++def _make_instance(+ *,+ connector_type: str = "crowdstrike",+ is_enabled: bool = True,+ auth_config: dict[str, Any] | None = None,+ connector_config: dict[str, Any] | None = None,+) -> ConnectorInstance:+ return ConnectorInstance(+ id=uuid.uuid4(),+ tenant_id=uuid.uuid4(),+ name="test",+ connector_type=connector_type,+ is_enabled=is_enabled,+ auth_config=auth_config or {"client_id": "id", "client_secret": "secret"},+ connector_config=connector_config or {},+ health_status="healthy",+ last_sync=None,+ events_ingested=0,+ error_count=0,+ )+++# ---------------------------------------------------------------------------+# poll_interval coercion+# ---------------------------------------------------------------------------+++def test_poll_interval_default():+ assert _coerce_poll_interval({}) == 300+++def test_poll_interval_clamps_low():+ assert _coerce_poll_interval({"poll_interval_seconds": 5}) == 30+++def test_poll_interval_clamps_high():+ assert _coerce_poll_interval({"poll_interval_seconds": 999_999}) == 86400+++def test_poll_interval_invalid_falls_back():+ assert _coerce_poll_interval({"poll_interval_seconds": "not-a-number"}) == 300+++def test_poll_interval_passes_through_valid():+ assert _coerce_poll_interval({"poll_interval_seconds": 600}) == 600+++# ---------------------------------------------------------------------------+# _poll_one happy path+# ---------------------------------------------------------------------------+++async def test_poll_one_happy_path(monkeypatch):+ inst = _make_instance(+ connector_type="fake_connector",+ connector_config={"poll_interval_seconds": 60},+ )++ fake_connector = MagicMock()+ fake_connector.fetch_alerts = AsyncMock(return_value=[{"raw": "event"}])+ fake_connector.normalize = MagicMock(side_effect=lambda e: {"normalized": True, **e})++ fake_connector_class = MagicMock(return_value=fake_connector)++ monkeypatch.setattr(+ "app.scheduler.CONNECTOR_REGISTRY",+ {"fake_connector": fake_connector_class},+ )++ fake_engine = _FakeEngine([inst])+ fake_vault = _FakeVault()+ fake_ingest = _FakeIngestClient(accepted=1)++ scheduler = ConnectorScheduler(+ engine=fake_engine, ingest_client=fake_ingest, vault=fake_vault+ )+ # We don't call start() because we don't want APScheduler running for+ # this test — _poll_one operates on the injected fakes directly.++ await scheduler._poll_one(connector_id=inst.id)++ fake_connector_class.assert_called_once_with(+ client_id="id", client_secret="secret"+ )+ fake_connector.fetch_alerts.assert_awaited_once_with(since_seconds=60)+ assert fake_ingest.calls == 1+ pushed = fake_ingest.last_payload+ assert pushed["events"] == [{"normalized": True, "raw": "event"}]+ assert fake_engine.success_calls == [inst.id]+ assert fake_engine.failure_calls == []+++# ---------------------------------------------------------------------------+# Failure paths each flip the row to unhealthy without raising+# ---------------------------------------------------------------------------+++async def test_poll_one_unknown_connector_type(monkeypatch):+ inst = _make_instance(connector_type="not_in_registry")+ monkeypatch.setattr("app.scheduler.CONNECTOR_REGISTRY", {})++ scheduler = ConnectorScheduler(+ engine=_FakeEngine([inst]), ingest_client=_FakeIngestClient(), vault=_FakeVault()+ )+ await scheduler._poll_one(connector_id=inst.id)++ engine: _FakeEngine = scheduler._engine # type: ignore[assignment]+ assert engine.failure_calls == [inst.id]+++async def test_poll_one_decrypt_failure(monkeypatch):+ inst = _make_instance(connector_type="fake_connector")+ monkeypatch.setattr(+ "app.scheduler.CONNECTOR_REGISTRY", {"fake_connector": MagicMock()}+ )++ bad_vault = _FakeVault()+ bad_vault.fail = True++ scheduler = ConnectorScheduler(+ engine=_FakeEngine([inst]), ingest_client=_FakeIngestClient(), vault=bad_vault+ )+ await scheduler._poll_one(connector_id=inst.id)++ engine: _FakeEngine = scheduler._engine # type: ignore[assignment]+ assert engine.failure_calls == [inst.id]+++async def test_poll_one_constructor_typeerror(monkeypatch):+ inst = _make_instance(+ connector_type="fake_connector",+ auth_config={"unexpected_kwarg": "x"},+ )++ def bad_ctor(**_kwargs: Any) -> Any:+ raise TypeError("got unexpected keyword argument")++ monkeypatch.setattr(+ "app.scheduler.CONNECTOR_REGISTRY", {"fake_connector": bad_ctor}+ )++ scheduler = ConnectorScheduler(+ engine=_FakeEngine([inst]), ingest_client=_FakeIngestClient(), vault=_FakeVault()+ )+ await scheduler._poll_one(connector_id=inst.id)++ engine: _FakeEngine = scheduler._engine # type: ignore[assignment]+ assert engine.failure_calls == [inst.id]+++async def test_poll_one_fetch_raises(monkeypatch):+ inst = _make_instance(connector_type="fake_connector")+ fake_connector = MagicMock()+ fake_connector.fetch_alerts = AsyncMock(side_effect=RuntimeError("api boom"))+ monkeypatch.setattr(+ "app.scheduler.CONNECTOR_REGISTRY",+ {"fake_connector": MagicMock(return_value=fake_connector)},+ )++ scheduler = ConnectorScheduler(+ engine=_FakeEngine([inst]), ingest_client=_FakeIngestClient(), vault=_FakeVault()+ )+ await scheduler._poll_one(connector_id=inst.id)++ engine: _FakeEngine = scheduler._engine # type: ignore[assignment]+ assert engine.failure_calls == [inst.id]+++async def test_poll_one_ingest_raises(monkeypatch):+ from app.ingest_client import IngestClientError++ inst = _make_instance(connector_type="fake_connector")+ fake_connector = MagicMock()+ fake_connector.fetch_alerts = AsyncMock(return_value=[{"x": 1}])+ fake_connector.normalize = MagicMock(side_effect=lambda e: e)+ monkeypatch.setattr(+ "app.scheduler.CONNECTOR_REGISTRY",+ {"fake_connector": MagicMock(return_value=fake_connector)},+ )++ bad_ingest = _FakeIngestClient()+ bad_ingest.exc = IngestClientError("ingest down")++ scheduler = ConnectorScheduler(+ engine=_FakeEngine([inst]), ingest_client=bad_ingest, vault=_FakeVault()+ )+ await scheduler._poll_one(connector_id=inst.id)++ engine: _FakeEngine = scheduler._engine # type: ignore[assignment]+ assert engine.failure_calls == [inst.id]+++async def test_poll_one_instance_disappeared(monkeypatch):+ """If the connector row was deleted between reload and poll, we no-op."""+ monkeypatch.setattr("app.scheduler.CONNECTOR_REGISTRY", {})+ scheduler = ConnectorScheduler(+ engine=_FakeEngine([]), ingest_client=_FakeIngestClient(), vault=_FakeVault()+ )+ await scheduler._poll_one(connector_id=uuid.uuid4())++ engine: _FakeEngine = scheduler._engine # type: ignore[assignment]+ assert engine.success_calls == []+ assert engine.failure_calls == []+++# ---------------------------------------------------------------------------+# reload_jobs sync logic+# ---------------------------------------------------------------------------+++async def test_reload_adds_and_drops_jobs(monkeypatch):+ monkeypatch.setattr("app.scheduler.CONNECTOR_REGISTRY", {})+ a = _make_instance()+ b = _make_instance()+ engine = _FakeEngine([a, b])+ scheduler = ConnectorScheduler(+ engine=engine, ingest_client=_FakeIngestClient(), vault=_FakeVault()+ )+ fake_aps = _FakeAPScheduler()+ scheduler._scheduler = fake_aps++ await scheduler.reload_jobs()+ added_first = {kw["id"] for kw in fake_aps.added}+ assert f"connector:{a.id}" in added_first+ assert f"connector:{b.id}" in added_first++ # Drop b.+ engine.instances = [a]+ await scheduler.reload_jobs()+ assert f"connector:{b.id}" in fake_aps.removed+++async def test_reload_skips_unchanged_jobs(monkeypatch):+ monkeypatch.setattr("app.scheduler.CONNECTOR_REGISTRY", {})+ inst = _make_instance(connector_config={"poll_interval_seconds": 120})+ engine = _FakeEngine([inst])+ scheduler = ConnectorScheduler(+ engine=engine, ingest_client=_FakeIngestClient(), vault=_FakeVault()+ )+ fake_aps = _FakeAPScheduler()+ scheduler._scheduler = fake_aps++ await scheduler.reload_jobs()+ assert len(fake_aps.added) == 1++ # Second reload with no changes — no add_job should fire again.+ await scheduler.reload_jobs()+ assert len(fake_aps.added) == 1+++async def test_reload_reschedules_when_interval_changes(monkeypatch):+ monkeypatch.setattr("app.scheduler.CONNECTOR_REGISTRY", {})+ inst = _make_instance(connector_config={"poll_interval_seconds": 120})+ engine = _FakeEngine([inst])+ scheduler = ConnectorScheduler(+ engine=engine, ingest_client=_FakeIngestClient(), vault=_FakeVault()+ )+ fake_aps = _FakeAPScheduler()+ scheduler._scheduler = fake_aps++ await scheduler.reload_jobs()+ assert len(fake_aps.added) == 1+ first_seconds = fake_aps.added[0]["seconds"]+ assert first_seconds == 120++ # Mutate the instance to bump the interval; reload should add_job again+ # (with replace_existing=True, which APScheduler treats as reschedule).+ inst.connector_config = {"poll_interval_seconds": 600}+ await scheduler.reload_jobs()+ assert len(fake_aps.added) == 2+ assert fake_aps.added[-1]["seconds"] == 600+++# ---------------------------------------------------------------------------+# Test fakes+# ---------------------------------------------------------------------------+++class _FakeAPScheduler:+ def __init__(self) -> None:+ self.added: list[dict[str, Any]] = []+ self.removed: list[str] = []++ def add_job(+ self,+ func: Any,+ trigger: str,+ *,+ seconds: int,+ id: str,+ replace_existing: bool,+ max_instances: int,+ coalesce: bool,+ next_run_time: Any = None,+ kwargs: dict[str, Any] | None = None,+ ) -> None:+ self.added.append(+ {+ "func": func,+ "trigger": trigger,+ "seconds": seconds,+ "id": id,+ "kwargs": kwargs or {},+ }+ )++ def remove_job(self, job_id: str) -> None:+ self.removed.append(job_id)+++class _FakeAsyncContextManager:+ """Minimal stand-in for ``async with engine.begin() as conn``."""++ def __init__(self, conn: Any) -> None:+ self._conn = conn++ async def __aenter__(self) -> Any:+ return self._conn++ async def __aexit__(self, *_args: Any) -> None:+ return None+++class _FakeEngine:+ """Async engine fake that records record_poll_success/failure calls."""++ def __init__(self, instances: list[ConnectorInstance]) -> None:+ self.instances = list(instances)+ self.success_calls: list[uuid.UUID] = []+ self.failure_calls: list[uuid.UUID] = []++ def begin(self) -> _FakeAsyncContextManager:+ return _FakeAsyncContextManager(_FakeConnection(self))+++class _FakeConnection:+ def __init__(self, engine: _FakeEngine) -> None:+ self.engine = engine+++class _FakeVault:+ def __init__(self) -> None:+ self.fail = False++ def decrypt_dict(self, payload: dict[str, Any]) -> dict[str, Any]:+ if self.fail:+ from app.security.credential_vault import CredentialVaultError++ raise CredentialVaultError("decrypt failed")+ return dict(payload)+++class _FakeIngestClient:+ def __init__(self, accepted: int = 0) -> None:+ self.calls = 0+ self.accepted = accepted+ self.last_payload: dict[str, Any] | None = None+ self.exc: BaseException | None = None++ async def push_events(self, **kwargs: Any) -> dict[str, Any]:+ self.calls += 1+ self.last_payload = kwargs+ if self.exc is not None:+ raise self.exc+ return {"accepted": self.accepted, "rejected": 0}++ async def aclose(self) -> None:+ return None+++# Patch the connector_repo db functions to operate on the fake engine+# attached to each connection. We do this once at module import via a+# pytest fixture autouse=True so individual tests don't need to+# remember.++[email protected](autouse=True)+def _patch_repo_calls(monkeypatch):+ async def fake_fetch_enabled_connectors(conn: Any) -> list[ConnectorInstance]:+ return list(conn.engine.instances)++ async def fake_record_poll_success(conn: Any, connector_id: uuid.UUID, *, events_added: int) -> None:+ conn.engine.success_calls.append(connector_id)++ async def fake_record_poll_failure(conn: Any, connector_id: uuid.UUID) -> None:+ conn.engine.failure_calls.append(connector_id)++ monkeypatch.setattr("app.scheduler.fetch_enabled_connectors", fake_fetch_enabled_connectors)+ monkeypatch.setattr("app.scheduler.record_poll_success", fake_record_poll_success)+ monkeypatch.setattr("app.scheduler.record_poll_failure", fake_record_poll_failure)diff --git a/services/connectors/tests/test_schemas.py b/services/connectors/tests/test_schemas.pynew file mode 100644index 000000000..13d455478--- /dev/null+++ b/services/connectors/tests/test_schemas.py@@ -0,0 +1,111 @@+"""+Contract tests for connector schemas.++These run against every connector registered in+``app.connectors.CONNECTOR_REGISTRY`` so a new connector cannot land+without a well-formed schema.+"""++from __future__ import annotations++import pytest++from app.connectors import CONNECTOR_REGISTRY, list_connector_schemas+from app.connectors.base import BaseConnector, ConnectorSchema, Field++ALLOWED_CATEGORIES = {"siem", "edr", "cloud", "iam", "saas", "audit", "vcs", "ndr"}+ALLOWED_FIELD_TYPES = {"string", "secret", "select", "textarea", "boolean", "number"}++[email protected](scope="module")+def registry() -> dict[str, type[BaseConnector]]:+ assert CONNECTOR_REGISTRY, "registry is empty — no connectors registered"+ return CONNECTOR_REGISTRY+++def test_registry_has_expected_baseline_connectors(registry):+ # These five existed before the housekeeping pass; protect against accidental removal.+ for required in (+ "crowdstrike",+ "splunk",+ "aws_security_hub",+ "okta",+ "microsoft_sentinel",+ ):+ assert required in registry, f"baseline connector '{required}' missing from registry"+++def test_every_connector_returns_a_schema(registry):+ for connector_id, cls in registry.items():+ schema = cls.schema()+ assert isinstance(schema, ConnectorSchema), (+ f"{cls.__name__}.schema() must return ConnectorSchema, got {type(schema).__name__}"+ )+ assert schema.connector_id == connector_id, (+ f"{cls.__name__}.schema().connector_id ({schema.connector_id!r}) "+ f"does not match registry key ({connector_id!r})"+ )+++def test_schema_metadata_is_well_formed(registry):+ for cls in registry.values():+ schema = cls.schema()+ assert schema.connector_name, f"{cls.__name__}: connector_name is empty"+ assert schema.category in ALLOWED_CATEGORIES, (+ f"{cls.__name__}: category {schema.category!r} not in {ALLOWED_CATEGORIES}"+ )+ assert schema.description, f"{cls.__name__}: description is empty"+ assert schema.fields, f"{cls.__name__}: at least one field is required"+++def test_schema_fields_are_well_formed(registry):+ for cls in registry.values():+ schema = cls.schema()+ seen_names: set[str] = set()+ for field in schema.fields:+ assert isinstance(field, Field)+ assert field.name, f"{cls.__name__}: field with empty name"+ assert field.name not in seen_names, (+ f"{cls.__name__}: duplicate field name {field.name!r}"+ )+ seen_names.add(field.name)+ assert field.label, f"{cls.__name__}.{field.name}: label is empty"+ assert field.type in ALLOWED_FIELD_TYPES, (+ f"{cls.__name__}.{field.name}: unknown field type {field.type!r}"+ )+ if field.type == "select":+ assert field.options, (+ f"{cls.__name__}.{field.name}: select fields require options"+ )+++def test_schemas_are_json_serialisable():+ import json++ payload = list_connector_schemas()+ # Will raise if any value can't be serialised.+ json.dumps(payload)+ assert isinstance(payload, list)+ assert all(isinstance(entry, dict) for entry in payload)+++def test_schema_to_dict_round_trips_required_keys(registry):+ for cls in registry.values():+ d = cls.schema().to_dict()+ for key in ("connector_id", "connector_name", "category", "description", "fields"):+ assert key in d, f"{cls.__name__}: schema dict missing '{key}'"+ assert isinstance(d["fields"], list)+++def test_secret_fields_are_marked_secret(registry):+ """Belt-and-braces check: anything whose name screams 'credential' should+ be a secret field so the wizard renders a masked input and the vault+ encrypts it."""+ suspicious = ("password", "secret", "token", "private_key", "api_key")+ for cls in registry.values():+ for field in cls.schema().fields:+ if any(s in field.name.lower() for s in suspicious):+ assert field.type == "secret", (+ f"{cls.__name__}.{field.name}: looks like a credential but type is "+ f"{field.type!r}, expected 'secret'"+ )