Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.
Python agentic-aiai-securityalert-triageblue-teamcybersecuritydetection-engineeringincident-responselanggraphllm-agentsmcpmitre-attackmodel-context-protocolpurple-teamsecurity-operationsself-hostedsiemsoarsocthreat-detectionthreat-intelligence
fix(connectors): schedule polling jobs active, not paused (#527) (#530)
Beenu Aroraea218ca8f216
diff --git a/services/connectors/app/scheduler.py b/services/connectors/app/scheduler.pyindex 2fcb305fe..20d00b7f9 100644--- a/services/connectors/app/scheduler.py+++ b/services/connectors/app/scheduler.py@@ -52,7 +52,7 @@ from __future__ import annotationsimport loggingimport osimport uuid-from datetime import UTC, datetime+from datetime import UTC, datetime, timedeltafrom typing import Anyfrom apscheduler.schedulers.asyncio import AsyncIOScheduler@@ -262,6 +262,17 @@ class ConnectorScheduler:continueinterval = _coerce_poll_interval(inst.connector_config)+ # APScheduler 3.x treats ``next_run_time=None`` as "add the job+ # PAUSED": it registers but never fires and nothing resumes it, so+ # enabled connectors silently never auto-poll (#527). Compute an+ # explicit, timezone-aware first-run time instead. The offset is a+ # STABLE per-connector jitter derived from the UUID (deterministic+ # across reloads so a config change doesn't randomly re-phase the+ # job) that spreads first-fire to avoid a stampede when many+ # connectors load at once. It's bounded by the poll interval, and+ # capped at 60s so long-interval connectors still start promptly.+ jitter_window = max(1, min(interval, 60))+ first_run = datetime.now(UTC) + timedelta(seconds=inst.id.int % jitter_window)self._scheduler.add_job(self._poll_one,"interval",@@ -271,9 +282,7 @@ class ConnectorScheduler:max_instances=1,# Don't pile up missed polls if the source was slow/dead.coalesce=True,- # Spread first-fire by seconds-since-epoch hash so 100- # connectors don't all stampede at the same instant.- next_run_time=None,+ next_run_time=first_run,kwargs={"connector_id": inst.id},)self._known_signatures[cid] = sig@@ -297,6 +306,31 @@ class ConnectorScheduler:self._known_signatures.pop(cid, None)logger.info("connector.scheduler.unscheduled id=%s", cid)+ def job_diagnostics(self) -> list[dict[str, Any]]:+ """Return per-job scheduling diagnostics.++ Each entry reports the job id and its ``next_run_time``; a null+ ``next_run_time`` means APScheduler has the job PAUSED (the #527+ failure mode), so operators / tests can assert every polling job is+ actually active. Returns an empty list when the scheduler isn't+ running.+ """+ if self._scheduler is None:+ return []+ out: list[dict[str, Any]] = []+ for job in self._scheduler.get_jobs():+ if not job.id.startswith("connector:"):+ continue+ nrt = getattr(job, "next_run_time", None)+ out.append(+ {+ "job_id": job.id,+ "next_run_time": nrt.isoformat() if nrt is not None else None,+ "paused": nrt is None,+ }+ )+ return out+@staticmethoddef _signature(inst: ConnectorInstance) -> str:"""Cheap fingerprint for change detection.diff --git a/services/connectors/tests/test_scheduler.py b/services/connectors/tests/test_scheduler.pyindex 0db050eaf..05f745888 100644--- a/services/connectors/tests/test_scheduler.py+++ b/services/connectors/tests/test_scheduler.py@@ -927,3 +927,76 @@ def _patch_repo_calls(monkeypatch):monkeypatch.setattr("app.scheduler.record_poll_failure", fake_record_poll_failure)monkeypatch.setattr("app.scheduler.record_schema_drift", fake_record_schema_drift)monkeypatch.setattr("app.scheduler.record_backfill_run", fake_record_backfill_run)+++# ---------------------------------------------------------------------------+# #527 — polling jobs must be registered ACTIVE, never paused.+#+# These use a REAL AsyncIOScheduler (not the fake) because the regression is a+# pure APScheduler semantic: passing ``next_run_time=None`` adds the job in a+# paused state, which the fake scheduler cannot reproduce.+# ---------------------------------------------------------------------------+++async def test_reload_registers_active_not_paused_job():+ """A newly enabled connector gets a non-null next_run_time (not paused)."""+ from apscheduler.schedulers.asyncio import AsyncIOScheduler++ inst = _make_instance(connector_type="crowdstrike", connector_config={"poll_interval_seconds": 60})+ sched = ConnectorScheduler(engine=_FakeEngine([inst]), ingest_client=_FakeIngestClient(), vault=_FakeVault())+ # Keep the test hermetic — the job must never actually reach the network.+ sched._poll_one = AsyncMock()++ aps = AsyncIOScheduler(timezone="UTC")+ aps.start(paused=False)+ sched._scheduler = aps+ try:+ await sched.reload_jobs()++ job = aps.get_job(f"connector:{inst.id}")+ assert job is not None, "polling job was not scheduled"+ # The #527 failure mode: a paused job has next_run_time is None.+ assert job.next_run_time is not None, "polling job registered PAUSED (#527 regression)"++ # First-run jitter stays bounded by the poll interval (capped at 60s).+ delay = (job.next_run_time - datetime.now(UTC)).total_seconds()+ assert -1 <= delay <= 61, f"first-run delay {delay}s outside jitter bound"++ diag = sched.job_diagnostics()+ assert diag, "job_diagnostics returned nothing"+ assert all(d["paused"] is False for d in diag), f"a job is paused: {diag}"+ finally:+ aps.shutdown(wait=False)+++async def test_scheduled_job_executes_without_manual_resume():+ """An active job fires on its own — no explicit ``resume()`` needed (#527)."""+ import asyncio as _asyncio++ from apscheduler.schedulers.asyncio import AsyncIOScheduler++ inst = _make_instance(connector_type="crowdstrike", connector_config={"poll_interval_seconds": 30})+ # UUID int == 0 → jitter (id.int % window) == 0 → first_run == now, so the+ # job is due immediately once the scheduler is running.+ inst.id = uuid.UUID(int=0)+ sched = ConnectorScheduler(engine=_FakeEngine([inst]), ingest_client=_FakeIngestClient(), vault=_FakeVault())++ ran = _asyncio.Event()++ async def _fake_poll(*, connector_id, **_kw): # noqa: ANN001+ ran.set()++ sched._poll_one = _fake_poll++ aps = AsyncIOScheduler(timezone="UTC")+ aps.start(paused=False)+ sched._scheduler = aps+ try:+ await sched.reload_jobs()+ await _asyncio.wait_for(ran.wait(), timeout=5)+ finally:+ aps.shutdown(wait=False)++ assert ran.is_set(), "scheduled polling job did not execute on its own"