Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(connectors): schedule polling jobs active, not paused (#527) (#530)

Beenu Aroraea218ca8f216
diff --git a/services/connectors/app/scheduler.py b/services/connectors/app/scheduler.py
index 2fcb305fe..20d00b7f9 100644
--- a/services/connectors/app/scheduler.py
+++ b/services/connectors/app/scheduler.py
@@ -52,7 +52,7 @@ from __future__ import annotations
import logging
import os
import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from typing import Any
from apscheduler.schedulers.asyncio import AsyncIOScheduler
@@ -262,6 +262,17 @@ class ConnectorScheduler:
continue
interval = _coerce_poll_interval(inst.connector_config)
+ # APScheduler 3.x treats ``next_run_time=None`` as "add the job
+ # PAUSED": it registers but never fires and nothing resumes it, so
+ # enabled connectors silently never auto-poll (#527). Compute an
+ # explicit, timezone-aware first-run time instead. The offset is a
+ # STABLE per-connector jitter derived from the UUID (deterministic
+ # across reloads so a config change doesn't randomly re-phase the
+ # job) that spreads first-fire to avoid a stampede when many
+ # connectors load at once. It's bounded by the poll interval, and
+ # capped at 60s so long-interval connectors still start promptly.
+ jitter_window = max(1, min(interval, 60))
+ first_run = datetime.now(UTC) + timedelta(seconds=inst.id.int % jitter_window)
self._scheduler.add_job(
self._poll_one,
"interval",
@@ -271,9 +282,7 @@ class ConnectorScheduler:
max_instances=1,
# Don't pile up missed polls if the source was slow/dead.
coalesce=True,
- # Spread first-fire by seconds-since-epoch hash so 100
- # connectors don't all stampede at the same instant.
- next_run_time=None,
+ next_run_time=first_run,
kwargs={"connector_id": inst.id},
)
self._known_signatures[cid] = sig
@@ -297,6 +306,31 @@ class ConnectorScheduler:
self._known_signatures.pop(cid, None)
logger.info("connector.scheduler.unscheduled id=%s", cid)
+ def job_diagnostics(self) -> list[dict[str, Any]]:
+ """Return per-job scheduling diagnostics.
+
+ Each entry reports the job id and its ``next_run_time``; a null
+ ``next_run_time`` means APScheduler has the job PAUSED (the #527
+ failure mode), so operators / tests can assert every polling job is
+ actually active. Returns an empty list when the scheduler isn't
+ running.
+ """
+ if self._scheduler is None:
+ return []
+ out: list[dict[str, Any]] = []
+ for job in self._scheduler.get_jobs():
+ if not job.id.startswith("connector:"):
+ continue
+ nrt = getattr(job, "next_run_time", None)
+ out.append(
+ {
+ "job_id": job.id,
+ "next_run_time": nrt.isoformat() if nrt is not None else None,
+ "paused": nrt is None,
+ }
+ )
+ return out
+
@staticmethod
def _signature(inst: ConnectorInstance) -> str:
"""Cheap fingerprint for change detection.
diff --git a/services/connectors/tests/test_scheduler.py b/services/connectors/tests/test_scheduler.py
index 0db050eaf..05f745888 100644
--- a/services/connectors/tests/test_scheduler.py
+++ b/services/connectors/tests/test_scheduler.py
@@ -927,3 +927,76 @@ def _patch_repo_calls(monkeypatch):
monkeypatch.setattr("app.scheduler.record_poll_failure", fake_record_poll_failure)
monkeypatch.setattr("app.scheduler.record_schema_drift", fake_record_schema_drift)
monkeypatch.setattr("app.scheduler.record_backfill_run", fake_record_backfill_run)
+
+
+# ---------------------------------------------------------------------------
+# #527 — polling jobs must be registered ACTIVE, never paused.
+#
+# These use a REAL AsyncIOScheduler (not the fake) because the regression is a
+# pure APScheduler semantic: passing ``next_run_time=None`` adds the job in a
+# paused state, which the fake scheduler cannot reproduce.
+# ---------------------------------------------------------------------------
+
+
+async def test_reload_registers_active_not_paused_job():
+ """A newly enabled connector gets a non-null next_run_time (not paused)."""
+ from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
+ inst = _make_instance(connector_type="crowdstrike", connector_config={"poll_interval_seconds": 60})
+ sched = ConnectorScheduler(engine=_FakeEngine([inst]), ingest_client=_FakeIngestClient(), vault=_FakeVault())
+ # Keep the test hermetic — the job must never actually reach the network.
+ sched._poll_one = AsyncMock()
+
+ aps = AsyncIOScheduler(timezone="UTC")
+ aps.start(paused=False)
+ sched._scheduler = aps
+ try:
+ await sched.reload_jobs()
+
+ job = aps.get_job(f"connector:{inst.id}")
+ assert job is not None, "polling job was not scheduled"
+ # The #527 failure mode: a paused job has next_run_time is None.
+ assert job.next_run_time is not None, "polling job registered PAUSED (#527 regression)"
+
+ # First-run jitter stays bounded by the poll interval (capped at 60s).
+ delay = (job.next_run_time - datetime.now(UTC)).total_seconds()
+ assert -1 <= delay <= 61, f"first-run delay {delay}s outside jitter bound"
+
+ diag = sched.job_diagnostics()
+ assert diag, "job_diagnostics returned nothing"
+ assert all(d["paused"] is False for d in diag), f"a job is paused: {diag}"
+ finally:
+ aps.shutdown(wait=False)
+
+
+async def test_scheduled_job_executes_without_manual_resume():
+ """An active job fires on its own — no explicit ``resume()`` needed (#527)."""
+ import asyncio as _asyncio
+
+ from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
+ inst = _make_instance(connector_type="crowdstrike", connector_config={"poll_interval_seconds": 30})
+ # UUID int == 0 → jitter (id.int % window) == 0 → first_run == now, so the
+ # job is due immediately once the scheduler is running.
+ inst.id = uuid.UUID(int=0)
+ sched = ConnectorScheduler(engine=_FakeEngine([inst]), ingest_client=_FakeIngestClient(), vault=_FakeVault())
+
+ ran = _asyncio.Event()
+
+ async def _fake_poll(*, connector_id, **_kw): # noqa: ANN001
+ ran.set()
+
+ sched._poll_one = _fake_poll
+
+ aps = AsyncIOScheduler(timezone="UTC")
+ aps.start(paused=False)
+ sched._scheduler = aps
+ try:
+ await sched.reload_jobs()
+ await _asyncio.wait_for(ran.wait(), timeout=5)
+ finally:
+ aps.shutdown(wait=False)
+
+ assert ran.is_set(), "scheduled polling job did not execute on its own"