Skip to content

armgit / AiSOC

Public Imported from GitHub
0

Open-source AI Security Operations Center: alert fusion, LLM-agent triage, MITRE ATT&CK investigation, and a replayable decision ledger for every agent step. Self-hostable, runs with no API keys, MIT licensed. Ships an MCP server for Claude, Cursor and Continue.

fix(agents): retry failed scheduled hunts (#1567)

Shreya Jha412eee4fd617
diff --git a/services/agents/app/hunt/scheduler.py b/services/agents/app/hunt/scheduler.py
index a2c7f6c8a..b88c023fa 100644
--- a/services/agents/app/hunt/scheduler.py
+++ b/services/agents/app/hunt/scheduler.py
@@ -130,7 +130,9 @@ class HuntScheduler:
await self._run_hunt(hunt, events)
except Exception as exc: # noqa: BLE001
logger.warning("hunt.scheduler.run_failed", hunt_id=hunt.id, error=str(exc))
- finally:
+ else:
+ # Only a completed run advances the cadence. On failure, leave
+ # the hunt due so the next scheduler tick can retry it.
self._last_run[hunt.id] = now
async def _run_hunt(self, hunt: HuntDefinition, events: list[dict[str, Any]]) -> None:
diff --git a/services/agents/tests/test_hunt_scheduler.py b/services/agents/tests/test_hunt_scheduler.py
new file mode 100644
index 000000000..b404c0773
--- /dev/null
+++ b/services/agents/tests/test_hunt_scheduler.py
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+from app.hunt import scheduler as scheduler_module
+from app.hunt.loader import HuntDefinition, HuntSchedule
+from app.hunt.scheduler import HuntScheduler
+
+
+class _Corpus:
+ def __init__(self, hunt: HuntDefinition) -> None:
+ self._hunt = hunt
+
+ def list(self) -> list[HuntDefinition]:
+ return [self._hunt]
+
+
+async def test_failed_hunt_is_retried_on_next_tick(monkeypatch: pytest.MonkeyPatch) -> None:
+ hunt = HuntDefinition(
+ id="retry-me",
+ name="Retry me",
+ schedule=HuntSchedule(enabled=True, interval_minutes=60),
+ )
+ scheduler = HuntScheduler(corpus=_Corpus(hunt)) # type: ignore[arg-type]
+ times = iter((1_000.0, 1_030.0, 1_040.0))
+ monkeypatch.setattr(scheduler_module, "time", SimpleNamespace(time=lambda: next(times)))
+ monkeypatch.setattr(scheduler_module, "_load_telemetry", lambda _tenant_ref: [])
+
+ attempts = 0
+
+ async def fail_once(_hunt: HuntDefinition, _events: list[dict[str, Any]]) -> None:
+ nonlocal attempts
+ attempts += 1
+ if attempts == 1:
+ raise RuntimeError("temporary persistence failure")
+
+ monkeypatch.setattr(scheduler, "_run_hunt", fail_once)
+
+ await scheduler._tick()
+ await scheduler._tick()
+ await scheduler._tick()
+
+ assert attempts == 2
+ assert scheduler._last_run[hunt.id] == 1_030.0