Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): make Python webhook trace appends atomic (#19091)

Guofang.Tang984d8ab510d8
diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.py
index 4194071198..faf506872a 100644
--- a/api/apps/restful_apis/agent_api.py
+++ b/api/apps/restful_apis/agent_api.py
@@ -23,6 +23,7 @@ import inspect
import ipaddress
import json
import logging
+import math
import time
from functools import partial, wraps
from typing import Set
@@ -70,10 +71,51 @@ from common.ssrf_guard import assert_host_is_safe
from common.constants import RetCode
from common.misc_utils import get_uuid, thread_pool_exec
from peewee import MySQLDatabase, PostgresqlDatabase
+from valkey.exceptions import WatchError
# Keeps strong references to fire-and-forget tasks so they are not GC'd before completion.
_background_tasks: Set[asyncio.Task] = set()
+_WEBHOOK_TRACE_MAX_RETRIES = 3
+_WEBHOOK_TRACE_TTL_SECONDS = 600
+
+
+def _append_webhook_trace(redis_client, agent_id: str, start_ts: float, event: dict, ttl: int = _WEBHOOK_TRACE_TTL_SECONDS) -> None:
+ """Atomically append one event to a webhook trace."""
+ key = f"webhook-trace-{agent_id}-logs"
+ run_id = str(start_ts)
+
+ for _ in range(_WEBHOOK_TRACE_MAX_RETRIES):
+ with redis_client.pipeline() as pipeline:
+ try:
+ pipeline.watch(key)
+ raw = pipeline.get(key)
+ obj = json.loads(raw) if raw else {"webhooks": {}}
+ webhooks = obj.setdefault("webhooks", {})
+ run = webhooks.setdefault(run_id, {"start_ts": start_ts, "events": []})
+ events = run.setdefault("events", [])
+
+ event_ts = time.time()
+ latest_ts = max(
+ (stored.get("ts", 0) for stored in events if isinstance(stored, dict) and isinstance(stored.get("ts"), (int, float))),
+ default=0,
+ )
+ if event_ts <= latest_ts:
+ event_ts = math.nextafter(latest_ts, math.inf)
+
+ record = dict(event)
+ record["ts"] = event_ts
+ events.append(record)
+
+ pipeline.multi()
+ pipeline.set(key, json.dumps(obj, ensure_ascii=False), ex=ttl)
+ pipeline.execute()
+ return
+ except WatchError:
+ continue
+
+ raise RuntimeError(f"Failed to update webhook trace after {_WEBHOOK_TRACE_MAX_RETRIES} retries")
+
def _canvas_json_default(obj):
"""Fallback serializer for canvas SSE events.
@@ -2348,19 +2390,14 @@ async def _webhook_impl(agent_id: str, is_test: bool):
execution_mode = webhook_cfg.get("execution_mode", "Immediately")
response_cfg = webhook_cfg.get("response", {})
- def append_webhook_trace(agent_id: str, start_ts: float, event: dict, ttl=600):
+ def append_webhook_trace(agent_id: str, start_ts: float, event: dict, ttl: int = _WEBHOOK_TRACE_TTL_SECONDS) -> None:
from rag.utils.redis_conn import REDIS_CONN
- key = f"webhook-trace-{agent_id}-logs"
-
- raw = REDIS_CONN.get(key)
- obj = json.loads(raw) if raw else {"webhooks": {}}
-
- ws = obj["webhooks"].setdefault(str(start_ts), {"start_ts": start_ts, "events": []})
-
- ws["events"].append({"ts": time.time(), **event})
-
- REDIS_CONN.set_obj(key, obj, ttl)
+ try:
+ _append_webhook_trace(REDIS_CONN.REDIS, agent_id, start_ts, event, ttl)
+ except Exception:
+ # Trace persistence is best-effort and must not fail the Agent run.
+ logging.exception("Failed to append webhook trace")
if execution_mode == "Immediately":
status = response_cfg.get("status", 200)
diff --git a/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py b/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py
index 1b558fb4e7..b556b172a4 100644
--- a/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py
+++ b/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py
@@ -115,6 +115,10 @@ class _StubCanvas:
async def get_files_async(self, desc):
return {"files": desc}
+ @staticmethod
+ def validate_component_parameters(_dsl):
+ return {}
+
def __str__(self):
return "{}"
@@ -137,6 +141,57 @@ class _StubRedisConn:
return None
+class _FakeRedisPipeline:
+ def __init__(self, client):
+ self.client = client
+ self.pending = None
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def watch(self, _key):
+ return None
+
+ def get(self, key):
+ return self.client.values.get(key)
+
+ def multi(self):
+ return None
+
+ def set(self, key, value, *, ex):
+ self.pending = (key, value, ex)
+
+ def execute(self):
+ self.client.execute_calls += 1
+ if self.client.conflicts_remaining:
+ if self.client.conflict_value is not None:
+ key, conflict_value = self.client.conflict_value
+ self.client.values[key] = conflict_value
+ self.client.conflicts_remaining -= 1
+ raise self.client.watch_error()
+
+ key, value, ttl = self.pending
+ self.client.values[key] = value
+ self.client.ttls[key] = ttl
+ return [True]
+
+
+class _FakeRedisClient:
+ def __init__(self, *, conflict_value=None, conflicts_remaining=None, watch_error=RuntimeError):
+ self.values = {}
+ self.ttls = {}
+ self.execute_calls = 0
+ self.conflict_value = conflict_value
+ self.conflicts_remaining = int(conflict_value is not None) if conflicts_remaining is None else conflicts_remaining
+ self.watch_error = watch_error
+
+ def pipeline(self):
+ return _FakeRedisPipeline(self)
+
+
def _run(coro):
return asyncio.run(coro)
@@ -289,6 +344,15 @@ def _load_agents_app(monkeypatch, *, target="rest"):
monkeypatch.setitem(sys.modules, "api.db.services.canvas_service", canvas_service_mod)
services_pkg.canvas_service = canvas_service_mod
+ compilation_template_group_service_mod = ModuleType("api.db.services.compilation_template_group_service")
+ compilation_template_group_service_mod.CompilationTemplateGroupService = type(
+ "_StubCompilationTemplateGroupService",
+ (),
+ {"list_saved": staticmethod(lambda *_args, **_kwargs: [])},
+ )
+ monkeypatch.setitem(sys.modules, "api.db.services.compilation_template_group_service", compilation_template_group_service_mod)
+ services_pkg.compilation_template_group_service = compilation_template_group_service_mod
+
api_service_mod = ModuleType("api.db.services.api_service")
class _StubAPI4ConversationService:
@@ -420,9 +484,14 @@ def _load_agents_app(monkeypatch, *, target="rest"):
# (it triggers heavy imports like quart, settings, DB connections).
api_apps_pkg = ModuleType("api.apps")
api_apps_pkg.__path__ = []
+ api_apps_pkg.AUTH_API = "api"
+ api_apps_pkg.AUTH_BETA = "beta"
+ api_apps_pkg.AUTH_JWT = "jwt"
api_apps_pkg.current_user = SimpleNamespace(id="tenant-1")
- def _identity_decorator(func):
+ def _identity_decorator(func=None, **_kwargs):
+ if func is None:
+ return lambda wrapped: wrapped
return func
api_apps_pkg.login_required = _identity_decorator
@@ -494,6 +563,7 @@ def _load_agents_app(monkeypatch, *, target="rest"):
module = importlib.util.module_from_spec(spec)
module.manager = _DummyManager()
spec.loader.exec_module(module)
+ module.REDIS_CONN = redis_obj
return module
@@ -1028,7 +1098,7 @@ def test_webhook_canvas_constructor_exception(monkeypatch):
"request",
_DummyRequest(headers={"Content-Type": "application/json"}, json_body={}),
)
- monkeypatch.setattr(module, "Canvas", lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("canvas init failed")))
+ monkeypatch.setattr(sys.modules["agent.canvas"], "Canvas", lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("canvas init failed")))
def fake_error_result(*, code, message):
return SimpleNamespace(code=code, message=message)
@@ -1352,19 +1422,71 @@ def test_webhook_immediate_response_status_and_template_validation(monkeypatch):
@pytest.mark.p2
-def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):
+def test_append_webhook_trace_retries_conflicts_without_losing_events(monkeypatch):
module = _load_agents_app(monkeypatch)
+ start_ts = 101.0
+ key = "webhook-trace-agent-1-logs"
+ concurrent_value = json.dumps(
+ {
+ "webhooks": {
+ str(start_ts): {
+ "start_ts": start_ts,
+ "events": [{"event": "first", "ts": 200.0}],
+ },
+ "102.0": {
+ "start_ts": 102.0,
+ "events": [{"event": "other-run", "ts": 201.0}],
+ },
+ }
+ }
+ )
+ redis_client = _FakeRedisClient(
+ conflict_value=(key, concurrent_value),
+ watch_error=module.WatchError,
+ )
+ monkeypatch.setattr(module.time, "time", lambda: 100.0)
- redis_store = {}
+ module._append_webhook_trace(
+ redis_client,
+ "agent-1",
+ start_ts,
+ {"event": "second", "ts": 1.0},
+ )
- def redis_get(key):
- return redis_store.get(key)
+ trace = json.loads(redis_client.values[key])
+ events = trace["webhooks"][str(start_ts)]["events"]
+ assert [event["event"] for event in events] == ["first", "second"]
+ assert events[1]["ts"] > events[0]["ts"]
+ assert "102.0" in trace["webhooks"]
+ assert redis_client.execute_calls == 2
+ assert redis_client.ttls[key] == 600
- def redis_set_obj(key, obj, _ttl):
- redis_store[key] = json.dumps(obj)
- monkeypatch.setattr(module.REDIS_CONN, "get", redis_get)
- monkeypatch.setattr(module.REDIS_CONN, "set_obj", redis_set_obj)
+def test_append_webhook_trace_stops_after_retry_budget(monkeypatch):
+ module = _load_agents_app(monkeypatch)
+ redis_client = _FakeRedisClient(
+ conflicts_remaining=module._WEBHOOK_TRACE_MAX_RETRIES,
+ watch_error=module.WatchError,
+ )
+
+ with pytest.raises(RuntimeError, match="after 3 retries"):
+ module._append_webhook_trace(
+ redis_client,
+ "agent-1",
+ 101.0,
+ {"event": "message"},
+ )
+
+ assert redis_client.execute_calls == 3
+ assert redis_client.values == {}
+
+
+def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):
+ module = _load_agents_app(monkeypatch)
+ redis_client = _FakeRedisClient()
+ module.REDIS_CONN.REDIS = redis_client
update_calls = []
monkeypatch.setattr(module.UserCanvasService, "update_by_id", lambda *_args, **_kwargs: update_calls.append(True))
@@ -1384,7 +1506,7 @@ def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):
def __str__(self):
return "{}"
- monkeypatch.setattr(module, "Canvas", _CanvasSuccess)
+ monkeypatch.setattr(sys.modules["agent.canvas"], "Canvas", _CanvasSuccess)
params = _default_webhook_params(security=_anonymous_security(), content_types="application/json")
cvs = _make_webhook_cvs(module, params=params)
@@ -1403,7 +1525,7 @@ def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):
assert update_calls == [True]
key = "webhook-trace-agent-1-logs"
- trace_obj = json.loads(redis_store[key])
+ trace_obj = json.loads(redis_client.values[key])
ws = next(iter(trace_obj["webhooks"].values()))
events = ws["events"]
assert any(event.get("event") == "message" for event in events)
@@ -1414,16 +1536,16 @@ def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):
raise RuntimeError("run failed")
yield {}
- monkeypatch.setattr(module, "Canvas", _CanvasError)
+ monkeypatch.setattr(sys.modules["agent.canvas"], "Canvas", _CanvasError)
tasks.clear()
- redis_store.clear()
+ redis_client.values.clear()
cvs = _make_webhook_cvs(module, params=params)
monkeypatch.setattr(module.UserCanvasService, "query", lambda **_kwargs: [cvs])
monkeypatch.setattr(module.UserCanvasService, "get_by_id", lambda _id, _cvs=cvs: (True, _cvs))
res = _run(module.webhook_test(agent_id="agent-1"))
assert res.status_code == 200
_run(tasks.pop(0))
- trace_obj = json.loads(redis_store[key])
+ trace_obj = json.loads(redis_client.values[key])
ws = next(iter(trace_obj["webhooks"].values()))
events = ws["events"]
assert any(event.get("event") == "error" for event in events)
@@ -1431,8 +1553,7 @@ def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):
log_messages = []
monkeypatch.setattr(module.logging, "exception", lambda msg, *_args, **_kwargs: log_messages.append(str(msg)))
- monkeypatch.setattr(module.REDIS_CONN, "get", lambda _key: "{")
- monkeypatch.setattr(module.REDIS_CONN, "set_obj", lambda *_args, **_kwargs: None)
+ redis_client.values[key] = "{"
tasks.clear()
cvs = _make_webhook_cvs(module, params=params)
monkeypatch.setattr(module.UserCanvasService, "query", lambda **_kwargs: [cvs])
@@ -1445,10 +1566,7 @@ def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):
@pytest.mark.p2
def test_webhook_sse_success_and_exception_paths(monkeypatch):
module = _load_agents_app(monkeypatch)
-
- redis_store = {}
- monkeypatch.setattr(module.REDIS_CONN, "get", lambda key: redis_store.get(key))
- monkeypatch.setattr(module.REDIS_CONN, "set_obj", lambda key, obj, _ttl: redis_store.__setitem__(key, json.dumps(obj)))
+ module.REDIS_CONN.REDIS = _FakeRedisClient()
params = _default_webhook_params(
security=_anonymous_security(),
@@ -1466,7 +1584,7 @@ def test_webhook_sse_success_and_exception_paths(monkeypatch):
yield {"event": "message", "data": {"content": "Hello"}}
yield {"event": "message_end", "data": {"status": "201"}}
- monkeypatch.setattr(module, "Canvas", _CanvasSSESuccess)
+ monkeypatch.setattr(sys.modules["agent.canvas"], "Canvas", _CanvasSSESuccess)
monkeypatch.setattr(
module,
"request",
@@ -1482,7 +1600,7 @@ def test_webhook_sse_success_and_exception_paths(monkeypatch):
raise RuntimeError("sse failed")
yield {}
- monkeypatch.setattr(module, "Canvas", _CanvasSSEError)
+ monkeypatch.setattr(sys.modules["agent.canvas"], "Canvas", _CanvasSSEError)
monkeypatch.setattr(
module,
"request",