RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): make Python webhook trace appends atomic (#19091)
Guofang.Tang984d8ab510d8
diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.pyindex 4194071198..faf506872a 100644--- a/api/apps/restful_apis/agent_api.py+++ b/api/apps/restful_apis/agent_api.py@@ -23,6 +23,7 @@ import inspectimport ipaddressimport jsonimport logging+import mathimport timefrom functools import partial, wrapsfrom typing import Set@@ -70,10 +71,51 @@ from common.ssrf_guard import assert_host_is_safefrom common.constants import RetCodefrom common.misc_utils import get_uuid, thread_pool_execfrom peewee import MySQLDatabase, PostgresqlDatabase+from valkey.exceptions import WatchError# Keeps strong references to fire-and-forget tasks so they are not GC'd before completion._background_tasks: Set[asyncio.Task] = set()+_WEBHOOK_TRACE_MAX_RETRIES = 3+_WEBHOOK_TRACE_TTL_SECONDS = 600+++def _append_webhook_trace(redis_client, agent_id: str, start_ts: float, event: dict, ttl: int = _WEBHOOK_TRACE_TTL_SECONDS) -> None:+ """Atomically append one event to a webhook trace."""+ key = f"webhook-trace-{agent_id}-logs"+ run_id = str(start_ts)++ for _ in range(_WEBHOOK_TRACE_MAX_RETRIES):+ with redis_client.pipeline() as pipeline:+ try:+ pipeline.watch(key)+ raw = pipeline.get(key)+ obj = json.loads(raw) if raw else {"webhooks": {}}+ webhooks = obj.setdefault("webhooks", {})+ run = webhooks.setdefault(run_id, {"start_ts": start_ts, "events": []})+ events = run.setdefault("events", [])++ event_ts = time.time()+ latest_ts = max(+ (stored.get("ts", 0) for stored in events if isinstance(stored, dict) and isinstance(stored.get("ts"), (int, float))),+ default=0,+ )+ if event_ts <= latest_ts:+ event_ts = math.nextafter(latest_ts, math.inf)++ record = dict(event)+ record["ts"] = event_ts+ events.append(record)++ pipeline.multi()+ pipeline.set(key, json.dumps(obj, ensure_ascii=False), ex=ttl)+ pipeline.execute()+ return+ except WatchError:+ continue++ raise RuntimeError(f"Failed to update webhook trace after {_WEBHOOK_TRACE_MAX_RETRIES} retries")+def _canvas_json_default(obj):"""Fallback serializer for canvas SSE events.@@ -2348,19 +2390,14 @@ async def _webhook_impl(agent_id: str, is_test: bool):execution_mode = webhook_cfg.get("execution_mode", "Immediately")response_cfg = webhook_cfg.get("response", {})- def append_webhook_trace(agent_id: str, start_ts: float, event: dict, ttl=600):+ def append_webhook_trace(agent_id: str, start_ts: float, event: dict, ttl: int = _WEBHOOK_TRACE_TTL_SECONDS) -> None:from rag.utils.redis_conn import REDIS_CONN- key = f"webhook-trace-{agent_id}-logs"-- raw = REDIS_CONN.get(key)- obj = json.loads(raw) if raw else {"webhooks": {}}-- ws = obj["webhooks"].setdefault(str(start_ts), {"start_ts": start_ts, "events": []})-- ws["events"].append({"ts": time.time(), **event})-- REDIS_CONN.set_obj(key, obj, ttl)+ try:+ _append_webhook_trace(REDIS_CONN.REDIS, agent_id, start_ts, event, ttl)+ except Exception:+ # Trace persistence is best-effort and must not fail the Agent run.+ logging.exception("Failed to append webhook trace")if execution_mode == "Immediately":status = response_cfg.get("status", 200)diff --git a/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py b/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.pyindex 1b558fb4e7..b556b172a4 100644--- a/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py+++ b/test/testcases/test_web_api/test_agent_app/test_agents_webhook_unit.py@@ -115,6 +115,10 @@ class _StubCanvas:async def get_files_async(self, desc):return {"files": desc}+ @staticmethod+ def validate_component_parameters(_dsl):+ return {}+def __str__(self):return "{}"@@ -137,6 +141,57 @@ class _StubRedisConn:return None+class _FakeRedisPipeline:+ def __init__(self, client):+ self.client = client+ self.pending = None++ def __enter__(self):+ return self++ def __exit__(self, *_args):+ return False++ def watch(self, _key):+ return None++ def get(self, key):+ return self.client.values.get(key)++ def multi(self):+ return None++ def set(self, key, value, *, ex):+ self.pending = (key, value, ex)++ def execute(self):+ self.client.execute_calls += 1+ if self.client.conflicts_remaining:+ if self.client.conflict_value is not None:+ key, conflict_value = self.client.conflict_value+ self.client.values[key] = conflict_value+ self.client.conflicts_remaining -= 1+ raise self.client.watch_error()++ key, value, ttl = self.pending+ self.client.values[key] = value+ self.client.ttls[key] = ttl+ return [True]+++class _FakeRedisClient:+ def __init__(self, *, conflict_value=None, conflicts_remaining=None, watch_error=RuntimeError):+ self.values = {}+ self.ttls = {}+ self.execute_calls = 0+ self.conflict_value = conflict_value+ self.conflicts_remaining = int(conflict_value is not None) if conflicts_remaining is None else conflicts_remaining+ self.watch_error = watch_error++ def pipeline(self):+ return _FakeRedisPipeline(self)++def _run(coro):return asyncio.run(coro)@@ -289,6 +344,15 @@ def _load_agents_app(monkeypatch, *, target="rest"):monkeypatch.setitem(sys.modules, "api.db.services.canvas_service", canvas_service_mod)services_pkg.canvas_service = canvas_service_mod+ compilation_template_group_service_mod = ModuleType("api.db.services.compilation_template_group_service")+ compilation_template_group_service_mod.CompilationTemplateGroupService = type(+ "_StubCompilationTemplateGroupService",+ (),+ {"list_saved": staticmethod(lambda *_args, **_kwargs: [])},+ )+ monkeypatch.setitem(sys.modules, "api.db.services.compilation_template_group_service", compilation_template_group_service_mod)+ services_pkg.compilation_template_group_service = compilation_template_group_service_mod+api_service_mod = ModuleType("api.db.services.api_service")class _StubAPI4ConversationService:@@ -420,9 +484,14 @@ def _load_agents_app(monkeypatch, *, target="rest"):# (it triggers heavy imports like quart, settings, DB connections).api_apps_pkg = ModuleType("api.apps")api_apps_pkg.__path__ = []+ api_apps_pkg.AUTH_API = "api"+ api_apps_pkg.AUTH_BETA = "beta"+ api_apps_pkg.AUTH_JWT = "jwt"api_apps_pkg.current_user = SimpleNamespace(id="tenant-1")- def _identity_decorator(func):+ def _identity_decorator(func=None, **_kwargs):+ if func is None:+ return lambda wrapped: wrappedreturn funcapi_apps_pkg.login_required = _identity_decorator@@ -494,6 +563,7 @@ def _load_agents_app(monkeypatch, *, target="rest"):module = importlib.util.module_from_spec(spec)module.manager = _DummyManager()spec.loader.exec_module(module)+ module.REDIS_CONN = redis_objreturn module@@ -1028,7 +1098,7 @@ def test_webhook_canvas_constructor_exception(monkeypatch):"request",_DummyRequest(headers={"Content-Type": "application/json"}, json_body={}),)- monkeypatch.setattr(module, "Canvas", lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("canvas init failed")))+ monkeypatch.setattr(sys.modules["agent.canvas"], "Canvas", lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("canvas init failed")))def fake_error_result(*, code, message):return SimpleNamespace(code=code, message=message)@@ -1352,19 +1422,71 @@ def test_webhook_immediate_response_status_and_template_validation(monkeypatch):@pytest.mark.p2-def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):+def test_append_webhook_trace_retries_conflicts_without_losing_events(monkeypatch):module = _load_agents_app(monkeypatch)+ start_ts = 101.0+ key = "webhook-trace-agent-1-logs"+ concurrent_value = json.dumps(+ {+ "webhooks": {+ str(start_ts): {+ "start_ts": start_ts,+ "events": [{"event": "first", "ts": 200.0}],+ },+ "102.0": {+ "start_ts": 102.0,+ "events": [{"event": "other-run", "ts": 201.0}],+ },+ }+ }+ )+ redis_client = _FakeRedisClient(+ conflict_value=(key, concurrent_value),+ watch_error=module.WatchError,+ )+ monkeypatch.setattr(module.time, "time", lambda: 100.0)- redis_store = {}+ module._append_webhook_trace(+ redis_client,+ "agent-1",+ start_ts,+ {"event": "second", "ts": 1.0},+ )- def redis_get(key):- return redis_store.get(key)+ trace = json.loads(redis_client.values[key])+ events = trace["webhooks"][str(start_ts)]["events"]+ assert [event["event"] for event in events] == ["first", "second"]+ assert events[1]["ts"] > events[0]["ts"]+ assert "102.0" in trace["webhooks"]+ assert redis_client.execute_calls == 2+ assert redis_client.ttls[key] == 600- def redis_set_obj(key, obj, _ttl):- redis_store[key] = json.dumps(obj)- monkeypatch.setattr(module.REDIS_CONN, "get", redis_get)- monkeypatch.setattr(module.REDIS_CONN, "set_obj", redis_set_obj)+def test_append_webhook_trace_stops_after_retry_budget(monkeypatch):+ module = _load_agents_app(monkeypatch)+ redis_client = _FakeRedisClient(+ conflicts_remaining=module._WEBHOOK_TRACE_MAX_RETRIES,+ watch_error=module.WatchError,+ )++ with pytest.raises(RuntimeError, match="after 3 retries"):+ module._append_webhook_trace(+ redis_client,+ "agent-1",+ 101.0,+ {"event": "message"},+ )++ assert redis_client.execute_calls == 3+ assert redis_client.values == {}+++def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):+ module = _load_agents_app(monkeypatch)+ redis_client = _FakeRedisClient()+ module.REDIS_CONN.REDIS = redis_clientupdate_calls = []monkeypatch.setattr(module.UserCanvasService, "update_by_id", lambda *_args, **_kwargs: update_calls.append(True))@@ -1384,7 +1506,7 @@ def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):def __str__(self):return "{}"- monkeypatch.setattr(module, "Canvas", _CanvasSuccess)+ monkeypatch.setattr(sys.modules["agent.canvas"], "Canvas", _CanvasSuccess)params = _default_webhook_params(security=_anonymous_security(), content_types="application/json")cvs = _make_webhook_cvs(module, params=params)@@ -1403,7 +1525,7 @@ def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):assert update_calls == [True]key = "webhook-trace-agent-1-logs"- trace_obj = json.loads(redis_store[key])+ trace_obj = json.loads(redis_client.values[key])ws = next(iter(trace_obj["webhooks"].values()))events = ws["events"]assert any(event.get("event") == "message" for event in events)@@ -1414,16 +1536,16 @@ def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):raise RuntimeError("run failed")yield {}- monkeypatch.setattr(module, "Canvas", _CanvasError)+ monkeypatch.setattr(sys.modules["agent.canvas"], "Canvas", _CanvasError)tasks.clear()- redis_store.clear()+ redis_client.values.clear()cvs = _make_webhook_cvs(module, params=params)monkeypatch.setattr(module.UserCanvasService, "query", lambda **_kwargs: [cvs])monkeypatch.setattr(module.UserCanvasService, "get_by_id", lambda _id, _cvs=cvs: (True, _cvs))res = _run(module.webhook_test(agent_id="agent-1"))assert res.status_code == 200_run(tasks.pop(0))- trace_obj = json.loads(redis_store[key])+ trace_obj = json.loads(redis_client.values[key])ws = next(iter(trace_obj["webhooks"].values()))events = ws["events"]assert any(event.get("event") == "error" for event in events)@@ -1431,8 +1553,7 @@ def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):log_messages = []monkeypatch.setattr(module.logging, "exception", lambda msg, *_args, **_kwargs: log_messages.append(str(msg)))- monkeypatch.setattr(module.REDIS_CONN, "get", lambda _key: "{")- monkeypatch.setattr(module.REDIS_CONN, "set_obj", lambda *_args, **_kwargs: None)+ redis_client.values[key] = "{"tasks.clear()cvs = _make_webhook_cvs(module, params=params)monkeypatch.setattr(module.UserCanvasService, "query", lambda **_kwargs: [cvs])@@ -1445,10 +1566,7 @@ def test_webhook_background_run_success_and_error_trace_paths(monkeypatch):@pytest.mark.p2def test_webhook_sse_success_and_exception_paths(monkeypatch):module = _load_agents_app(monkeypatch)-- redis_store = {}- monkeypatch.setattr(module.REDIS_CONN, "get", lambda key: redis_store.get(key))- monkeypatch.setattr(module.REDIS_CONN, "set_obj", lambda key, obj, _ttl: redis_store.__setitem__(key, json.dumps(obj)))+ module.REDIS_CONN.REDIS = _FakeRedisClient()params = _default_webhook_params(security=_anonymous_security(),@@ -1466,7 +1584,7 @@ def test_webhook_sse_success_and_exception_paths(monkeypatch):yield {"event": "message", "data": {"content": "Hello"}}yield {"event": "message_end", "data": {"status": "201"}}- monkeypatch.setattr(module, "Canvas", _CanvasSSESuccess)+ monkeypatch.setattr(sys.modules["agent.canvas"], "Canvas", _CanvasSSESuccess)monkeypatch.setattr(module,"request",@@ -1482,7 +1600,7 @@ def test_webhook_sse_success_and_exception_paths(monkeypatch):raise RuntimeError("sse failed")yield {}- monkeypatch.setattr(module, "Canvas", _CanvasSSEError)+ monkeypatch.setattr(sys.modules["agent.canvas"], "Canvas", _CanvasSSEError)monkeypatch.setattr(module,"request",