Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): prevent concurrent webhook trace overwrites (#19026)

Guofang.Tang8cbe55c896a3
diff --git a/internal/handler/agent_webhook.go b/internal/handler/agent_webhook.go
index bc1028d501..99f08962ed 100644
--- a/internal/handler/agent_webhook.go
+++ b/internal/handler/agent_webhook.go
@@ -64,6 +64,7 @@ import (
"errors"
"fmt"
"io"
+ "math"
"net/http"
"ragflow/internal/agent/canvas"
"ragflow/internal/common"
@@ -75,6 +76,7 @@ import (
"time"
"github.com/gin-gonic/gin"
+ goredis "github.com/redis/go-redis/v9"
"go.uber.org/zap"
"ragflow/internal/dao"
@@ -742,49 +744,30 @@ func mustJSON(v any) string {
return buf.String()
}
+const (
+ webhookTraceTTL = 10 * time.Minute
+ webhookTraceMaxRetries = 100
+)
+
// appendWebhookTrace appends a single RunEvent to the per-canvas trace
-// key in Redis.
-// The trace key is `webhook-trace-<agent_id>-logs` with a 600 s TTL.
-// Each event is recorded as {"ts": <float>, "event": <type>, ...}.
+// key in Redis. Each event is recorded as {"ts": <float>, "event": <type>, ...}.
func appendWebhookTrace(ctx context.Context, agentID string, startTs time.Time, ev canvas.RunEvent) {
rdb := rediscli.Get()
- if rdb == nil {
+ if rdb == nil || rdb.GetClient() == nil {
return
}
- appendWebhookTraceWithWriter(ctx, rdb, agentID, startTs, ev)
-}
-
-type webhookTraceWriter interface {
- Get(context.Context, string) (string, error)
- SetObj(context.Context, string, any, time.Duration) bool
+ if err := appendWebhookTraceWithClient(ctx, rdb.GetClient(), agentID, startTs, ev); err != nil {
+ common.Warn("webhook trace append failed", zap.String("agent_id", agentID), zap.Error(err))
+ }
}
-func appendWebhookTraceWithWriter(
- ctx context.Context, writer webhookTraceWriter, agentID string, startTs time.Time, ev canvas.RunEvent,
-) {
- key := fmt.Sprintf("webhook-trace-%s-logs", agentID)
- raw, _ := writer.Get(ctx, key)
- obj := map[string]any{}
- if raw != "" {
- _ = json.Unmarshal([]byte(raw), &obj)
- }
- whs, _ := obj["webhooks"].(map[string]any)
- if whs == nil {
- whs = map[string]any{}
- obj["webhooks"] = whs
- }
- entryKey := strconv.FormatFloat(float64(startTs.UnixNano())/1e9, 'f', -1, 64)
- entry, _ := whs[entryKey].(map[string]any)
- if entry == nil {
- entry = map[string]any{
- "start_ts": float64(startTs.UnixNano()) / 1e9,
- "events": []any{},
- }
- whs[entryKey] = entry
- }
- events, _ := entry["events"].([]any)
+func appendWebhookTraceWithClient(
+ ctx context.Context, client *goredis.Client, agentID string, startTs time.Time, ev canvas.RunEvent,
+) error {
+ key := "webhook-trace-" + agentID + "-logs"
+ startSeconds := float64(startTs.UnixNano()) / 1e9
+ entryKey := strconv.FormatFloat(startSeconds, 'f', -1, 64)
eventRecord := map[string]any{
- "ts": float64(time.Now().UnixNano()) / 1e9,
"event": ev.Type,
}
if ev.Data != "" {
@@ -797,7 +780,59 @@ func appendWebhookTraceWithWriter(
eventRecord["task_id"] = ev.SessionID
eventRecord["session_id"] = ev.SessionID
}
- entry["events"] = append(events, eventRecord)
- writer.SetObj(ctx, key, obj, 600*time.Second)
+ for range webhookTraceMaxRetries {
+ err := client.Watch(ctx, func(tx *goredis.Tx) error {
+ store := webhookTraceStore{Webhooks: make(map[string]webhookTraceRun)}
+ raw, getErr := tx.Get(ctx, key).Bytes()
+ switch {
+ case getErr == nil:
+ if err := json.Unmarshal(raw, &store); err != nil {
+ return fmt.Errorf("decode webhook trace: %w", err)
+ }
+ if store.Webhooks == nil {
+ store.Webhooks = make(map[string]webhookTraceRun)
+ }
+ case errors.Is(getErr, goredis.Nil):
+ default:
+ return fmt.Errorf("read webhook trace: %w", getErr)
+ }
+
+ run, ok := store.Webhooks[entryKey]
+ if !ok {
+ run = webhookTraceRun{StartTS: startSeconds}
+ }
+ eventTimestamp := float64(time.Now().UnixNano()) / 1e9
+ latestTimestamp := float64(0)
+ for _, existingEvent := range run.Events {
+ latestTimestamp = max(latestTimestamp, webhookTraceEventTimestamp(existingEvent))
+ }
+ if eventTimestamp <= latestTimestamp {
+ eventTimestamp = math.Nextafter(latestTimestamp, math.Inf(1))
+ }
+ eventRecord["ts"] = eventTimestamp
+ run.Events = append(run.Events, eventRecord)
+ store.Webhooks[entryKey] = run
+
+ payload, err := json.Marshal(store)
+ if err != nil {
+ return fmt.Errorf("encode webhook trace: %w", err)
+ }
+ _, err = tx.TxPipelined(ctx, func(pipe goredis.Pipeliner) error {
+ pipe.Set(ctx, key, payload, webhookTraceTTL)
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("write webhook trace: %w", err)
+ }
+ return nil
+ }, key)
+ if err == nil {
+ return nil
+ }
+ if !errors.Is(err, goredis.TxFailedErr) {
+ return err
+ }
+ }
+ return fmt.Errorf("update webhook trace: transaction failed after %d retries", webhookTraceMaxRetries)
}
diff --git a/internal/handler/agent_webhook_trace.go b/internal/handler/agent_webhook_trace.go
index 6e573642e0..480f007099 100644
--- a/internal/handler/agent_webhook_trace.go
+++ b/internal/handler/agent_webhook_trace.go
@@ -39,7 +39,8 @@ type webhookTraceStore struct {
}
type webhookTraceRun struct {
- Events []map[string]any `json:"events"`
+ StartTS float64 `json:"start_ts"`
+ Events []map[string]any `json:"events"`
}
// parseWebhookSinceTS parses an optional finite timestamp cursor.
diff --git a/internal/handler/agent_webhook_trace_test.go b/internal/handler/agent_webhook_trace_test.go
index d2d019746d..9e58c24355 100644
--- a/internal/handler/agent_webhook_trace_test.go
+++ b/internal/handler/agent_webhook_trace_test.go
@@ -15,13 +15,13 @@
package handler
import (
- "context"
"encoding/json"
"errors"
"net/http/httptest"
"net/url"
"strconv"
"strings"
+ "sync"
"testing"
"time"
@@ -49,26 +49,6 @@ type webhookTraceResponse struct {
Message string `json:"message"`
}
-type miniredisWebhookTraceWriter struct {
- client *goredis.Client
-}
-
-func (w miniredisWebhookTraceWriter) Get(ctx context.Context, key string) (string, error) {
- value, err := w.client.Get(ctx, key).Result()
- if errors.Is(err, goredis.Nil) {
- return "", nil
- }
- return value, err
-}
-
-func (w miniredisWebhookTraceWriter) SetObj(ctx context.Context, key string, value any, ttl time.Duration) bool {
- payload, err := json.Marshal(value)
- if err != nil {
- return false
- }
- return w.client.Set(ctx, key, payload, ttl).Err() == nil
-}
-
// newWebhookTraceTestHandler wires ownership storage and miniredis for HTTP tests.
func newWebhookTraceTestHandler(t *testing.T) (*AgentHandler, *goredis.Client) {
t.Helper()
@@ -137,6 +117,7 @@ func seedWebhookTrace(t *testing.T, rdb *goredis.Client, canvasID string, webhoo
}
}
+// TestAppendWebhookTracePersistsReadableEvents covers sequential appends and polling.
func TestAppendWebhookTracePersistsReadableEvents(t *testing.T) {
mr, err := miniredis.Run()
if err != nil {
@@ -147,17 +128,20 @@ func TestAppendWebhookTracePersistsReadableEvents(t *testing.T) {
t.Cleanup(func() { _ = rdb.Close() })
ctx := t.Context()
- writer := miniredisWebhookTraceWriter{client: rdb}
start := time.Unix(1_700_000_000, 0)
- appendWebhookTraceWithWriter(ctx, writer, "c1", start, canvas.RunEvent{
+ if err = appendWebhookTraceWithClient(ctx, rdb, "c1", start, canvas.RunEvent{
Type: "message",
Data: `{"content":"hello"}`,
SessionID: "task-1",
- })
- appendWebhookTraceWithWriter(ctx, writer, "c1", start, canvas.RunEvent{
+ }); err != nil {
+ t.Fatalf("append message trace: %v", err)
+ }
+ if err = appendWebhookTraceWithClient(ctx, rdb, "c1", start, canvas.RunEvent{
Type: "finished",
Data: `{"success":true}`,
- })
+ }); err != nil {
+ t.Fatalf("append finished trace: %v", err)
+ }
const key = "webhook-trace-c1-logs"
raw, err := rdb.Get(ctx, key).Result()
@@ -179,11 +163,122 @@ func TestAppendWebhookTracePersistsReadableEvents(t *testing.T) {
if !poll.Finished {
t.Fatal("persisted trace should be finished")
}
- if ttl := mr.TTL(key); ttl != 600*time.Second {
+ if ttl := mr.TTL(key); ttl != webhookTraceTTL {
+ t.Fatalf("trace TTL = %s, want 10m0s", ttl)
+ }
+}
+
+// TestAppendWebhookTracePreservesConcurrentRuns covers overlapping runs on one agent key.
+func TestAppendWebhookTracePreservesConcurrentRuns(t *testing.T) {
+ mr, err := miniredis.Run()
+ if err != nil {
+ t.Fatalf("miniredis.Run: %v", err)
+ }
+ t.Cleanup(mr.Close)
+ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ const runCount = 32
+ ctx := t.Context()
+ errorsByRun := make(chan error, runCount)
+ var wg sync.WaitGroup
+ for index := range runCount {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ start := time.Unix(1_700_000_000, int64(index)*int64(time.Millisecond))
+ errorsByRun <- appendWebhookTraceWithClient(ctx, rdb, "c1", start, canvas.RunEvent{
+ Type: "message",
+ SessionID: "task-" + strconv.Itoa(index),
+ })
+ }()
+ }
+ wg.Wait()
+ close(errorsByRun)
+ for appendErr := range errorsByRun {
+ if appendErr != nil {
+ t.Fatalf("append concurrent trace: %v", appendErr)
+ }
+ }
+
+ const key = "webhook-trace-c1-logs"
+ raw, err := rdb.Get(ctx, key).Result()
+ if err != nil {
+ t.Fatalf("read concurrent trace: %v", err)
+ }
+ var persisted webhookTraceStore
+ if err = json.Unmarshal([]byte(raw), &persisted); err != nil {
+ t.Fatalf("decode concurrent trace: %v", err)
+ }
+ if got := len(persisted.Webhooks); got != runCount {
+ t.Fatalf("persisted runs = %d, want %d", got, runCount)
+ }
+ for runID, run := range persisted.Webhooks {
+ if len(run.Events) != 1 {
+ t.Errorf("run %s events = %d, want 1", runID, len(run.Events))
+ }
+ }
+ if ttl := mr.TTL(key); ttl != webhookTraceTTL {
t.Fatalf("trace TTL = %s, want 10m0s", ttl)
}
}
+// TestAppendWebhookTraceOrdersConcurrentEvents protects incremental polling cursors.
+func TestAppendWebhookTraceOrdersConcurrentEvents(t *testing.T) {
+ mr, err := miniredis.Run()
+ if err != nil {
+ t.Fatalf("miniredis.Run: %v", err)
+ }
+ t.Cleanup(mr.Close)
+ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})
+ t.Cleanup(func() { _ = rdb.Close() })
+
+ const eventCount = 32
+ ctx := t.Context()
+ start := time.Unix(1_700_000_000, 0)
+ errorsByEvent := make(chan error, eventCount)
+ var wg sync.WaitGroup
+ for index := range eventCount {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ errorsByEvent <- appendWebhookTraceWithClient(ctx, rdb, "c1", start, canvas.RunEvent{
+ Type: "message",
+ SessionID: "task-" + strconv.Itoa(index),
+ })
+ }()
+ }
+ wg.Wait()
+ close(errorsByEvent)
+ for appendErr := range errorsByEvent {
+ if appendErr != nil {
+ t.Fatalf("append concurrent event: %v", appendErr)
+ }
+ }
+
+ raw, err := rdb.Get(ctx, "webhook-trace-c1-logs").Result()
+ if err != nil {
+ t.Fatalf("read concurrent events: %v", err)
+ }
+ var persisted webhookTraceStore
+ if err = json.Unmarshal([]byte(raw), &persisted); err != nil {
+ t.Fatalf("decode concurrent events: %v", err)
+ }
+ runID := strconv.FormatFloat(float64(start.UnixNano())/1e9, 'f', -1, 64)
+ events := persisted.Webhooks[runID].Events
+ if len(events) != eventCount {
+ t.Fatalf("persisted events = %d, want %d", len(events), eventCount)
+ }
+ previousTimestamp := float64(0)
+ for index, event := range events {
+ timestamp := webhookTraceEventTimestamp(event)
+ if timestamp <= previousTimestamp {
+ t.Fatalf("event %d timestamp = %v, want greater than %v", index, timestamp, previousTimestamp)
+ }
+ previousTimestamp = timestamp
+ }
+}
+
// TestGetAgentWebhookLogsPollsTraceIncrementally covers the complete UI poll flow.
func TestGetAgentWebhookLogsPollsTraceIncrementally(t *testing.T) {
h, rdb := newWebhookTraceTestHandler(t)