RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): prevent concurrent webhook trace overwrites (#19026)
Guofang.Tang8cbe55c896a3
diff --git a/internal/handler/agent_webhook.go b/internal/handler/agent_webhook.goindex bc1028d501..99f08962ed 100644--- a/internal/handler/agent_webhook.go+++ b/internal/handler/agent_webhook.go@@ -64,6 +64,7 @@ import ("errors""fmt""io"+ "math""net/http""ragflow/internal/agent/canvas""ragflow/internal/common"@@ -75,6 +76,7 @@ import ("time""github.com/gin-gonic/gin"+ goredis "github.com/redis/go-redis/v9""go.uber.org/zap""ragflow/internal/dao"@@ -742,49 +744,30 @@ func mustJSON(v any) string {return buf.String()}+const (+ webhookTraceTTL = 10 * time.Minute+ webhookTraceMaxRetries = 100+)+// appendWebhookTrace appends a single RunEvent to the per-canvas trace-// key in Redis.-// The trace key is `webhook-trace-<agent_id>-logs` with a 600 s TTL.-// Each event is recorded as {"ts": <float>, "event": <type>, ...}.+// key in Redis. Each event is recorded as {"ts": <float>, "event": <type>, ...}.func appendWebhookTrace(ctx context.Context, agentID string, startTs time.Time, ev canvas.RunEvent) {rdb := rediscli.Get()- if rdb == nil {+ if rdb == nil || rdb.GetClient() == nil {return}- appendWebhookTraceWithWriter(ctx, rdb, agentID, startTs, ev)-}--type webhookTraceWriter interface {- Get(context.Context, string) (string, error)- SetObj(context.Context, string, any, time.Duration) bool+ if err := appendWebhookTraceWithClient(ctx, rdb.GetClient(), agentID, startTs, ev); err != nil {+ common.Warn("webhook trace append failed", zap.String("agent_id", agentID), zap.Error(err))+ }}-func appendWebhookTraceWithWriter(- ctx context.Context, writer webhookTraceWriter, agentID string, startTs time.Time, ev canvas.RunEvent,-) {- key := fmt.Sprintf("webhook-trace-%s-logs", agentID)- raw, _ := writer.Get(ctx, key)- obj := map[string]any{}- if raw != "" {- _ = json.Unmarshal([]byte(raw), &obj)- }- whs, _ := obj["webhooks"].(map[string]any)- if whs == nil {- whs = map[string]any{}- obj["webhooks"] = whs- }- entryKey := strconv.FormatFloat(float64(startTs.UnixNano())/1e9, 'f', -1, 64)- entry, _ := whs[entryKey].(map[string]any)- if entry == nil {- entry = map[string]any{- "start_ts": float64(startTs.UnixNano()) / 1e9,- "events": []any{},- }- whs[entryKey] = entry- }- events, _ := entry["events"].([]any)+func appendWebhookTraceWithClient(+ ctx context.Context, client *goredis.Client, agentID string, startTs time.Time, ev canvas.RunEvent,+) error {+ key := "webhook-trace-" + agentID + "-logs"+ startSeconds := float64(startTs.UnixNano()) / 1e9+ entryKey := strconv.FormatFloat(startSeconds, 'f', -1, 64)eventRecord := map[string]any{- "ts": float64(time.Now().UnixNano()) / 1e9,"event": ev.Type,}if ev.Data != "" {@@ -797,7 +780,59 @@ func appendWebhookTraceWithWriter(eventRecord["task_id"] = ev.SessionIDeventRecord["session_id"] = ev.SessionID}- entry["events"] = append(events, eventRecord)- writer.SetObj(ctx, key, obj, 600*time.Second)+ for range webhookTraceMaxRetries {+ err := client.Watch(ctx, func(tx *goredis.Tx) error {+ store := webhookTraceStore{Webhooks: make(map[string]webhookTraceRun)}+ raw, getErr := tx.Get(ctx, key).Bytes()+ switch {+ case getErr == nil:+ if err := json.Unmarshal(raw, &store); err != nil {+ return fmt.Errorf("decode webhook trace: %w", err)+ }+ if store.Webhooks == nil {+ store.Webhooks = make(map[string]webhookTraceRun)+ }+ case errors.Is(getErr, goredis.Nil):+ default:+ return fmt.Errorf("read webhook trace: %w", getErr)+ }++ run, ok := store.Webhooks[entryKey]+ if !ok {+ run = webhookTraceRun{StartTS: startSeconds}+ }+ eventTimestamp := float64(time.Now().UnixNano()) / 1e9+ latestTimestamp := float64(0)+ for _, existingEvent := range run.Events {+ latestTimestamp = max(latestTimestamp, webhookTraceEventTimestamp(existingEvent))+ }+ if eventTimestamp <= latestTimestamp {+ eventTimestamp = math.Nextafter(latestTimestamp, math.Inf(1))+ }+ eventRecord["ts"] = eventTimestamp+ run.Events = append(run.Events, eventRecord)+ store.Webhooks[entryKey] = run++ payload, err := json.Marshal(store)+ if err != nil {+ return fmt.Errorf("encode webhook trace: %w", err)+ }+ _, err = tx.TxPipelined(ctx, func(pipe goredis.Pipeliner) error {+ pipe.Set(ctx, key, payload, webhookTraceTTL)+ return nil+ })+ if err != nil {+ return fmt.Errorf("write webhook trace: %w", err)+ }+ return nil+ }, key)+ if err == nil {+ return nil+ }+ if !errors.Is(err, goredis.TxFailedErr) {+ return err+ }+ }+ return fmt.Errorf("update webhook trace: transaction failed after %d retries", webhookTraceMaxRetries)}diff --git a/internal/handler/agent_webhook_trace.go b/internal/handler/agent_webhook_trace.goindex 6e573642e0..480f007099 100644--- a/internal/handler/agent_webhook_trace.go+++ b/internal/handler/agent_webhook_trace.go@@ -39,7 +39,8 @@ type webhookTraceStore struct {}type webhookTraceRun struct {- Events []map[string]any `json:"events"`+ StartTS float64 `json:"start_ts"`+ Events []map[string]any `json:"events"`}// parseWebhookSinceTS parses an optional finite timestamp cursor.diff --git a/internal/handler/agent_webhook_trace_test.go b/internal/handler/agent_webhook_trace_test.goindex d2d019746d..9e58c24355 100644--- a/internal/handler/agent_webhook_trace_test.go+++ b/internal/handler/agent_webhook_trace_test.go@@ -15,13 +15,13 @@package handlerimport (- "context""encoding/json""errors""net/http/httptest""net/url""strconv""strings"+ "sync""testing""time"@@ -49,26 +49,6 @@ type webhookTraceResponse struct {Message string `json:"message"`}-type miniredisWebhookTraceWriter struct {- client *goredis.Client-}--func (w miniredisWebhookTraceWriter) Get(ctx context.Context, key string) (string, error) {- value, err := w.client.Get(ctx, key).Result()- if errors.Is(err, goredis.Nil) {- return "", nil- }- return value, err-}--func (w miniredisWebhookTraceWriter) SetObj(ctx context.Context, key string, value any, ttl time.Duration) bool {- payload, err := json.Marshal(value)- if err != nil {- return false- }- return w.client.Set(ctx, key, payload, ttl).Err() == nil-}-// newWebhookTraceTestHandler wires ownership storage and miniredis for HTTP tests.func newWebhookTraceTestHandler(t *testing.T) (*AgentHandler, *goredis.Client) {t.Helper()@@ -137,6 +117,7 @@ func seedWebhookTrace(t *testing.T, rdb *goredis.Client, canvasID string, webhoo}}+// TestAppendWebhookTracePersistsReadableEvents covers sequential appends and polling.func TestAppendWebhookTracePersistsReadableEvents(t *testing.T) {mr, err := miniredis.Run()if err != nil {@@ -147,17 +128,20 @@ func TestAppendWebhookTracePersistsReadableEvents(t *testing.T) {t.Cleanup(func() { _ = rdb.Close() })ctx := t.Context()- writer := miniredisWebhookTraceWriter{client: rdb}start := time.Unix(1_700_000_000, 0)- appendWebhookTraceWithWriter(ctx, writer, "c1", start, canvas.RunEvent{+ if err = appendWebhookTraceWithClient(ctx, rdb, "c1", start, canvas.RunEvent{Type: "message",Data: `{"content":"hello"}`,SessionID: "task-1",- })- appendWebhookTraceWithWriter(ctx, writer, "c1", start, canvas.RunEvent{+ }); err != nil {+ t.Fatalf("append message trace: %v", err)+ }+ if err = appendWebhookTraceWithClient(ctx, rdb, "c1", start, canvas.RunEvent{Type: "finished",Data: `{"success":true}`,- })+ }); err != nil {+ t.Fatalf("append finished trace: %v", err)+ }const key = "webhook-trace-c1-logs"raw, err := rdb.Get(ctx, key).Result()@@ -179,11 +163,122 @@ func TestAppendWebhookTracePersistsReadableEvents(t *testing.T) {if !poll.Finished {t.Fatal("persisted trace should be finished")}- if ttl := mr.TTL(key); ttl != 600*time.Second {+ if ttl := mr.TTL(key); ttl != webhookTraceTTL {+ t.Fatalf("trace TTL = %s, want 10m0s", ttl)+ }+}++// TestAppendWebhookTracePreservesConcurrentRuns covers overlapping runs on one agent key.+func TestAppendWebhookTracePreservesConcurrentRuns(t *testing.T) {+ mr, err := miniredis.Run()+ if err != nil {+ t.Fatalf("miniredis.Run: %v", err)+ }+ t.Cleanup(mr.Close)+ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ const runCount = 32+ ctx := t.Context()+ errorsByRun := make(chan error, runCount)+ var wg sync.WaitGroup+ for index := range runCount {+ wg.Add(1)+ go func() {+ defer wg.Done()+ start := time.Unix(1_700_000_000, int64(index)*int64(time.Millisecond))+ errorsByRun <- appendWebhookTraceWithClient(ctx, rdb, "c1", start, canvas.RunEvent{+ Type: "message",+ SessionID: "task-" + strconv.Itoa(index),+ })+ }()+ }+ wg.Wait()+ close(errorsByRun)+ for appendErr := range errorsByRun {+ if appendErr != nil {+ t.Fatalf("append concurrent trace: %v", appendErr)+ }+ }++ const key = "webhook-trace-c1-logs"+ raw, err := rdb.Get(ctx, key).Result()+ if err != nil {+ t.Fatalf("read concurrent trace: %v", err)+ }+ var persisted webhookTraceStore+ if err = json.Unmarshal([]byte(raw), &persisted); err != nil {+ t.Fatalf("decode concurrent trace: %v", err)+ }+ if got := len(persisted.Webhooks); got != runCount {+ t.Fatalf("persisted runs = %d, want %d", got, runCount)+ }+ for runID, run := range persisted.Webhooks {+ if len(run.Events) != 1 {+ t.Errorf("run %s events = %d, want 1", runID, len(run.Events))+ }+ }+ if ttl := mr.TTL(key); ttl != webhookTraceTTL {t.Fatalf("trace TTL = %s, want 10m0s", ttl)}}+// TestAppendWebhookTraceOrdersConcurrentEvents protects incremental polling cursors.+func TestAppendWebhookTraceOrdersConcurrentEvents(t *testing.T) {+ mr, err := miniredis.Run()+ if err != nil {+ t.Fatalf("miniredis.Run: %v", err)+ }+ t.Cleanup(mr.Close)+ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ const eventCount = 32+ ctx := t.Context()+ start := time.Unix(1_700_000_000, 0)+ errorsByEvent := make(chan error, eventCount)+ var wg sync.WaitGroup+ for index := range eventCount {+ wg.Add(1)+ go func() {+ defer wg.Done()+ errorsByEvent <- appendWebhookTraceWithClient(ctx, rdb, "c1", start, canvas.RunEvent{+ Type: "message",+ SessionID: "task-" + strconv.Itoa(index),+ })+ }()+ }+ wg.Wait()+ close(errorsByEvent)+ for appendErr := range errorsByEvent {+ if appendErr != nil {+ t.Fatalf("append concurrent event: %v", appendErr)+ }+ }++ raw, err := rdb.Get(ctx, "webhook-trace-c1-logs").Result()+ if err != nil {+ t.Fatalf("read concurrent events: %v", err)+ }+ var persisted webhookTraceStore+ if err = json.Unmarshal([]byte(raw), &persisted); err != nil {+ t.Fatalf("decode concurrent events: %v", err)+ }+ runID := strconv.FormatFloat(float64(start.UnixNano())/1e9, 'f', -1, 64)+ events := persisted.Webhooks[runID].Events+ if len(events) != eventCount {+ t.Fatalf("persisted events = %d, want %d", len(events), eventCount)+ }+ previousTimestamp := float64(0)+ for index, event := range events {+ timestamp := webhookTraceEventTimestamp(event)+ if timestamp <= previousTimestamp {+ t.Fatalf("event %d timestamp = %v, want greater than %v", index, timestamp, previousTimestamp)+ }+ previousTimestamp = timestamp+ }+}+// TestGetAgentWebhookLogsPollsTraceIncrementally covers the complete UI poll flow.func TestGetAgentWebhookLogsPollsTraceIncrementally(t *testing.T) {h, rdb := newWebhookTraceTestHandler(t)