RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): implement webhook trace polling (#18815)
Guofang.Tang1cc2761f6d3a
diff --git a/internal/handler/agent.go b/internal/handler/agent.goindex 1bcee2a1f6..4240a49533 100644--- a/internal/handler/agent.go+++ b/internal/handler/agent.go@@ -96,8 +96,8 @@ type AgentHandler struct {documentService documentAccessChecker// redisGet fetches a raw string from Redis. Defaults to the global// client (redis.Get) so production behaviour is unchanged; tests inject- // a miniredis-backed getter to exercise GetAgentLogs without a live- // Redis (mirrors the newExecutor injection pattern).+ // a miniredis-backed getter to exercise Agent log endpoints without a+ // live Redis (mirrors the newExecutor injection pattern).redisGet func(key string) (string, error)// redisStore writes the debug-run log array. Defaults to the global// client (redis.Get, which satisfies task.DebugLogStore); tests inject a@@ -146,9 +146,9 @@ func NewAgentHandler(ctx context.Context, agentService *service.AgentService, fi}}-// WithRedisGetter overrides the Redis string getter used by GetAgentLogs.-// Tests pass a miniredis-backed getter so the debug-log polling endpoint can-// be exercised end-to-end without a live Redis.+// WithRedisGetter overrides the Redis string getter used by Agent log endpoints.+// Tests pass a miniredis-backed getter so polling can be exercised end-to-end+// without a live Redis.func (h *AgentHandler) WithRedisGetter(f func(key string) (string, error)) *AgentHandler {h.redisGet = freturn h@@ -1605,7 +1605,9 @@ func (h *AgentHandler) GetAgentLogs(c *gin.Context) {// id does not resolve to a canvas owned by the caller (see// api/apps/restful_apis/agent_api.py webhook_trace). We replicate// that envelope here so the front-end poll does not surface a 500-// for unknown / foreign canvas ids.+// for unknown / foreign canvas ids. Polling follows the same cursor+// protocol: establish since_ts, discover a run, then fetch events by+// webhook_id until a finished event arrives.func (h *AgentHandler) GetAgentWebhookLogs(c *gin.Context) {user, code, msg := GetUser(c)if code != common.CodeSuccess {@@ -1621,17 +1623,33 @@ func (h *AgentHandler) GetAgentWebhookLogs(c *gin.Context) {// indistinguishable for missing vs foreign, so collapse// both into 102 "Canvas not found." here.if err != nil && !errors.Is(err, dao.ErrUserCanvasNotFound) {- common.ResponseWithCodeData(c, common.CodeServerError, nil, err.Error())+ jsonInternalError(c, err)return}common.ResponseWithCodeData(c, common.CodeDataError, nil, "Canvas not found.")return}- common.SuccessWithData(c, gin.H{- "events": []interface{}{},- "finished": false,- "next_since_ts": 0,- }, "success")++ sinceTS, hasSinceTS := parseWebhookSinceTS(c.Query("since_ts"))+ if !hasSinceTS {+ common.SuccessWithData(c, newWebhookTracePoll(nil, float64(time.Now().UnixNano())/1e9, false), "success")+ return+ }+ if h.redisGet == nil {+ jsonInternalError(c, errors.New("agent webhook trace: redis getter not configured"))+ return+ }+ payload, err := h.redisGet(fmt.Sprintf("webhook-trace-%s-logs", canvasID))+ if err != nil {+ jsonInternalError(c, fmt.Errorf("read agent webhook trace: %w", err))+ return+ }+ result, err := pollWebhookTrace(payload, sinceTS, c.Query("webhook_id"))+ if err != nil {+ jsonInternalError(c, err)+ return+ }+ common.SuccessWithData(c, result, "success")}// checkCanvasAccessForHandler is the shared 103 envelope helper fordiff --git a/internal/handler/agent_test.go b/internal/handler/agent_test.goindex fe9b054f0b..27025305c7 100644--- a/internal/handler/agent_test.go+++ b/internal/handler/agent_test.go@@ -1686,48 +1686,6 @@ func TestPromptsReturnsHardcodedFields(t *testing.T) {}}-// TestGetAgentWebhookLogsReturnsEmptyPoll covers the contract: the-// payload must be {events:[], finished:false, next_since_ts:0}. This-// test exercises the handler's auth + response-shape path against a-// real (in-memory) user_canvas row.-func TestGetAgentWebhookLogsReturnsEmptyPoll(t *testing.T) {- gin.SetMode(gin.TestMode)-- db := setupHandlerAgentsTestDB(t)- orig := dao.DB- dao.DB = db- t.Cleanup(func() { dao.DB = orig })-- db.Create(&entity.UserCanvas{ID: "c1", UserID: "u1", Title: sptr("Test")})-- w := httptest.NewRecorder()- c, _ := gin.CreateTestContext(w)- c.Request = httptest.NewRequest("GET", "/api/v1/agents/c1/webhook/logs?since_ts=0", nil)- c.Set("user", &entity.User{ID: "u1"})- c.Set("user_id", "u1")- c.Params = gin.Params{{Key: "canvas_id", Value: "c1"}}-- ctx := t.Context()- h := NewAgentHandler(ctx, service.NewAgentService(), nil)- h.GetAgentWebhookLogs(c)-- var resp map[string]interface{}- _ = json.Unmarshal(w.Body.Bytes(), &resp)- if code, _ := resp["code"].(float64); code != float64(common.CodeSuccess) {- t.Fatalf("code = %v, want 0; body=%s", code, w.Body.String())- }- data, _ := resp["data"].(map[string]interface{})- if events, _ := data["events"].([]interface{}); len(events) != 0 {- t.Errorf("events = %v, want []", events)- }- if finished, _ := data["finished"].(bool); finished {- t.Errorf("finished = true, want false")- }- if _, ok := data["next_since_ts"]; !ok {- t.Errorf("missing next_since_ts key")- }-}-// TestRerunAgent_RejectsInaccessibleDocument mirrors PR #15145:// POST /api/v1/agents/rerun gates on DocumentService.accessible// (the python "is the document reachable by this tenant" check)diff --git a/internal/handler/agent_webhook_trace.go b/internal/handler/agent_webhook_trace.gonew file mode 100644index 0000000000..6e573642e0--- /dev/null+++ b/internal/handler/agent_webhook_trace.go@@ -0,0 +1,164 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package handler++import (+ "crypto/hmac"+ "crypto/sha256"+ "encoding/base64"+ "encoding/json"+ "fmt"+ "math"+ "strconv"+ "strings"+)++const webhookIDSecret = "webhook_id_secret"++type webhookTracePoll struct {+ WebhookID *string `json:"webhook_id"`+ Events []map[string]any `json:"events"`+ NextSinceTS float64 `json:"next_since_ts"`+ Finished bool `json:"finished"`+}++type webhookTraceStore struct {+ Webhooks map[string]webhookTraceRun `json:"webhooks"`+}++type webhookTraceRun struct {+ Events []map[string]any `json:"events"`+}++// parseWebhookSinceTS parses an optional finite timestamp cursor.+func parseWebhookSinceTS(raw string) (float64, bool) {+ if strings.TrimSpace(raw) == "" {+ return 0, false+ }+ value, err := strconv.ParseFloat(raw, 64)+ if err != nil || math.IsNaN(value) || math.IsInf(value, 0) {+ return 0, false+ }+ return value, true+}++// pollWebhookTrace advances one webhook trace polling step.+func pollWebhookTrace(raw string, sinceTS float64, webhookID string) (webhookTracePoll, error) {+ empty := newWebhookTracePoll(nil, sinceTS, false)+ if strings.TrimSpace(raw) == "" {+ return empty, nil+ }++ var store webhookTraceStore+ if err := json.Unmarshal([]byte(raw), &store); err != nil {+ return webhookTracePoll{}, fmt.Errorf("decode webhook trace: %w", err)+ }+ if webhookID == "" {+ startKey, startTS, ok := nextWebhookTraceRun(store.Webhooks, sinceTS)+ if !ok {+ return empty, nil+ }+ encodedID := encodeWebhookID(startKey)+ return newWebhookTracePoll(&encodedID, startTS, false), nil+ }++ realID, ok := decodeWebhookID(webhookID, store.Webhooks)+ if !ok {+ return newWebhookTracePoll(&webhookID, sinceTS, true), nil+ }+ return collectWebhookTraceEvents(store.Webhooks[realID], sinceTS, webhookID), nil+}++// newWebhookTracePoll keeps empty event lists serialized as [] instead of null.+func newWebhookTracePoll(webhookID *string, nextSinceTS float64, finished bool) webhookTracePoll {+ return webhookTracePoll{+ WebhookID: webhookID,+ Events: make([]map[string]any, 0),+ NextSinceTS: nextSinceTS,+ Finished: finished,+ }+}++// nextWebhookTraceRun selects the earliest run that started after the cursor.+func nextWebhookTraceRun(webhooks map[string]webhookTraceRun, sinceTS float64) (string, float64, bool) {+ var selectedKey string+ var selectedTS float64+ found := false+ for startKey := range webhooks {+ startTS, err := strconv.ParseFloat(startKey, 64)+ if err != nil || startTS <= sinceTS || (found && startTS >= selectedTS) {+ continue+ }+ selectedKey, selectedTS, found = startKey, startTS, true+ }+ return selectedKey, selectedTS, found+}++// encodeWebhookID creates an opaque run cursor. Canvas ownership is checked+// before the cursor is decoded.+func encodeWebhookID(startTS string) string {+ mac := hmac.New(sha256.New, []byte(webhookIDSecret))+ _, _ = mac.Write([]byte(startTS))+ return base64.RawURLEncoding.EncodeToString(mac.Sum(nil))+}++// decodeWebhookID resolves an opaque cursor without exposing trace timestamps.+func decodeWebhookID(webhookID string, webhooks map[string]webhookTraceRun) (string, bool) {+ for startTS := range webhooks {+ if hmac.Equal([]byte(encodeWebhookID(startTS)), []byte(webhookID)) {+ return startTS, true+ }+ }+ return "", false+}++// collectWebhookTraceEvents returns only events newer than the supplied cursor.+func collectWebhookTraceEvents(run webhookTraceRun, sinceTS float64, webhookID string) webhookTracePoll {+ result := newWebhookTracePoll(&webhookID, sinceTS, false)+ for _, event := range run.Events {+ eventTS := webhookTraceEventTimestamp(event)+ if eventTS <= sinceTS {+ continue+ }+ result.Events = append(result.Events, event)+ if eventTS > result.NextSinceTS {+ result.NextSinceTS = eventTS+ }+ if eventType, _ := event["event"].(string); eventType == "finished" {+ result.Finished = true+ break+ }+ }+ return result+}++// webhookTraceEventTimestamp returns zero when an event has no numeric timestamp.+func webhookTraceEventTimestamp(event map[string]any) float64 {+ switch value := event["ts"].(type) {+ case float64:+ return value+ case float32:+ return float64(value)+ case int:+ return float64(value)+ case int64:+ return float64(value)+ case json.Number:+ parsed, _ := value.Float64()+ return parsed+ default:+ return 0+ }+}diff --git a/internal/handler/agent_webhook_trace_test.go b/internal/handler/agent_webhook_trace_test.gonew file mode 100644index 0000000000..d6acce0291--- /dev/null+++ b/internal/handler/agent_webhook_trace_test.go@@ -0,0 +1,283 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package handler++import (+ "encoding/json"+ "errors"+ "net/http/httptest"+ "net/url"+ "strconv"+ "strings"+ "testing"+ "time"++ "github.com/alicebob/miniredis/v2"+ "github.com/gin-gonic/gin"+ goredis "github.com/redis/go-redis/v9"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "ragflow/internal/service"+)++type webhookTraceResponseData struct {+ WebhookID *string `json:"webhook_id"`+ Events []map[string]any `json:"events"`+ NextSinceTS float64 `json:"next_since_ts"`+ Finished bool `json:"finished"`+}++type webhookTraceResponse struct {+ Code int `json:"code"`+ Data *webhookTraceResponseData `json:"data"`+ Message string `json:"message"`+}++// newWebhookTraceTestHandler wires ownership storage and miniredis for HTTP tests.+func newWebhookTraceTestHandler(t *testing.T) (*AgentHandler, *goredis.Client) {+ t.Helper()+ gin.SetMode(gin.TestMode)++ db := setupHandlerAgentsTestDB(t)+ originalDB := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = originalDB })+ if err := db.Create(&entity.UserCanvas{ID: "c1", UserID: "u1", Title: sptr("Test")}).Error; err != nil {+ t.Fatalf("create canvas: %v", err)+ }++ mr, err := miniredis.Run()+ if err != nil {+ t.Fatalf("miniredis.Run: %v", err)+ }+ t.Cleanup(mr.Close)+ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ ctx := t.Context()+ h := NewAgentHandler(ctx, service.NewAgentService(), nil).+ WithRedisGetter(func(key string) (string, error) {+ value, getErr := rdb.Get(ctx, key).Result()+ if errors.Is(getErr, goredis.Nil) {+ return "", nil+ }+ return value, getErr+ })+ return h, rdb+}++// requestWebhookTrace invokes the real Gin handler and decodes its envelope.+func requestWebhookTrace(t *testing.T, h *AgentHandler, canvasID, userID string, query url.Values) webhookTraceResponse {+ t.Helper()+ path := "/api/v1/agents/" + canvasID + "/webhook/logs"+ if encoded := query.Encode(); encoded != "" {+ path += "?" + encoded+ }+ w := httptest.NewRecorder()+ c, _ := gin.CreateTestContext(w)+ c.Request = httptest.NewRequest("GET", path, nil)+ c.Set("user", &entity.User{ID: userID})+ c.Set("user_id", userID)+ c.Params = gin.Params{{Key: "canvas_id", Value: canvasID}}++ h.GetAgentWebhookLogs(c)++ var response webhookTraceResponse+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {+ t.Fatalf("decode response: %v; body=%s", err, w.Body.String())+ }+ return response+}++// seedWebhookTrace writes the same Redis shape produced by appendWebhookTrace.+func seedWebhookTrace(t *testing.T, rdb *goredis.Client, canvasID string, webhooks map[string]any) {+ t.Helper()+ payload, err := json.Marshal(map[string]any{"webhooks": webhooks})+ if err != nil {+ t.Fatalf("marshal trace: %v", err)+ }+ if err = rdb.Set(t.Context(), "webhook-trace-"+canvasID+"-logs", payload, 0).Err(); err != nil {+ t.Fatalf("seed trace: %v", err)+ }+}++// TestGetAgentWebhookLogsPollsTraceIncrementally covers the complete UI poll flow.+func TestGetAgentWebhookLogsPollsTraceIncrementally(t *testing.T) {+ h, rdb := newWebhookTraceTestHandler(t)++ before := float64(time.Now().UnixNano()) / 1e9+ initial := requestWebhookTrace(t, h, "c1", "u1", url.Values{})+ after := float64(time.Now().UnixNano()) / 1e9+ if initial.Code != int(common.CodeSuccess) || initial.Data == nil {+ t.Fatalf("initial response = %+v", initial)+ }+ if initial.Data.WebhookID != nil || len(initial.Data.Events) != 0 || initial.Data.Finished {+ t.Fatalf("initial data = %+v, want empty unfinished cursor", initial.Data)+ }+ if initial.Data.NextSinceTS < before || initial.Data.NextSinceTS > after {+ t.Fatalf("initial next_since_ts = %f, want between %f and %f", initial.Data.NextSinceTS, before, after)+ }++ startTS := initial.Data.NextSinceTS + 1+ messageTS := startTS + 1+ finishedTS := startTS + 2+ trailingTS := startTS + 3+ startKey := strconv.FormatFloat(startTS, 'f', -1, 64)+ laterKey := strconv.FormatFloat(startTS+10, 'f', -1, 64)+ seedWebhookTrace(t, rdb, "c1", map[string]any{+ laterKey: map[string]any{+ "start_ts": startTS + 10,+ "events": []any{},+ },+ startKey: map[string]any{+ "start_ts": startTS,+ "events": []any{+ map[string]any{+ "ts": messageTS,+ "event": "message",+ "data": map[string]any{"content": "done"},+ "task_id": "task-1",+ },+ map[string]any{+ "ts": finishedTS,+ "event": "finished",+ "data": map[string]any{"success": true},+ },+ map[string]any{+ "ts": trailingTS,+ "event": "message",+ "data": map[string]any{"content": "late"},+ },+ },+ },+ })++ discovery := requestWebhookTrace(t, h, "c1", "u1", url.Values{+ "since_ts": {strconv.FormatFloat(initial.Data.NextSinceTS, 'f', -1, 64)},+ })+ if discovery.Data == nil || discovery.Data.WebhookID == nil {+ t.Fatalf("discovery data = %+v, want webhook id", discovery.Data)+ }+ if discovery.Data.NextSinceTS != startTS || len(discovery.Data.Events) != 0 || discovery.Data.Finished {+ t.Fatalf("discovery data = %+v, want earliest run cursor", discovery.Data)+ }++ poll := requestWebhookTrace(t, h, "c1", "u1", url.Values{+ "since_ts": {strconv.FormatFloat(initial.Data.NextSinceTS, 'f', -1, 64)},+ "webhook_id": {*discovery.Data.WebhookID},+ })+ if poll.Data == nil ||+ len(poll.Data.Events) != 2 ||+ poll.Data.Events[1]["event"] != "finished" ||+ !poll.Data.Finished {+ t.Fatalf("poll data = %+v, want two events and finished", poll.Data)+ }+ if poll.Data.NextSinceTS != finishedTS {+ t.Errorf("poll next_since_ts = %f, want %f", poll.Data.NextSinceTS, finishedTS)+ }+ messageData, ok := poll.Data.Events[0]["data"].(map[string]any)+ if !ok {+ t.Fatalf("message data = %T, want object", poll.Data.Events[0]["data"])+ }+ if content, _ := messageData["content"].(string); content != "done" {+ t.Errorf("message content = %q, want done", content)+ }++ incremental := requestWebhookTrace(t, h, "c1", "u1", url.Values{+ "since_ts": {strconv.FormatFloat(messageTS, 'f', -1, 64)},+ "webhook_id": {*discovery.Data.WebhookID},+ })+ if incremental.Data == nil || len(incremental.Data.Events) != 1 || incremental.Data.Events[0]["event"] != "finished" {+ t.Fatalf("incremental data = %+v, want only finished event", incremental.Data)+ }+ if !incremental.Data.Finished || incremental.Data.NextSinceTS != finishedTS {+ t.Fatalf("incremental completion = %+v", incremental.Data)+ }+}++// TestGetAgentWebhookLogsHandlesMissingAndInvalidState covers empty and stale cursors.+func TestGetAgentWebhookLogsHandlesMissingAndInvalidState(t *testing.T) {+ h, rdb := newWebhookTraceTestHandler(t)++ missing := requestWebhookTrace(t, h, "c1", "u1", url.Values{"since_ts": {"42"}})+ if missing.Data == nil || missing.Data.WebhookID != nil || missing.Data.NextSinceTS != 42 || missing.Data.Finished {+ t.Fatalf("missing trace data = %+v", missing.Data)+ }++ seedWebhookTrace(t, rdb, "c1", map[string]any{+ "50": map[string]any{"start_ts": 50, "events": []any{}},+ })+ forged := requestWebhookTrace(t, h, "c1", "u1", url.Values{+ "since_ts": {"42"},+ "webhook_id": {"forged-id"},+ })+ if forged.Data == nil || forged.Data.WebhookID == nil || *forged.Data.WebhookID != "forged-id" || !forged.Data.Finished {+ t.Fatalf("forged id data = %+v, want finished invalid cursor", forged.Data)+ }++ invalidSince := requestWebhookTrace(t, h, "c1", "u1", url.Values{"since_ts": {"not-a-number"}})+ if invalidSince.Data == nil || invalidSince.Data.NextSinceTS <= 0 || invalidSince.Data.Finished {+ t.Fatalf("invalid since_ts data = %+v, want a fresh cursor", invalidSince.Data)+ }+}++// TestGetAgentWebhookLogsRedactsRedisFailures covers corrupt data and backend errors.+func TestGetAgentWebhookLogsRedactsRedisFailures(t *testing.T) {+ h, rdb := newWebhookTraceTestHandler(t)+ if err := rdb.Set(t.Context(), "webhook-trace-c1-logs", `{"webhooks":`, 0).Err(); err != nil {+ t.Fatalf("seed corrupt trace: %v", err)+ }++ corrupt := requestWebhookTrace(t, h, "c1", "u1", url.Values{"since_ts": {"0"}})+ if corrupt.Code != int(common.CodeServerError) || corrupt.Message != common.CodeServerError.Message() {+ t.Fatalf("corrupt response = %+v", corrupt)+ }++ h.WithRedisGetter(func(string) (string, error) {+ return "", errors.New("redis password=secret")+ })+ failed := requestWebhookTrace(t, h, "c1", "u1", url.Values{"since_ts": {"0"}})+ if failed.Code != int(common.CodeServerError) || strings.Contains(failed.Message, "password=secret") {+ t.Fatalf("redis failure response = %+v", failed)+ }+}++// TestGetAgentWebhookLogsChecksOwnershipBeforeRedis prevents cross-user trace probes.+func TestGetAgentWebhookLogsChecksOwnershipBeforeRedis(t *testing.T) {+ h, _ := newWebhookTraceTestHandler(t)+ redisCalls := 0+ h.WithRedisGetter(func(string) (string, error) {+ redisCalls+++ return "", nil+ })++ response := requestWebhookTrace(t, h, "c1", "other-user", url.Values{"since_ts": {"0"}})+ if response.Code != int(common.CodeDataError) || response.Message != "Canvas not found." {+ t.Fatalf("ownership response = %+v", response)+ }+ if redisCalls != 0 {+ t.Fatalf("redis calls = %d, want 0 before ownership succeeds", redisCalls)+ }+}++// TestEncodeWebhookIDUsesStableEncoding pins deterministic cursor output.+func TestEncodeWebhookIDUsesStableEncoding(t *testing.T) {+ const want = "7a7-Rfe0PSB5OwV10qD7SWcmrtbFhfQKTZajRny8STM"+ if got := encodeWebhookID("123.5"); got != want {+ t.Fatalf("encodeWebhookID = %q, want %q", got, want)+ }+}