RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(agent): finalize interrupted webhook traces (#19089)
Guofang.Tang09586c5ddb21
diff --git a/internal/handler/agent_webhook.go b/internal/handler/agent_webhook.goindex 99f08962ed..14a6077932 100644--- a/internal/handler/agent_webhook.go+++ b/internal/handler/agent_webhook.go@@ -515,12 +515,25 @@ func (h *AgentHandler) runWebhookDetached(parent = service.WithAgentSessionID(parent, sessionID)ctx, cancel := context.WithTimeout(context.WithoutCancel(parent), 5*time.Minute)defer cancel()+ h.runWebhookDetachedWithContext(ctx, cv, payload, isTest, startTs, sessionID)+}+func (h *AgentHandler) runWebhookDetachedWithContext(+ ctx context.Context, cv *entity.UserCanvas, payload map[string]any,+ isTest bool, startTs time.Time, sessionID string,+) {events, err := h.loader.RunAgentWithWebhook(ctx, cv.UserID, cv.ID, payload)if err != nil {common.Warn("webhook detached run start failed",zap.String("canvas", cv.ID),zap.Error(err))+ if ctxErr := ctx.Err(); ctxErr != nil {+ if isTest {+ event := webhookContextTerminalEvent(ctxErr, sessionID)+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)+ }+ return+ }if isTest {h.appendWebhookRunTrace(ctx, cv.ID, startTs, webhookStartErrorEvent(err, sessionID))h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, false)@@ -532,15 +545,28 @@ func (h *AgentHandler) runWebhookDetached(ev.SessionID = sessionID}terminal := isWebhookTerminalEvent(ev)- if isTest {- h.appendWebhookRunTrace(ctx, cv.ID, startTs, sanitizeWebhookTraceEvent(ev))- }if terminal {if isTest {- h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, false)+ event := sanitizeWebhookTraceEvent(ev)+ if ctx.Err() != nil {+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)+ } else {+ h.appendWebhookRunTrace(ctx, cv.ID, startTs, event)+ h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, false)+ }}return}+ if isTest {+ h.appendWebhookRunTrace(ctx, cv.ID, startTs, sanitizeWebhookTraceEvent(ev))+ }+ }+ if err = ctx.Err(); err != nil {+ if isTest {+ event := webhookContextTerminalEvent(err, sessionID)+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)+ }+ return}if isTest {h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, true)@@ -565,6 +591,14 @@ func (h *AgentHandler) runWebhookSync(ctx = service.WithAgentSessionID(ctx, sessionID)events, err := h.loader.RunAgentWithWebhook(ctx, cv.UserID, cv.ID, payload)if err != nil {+ if ctxErr := ctx.Err(); ctxErr != nil {+ event := webhookContextTerminalEvent(ctxErr, sessionID)+ if isTest {+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)+ }+ result, _ := webhookTerminalResult(event, sessionID)+ return result+ }errorEvent := webhookStartErrorEvent(err, sessionID)if isTest {h.appendWebhookRunTrace(ctx, cv.ID, startTs, errorEvent)@@ -581,8 +615,13 @@ func (h *AgentHandler) runWebhookSync(}if result, terminal := webhookTerminalResult(ev, sessionID); terminal {if isTest {- h.appendWebhookRunTrace(ctx, cv.ID, startTs, sanitizeWebhookTraceEvent(ev))- h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, false)+ event := sanitizeWebhookTraceEvent(ev)+ if ctx.Err() != nil {+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)+ } else {+ h.appendWebhookRunTrace(ctx, cv.ID, startTs, event)+ h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, false)+ }}return result}@@ -616,6 +655,14 @@ func (h *AgentHandler) runWebhookSync(}}}+ if err = ctx.Err(); err != nil {+ event := webhookContextTerminalEvent(err, sessionID)+ if isTest {+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)+ }+ result, _ := webhookTerminalResult(event, sessionID)+ return result+ }final := strings.Join(contents, "")if isTest {h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, true)@@ -699,6 +746,20 @@ func isWebhookTerminalEvent(ev canvas.RunEvent) bool {}}+func webhookContextTerminalEvent(err error, sessionID string) canvas.RunEvent {+ eventType := "cancelled"+ message := "Agent run was cancelled."+ if errors.Is(err, context.DeadlineExceeded) {+ eventType = "error"+ message = "Agent run timed out."+ }+ return canvas.RunEvent{+ Type: eventType,+ SessionID: sessionID,+ Data: mustJSON(canvas.ErrorEvent{Message: message}),+ }+}+func sanitizeWebhookTraceEvent(ev canvas.RunEvent) canvas.RunEvent {if ev.Type != "error" {return ev@@ -735,6 +796,15 @@ func (h *AgentHandler) appendWebhookFinishedTrace(})}+func (h *AgentHandler) appendWebhookInterruptedTrace(+ ctx context.Context, agentID string, startTs time.Time, sessionID string, event canvas.RunEvent,+) {+ traceCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)+ defer cancel()+ h.appendWebhookRunTrace(traceCtx, agentID, startTs, event)+ h.appendWebhookFinishedTrace(traceCtx, agentID, startTs, sessionID, false)+}+// mustJSON marshals v to a JSON object string. Used by trace appenders;// panics on marshal failure (acceptable because we only marshal// statically-typed map[string]any values).diff --git a/internal/handler/agent_webhook_test.go b/internal/handler/agent_webhook_test.goindex 7408cdab3a..ccf5d9d572 100644--- a/internal/handler/agent_webhook_test.go+++ b/internal/handler/agent_webhook_test.go@@ -40,10 +40,11 @@ import (// without touching the database. Mirrors the pattern used by// waitFakeAgentService at internal/handler/agent_wait_for_user_test.go.type fakeCanvasLoader struct {- canvas *entity.UserCanvas- err error- runErr error- events []canvas.RunEvent+ canvas *entity.UserCanvas+ err error+ runErr error+ events []canvas.RunEvent+ waitForCancellation bool}func (f *fakeCanvasLoader) LoadCanvasByID(_ context.Context, _, _ string) (*entity.UserCanvas, error) {@@ -53,10 +54,18 @@ func (f *fakeCanvasLoader) LoadCanvasByID(_ context.Context, _, _ string) (*entireturn f.canvas, nil}-func (f *fakeCanvasLoader) RunAgentWithWebhook(_ context.Context, _, _ string, _ map[string]any) (<-chan canvas.RunEvent, error) {+func (f *fakeCanvasLoader) RunAgentWithWebhook(ctx context.Context, _, _ string, _ map[string]any) (<-chan canvas.RunEvent, error) {if f.runErr != nil {return nil, f.runErr}+ if f.waitForCancellation {+ out := make(chan canvas.RunEvent)+ go func() {+ <-ctx.Done()+ close(out)+ }()+ return out, nil+ }out := make(chan canvas.RunEvent, len(f.events))for _, e := range f.events {out <- e@@ -659,6 +668,120 @@ func TestRunWebhookDetachedAppendsFinishedTrace(t *testing.T) {}}+func TestRunWebhookDetachedRecordsTimeoutTrace(t *testing.T) {+ tests := []struct {+ name string+ loader *fakeCanvasLoader+ }{+ {name: "during start", loader: &fakeCanvasLoader{runErr: context.DeadlineExceeded}},+ {name: "during run", loader: &fakeCanvasLoader{waitForCancellation: true}},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ cv := makeWebhookCanvas("c1", "u-1", "Webhook", nil)+ tt.loader.canvas = cv+ h := &AgentHandler{loader: tt.loader}+ var trace []canvas.RunEvent+ h.webhookTraceAppender = func(ctx context.Context, _ string, _ time.Time, ev canvas.RunEvent) {+ if err := ctx.Err(); err != nil {+ t.Errorf("trace context error = %v, want nil", err)+ }+ trace = append(trace, ev)+ }++ ctx, cancel := context.WithTimeout(+ service.WithAgentSessionID(t.Context(), "session-1"),+ time.Nanosecond,+ )+ defer cancel()+ <-ctx.Done()+ h.runWebhookDetachedWithContext(ctx, cv, map[string]any{}, true, time.Now(), "session-1")++ if len(trace) < 2 || trace[0].Type != "error" || !strings.Contains(trace[0].Data, "timed out") {+ t.Fatalf("timeout trace = %+v, want error followed by failed finished event", trace)+ }+ assertWebhookFinishedTrace(t, trace, false)+ })+ }+}++func TestRunWebhookSyncRecordsCancelledTrace(t *testing.T) {+ cv := makeWebhookCanvas("c1", "u-1", "Webhook", nil)+ h := &AgentHandler{loader: &fakeCanvasLoader{canvas: cv, waitForCancellation: true}}+ var trace []canvas.RunEvent+ h.webhookTraceAppender = func(ctx context.Context, _ string, _ time.Time, ev canvas.RunEvent) {+ if err := ctx.Err(); err != nil {+ t.Errorf("trace context error = %v, want nil", err)+ }+ trace = append(trace, ev)+ }++ ctx, cancel := context.WithCancel(t.Context())+ cancel()+ result := h.runWebhookSync(ctx, cv, map[string]any{}, true, time.Now())++ if result.status != http.StatusConflict {+ t.Errorf("status = %d, want %d", result.status, http.StatusConflict)+ }+ if len(trace) < 2 || trace[0].Type != "cancelled" || !strings.Contains(trace[0].Data, "cancelled") {+ t.Fatalf("cancelled trace = %+v, want cancelled followed by failed finished event", trace)+ }+ assertWebhookFinishedTrace(t, trace, false)+}++func TestWebhookBufferedTerminalEventAfterCancellationUsesCleanupContext(t *testing.T) {+ tests := []struct {+ name string+ run func(*testing.T, *AgentHandler, context.Context, *entity.UserCanvas)+ }{+ {+ name: "detached",+ run: func(_ *testing.T, h *AgentHandler, ctx context.Context, cv *entity.UserCanvas) {+ h.runWebhookDetachedWithContext(ctx, cv, map[string]any{}, true, time.Now(), "session-1")+ },+ },+ {+ name: "sync",+ run: func(t *testing.T, h *AgentHandler, ctx context.Context, cv *entity.UserCanvas) {+ result := h.runWebhookSync(ctx, cv, map[string]any{}, true, time.Now())+ if result.status != http.StatusConflict {+ t.Errorf("status = %d, want %d", result.status, http.StatusConflict)+ }+ },+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ cv := makeWebhookCanvas("c1", "u-1", "Webhook", nil)+ loader := &fakeCanvasLoader{+ canvas: cv,+ events: []canvas.RunEvent{{+ Type: "cancelled",+ Data: `{"message":"Agent run was cancelled."}`,+ }},+ }+ h := &AgentHandler{loader: loader}+ var trace []canvas.RunEvent+ h.webhookTraceAppender = func(ctx context.Context, _ string, _ time.Time, ev canvas.RunEvent) {+ if err := ctx.Err(); err != nil {+ t.Errorf("trace context error = %v, want nil", err)+ }+ trace = append(trace, ev)+ }++ ctx, cancel := context.WithCancel(t.Context())+ cancel()+ tt.run(t, h, ctx, cv)++ if len(trace) < 2 || trace[0].Type != "cancelled" {+ t.Fatalf("cancelled trace = %+v, want cancelled followed by failed finished event", trace)+ }+ assertWebhookFinishedTrace(t, trace, false)+ })+ }+}+func assertWebhookFinishedTrace(t *testing.T, events []canvas.RunEvent, wantSuccess bool) {t.Helper()if len(events) == 0 || events[len(events)-1].Type != "finished" {