Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): finalize interrupted webhook traces (#19089)

Guofang.Tang09586c5ddb21
diff --git a/internal/handler/agent_webhook.go b/internal/handler/agent_webhook.go
index 99f08962ed..14a6077932 100644
--- a/internal/handler/agent_webhook.go
+++ b/internal/handler/agent_webhook.go
@@ -515,12 +515,25 @@ func (h *AgentHandler) runWebhookDetached(
parent = service.WithAgentSessionID(parent, sessionID)
ctx, cancel := context.WithTimeout(context.WithoutCancel(parent), 5*time.Minute)
defer cancel()
+ h.runWebhookDetachedWithContext(ctx, cv, payload, isTest, startTs, sessionID)
+}
+func (h *AgentHandler) runWebhookDetachedWithContext(
+ ctx context.Context, cv *entity.UserCanvas, payload map[string]any,
+ isTest bool, startTs time.Time, sessionID string,
+) {
events, err := h.loader.RunAgentWithWebhook(ctx, cv.UserID, cv.ID, payload)
if err != nil {
common.Warn("webhook detached run start failed",
zap.String("canvas", cv.ID),
zap.Error(err))
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ if isTest {
+ event := webhookContextTerminalEvent(ctxErr, sessionID)
+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)
+ }
+ return
+ }
if isTest {
h.appendWebhookRunTrace(ctx, cv.ID, startTs, webhookStartErrorEvent(err, sessionID))
h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, false)
@@ -532,15 +545,28 @@ func (h *AgentHandler) runWebhookDetached(
ev.SessionID = sessionID
}
terminal := isWebhookTerminalEvent(ev)
- if isTest {
- h.appendWebhookRunTrace(ctx, cv.ID, startTs, sanitizeWebhookTraceEvent(ev))
- }
if terminal {
if isTest {
- h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, false)
+ event := sanitizeWebhookTraceEvent(ev)
+ if ctx.Err() != nil {
+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)
+ } else {
+ h.appendWebhookRunTrace(ctx, cv.ID, startTs, event)
+ h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, false)
+ }
}
return
}
+ if isTest {
+ h.appendWebhookRunTrace(ctx, cv.ID, startTs, sanitizeWebhookTraceEvent(ev))
+ }
+ }
+ if err = ctx.Err(); err != nil {
+ if isTest {
+ event := webhookContextTerminalEvent(err, sessionID)
+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)
+ }
+ return
}
if isTest {
h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, true)
@@ -565,6 +591,14 @@ func (h *AgentHandler) runWebhookSync(
ctx = service.WithAgentSessionID(ctx, sessionID)
events, err := h.loader.RunAgentWithWebhook(ctx, cv.UserID, cv.ID, payload)
if err != nil {
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ event := webhookContextTerminalEvent(ctxErr, sessionID)
+ if isTest {
+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)
+ }
+ result, _ := webhookTerminalResult(event, sessionID)
+ return result
+ }
errorEvent := webhookStartErrorEvent(err, sessionID)
if isTest {
h.appendWebhookRunTrace(ctx, cv.ID, startTs, errorEvent)
@@ -581,8 +615,13 @@ func (h *AgentHandler) runWebhookSync(
}
if result, terminal := webhookTerminalResult(ev, sessionID); terminal {
if isTest {
- h.appendWebhookRunTrace(ctx, cv.ID, startTs, sanitizeWebhookTraceEvent(ev))
- h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, false)
+ event := sanitizeWebhookTraceEvent(ev)
+ if ctx.Err() != nil {
+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)
+ } else {
+ h.appendWebhookRunTrace(ctx, cv.ID, startTs, event)
+ h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, false)
+ }
}
return result
}
@@ -616,6 +655,14 @@ func (h *AgentHandler) runWebhookSync(
}
}
}
+ if err = ctx.Err(); err != nil {
+ event := webhookContextTerminalEvent(err, sessionID)
+ if isTest {
+ h.appendWebhookInterruptedTrace(ctx, cv.ID, startTs, sessionID, event)
+ }
+ result, _ := webhookTerminalResult(event, sessionID)
+ return result
+ }
final := strings.Join(contents, "")
if isTest {
h.appendWebhookFinishedTrace(ctx, cv.ID, startTs, sessionID, true)
@@ -699,6 +746,20 @@ func isWebhookTerminalEvent(ev canvas.RunEvent) bool {
}
}
+func webhookContextTerminalEvent(err error, sessionID string) canvas.RunEvent {
+ eventType := "cancelled"
+ message := "Agent run was cancelled."
+ if errors.Is(err, context.DeadlineExceeded) {
+ eventType = "error"
+ message = "Agent run timed out."
+ }
+ return canvas.RunEvent{
+ Type: eventType,
+ SessionID: sessionID,
+ Data: mustJSON(canvas.ErrorEvent{Message: message}),
+ }
+}
+
func sanitizeWebhookTraceEvent(ev canvas.RunEvent) canvas.RunEvent {
if ev.Type != "error" {
return ev
@@ -735,6 +796,15 @@ func (h *AgentHandler) appendWebhookFinishedTrace(
})
}
+func (h *AgentHandler) appendWebhookInterruptedTrace(
+ ctx context.Context, agentID string, startTs time.Time, sessionID string, event canvas.RunEvent,
+) {
+ traceCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+ defer cancel()
+ h.appendWebhookRunTrace(traceCtx, agentID, startTs, event)
+ h.appendWebhookFinishedTrace(traceCtx, agentID, startTs, sessionID, false)
+}
+
// mustJSON marshals v to a JSON object string. Used by trace appenders;
// panics on marshal failure (acceptable because we only marshal
// statically-typed map[string]any values).
diff --git a/internal/handler/agent_webhook_test.go b/internal/handler/agent_webhook_test.go
index 7408cdab3a..ccf5d9d572 100644
--- a/internal/handler/agent_webhook_test.go
+++ b/internal/handler/agent_webhook_test.go
@@ -40,10 +40,11 @@ import (
// without touching the database. Mirrors the pattern used by
// waitFakeAgentService at internal/handler/agent_wait_for_user_test.go.
type fakeCanvasLoader struct {
- canvas *entity.UserCanvas
- err error
- runErr error
- events []canvas.RunEvent
+ canvas *entity.UserCanvas
+ err error
+ runErr error
+ events []canvas.RunEvent
+ waitForCancellation bool
}
func (f *fakeCanvasLoader) LoadCanvasByID(_ context.Context, _, _ string) (*entity.UserCanvas, error) {
@@ -53,10 +54,18 @@ func (f *fakeCanvasLoader) LoadCanvasByID(_ context.Context, _, _ string) (*enti
return f.canvas, nil
}
-func (f *fakeCanvasLoader) RunAgentWithWebhook(_ context.Context, _, _ string, _ map[string]any) (<-chan canvas.RunEvent, error) {
+func (f *fakeCanvasLoader) RunAgentWithWebhook(ctx context.Context, _, _ string, _ map[string]any) (<-chan canvas.RunEvent, error) {
if f.runErr != nil {
return nil, f.runErr
}
+ if f.waitForCancellation {
+ out := make(chan canvas.RunEvent)
+ go func() {
+ <-ctx.Done()
+ close(out)
+ }()
+ return out, nil
+ }
out := make(chan canvas.RunEvent, len(f.events))
for _, e := range f.events {
out <- e
@@ -659,6 +668,120 @@ func TestRunWebhookDetachedAppendsFinishedTrace(t *testing.T) {
}
}
+func TestRunWebhookDetachedRecordsTimeoutTrace(t *testing.T) {
+ tests := []struct {
+ name string
+ loader *fakeCanvasLoader
+ }{
+ {name: "during start", loader: &fakeCanvasLoader{runErr: context.DeadlineExceeded}},
+ {name: "during run", loader: &fakeCanvasLoader{waitForCancellation: true}},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ cv := makeWebhookCanvas("c1", "u-1", "Webhook", nil)
+ tt.loader.canvas = cv
+ h := &AgentHandler{loader: tt.loader}
+ var trace []canvas.RunEvent
+ h.webhookTraceAppender = func(ctx context.Context, _ string, _ time.Time, ev canvas.RunEvent) {
+ if err := ctx.Err(); err != nil {
+ t.Errorf("trace context error = %v, want nil", err)
+ }
+ trace = append(trace, ev)
+ }
+
+ ctx, cancel := context.WithTimeout(
+ service.WithAgentSessionID(t.Context(), "session-1"),
+ time.Nanosecond,
+ )
+ defer cancel()
+ <-ctx.Done()
+ h.runWebhookDetachedWithContext(ctx, cv, map[string]any{}, true, time.Now(), "session-1")
+
+ if len(trace) < 2 || trace[0].Type != "error" || !strings.Contains(trace[0].Data, "timed out") {
+ t.Fatalf("timeout trace = %+v, want error followed by failed finished event", trace)
+ }
+ assertWebhookFinishedTrace(t, trace, false)
+ })
+ }
+}
+
+func TestRunWebhookSyncRecordsCancelledTrace(t *testing.T) {
+ cv := makeWebhookCanvas("c1", "u-1", "Webhook", nil)
+ h := &AgentHandler{loader: &fakeCanvasLoader{canvas: cv, waitForCancellation: true}}
+ var trace []canvas.RunEvent
+ h.webhookTraceAppender = func(ctx context.Context, _ string, _ time.Time, ev canvas.RunEvent) {
+ if err := ctx.Err(); err != nil {
+ t.Errorf("trace context error = %v, want nil", err)
+ }
+ trace = append(trace, ev)
+ }
+
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ result := h.runWebhookSync(ctx, cv, map[string]any{}, true, time.Now())
+
+ if result.status != http.StatusConflict {
+ t.Errorf("status = %d, want %d", result.status, http.StatusConflict)
+ }
+ if len(trace) < 2 || trace[0].Type != "cancelled" || !strings.Contains(trace[0].Data, "cancelled") {
+ t.Fatalf("cancelled trace = %+v, want cancelled followed by failed finished event", trace)
+ }
+ assertWebhookFinishedTrace(t, trace, false)
+}
+
+func TestWebhookBufferedTerminalEventAfterCancellationUsesCleanupContext(t *testing.T) {
+ tests := []struct {
+ name string
+ run func(*testing.T, *AgentHandler, context.Context, *entity.UserCanvas)
+ }{
+ {
+ name: "detached",
+ run: func(_ *testing.T, h *AgentHandler, ctx context.Context, cv *entity.UserCanvas) {
+ h.runWebhookDetachedWithContext(ctx, cv, map[string]any{}, true, time.Now(), "session-1")
+ },
+ },
+ {
+ name: "sync",
+ run: func(t *testing.T, h *AgentHandler, ctx context.Context, cv *entity.UserCanvas) {
+ result := h.runWebhookSync(ctx, cv, map[string]any{}, true, time.Now())
+ if result.status != http.StatusConflict {
+ t.Errorf("status = %d, want %d", result.status, http.StatusConflict)
+ }
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ cv := makeWebhookCanvas("c1", "u-1", "Webhook", nil)
+ loader := &fakeCanvasLoader{
+ canvas: cv,
+ events: []canvas.RunEvent{{
+ Type: "cancelled",
+ Data: `{"message":"Agent run was cancelled."}`,
+ }},
+ }
+ h := &AgentHandler{loader: loader}
+ var trace []canvas.RunEvent
+ h.webhookTraceAppender = func(ctx context.Context, _ string, _ time.Time, ev canvas.RunEvent) {
+ if err := ctx.Err(); err != nil {
+ t.Errorf("trace context error = %v, want nil", err)
+ }
+ trace = append(trace, ev)
+ }
+
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ tt.run(t, h, ctx, cv)
+
+ if len(trace) < 2 || trace[0].Type != "cancelled" {
+ t.Fatalf("cancelled trace = %+v, want cancelled followed by failed finished event", trace)
+ }
+ assertWebhookFinishedTrace(t, trace, false)
+ })
+ }
+}
+
func assertWebhookFinishedTrace(t *testing.T, events []canvas.RunEvent, wantSuccess bool) {
t.Helper()
if len(events) == 0 || events[len(events)-1].Type != "finished" {