Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(agent): redact internal terminal errors (#18743)

Guofang.Tang80f74216d84e
diff --git a/internal/agent/canvas/run_error.go b/internal/agent/canvas/run_error.go
new file mode 100644
index 0000000000..3f1b3367e6
--- /dev/null
+++ b/internal/agent/canvas/run_error.go
@@ -0,0 +1,59 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package canvas
+
+import "errors"
+
+const (
+ // RunErrorKindInternal marks an error whose cause is safe for server logs
+ // but must not be exposed through Agent event streams.
+ RunErrorKindInternal = "internal"
+
+ // InternalRunErrorMessage is the public message used for internal Agent
+ // execution failures.
+ InternalRunErrorMessage = "Internal storage error while accessing the agent."
+)
+
+type internalRunError struct {
+ cause error
+}
+
+// NewInternalRunError preserves cause for server-side logging and errors.Is
+// while exposing only InternalRunErrorMessage in the terminal RunEvent.
+func NewInternalRunError(cause error) error {
+ if cause == nil {
+ cause = errors.New("internal agent run error")
+ }
+ return &internalRunError{cause: cause}
+}
+
+func (e *internalRunError) Error() string {
+ return e.cause.Error()
+}
+
+func (e *internalRunError) Unwrap() error {
+ return e.cause
+}
+
+func runErrorEvent(err error) ErrorEvent {
+ var internalErr *internalRunError
+ if errors.As(err, &internalErr) {
+ return ErrorEvent{
+ Message: InternalRunErrorMessage,
+ Kind: RunErrorKindInternal,
+ }
+ }
+ return ErrorEvent{Message: err.Error()}
+}
diff --git a/internal/agent/canvas/run_error_test.go b/internal/agent/canvas/run_error_test.go
new file mode 100644
index 0000000000..61ec997c73
--- /dev/null
+++ b/internal/agent/canvas/run_error_test.go
@@ -0,0 +1,71 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package canvas
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+)
+
+func TestRunnerRedactsInternalRunError(t *testing.T) {
+ cause := errors.New("dial mysql.internal:3306 password=secret")
+ runErr := NewInternalRunError(fmt.Errorf("persist session: %w", cause))
+ if !errors.Is(runErr, cause) {
+ t.Fatal("NewInternalRunError must preserve the original cause")
+ }
+
+ runner := NewRunner()
+ events := runner.Run(t.Context(), func(context.Context, map[string]any) (*CanvasState, error) {
+ return nil, runErr
+ }, "canvas", "session", nil, map[string]any{})
+
+ ev, ok := <-events
+ if !ok {
+ t.Fatal("runner closed without an error event")
+ }
+ if ev.Type != "error" {
+ t.Fatalf("event type = %q, want error", ev.Type)
+ }
+ var payload ErrorEvent
+ if err := json.Unmarshal([]byte(ev.Data), &payload); err != nil {
+ t.Fatalf("decode error event: %v", err)
+ }
+ if payload.Kind != RunErrorKindInternal {
+ t.Errorf("error kind = %q, want %q", payload.Kind, RunErrorKindInternal)
+ }
+ if payload.Message != InternalRunErrorMessage {
+ t.Errorf("error message = %q, want %q", payload.Message, InternalRunErrorMessage)
+ }
+ if strings.Contains(ev.Data, "mysql.internal") || strings.Contains(ev.Data, "password=secret") {
+ t.Fatalf("error event leaked its internal cause: %s", ev.Data)
+ }
+ if _, open := <-events; open {
+ t.Fatal("runner should close after the terminal error event")
+ }
+}
+
+func TestRunErrorEventPreservesPublicMessage(t *testing.T) {
+ payload := runErrorEvent(errors.New("invalid variable reference"))
+ if payload.Message != "invalid variable reference" {
+ t.Errorf("message = %q, want public runtime error", payload.Message)
+ }
+ if payload.Kind != "" {
+ t.Errorf("kind = %q, want empty for public runtime error", payload.Kind)
+ }
+}
diff --git a/internal/agent/canvas/runner.go b/internal/agent/canvas/runner.go
index b75068a1f8..b75790100c 100644
--- a/internal/agent/canvas/runner.go
+++ b/internal/agent/canvas/runner.go
@@ -46,7 +46,7 @@
// SSE wire contract (matches the handler envelope):
// - RunEvent.Type == "message" → {data: <string>}
// - RunEvent.Type == "waiting_for_user" → {cpn_id: <string>}
-// - RunEvent.Type == "error" → {message: <string>}
+// - RunEvent.Type == "error" → {message: <string>, kind?: <string>}
// - RunEvent.Type == "cancelled" → {message: <string>}
package canvas
@@ -135,9 +135,11 @@ type WaitingForUserEvent struct {
Inputs map[string]any `json:"inputs,omitempty"`
}
-// ErrorEvent is the JSON payload for Type=="error" frames.
+// ErrorEvent is the JSON payload for Type=="error" frames. Kind is present
+// only when adapters must apply special handling, such as internal redaction.
type ErrorEvent struct {
Message string `json:"message"`
+ Kind string `json:"kind,omitempty"`
}
// CancelledEvent is an alias for ErrorEvent because both terminal payloads
@@ -358,7 +360,19 @@ func (r *Runner) Run(
push(ctx, out, RunEvent{Type: "waiting_for_user", Data: safeEventJSON(WaitingForUserEvent{CpnID: runErr.Error()}), MessageID: messageID, CreatedAt: nowUnix(), SessionID: sessionID})
return
}
- pushErr(ctx, out, runErr.Error(), sessionID)
+ errorEvent := runErrorEvent(runErr)
+ if errorEvent.Kind == RunErrorKindInternal {
+ common.Error("canvas runner internal error", runErr,
+ zap.String("canvas", canvasID),
+ zap.String("session", sessionID))
+ }
+ push(ctx, out, RunEvent{
+ Type: "error",
+ Data: safeEventJSON(errorEvent),
+ MessageID: messageID,
+ CreatedAt: nowUnix(),
+ SessionID: sessionID,
+ })
return
}
}()
diff --git a/internal/handler/agent.go b/internal/handler/agent.go
index f08ae37895..123b4d4a17 100644
--- a/internal/handler/agent.go
+++ b/internal/handler/agent.go
@@ -669,23 +669,6 @@ func (h *AgentHandler) extractChatDebugFile(ctx context.Context, files []map[str
return name, data
}
-// sanitiseRunEventError passes through the error event payload
-// unchanged. The runner serialises canvas.ErrorEvent ({"message": ...})
-// before push, so when the payload round-trips through JSON the
-// message field is already preserved. Heuristic sanitisation is
-// disabled until the runner tags error events with a "kind"
-// field — without that, blanket rewriting every error to
-// "Internal storage error while accessing the agent." hides the
-// real failure from the front-end and the user (v3.6.1 diagnostic
-// regression: every canvas run failure surfaced as the same opaque
-// string).
-func sanitiseRunEventError(data string) string {
- if data == "" {
- return `{"message":"Unknown agent runtime error"}`
- }
- return data
-}
-
// CancelSessionRun cancels one ordinary Agent run by session id.
// @Summary Cancel Agent Session Run
// @Tags agents
diff --git a/internal/handler/agent_openai.go b/internal/handler/agent_openai.go
index a198b3446e..abb957071c 100644
--- a/internal/handler/agent_openai.go
+++ b/internal/handler/agent_openai.go
@@ -326,8 +326,13 @@ func openAICompatUsageForCompletion(promptTokens, completionTokens int) *openAIC
func openAICompatRunEventMessage(ev canvas.RunEvent, fallback string) string {
var payload canvas.ErrorEvent
- if err := json.Unmarshal([]byte(ev.Data), &payload); err == nil && payload.Message != "" {
- return payload.Message
+ if err := json.Unmarshal([]byte(ev.Data), &payload); err == nil {
+ if payload.Kind == canvas.RunErrorKindInternal {
+ return canvas.InternalRunErrorMessage
+ }
+ if payload.Message != "" {
+ return payload.Message
+ }
}
if message := strings.TrimSpace(ev.Data); message != "" {
return message
diff --git a/internal/handler/agent_test.go b/internal/handler/agent_test.go
index b9051c3e73..fe9b054f0b 100644
--- a/internal/handler/agent_test.go
+++ b/internal/handler/agent_test.go
@@ -1159,6 +1159,46 @@ func TestAgentChatCompletions_OpenAICompat_NonStreamReturnsRunnerError(t *testin
}
}
+func TestAgentChatCompletions_OpenAICompat_NonStreamRedactsInternalRunnerError(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = httptest.NewRequest("POST", "/api/v1/agents/chat/completions",
+ strings.NewReader(`{"agent_id":"a1","openai-compatible":true,"messages":[{"role":"user","content":"hi"}]}`))
+ c.Request.Header.Set("Content-Type", "application/json")
+ c.Set("user", &entity.User{ID: "u1"})
+ c.Set("user_id", "u1")
+
+ runner := &stubChatRunner{events: []canvas.RunEvent{{
+ Type: "error",
+ Data: `{"message":"dial mysql.internal:3306 password=secret","kind":"internal"}`,
+ }}}
+ h := &AgentHandler{chatRunner: runner}
+ h.AgentChatCompletions(c)
+
+ if w.Code != http.StatusInternalServerError {
+ t.Fatalf("status = %d, want 500: %s", w.Code, w.Body.String())
+ }
+ var response struct {
+ Error struct {
+ Message string `json:"message"`
+ Type string `json:"type"`
+ } `json:"error"`
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+ t.Fatalf("decode error response: %v", err)
+ }
+ if response.Error.Type != "server_error" {
+ t.Errorf("error.type = %q, want server_error", response.Error.Type)
+ }
+ if response.Error.Message != canvas.InternalRunErrorMessage {
+ t.Errorf("error.message = %q, want %q", response.Error.Message, canvas.InternalRunErrorMessage)
+ }
+ if strings.Contains(w.Body.String(), "mysql.internal") || strings.Contains(w.Body.String(), "password=secret") {
+ t.Fatalf("response leaked internal error details: %s", w.Body.String())
+ }
+}
+
func TestAgentChatCompletions_OpenAICompat_NonStreamReturnsWaitingForUser(t *testing.T) {
gin.SetMode(gin.TestMode)
w := httptest.NewRecorder()
@@ -1417,6 +1457,47 @@ func TestAgentChatCompletions_OpenAICompat_StreamSurfacesRunnerError(t *testing.
}
}
+func TestAgentChatCompletions_OpenAICompat_StreamRedactsInternalRunnerError(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = httptest.NewRequest("POST", "/api/v1/agents/chat/completions",
+ strings.NewReader(`{"agent_id":"a1","openai-compatible":true,"stream":true,"messages":[{"role":"user","content":"hi"}]}`))
+ c.Request.Header.Set("Content-Type", "application/json")
+ c.Set("user", &entity.User{ID: "u1"})
+ c.Set("user_id", "u1")
+
+ runner := &stubChatRunner{events: []canvas.RunEvent{{
+ Type: "error",
+ Data: `{"message":"dial mysql.internal:3306 password=secret","kind":"internal"}`,
+ }}}
+ h := &AgentHandler{chatRunner: runner}
+ h.AgentChatCompletions(c)
+
+ chunks, done := decodeOpenAICompatStream(t, w.Body.String())
+ if done {
+ t.Fatal("failed stream must not end with [DONE]")
+ }
+ if len(chunks) != 1 {
+ t.Fatalf("decoded %d chunks, want one internal error", len(chunks))
+ }
+ choice := chunks[0]["choices"].([]interface{})[0].(map[string]interface{})
+ if choice["finish_reason"] != "error" {
+ t.Errorf("finish_reason = %v, want error", choice["finish_reason"])
+ }
+ delta := choice["delta"].(map[string]interface{})
+ if delta["content"] != "**ERROR**: "+canvas.InternalRunErrorMessage {
+ t.Errorf("error content = %v, want redacted error", delta["content"])
+ }
+ errorPayload := delta["error"].(map[string]interface{})
+ if errorPayload["message"] != canvas.InternalRunErrorMessage {
+ t.Errorf("error payload = %v, want redacted message", errorPayload)
+ }
+ if strings.Contains(w.Body.String(), "mysql.internal") || strings.Contains(w.Body.String(), "password=secret") {
+ t.Fatalf("stream leaked internal error details: %s", w.Body.String())
+ }
+}
+
func TestAgentChatCompletions_OpenAICompat_StreamSurfacesWaitingForUser(t *testing.T) {
gin.SetMode(gin.TestMode)
w := httptest.NewRecorder()
diff --git a/internal/service/agent.go b/internal/service/agent.go
index 7b878c913d..a1dc3907b2 100644
--- a/internal/service/agent.go
+++ b/internal/service/agent.go
@@ -328,19 +328,11 @@ var ErrAgentNotOwner = errors.New("agent not owned by user")
// same Agent session before the current run reaches a terminal state.
var ErrAgentSessionBusy = errors.New("agent session is already running")
-// ErrAgentStorageError is returned by RunAgent when the underlying
-// version / canvas / tenant DAO surfaces a non-sentinel error (DB
-// connectivity, schema drift, deadlock, etc.). The handler's
-// mapAgentError recognises this sentinel and maps it to
-// common.CodeServerError (500) with a SANITIZED message — the raw
-// DAO error string is never echoed to the client, so internal
-// connection-string / table-name leaks are avoided.
-//
-// v3.5.2 follow-up: the prior af2ac2eda commit claimed "DB error ->
-// 500" in the branch table, but the handler's mapAgentError did not
-// actually classify those errors as CodeServerError — every DAO
-// failure fell through to CodeDataError with the raw err.Error()
-// string. This sentinel closes that gap.
+// ErrAgentStorageError identifies internal Agent service failures such as
+// database connectivity, schema drift, or persistence errors. Synchronous
+// callers map this sentinel to a sanitized 500 response; failures raised after
+// streaming starts are wrapped with canvas.NewInternalRunError so Runner emits
+// the same safe message in its terminal event.
var ErrAgentStorageError = errors.New("agent storage error")
// AgentService agent service
@@ -2099,7 +2091,9 @@ func (s *AgentService) buildRunFunc(canvasID string, versionRow *entity.UserCanv
zap.String("type", fmt.Sprintf("%T", err)),
zap.Error(err))
s.markRunFailed(ctx2, runID, "compile: "+err.Error())
- return nil, fmt.Errorf("canvas compile: %w: %w", ErrAgentStorageError, err)
+ return nil, canvas.NewInternalRunError(
+ fmt.Errorf("canvas compile: %w: %w", ErrAgentStorageError, err),
+ )
}
cpID := ""
@@ -2198,7 +2192,9 @@ func (s *AgentService) buildRunFunc(canvasID string, versionRow *entity.UserCanv
appendAssistantHistory(state, partialAssistantOutput(answer, downloads))
}
if persistErr := s.persistAgentRunSession(ctx, canvasID, userID, sessionID, messageID, userInput, answer, thinking, referencePayload, dsl, state, answer != ""); persistErr != nil {
- return nil, fmt.Errorf("persist interrupted agent session: %w: %w", persistErr, ErrAgentStorageError)
+ return nil, canvas.NewInternalRunError(
+ fmt.Errorf("persist interrupted agent session: %w: %w", persistErr, ErrAgentStorageError),
+ )
}
if answer != "" && shouldEmitMessage {
if !messageEventsEmitted {
@@ -2216,7 +2212,9 @@ func (s *AgentService) buildRunFunc(canvasID string, versionRow *entity.UserCanv
appendAssistantHistory(state, assistantOutput)
if persistErr := s.persistAgentRunSession(ctx, canvasID, userID, sessionID, messageID, userInput, answer, thinking, referencePayload, dsl, state, true); persistErr != nil {
s.markRunFailed(ctx2, runID, "persist session: "+persistErr.Error())
- return nil, fmt.Errorf("persist agent session: %w: %w", persistErr, ErrAgentStorageError)
+ return nil, canvas.NewInternalRunError(
+ fmt.Errorf("persist agent session: %w: %w", persistErr, ErrAgentStorageError),
+ )
}
if !messageEventsEmitted && shouldEmitMessage {
emitAgentMessageEvents(emit, answer, thinking, referencePayload)
@@ -2252,7 +2250,9 @@ func (s *AgentService) buildRunFunc(canvasID string, versionRow *entity.UserCanv
appendAssistantHistory(state, assistantOutput)
if persistErr := s.persistAgentRunSession(ctx, canvasID, userID, sessionID, messageID, userInput, answer, thinking, referencePayload, dsl, state, true); persistErr != nil {
s.markRunFailed(ctx2, runID, "persist session: "+persistErr.Error())
- return nil, fmt.Errorf("persist agent session: %w: %w", persistErr, ErrAgentStorageError)
+ return nil, canvas.NewInternalRunError(
+ fmt.Errorf("persist agent session: %w: %w", persistErr, ErrAgentStorageError),
+ )
}
if !messageEventsEmitted && shouldEmitMessage {
emitAgentMessageEvents(emit, answer, thinking, referencePayload)
diff --git a/internal/service/agent_run_e2e_test.go b/internal/service/agent_run_e2e_test.go
index 1b8808f790..b0b95b2ecc 100644
--- a/internal/service/agent_run_e2e_test.go
+++ b/internal/service/agent_run_e2e_test.go
@@ -1498,13 +1498,9 @@ func TestRunAgent_AllFixture_DataOps(t *testing.T) {
}
}
-// TestRunAgent_RealCanvas_CompileFails pins the schema-failure
-// branch: when the DSL references a component name that is not
-// registered against runtime.DefaultFactory, canvas.Compile fails
-// (buildNodeBody returns 'factory: component: unknown component'),
-// and RunAgent must surface that as a wrapped ErrAgentStorageError
-// so mapAgentError classifies it as CodeServerError (500) with a
-// sanitized message — NOT the raw build error string.
+// TestRunAgent_RealCanvas_CompileFails pins the asynchronous compile-failure
+// branch. Once streaming has started, Runner must emit a typed internal error
+// with a stable public message rather than the raw component factory failure.
func TestRunAgent_RealCanvas_CompileFails(t *testing.T) {
testDB := setupServiceTestDB(t)
if err := testDB.AutoMigrate(
@@ -1557,13 +1553,13 @@ func TestRunAgent_RealCanvas_CompileFails(t *testing.T) {
if len(errs) == 0 {
t.Fatal("expected error event from Compile of DSL with unknown component name")
}
- // The error message should mention ErrAgentStorageError but NOT
- // contain the raw factory error substring (sanitised at the
- // service layer). The factory error is wrapped inside the
- // buildNodeBody / BuildWorkflow chain — its full text is
- // preserved for the logs but not echoed as the SSE message.
- if !strings.Contains(errs[0].Message, "agent storage error") {
- t.Errorf("error message %q does not mention sanitised label", errs[0].Message)
+ // The raw factory error remains available to server logs, while the event
+ // carries an explicit internal kind and a stable public message.
+ if errs[0].Kind != canvas.RunErrorKindInternal {
+ t.Errorf("error kind = %q, want %q", errs[0].Kind, canvas.RunErrorKindInternal)
+ }
+ if errs[0].Message != canvas.InternalRunErrorMessage {
+ t.Errorf("error message = %q, want %q", errs[0].Message, canvas.InternalRunErrorMessage)
}
}
diff --git a/internal/service/bot_completion.go b/internal/service/bot_completion.go
index 120b14e8a9..44e6e3fc34 100644
--- a/internal/service/bot_completion.go
+++ b/internal/service/bot_completion.go
@@ -219,7 +219,9 @@ func WriteChatbotRunEvent(w http.ResponseWriter, ev canvas.RunEvent) error {
if ev.Type == "error" {
msg := "an internal error occurred"
if m, ok := data.(map[string]any); ok {
- if s, _ := m["message"].(string); s != "" {
+ if kind, _ := m["kind"].(string); kind == canvas.RunErrorKindInternal {
+ msg = canvas.InternalRunErrorMessage
+ } else if s, _ := m["message"].(string); s != "" {
msg = s
}
}
diff --git a/internal/service/bot_completion_history_test.go b/internal/service/bot_completion_history_test.go
index 2703006d28..b0e8c41cbf 100644
--- a/internal/service/bot_completion_history_test.go
+++ b/internal/service/bot_completion_history_test.go
@@ -358,6 +358,23 @@ func TestWriteChatbotRunEvent_ErrorCarriesSessionAlias(t *testing.T) {
}
}
+func TestWriteChatbotRunEvent_RedactsInternalError(t *testing.T) {
+ rec := &recordingResponseWriter{header: http.Header{}}
+ if err := WriteChatbotRunEvent(rec, canvas.RunEvent{
+ Type: "error",
+ Data: `{"message":"dial mysql.internal:3306 password=secret","kind":"internal"}`,
+ }); err != nil {
+ t.Fatalf("WriteChatbotRunEvent: %v", err)
+ }
+ body := rec.body.String()
+ if !strings.Contains(body, canvas.InternalRunErrorMessage) {
+ t.Errorf("body missing safe internal error message: %s", body)
+ }
+ if strings.Contains(body, "mysql.internal") || strings.Contains(body, "password=secret") {
+ t.Fatalf("body leaked internal error details: %s", body)
+ }
+}
+
// TestBotService_ChatbotCompletion_NewSessionSkipsLLM locks in the
// share-page handshake behaviour: the front-end opens a shared chat
// with an empty question and no session_id only to obtain a session