RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Feat(ingestion): add canvas pipeline debug (dry-run) mode with View result log (#17538)
Jack8546eb62bfa8
diff --git a/internal/handler/agent.go b/internal/handler/agent.goindex 9d1a51d0a4..6b451f4fe9 100644--- a/internal/handler/agent.go+++ b/internal/handler/agent.go@@ -29,12 +29,15 @@ import ("time""github.com/gin-gonic/gin"+ "github.com/google/uuid""go.uber.org/zap""ragflow/internal/agent/canvas""ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity"+ pipelinepkg "ragflow/internal/ingestion/pipeline"+ "ragflow/internal/ingestion/task""ragflow/internal/service""ragflow/internal/service/file""ragflow/internal/utility"@@ -91,6 +94,28 @@ type AgentHandler struct {// tenant-only authorization (i.e. cannot verify the doc, so the// check is skipped — same shape as the pre-port behaviour).documentService documentAccessChecker+ // redisGet fetches a raw string from Redis. Defaults to the global+ // client (redis.Get) so production behaviour is unchanged; tests inject+ // a miniredis-backed getter to exercise GetAgentLogs without a live+ // Redis (mirrors the newExecutor injection pattern).+ redisGet func(key string) (string, error)+ // redisStore writes the debug-run log array. Defaults to the global+ // client (redis.Get, which satisfies task.DebugLogStore); tests inject a+ // miniredis-backed writer so runCanvasPipelineDebug can be exercised without a+ // live Redis.+ redisStore task.DebugLogStore+ // newExecutor builds the pipeline executor for a debug run. Defaults to+ // task.NewPipelineExecutor; tests inject a fake so the debug path can be+ // driven without a real canvas/DSL.+ newExecutor func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error)+}++// debugExecutor is the minimal executor surface runCanvasPipelineDebug needs. It is an+// interface (not *task.PipelineExecutor) so tests can substitute a fake that+// emits progress through the attached sink. *task.PipelineExecutor satisfies it.+type debugExecutor interface {+ WithProgressSink(sink pipelinepkg.ProgressSink) *task.PipelineExecutor+ Execute(ctx context.Context) (*task.PipelineResult, error)}// WithDocumentService injects the document service used by@@ -110,9 +135,37 @@ func NewAgentHandler(agentService *service.AgentService, fileService *file.FileSchatRunner: agentService,fileService: fileService,loader: agentService,+ redisGet: func(key string) (string, error) { return redis.Get().Get(key) },+ redisStore: redis.Get(),+ newExecutor: func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error) {+ return task.NewPipelineExecutor(taskCtx, canvasID, docBulkSize)+ },}}+// WithRedisGetter overrides the Redis string getter used by GetAgentLogs.+// Tests pass a miniredis-backed getter so the debug-log polling endpoint can+// be exercised end-to-end without a live Redis.+func (h *AgentHandler) WithRedisGetter(f func(key string) (string, error)) *AgentHandler {+ h.redisGet = f+ return h+}++// WithRedisStore overrides the Redis writer used by runCanvasPipelineDebug to persist+// the debug-run log. Tests pass a miniredis-backed writer.+func (h *AgentHandler) WithRedisStore(s task.DebugLogStore) *AgentHandler {+ h.redisStore = s+ return h+}++// WithNewExecutor overrides the pipeline executor factory used by+// runCanvasPipelineDebug. Tests inject a fake that emits progress through the+// attached sink without a real canvas/DSL.+func (h *AgentHandler) WithNewExecutor(f func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error)) *AgentHandler {+ h.newExecutor = f+ return h+}+// ListAgents lists agent canvases for the current user.// @Summary List Agents// @Description List agent canvases accessible to the current user (Home dashboard tile)@@ -486,6 +539,127 @@ func readUserInput(c *gin.Context) string {return c.Query("user_input")}+// runCanvasPipelineDebug runs a canvas in pipeline dry-run (debug) mode: it builds+// an in-memory debug TaskContext and executes the pipeline synchronously,+// returning the produced chunks. The run is fully synchronous: the complete+// debug log (including the terminal END marker) is flushed to Redis BEFORE+// this method returns, so the message_id it yields is for *replay* —+// re-fetching the log after a page refresh or from a shared link — not for+// incremental streaming; the front-end's poll therefore receives a finished+// log on the first request.+//+// It performs no persistence (no MinIO upload, index insert, or pipeline_log)+// because the debug context carries no KB (KB.ID == ""), which is the single+// debug signal used across the ingestion pipeline: the tokenizer skips+// embedding and the executor skips the persist stage. kb_id == "" occurs ONLY+// in debug mode; production ingestion always supplies a KB, so this path is+// never taken in normal operation. Debug takes no kb_id at all: it never+// resolves an embedder or writes to a KB, and parse/chunk work without it.+// Callers extract the (fileName, fileData) pair from their own request shape+// and pass them in.+func (h *AgentHandler) runCanvasPipelineDebug(ctx context.Context, user *entity.User,+ canvasID, fileName string, fileData []byte) (*task.PipelineResult, error) {+ // messageID is the polling key the front-end reads from the run response+ // (see web/src/pages/agent/hooks/use-run-dataflow.ts:40) to fetch the debug+ // log. It is generated up-front so the response always carries a stable key.+ messageID := uuid.New().String()++ // Force an empty KB: a debug context has no knowledgebase, so kb_id == ""+ // holds and the debug signal is unambiguous.+ taskCtx := task.NewDebugTaskContext(user.ID, canvasID, fileName, fileData)++ exec, err := h.newExecutor(taskCtx, canvasID, 0)+ if err != nil {+ return nil, err+ }++ // Attach a sink that records component progress into the+ // [{component_id, trace}] array the front-end polls for.+ sink := task.NewDebugLogSink(canvasID, messageID, h.redisStore)+ exec.WithProgressSink(sink)++ result, runErr := exec.Execute(ctx)+ // Flush even on error so the front-end can poll the failure log; the END+ // marker carries the error and keeps the run detectable as finished.+ sink.Flush(ctx, runErr)+ if result == nil {+ // The executor may return (nil, err) on failure; synthesize a result so+ // the polling key survives to the response and the front-end can still+ // reach the failure log written just above.+ result = &task.PipelineResult{}+ }+ result.MessageID = messageID+ return result, runErr+}++// respondWithDebugResult writes the outcome of a dataflow debug run. On a run+// failure it keeps a non-success code (errors must not be masked as success)+// but still carries message_id in the error envelope's data, because+// runCanvasPipelineDebug already flushed the failure log to Redis — the front-end+// needs that key to poll the failure timeline. On success it returns the+// message_id (the polling key) together with the produced chunks. Empty chunk+// slices become `[]` rather than `null` so the response shape stays stable.+// This is the single wire-shape helper for the chat/completions DataFlow debug+// entry point, and it matches the contract the front-end expects:+// `data.data.message_id` drives the log-poll loop and `data.data.chunks`+// renders the debug output.+func respondWithDebugResult(c *gin.Context, result *task.PipelineResult, err error) {+ if err != nil {+ messageID := ""+ if result != nil {+ messageID = result.MessageID+ }+ common.ResponseWithCodeData(c, common.CodeServerError, gin.H{"message_id": messageID}, err.Error())+ return+ }+ messageID := ""+ var chunks []map[string]any+ if result != nil {+ messageID = result.MessageID+ chunks = result.Chunks+ }+ if len(chunks) == 0 {+ chunks = []map[string]any{}+ }+ common.SuccessWithData(c, gin.H{"message_id": messageID, "chunks": chunks}, "success")+}++// extractChatDebugFile pulls the first uploaded file from a chat/completions+// request into (fileName, bytes) for a dataflow debug run. Unlike a debug+// multipart upload, chat/completions `files` are storage references+// (id/name/created_by); the raw bytes are fetched from the per-user downloads+// bucket via the file service. When no usable file is present it returns a+// default name and a nil slice so the pipeline can still run file-less.+func (h *AgentHandler) extractChatDebugFile(ctx context.Context, files [][]map[string]interface{}, user *entity.User) (string, []byte) {+ if len(files) == 0 {+ return "debug", nil+ }+ fileList := files[0]+ if len(fileList) == 0 {+ return "debug", nil+ }+ fd := fileList[0]+ name, _ := fd["name"].(string)+ id, _ := fd["id"].(string)+ if id == "" {+ if name == "" {+ return "debug", nil+ }+ return name, nil+ }+ data, err := h.fileService.DownloadAgentFile(ctx, user.ID, id)+ if err != nil || len(data) == 0 {+ if name == "" {+ return "debug", nil+ }+ return name, nil+ }+ if name == "" {+ name = "debug"+ }+ return name, data+}+// sanitiseRunEventError passes through the error event payload// unchanged. The runner serialises canvas.ErrorEvent ({"message": ...})// before push, so when the payload round-trips through JSON the@@ -889,7 +1063,16 @@ type agentChatCompletionsRequest struct {Model string `json:"model"`Messages []map[string]interface{} `json:"messages"`ReturnTrace bool `json:"return_trace"`- Files []map[string]interface{} `json:"files"`+ // Files carries the uploaded file references for a run. The RAGFlow web+ // front-end wraps the file list one extra level (a list of per-turn file+ // lists), so the wire shape is `[[{id, name, ...}]]`. This mirrors the+ // Python agent_api.py:1611 contract `queue_dataflow(..., files[0], 0)`,+ // where `files[0]` (the first inner list) is the set of files for this+ // run. The dataflow debug extractor and the agent run path both unwrap+ // the outer layer: `Files[0]` reaches the file dicts. The web contract+ // is 2D, so a plain 1D `[]map[string]interface{}` would fail to decode+ // and 400 the whole request.+ Files [][]map[string]interface{} `json:"files"`}// extractLastUserContent returns the content of the last message in@@ -1028,6 +1211,29 @@ func (h *AgentHandler) AgentChatCompletions(c *gin.Context) {return}+ // DataFlow canvas: run a synchronous, side-effect-free debug (dry-run)+ // and return the parsed chunks inline — reusing this existing+ // chat/completions endpoint instead of a dedicated dataflow/debug route.+ // This mirrors the Python agent_api.py:1569 DataFlow branch, which is+ // reached only on the non-openai, non-session path (openai-compatible+ // requests return their choices shape above and never enter debug). The+ // The canvas is loaded only to read its category, and the debug path is+ // skipped when the loader is absent or the canvas is not a DataFlow. No+ // kb_id is involved: the debug context always runs KB-less (see+ // runCanvasPipelineDebug).+ if h.loader != nil {+ if cv, lerr := h.loader.LoadCanvasByID(c.Request.Context(), user.ID, req.AgentID); lerr == nil && cv.CanvasCategory == "dataflow_canvas" {+ fileName, fileData := h.extractChatDebugFile(c.Request.Context(), req.Files, user)+ if len(fileData) == 0 {+ common.ResponseWithCodeData(c, common.CodeDataError, nil, "dataflow debug requires an uploaded file")+ return+ }+ result, derr := h.runCanvasPipelineDebug(c.Request.Context(), user, req.AgentID, fileName, fileData)+ respondWithDebugResult(c, result, derr)+ return+ }+ }+// Real canvas run — derive userInput from `query` first, then fall// back to the last user message (covers the front-end that posts// running_hint_text without a top-level `query`).@@ -1052,7 +1258,16 @@ func (h *AgentHandler) AgentChatCompletions(c *gin.Context) {req.SessionID = utility.GenerateToken()}- events, err := h.chatRunner.RunAgent(c.Request.Context(), user.ID, req.AgentID, req.SessionID, "", userInput, req.Files)+ // The web contract wraps `files` one level ([[{...}]]); unwrap the+ // outer layer so RunAgent receives the inner 1D file list, matching the+ // Python canvas file component's `kwargs.get("file")[0]` unwrap. When no+ // files are present (the common agent-chat case) pass nil so RunAgent's+ // `len(files) > 0` guard sees an empty run.+ var chatFiles []map[string]interface{}+ if len(req.Files) > 0 {+ chatFiles = req.Files[0]+ }+ events, err := h.chatRunner.RunAgent(c.Request.Context(), user.ID, req.AgentID, req.SessionID, "", userInput, chatFiles)if err != nil {common.Warn("agent chat completions: RunAgent failed",append([]zap.Field{@@ -1314,12 +1529,35 @@ func (h *AgentHandler) TestDBConnection(c *gin.Context) {common.SuccessWithData(c, true, "success")}+// parseAgentLogs decodes the debug-log payload stored in Redis by the pipeline+// run. The Python pipeline (rag/flow/pipeline.py callback) writes a JSON+// *array* ([{component_id, trace:[...]}]) that the front-end's+// useFetchMessageTrace (web/src/hooks/use-agent-request.ts) requires as+// response.data. A missing, empty, or corrupt payload collapses to an empty+// object so the contract from get_agent_logs (api/apps/restful_apis/agent_api.py:1087)+// — `code == 0` and `data` is a (possibly empty) dict/array — stays satisfied+// instead of 500ing on bad bytes.+func parseAgentLogs(payload string) interface{} {+ if strings.TrimSpace(payload) == "" {+ return map[string]interface{}{}+ }+ var data interface{}+ if err := json.Unmarshal([]byte(payload), &data); err != nil {+ return map[string]interface{}{}+ }+ return data+}+// GetAgentLogs GET /api/v1/agents/:canvas_id/logs/:message_id//// Reads "{agent_id}-{message_id}-logs" from Redis (same key format-// used by the Python agent API in api/apps/restful_apis/agent_api.py-// line 920). Missing key returns an empty dict so the test contract-// `data is dict` and `code == 0` are both satisfied.+// used by the Python agent API in api/apps/restful_apis/agent_api.py) and+// returns it as-is. Because runCanvasPipelineDebug flushes the ENTIRE log (including+// the END marker) before returning the run response, this endpoint returns a+// completed snapshot on the first poll — it is a replay/fetch of an already+// finished debug log, not a streaming subscription. Missing key returns an+// empty dict so the test contract `data is dict` and `code == 0` are both+// satisfied when nothing has been written yet.func (h *AgentHandler) GetAgentLogs(c *gin.Context) {user, code, msg := GetUser(c)if code != common.CodeSuccess {@@ -1335,10 +1573,10 @@ func (h *AgentHandler) GetAgentLogs(c *gin.Context) {}key := fmt.Sprintf("%s-%s-logs", canvasID, messageID)- payload, rerr := redis.Get().Get(key)- data := map[string]interface{}{}- if rerr == nil && payload != "" {- _ = json.Unmarshal([]byte(payload), &data)+ payload, rerr := h.redisGet(key)+ var data interface{} = map[string]interface{}{}+ if rerr == nil {+ data = parseAgentLogs(payload)}common.SuccessWithData(c, data, "success")}diff --git a/internal/handler/agent_logs_test.go b/internal/handler/agent_logs_test.gonew file mode 100644index 0000000000..793b4fe9f0--- /dev/null+++ b/internal/handler/agent_logs_test.go@@ -0,0 +1,675 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package handler++import (+ "context"+ "encoding/json"+ "errors"+ "net/http/httptest"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/alicebob/miniredis/v2"+ "github.com/gin-gonic/gin"+ goredis "github.com/redis/go-redis/v9"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+ pipelinepkg "ragflow/internal/ingestion/pipeline"+ "ragflow/internal/ingestion/task"+ "ragflow/internal/service"+)++// TestParseAgentLogs pins the contract between GetAgentLogs and the+// front-end DataFlow Log box. The Python pipeline (rag/flow/pipeline.py+// callback) stores a JSON *array* ([{component_id, trace:[...]}]); the+// front-end's useFetchMessageTrace (web/src/hooks/use-agent-request.ts)+// requires response.data to be that array. The original handler unmarshalled+// into a map[string]interface{} and silently dropped the array, leaving the+// Log box stuck on an empty SkeletonCard. A missing/empty/corrupt payload must+// still collapse to an empty object to mirror the Python get_agent_logs+// missing-key contract (api/apps/restful_apis/agent_api.py:1087).+func TestParseAgentLogs(t *testing.T) {+ cases := []struct {+ name string+ payload string+ wantKind byte // '[' for array, '{' for object+ }{+ {+ name: "missing_key_returns_empty_object",+ payload: "",+ wantKind: '{',+ },+ {+ name: "present_array_returns_array",+ payload: `[` ++ `{"component_id":"File","trace":[` ++ `{"progress":1,"message":"parsed","datetime":"10:00:00","timestamp":1.0,"elapsed_time":0}` ++ `]},` ++ `{"component_id":"END","trace":[{"progress":1,"message":"done","datetime":"10:00:01","timestamp":2.0,"elapsed_time":1.0}]}` ++ `]`,+ wantKind: '[',+ },+ {+ name: "corrupt_payload_falls_back_to_empty_object",+ payload: `{"component_id":`,+ wantKind: '{',+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := parseAgentLogs(tc.payload)+ b, err := json.Marshal(got)+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ if len(b) == 0 || b[0] != tc.wantKind {+ t.Fatalf("want JSON kind %q, got %s", tc.wantKind, string(b))+ }+ if tc.wantKind == '[' {+ var arr []map[string]interface{}+ if err := json.Unmarshal(b, &arr); err != nil {+ t.Fatalf("round-trip: %v", err)+ }+ if arr[0]["component_id"] != "File" {+ t.Fatalf("component_id lost after round-trip: %v", arr[0])+ }+ if arr[len(arr)-1]["component_id"] != "END" {+ t.Fatalf("END marker lost after round-trip: %v", arr[len(arr)-1])+ }+ }+ })+ }+}++// TestGetAgentLogs_E2EViaMiniredis exercises the full HTTP handler path+// (auth -> canvas-access -> Redis fetch -> shape) against an in-memory+// miniredis so we can assert the wire response shape the front-end depends on:+// - present key -> response.data is the JSON array ([...]) with the END marker+// - missing key -> response.data is an empty object ("{}"), matching the+// Python get_agent_logs missing-key contract.+//+// The Redis client is injected via WithRedisGetter (a miniredis-backed+// go-redis client); the canvas-access gate is satisfied by a real in-memory+// UserCanvas row, mirroring TestGetAgentWebhookLogsReturnsEmptyPoll.+func TestGetAgentLogs_E2EViaMiniredis(t *testing.T) {+ gin.SetMode(gin.TestMode)++ db := setupHandlerAgentsTestDB(t)+ orig := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = orig })++ db.Create(&entity.UserCanvas{ID: "c1", UserID: "u1", Title: sptr("Test")})++ mr, err := miniredis.Run()+ if err != nil {+ t.Fatalf("miniredis.Run: %v", err)+ }+ t.Cleanup(mr.Close)++ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ logKey := "c1-msg1-logs"+ arrayPayload := `[` ++ `{"component_id":"File","trace":[{"progress":1,"message":"parsed","datetime":"10:00:00","timestamp":1.0,"elapsed_time":0}]},` ++ `{"component_id":"END","trace":[{"progress":1,"message":"done","datetime":"10:00:01","timestamp":2.0,"elapsed_time":1.0}]}` ++ `]`+ if err := rdb.Set(context.Background(), logKey, arrayPayload, 0).Err(); err != nil {+ t.Fatalf("seed redis: %v", err)+ }++ h := NewAgentHandler(service.NewAgentService(), nil).+ WithRedisGetter(func(key string) (string, error) {+ return rdb.Get(context.Background(), key).Result()+ })++ run := func(messageID string) map[string]interface{} {+ w := httptest.NewRecorder()+ c, _ := gin.CreateTestContext(w)+ c.Request = httptest.NewRequest("GET", "/api/v1/agents/c1/logs/"+messageID, nil)+ c.Set("user", &entity.User{ID: "u1"})+ c.Set("user_id", "u1")+ c.Params = gin.Params{+ {Key: "canvas_id", Value: "c1"},+ {Key: "message_id", Value: messageID},+ }+ h.GetAgentLogs(c)+ var resp map[string]interface{}+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatalf("decode (msg=%s): %v body=%s", messageID, err, w.Body.String())+ }+ return resp+ }++ // Present key -> data must be the array, with File first and END last.+ present := run("msg1")+ if code, _ := present["code"].(float64); int(code) != int(common.CodeSuccess) {+ t.Fatalf("present code=%v want 0 body=%s", code, mustJSON(present))+ }+ dataRaw, _ := json.Marshal(present["data"])+ trimmed := strings.TrimSpace(string(dataRaw))+ if len(trimmed) == 0 || trimmed[0] != '[' {+ t.Fatalf("present response.data must be a JSON array, got %s", string(dataRaw))+ }+ var arr []map[string]interface{}+ if err := json.Unmarshal(dataRaw, &arr); err != nil {+ t.Fatalf("present data not array: %v body=%s", err, string(dataRaw))+ }+ if arr[0]["component_id"] != "File" {+ t.Errorf("first component_id=%v want File", arr[0]["component_id"])+ }+ if arr[len(arr)-1]["component_id"] != "END" {+ t.Errorf("last component_id=%v want END", arr[len(arr)-1]["component_id"])+ }++ // Missing key -> data must be an empty object (Python parity), code 0.+ missing := run("missing")+ if code, _ := missing["code"].(float64); int(code) != int(common.CodeSuccess) {+ t.Fatalf("missing code=%v want 0 body=%s", code, mustJSON(missing))+ }+ dataRawMissing, _ := json.Marshal(missing["data"])+ if strings.TrimSpace(string(dataRawMissing)) != "{}" {+ t.Fatalf("missing response.data must be {} (object), got %s", string(dataRawMissing))+ }+}++// clientConsidersComplete replicates the front-end completion predicate from+// web/src/pages/agent/hooks/use-fetch-pipeline-log.ts: the run is "really+// done" only when the LAST array element has component_id == "END" AND its+// first trace entry carries a non-empty message. If any part of that signal+// is missing, the Log box keeps polling forever.+func clientConsidersComplete(arr []map[string]interface{}) bool {+ if len(arr) == 0 {+ return false+ }+ last := arr[len(arr)-1]+ if last["component_id"] != "END" {+ return false+ }+ trace, ok := last["trace"].([]interface{})+ if !ok || len(trace) == 0 {+ return false+ }+ first, ok := trace[0].(map[string]interface{})+ if !ok {+ return false+ }+ msg, _ := first["message"].(string)+ return strings.TrimSpace(msg) != ""+}++// TestGetAgentLogs_EndSignalCompletion pins the exact signal the front-end+// polls for: the response array must end with an END element whose first+// trace message is non-empty. The handler must preserve that signal+// byte-for-byte through JSON round-tripping.+func TestGetAgentLogs_EndSignalCompletion(t *testing.T) {+ gin.SetMode(gin.TestMode)++ db := setupHandlerAgentsTestDB(t)+ orig := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = orig })++ db.Create(&entity.UserCanvas{ID: "c1", UserID: "u1", Title: sptr("Test")})++ mr, err := miniredis.Run()+ if err != nil {+ t.Fatalf("miniredis.Run: %v", err)+ }+ t.Cleanup(mr.Close)++ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ // End element with a NON-empty message -> client must consider it done.+ goodPayload := `[` ++ `{"component_id":"File","trace":[{"progress":1,"message":"parsed","datetime":"10:00:00","timestamp":1.0,"elapsed_time":0}]},` ++ `{"component_id":"END","trace":[{"progress":1,"message":"run finished","datetime":"10:00:01","timestamp":2.0,"elapsed_time":1.0}]}` ++ `]`+ if err := rdb.Set(context.Background(), "c1-msg-good-logs", goodPayload, 0).Err(); err != nil {+ t.Fatalf("seed redis: %v", err)+ }++ // End element with an EMPTY message -> client must NOT consider it done+ // (would poll forever). Locks the contract that the debug writer must+ // emit a non-empty END message.+ badPayload := `[` ++ `{"component_id":"File","trace":[{"progress":1,"message":"parsed","datetime":"10:00:00","timestamp":1.0,"elapsed_time":0}]},` ++ `{"component_id":"END","trace":[{"progress":1,"message":"","datetime":"10:00:01","timestamp":2.0,"elapsed_time":1.0}]}` ++ `]`+ if err := rdb.Set(context.Background(), "c1-msg-bad-logs", badPayload, 0).Err(); err != nil {+ t.Fatalf("seed redis: %v", err)+ }++ h := NewAgentHandler(service.NewAgentService(), nil).+ WithRedisGetter(func(key string) (string, error) {+ return rdb.Get(context.Background(), key).Result()+ })++ call := func(messageID string) []map[string]interface{} {+ w := httptest.NewRecorder()+ c, _ := gin.CreateTestContext(w)+ c.Request = httptest.NewRequest("GET", "/api/v1/agents/c1/logs/"+messageID, nil)+ c.Set("user", &entity.User{ID: "u1"})+ c.Set("user_id", "u1")+ c.Params = gin.Params{+ {Key: "canvas_id", Value: "c1"},+ {Key: "message_id", Value: messageID},+ }+ h.GetAgentLogs(c)+ var resp map[string]interface{}+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatalf("decode (msg=%s): %v body=%s", messageID, err, w.Body.String())+ }+ if code, _ := resp["code"].(float64); int(code) != int(common.CodeSuccess) {+ t.Fatalf("code=%v want 0 body=%s", code, mustJSON(resp))+ }+ dataRaw, _ := json.Marshal(resp["data"])+ var arr []map[string]interface{}+ if err := json.Unmarshal(dataRaw, &arr); err != nil {+ t.Fatalf("data not array (msg=%s): %v body=%s", messageID, err, string(dataRaw))+ }+ return arr+ }++ good := call("msg-good")+ if !clientConsidersComplete(good) {+ t.Fatalf("expected client to consider run complete for non-empty END message; arr=%s", mustJSON(good))+ }++ bad := call("msg-bad")+ if clientConsidersComplete(bad) {+ t.Fatalf("client must NOT consider run complete for empty END message (would poll forever); arr=%s", mustJSON(bad))+ }+}++// capturedStore is an in-memory task.DebugLogStore used to assert that+// runCanvasPipelineDebug actually wrote the debug log array under the expected key.+type capturedStore struct {+ mu sync.Mutex+ data map[string]string+}++func (s *capturedStore) Set(key, value string, _ time.Duration) bool {+ s.mu.Lock()+ defer s.mu.Unlock()+ if s.data == nil {+ s.data = map[string]string{}+ }+ s.data[key] = value+ return true+}++func (s *capturedStore) get(key string) (string, bool) {+ s.mu.Lock()+ defer s.mu.Unlock()+ v, ok := s.data[key]+ return v, ok+}++// fakeDebugExecutor is a debugExecutor stand-in for runCanvasPipelineDebug. It+// captures the sink attached via WithProgressSink and replays a couple of+// component lifecycle events on Execute, mirroring what the real+// PipelineExecutor emits through its progress callback.+type fakeDebugExecutor struct {+ capturedSink pipelinepkg.ProgressSink+ result *task.PipelineResult+ runErr error+}++func (f *fakeDebugExecutor) WithProgressSink(sink pipelinepkg.ProgressSink) *task.PipelineExecutor {+ f.capturedSink = sink+ return nil // return ignored by runCanvasPipelineDebug+}++func (f *fakeDebugExecutor) Execute(ctx context.Context) (*task.PipelineResult, error) {+ if f.capturedSink != nil {+ f.capturedSink.OnComponentTotal(ctx, "t1", 1)+ f.capturedSink.OnComponentProgress(ctx, pipelinepkg.ProgressEvent{+ TaskID: "t1", Component: "File", Phase: 0, Message: "File Started",+ })+ f.capturedSink.OnComponentProgress(ctx, pipelinepkg.ProgressEvent{+ TaskID: "t1", Component: "File", Phase: 1, Message: "File Done",+ })+ }+ return f.result, f.runErr+}++// TestRespondWithDebugResult_Shape pins the wire contract the front-end reads+// after kicking off a debug run: response.data must carry both message_id (the+// polling key) and chunks (the rendered output). Empty chunks must stay an+// empty array (never null), and a run error must surface as a server-error+// code rather than success.+func TestRespondWithDebugResult_Shape(t *testing.T) {+ gin.SetMode(gin.TestMode)++ // Success with chunks: data.message_id + data.chunks both present.+ w := httptest.NewRecorder()+ c, _ := gin.CreateTestContext(w)+ respondWithDebugResult(c, &task.PipelineResult{+ MessageID: "m-abc",+ Chunks: []map[string]any{{"content": "hello"}},+ }, nil)+ var resp map[string]interface{}+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatalf("decode: %v body=%s", err, w.Body.String())+ }+ if code, _ := resp["code"].(float64); int(code) != int(common.CodeSuccess) {+ t.Fatalf("code=%v want 0 body=%s", code, mustJSON(resp))+ }+ data, ok := resp["data"].(map[string]interface{})+ if !ok {+ t.Fatalf("data not object: %v", resp)+ }+ if data["message_id"] != "m-abc" {+ t.Fatalf("data.message_id=%v want m-abc", data["message_id"])+ }+ chunks, ok := data["chunks"].([]interface{})+ if !ok {+ t.Fatalf("data.chunks not array: %v", data["chunks"])+ }+ if len(chunks) != 1 {+ t.Fatalf("data.chunks len=%d want 1", len(chunks))+ }++ // Empty chunks: still an object with message_id and an empty array.+ w2 := httptest.NewRecorder()+ c2, _ := gin.CreateTestContext(w2)+ respondWithDebugResult(c2, &task.PipelineResult{MessageID: "m-empty"}, nil)+ var resp2 map[string]interface{}+ if err := json.Unmarshal(w2.Body.Bytes(), &resp2); err != nil {+ t.Fatalf("decode empty: %v body=%s", err, w2.Body.String())+ }+ data2, ok := resp2["data"].(map[string]interface{})+ if !ok {+ t.Fatalf("data not object: %v", resp2)+ }+ if data2["message_id"] != "m-empty" {+ t.Fatalf("data.message_id=%v want m-empty", data2["message_id"])+ }+ chunks2, ok := data2["chunks"].([]interface{})+ if !ok {+ t.Fatalf("data.chunks not array: %v", data2["chunks"])+ }+ if len(chunks2) != 0 {+ t.Fatalf("data.chunks len=%d want 0", len(chunks2))+ }++ // Error: must NOT be a success code.+ w3 := httptest.NewRecorder()+ c3, _ := gin.CreateTestContext(w3)+ respondWithDebugResult(c3, nil, errors.New("boom"))+ var resp3 map[string]interface{}+ if err := json.Unmarshal(w3.Body.Bytes(), &resp3); err != nil {+ t.Fatalf("decode err: %v body=%s", err, w3.Body.String())+ }+ if code, _ := resp3["code"].(float64); int(code) == int(common.CodeSuccess) {+ t.Fatalf("error case must not be success: %v", resp3)+ }+}++// TestRespondWithDebugResult_ErrorCarriesMessageID pins that a run failure+// still surfaces the polling key: the response must keep a non-success code+// (the existing contract forbids masking errors as success) BUT the error+// envelope's data must carry message_id so the front-end can poll the failure+// timeline that DebugLogSink.Flush already wrote to Redis.+func TestRespondWithDebugResult_ErrorCarriesMessageID(t *testing.T) {+ gin.SetMode(gin.TestMode)++ w := httptest.NewRecorder()+ c, _ := gin.CreateTestContext(w)+ respondWithDebugResult(c, &task.PipelineResult{MessageID: "m-err"}, errors.New("boom"))+ var resp map[string]interface{}+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatalf("decode: %v body=%s", err, w.Body.String())+ }+ if code, _ := resp["code"].(float64); int(code) == int(common.CodeSuccess) {+ t.Fatalf("error case must not be success: %v", resp)+ }+ data, ok := resp["data"].(map[string]interface{})+ if !ok {+ t.Fatalf("error envelope data must be an object carrying message_id: %v", resp)+ }+ if data["message_id"] != "m-err" {+ t.Fatalf("error envelope data.message_id=%v want m-err", data["message_id"])+ }+}++// TestRunCanvasPipelineDebug_ErrorStillExposesMessageID asserts the full write-side+// wiring when the executor fails: runCanvasPipelineDebug must (a) still return a+// non-nil result carrying MessageID even though exec.Execute returned nil, and+// (b) have already flushed the failure log under "{canvasID}-{messageID}-logs"+// so the front-end can poll that exact key. This is the scenario that broke+// before — the failure log was written but unreachable because message_id was+// dropped on the error path.+func TestRunCanvasPipelineDebug_ErrorStillExposesMessageID(t *testing.T) {+ ctx := context.Background()+ store := &capturedStore{}++ h := NewAgentHandler(service.NewAgentService(), nil).+ WithRedisStore(store).+ WithNewExecutor(func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error) {+ return &fakeDebugExecutor{+ // Simulate an executor that fails and returns no result.+ result: nil,+ runErr: errors.New("executor blew up"),+ }, nil+ })++ result, err := h.runCanvasPipelineDebug(ctx, &entity.User{ID: "u1"}, "c1", "f.txt", []byte("data"))+ if err == nil {+ t.Fatalf("expected run error")+ }+ if result == nil {+ t.Fatalf("result is nil; front-end would have no polling key on failure")+ }+ if result.MessageID == "" {+ t.Fatalf("result.MessageID empty on failure; front-end cannot poll the failure log")+ }++ // The failure log must be written under the composed key.+ key := "c1-" + result.MessageID + "-logs"+ raw, ok := store.get(key)+ if !ok {+ t.Fatalf("failure log not written under key %q; store keys=%v", key, keysOf(store))+ }+ var arr []map[string]interface{}+ if err := json.Unmarshal([]byte(raw), &arr); err != nil {+ t.Fatalf("stored failure log not array: %v raw=%s", err, raw)+ }+ if !clientConsidersComplete(arr) {+ t.Fatalf("stored failure log not completion-complete (END last + non-empty msg); arr=%s", raw)+ }+}++// TestRunCanvasPipelineDebug_WiresMessageIDAndLog asserts the full write-side wiring+// of runCanvasPipelineDebug without a live Redis or real canvas: a fake executor+// emits component progress, the injected DebugLogSink flushes the+// [{component_id, trace}] array (with the END marker last) to the captured+// store under the key "{canvasID}-{messageID}-logs", and the returned result+// carries message_id so the front-end can poll that exact key. The array must+// satisfy the completion predicate (END last, non-empty END message) so the+// Log box stops polling.+func TestRunCanvasPipelineDebug_WiresMessageIDAndLog(t *testing.T) {+ ctx := context.Background()+ store := &capturedStore{}++ h := NewAgentHandler(service.NewAgentService(), nil).+ WithRedisStore(store).+ WithNewExecutor(func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error) {+ return &fakeDebugExecutor{+ result: &task.PipelineResult{+ Chunks: []map[string]any{{"content": "chunk-1"}},+ },+ }, nil+ })++ result, err := h.runCanvasPipelineDebug(ctx, &entity.User{ID: "u1"}, "c1", "f.txt", []byte("data"))+ if err != nil {+ t.Fatalf("runCanvasPipelineDebug: %v", err)+ }+ if result == nil {+ t.Fatalf("result is nil")+ }+ if result.MessageID == "" {+ t.Fatalf("result.MessageID empty; front-end would have no polling key")+ }++ // The log array must be written under the composed key.+ key := "c1-" + result.MessageID + "-logs"+ raw, ok := store.get(key)+ if !ok {+ t.Fatalf("log not written under key %q; store keys=%v", key, keysOf(store))+ }++ var arr []map[string]interface{}+ if err := json.Unmarshal([]byte(raw), &arr); err != nil {+ t.Fatalf("stored log not array: %v raw=%s", err, raw)+ }+ if !clientConsidersComplete(arr) {+ t.Fatalf("stored log not completion-complete (END last + non-empty msg); arr=%s", raw)+ }+}++func keysOf(s *capturedStore) []string {+ s.mu.Lock()+ defer s.mu.Unlock()+ keys := make([]string, 0, len(s.data))+ for k := range s.data {+ keys = append(keys, k)+ }+ return keys+}++// miniredisDebugStore adapts a go-redis client to task.DebugLogStore so the+// write path can be exercised against a real (in-memory) Redis in tests. It is+// the production-faithful counterpart of capturedStore: capturedStore proves+// the key shape, miniredisDebugStore proves the bytes survive a real Redis+// round-trip that the read path consumes.+type miniredisDebugStore struct {+ rdb *goredis.Client+}++func (s miniredisDebugStore) Set(key, value string, ttl time.Duration) bool {+ if err := s.rdb.Set(context.Background(), key, value, ttl).Err(); err != nil {+ return false+ }+ return true+}++// TestRunCanvasPipelineDebug_WriteThenReadViaMiniredis locks the write/read contract+// end-to-end: runCanvasPipelineDebug writes the debug log array under+// "{canvasID}-{messageID}-logs" into a real (miniredis) Redis via the injected+// DebugLogStore, and the SAME handler's GetAgentLogs reads it back through the+// injected redisGetter — proving the writer's key shape exactly matches the+// reader's and that the stored bytes round-trip into the [{component_id,+// trace}] array the front-end polls for (File first, END last, non-empty END+// message). This is the only test that would catch a key-format drift between+// the two sides, since the write test uses a capturedStore and the read test+// seeds Redis directly rather than going through the writer.+func TestRunCanvasPipelineDebug_WriteThenReadViaMiniredis(t *testing.T) {+ gin.SetMode(gin.TestMode)+ ctx := context.Background()++ db := setupHandlerAgentsTestDB(t)+ orig := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = orig })+ db.Create(&entity.UserCanvas{ID: "c1", UserID: "u1", Title: sptr("Test")})++ mr, err := miniredis.Run()+ if err != nil {+ t.Fatalf("miniredis.Run: %v", err)+ }+ t.Cleanup(mr.Close)+ rdb := goredis.NewClient(&goredis.Options{Addr: mr.Addr()})+ t.Cleanup(func() { _ = rdb.Close() })++ // Both seams point at the same miniredis: the writer stores via+ // WithRedisStore and the reader fetches via WithRedisGetter, mirroring+ // production where both hit one Redis.+ h := NewAgentHandler(service.NewAgentService(), nil).+ WithRedisStore(miniredisDebugStore{rdb: rdb}).+ WithRedisGetter(func(key string) (string, error) {+ return rdb.Get(ctx, key).Result()+ }).+ WithNewExecutor(func(taskCtx *task.TaskContext, canvasID string, docBulkSize int) (debugExecutor, error) {+ return &fakeDebugExecutor{+ result: &task.PipelineResult{Chunks: []map[string]any{{"content": "chunk-1"}}},+ }, nil+ })++ // Write side.+ writeResult, werr := h.runCanvasPipelineDebug(ctx, &entity.User{ID: "u1"}, "c1", "f.txt", []byte("data"))+ if werr != nil {+ t.Fatalf("runCanvasPipelineDebug: %v", werr)+ }+ if writeResult == nil || writeResult.MessageID == "" {+ t.Fatalf("runCanvasPipelineDebug returned no message_id")+ }+ messageID := writeResult.MessageID++ // The key must exist in the same miniredis the reader will query.+ if !mr.Exists("c1-" + messageID + "-logs") {+ t.Fatalf("log key c1-%s-logs not present in redis after write", messageID)+ }++ // Read side: same handler, same miniredis, driven through the real+ // GetAgentLogs HTTP path.+ w := httptest.NewRecorder()+ c, _ := gin.CreateTestContext(w)+ c.Request = httptest.NewRequest("GET", "/api/v1/agents/c1/logs/"+messageID, nil)+ c.Set("user", &entity.User{ID: "u1"})+ c.Set("user_id", "u1")+ c.Params = gin.Params{+ {Key: "canvas_id", Value: "c1"},+ {Key: "message_id", Value: messageID},+ }+ h.GetAgentLogs(c)++ var resp map[string]interface{}+ if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {+ t.Fatalf("decode read response: %v body=%s", err, w.Body.String())+ }+ if code, _ := resp["code"].(float64); int(code) != int(common.CodeSuccess) {+ t.Fatalf("read code=%v want 0 body=%s", code, mustJSON(resp))+ }+ dataRaw, _ := json.Marshal(resp["data"])+ trimmed := strings.TrimSpace(string(dataRaw))+ if len(trimmed) == 0 || trimmed[0] != '[' {+ t.Fatalf("read response.data must be a JSON array, got %s", string(dataRaw))+ }+ var arr []map[string]interface{}+ if err := json.Unmarshal(dataRaw, &arr); err != nil {+ t.Fatalf("read data not array: %v body=%s", err, string(dataRaw))+ }+ if arr[0]["component_id"] != "File" {+ t.Errorf("first component_id=%v want File", arr[0]["component_id"])+ }+ if arr[len(arr)-1]["component_id"] != "END" {+ t.Errorf("last component_id=%v want END", arr[len(arr)-1]["component_id"])+ }+ if !clientConsidersComplete(arr) {+ t.Errorf("stored log not completion-complete (END last + non-empty msg); arr=%s", string(dataRaw))+ }+}diff --git a/internal/handler/agent_pipeline_debug_test.go b/internal/handler/agent_pipeline_debug_test.gonew file mode 100644index 0000000000..21d961b0a2--- /dev/null+++ b/internal/handler/agent_pipeline_debug_test.go@@ -0,0 +1,92 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package handler++import (+ "encoding/json"+ "net/http"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/task"+)++func dataflowCanvas(id, userID string) *entity.UserCanvas {+ cv := makeWebhookCanvas(id, userID, "Webhook", nil)+ cv.CanvasCategory = "dataflow_canvas"+ return cv+}++func decodeSuccess(t *testing.T, body []byte) (int, string, []map[string]any, string) {+ t.Helper()+ var resp struct {+ Code int `json:"code"`+ Data struct {+ MessageID string `json:"message_id"`+ Chunks []map[string]any `json:"chunks"`+ } `json:"data"`+ Message string `json:"message"`+ }+ if err := json.Unmarshal(body, &resp); err != nil {+ t.Fatalf("decode response: %v (body=%s)", err, body)+ }+ return resp.Code, resp.Data.MessageID, resp.Data.Chunks, resp.Message+}++// TestAgentChatCompletions_DataFlowReturnsChunks pins that the existing+// chat/completions endpoint serves the pipeline debug (dry-run) for DataFlow+// canvases: it returns the parsed chunks inline, synchronously, without a+// dedicated debug route. The webhook entry point is intentionally NOT a+// debug surface — see TestWebhook_DataFlowRejected in agent_webhook_test.go.+func TestAgentChatCompletions_DataFlowReturnsChunks(t *testing.T) {+ h := &AgentHandler{+ loader: &fakeCanvasLoader{canvas: dataflowCanvas("c1", "u-1")},+ fileService: &fakeAgentFileService{blob: []byte("file-bytes")},+ }+ h.WithNewExecutor(func(_ *task.TaskContext, _ string, _ int) (debugExecutor, error) {+ return &fakeDebugExecutor{result: &task.PipelineResult{Chunks: []map[string]any{{"text": "dbg-chunk"}}}}, nil+ })+ // runCanvasPipelineDebug flushes the debug log via redisStore; inject a no-op+ // store so the (skipped in this test) log write does not nil-panic.+ h.WithRedisStore(&capturedStore{})++ // The web contract sends the file list wrapped one level:+ // `files: [[{id, name}]]` (see use-run-dataflow.ts:34). The pipeline+ // debug path unwraps both layers and downloads the referenced bytes.+ c, w := webhookCtx("POST", "/api/v1/agents/c1/chat/completions",+ `{"agent_id":"c1","files":[[{"id":"f1","name":"doc.txt"}]]}`, "application/json")++ h.AgentChatCompletions(c)++ if w.Code != http.StatusOK {+ t.Fatalf("status = %d, want 200 (body=%s)", w.Code, w.Body.String())+ }+ code, msgID, data, msg := decodeSuccess(t, w.Body.Bytes())+ if code != int(common.CodeSuccess) {+ t.Errorf("code = %d, want %d", code, common.CodeSuccess)+ }+ if msg != "success" {+ t.Errorf("message = %q, want %q", msg, "success")+ }+ if msgID == "" {+ t.Errorf("message_id empty; front-end needs it to poll the debug log")+ }+ if len(data) != 1 || data[0]["text"] != "dbg-chunk" {+ t.Errorf("data.chunks = %#v, want one chunk with text %q", data, "dbg-chunk")+ }+}diff --git a/internal/handler/agent_test.go b/internal/handler/agent_test.goindex bfd2d4301c..8e7501fdb1 100644--- a/internal/handler/agent_test.go+++ b/internal/handler/agent_test.go@@ -1228,9 +1228,11 @@ func (s *stubDocService) Accessible(_, _ string) bool {}// TestAgentChatCompletions_FilesDeserialized verifies that when the-// JSON request body contains a `files` field, the-// agentChatCompletionsRequest struct deserializes it correctly.-// Mirrors Python's req.get("files", []) at agent_api.py:1313.+// JSON request body contains the web-contract 2D `files` field+// (`[[{...}]]`), the agentChatCompletionsRequest struct deserializes it+// and the inner file list reaches RunAgent. Mirrors Python's+// agent_api.py:1611 `queue_dataflow(..., files[0], 0)`, where the first+// inner list is the set of files for the run.func TestAgentChatCompletions_FilesDeserialized(t *testing.T) {gin.SetMode(gin.TestMode)w := httptest.NewRecorder()@@ -1238,9 +1240,9 @@ func TestAgentChatCompletions_FilesDeserialized(t *testing.T) {body := `{"agent_id": "a1","query": "hi",- "files": [+ "files": [[{"id": "file-1", "name": "resume.txt", "mime_type": "text/plain", "created_by": "u1"}- ]+ ]]}`c.Request = httptest.NewRequest("POST", "/api/v1/agents/chat/completions",strings.NewReader(body))diff --git a/internal/handler/agent_webhook.go b/internal/handler/agent_webhook.goindex 2173533300..ef2ded7725 100644--- a/internal/handler/agent_webhook.go+++ b/internal/handler/agent_webhook.go@@ -133,8 +133,11 @@ func (h *AgentHandler) Webhook(c *gin.Context) {return}- // 2. Reject DataFlow.- if cv.CanvasCategory == "DataFlow" {+ // 2. Reject DataFlow. DataFlow canvases are ingestion pipelines, not+ // interactive agents, and must not be triggered by an external webhook.+ // Mirrors Python agent_api.py:1786, which returns+ // "Dataflow can not be triggered by webhook." for the same case.+ if cv.CanvasCategory == "dataflow_canvas" {common.ResponseWithCodeData(c, common.CodeDataError, nil, "Dataflow can not be triggered by webhook.")return}diff --git a/internal/handler/agent_webhook_test.go b/internal/handler/agent_webhook_test.goindex ff4a768d4d..3f08cad01e 100644--- a/internal/handler/agent_webhook_test.go+++ b/internal/handler/agent_webhook_test.go@@ -153,12 +153,14 @@ func TestWebhook_RejectsUnknownCanvas(t *testing.T) {}}-// TestWebhook_RejectsDataFlowCanvas covers the second guard: the canvas-// exists but its category is DataFlow (which python-// agent_api.py:1575 explicitly rejects).-func TestWebhook_RejectsDataFlowCanvas(t *testing.T) {+// TestWebhook_DataFlowRejected pins the second guard: a canvas whose category+// is DataFlow must NOT be triggered by an external webhook. The handler returns+// CodeDataError with "Dataflow can not be triggered by webhook." — mirroring+// Python agent_api.py:1786. DataFlow canvases are ingestion pipelines, not+// interactive agents, and have no chat/debug surface over the webhook entry.+func TestWebhook_DataFlowRejected(t *testing.T) {cv := makeWebhookCanvas("c1", "u-1", "Webhook", nil)- cv.CanvasCategory = "DataFlow"+ cv.CanvasCategory = "dataflow_canvas"h := &AgentHandler{loader: &fakeCanvasLoader{canvas: cv}}c, w := webhookCtx("POST", "/api/v1/agents/c1/webhook", `{}`, "application/json")@@ -166,7 +168,7 @@ func TestWebhook_RejectsDataFlowCanvas(t *testing.T) {code, msg := errBody(t, w.Body.Bytes())if code != int(common.CodeDataError) {- t.Errorf("code = %d, want %d", code, common.CodeDataError)+ t.Errorf("code = %d, want %d (CodeDataError)", code, common.CodeDataError)}if msg != "Dataflow can not be triggered by webhook." {t.Errorf("message = %q, want %q", msg, "Dataflow can not be triggered by webhook.")diff --git a/internal/ingestion/component/chunker/image_upload_test.go b/internal/ingestion/component/chunker/image_upload_test.goindex 2a11e05e65..261b105bb1 100644--- a/internal/ingestion/component/chunker/image_upload_test.go+++ b/internal/ingestion/component/chunker/image_upload_test.go@@ -276,3 +276,48 @@ func waitForInflight(t *testing.T, counter *int64, target int64) {}}}++// TestImageUploadDecorator_DebugSkipsUpload verifies that a debug/dry-run run+// (empty kb_id) does NOT upload the chunk image to storage, while still+// dropping the raw image bytes. This keeps the debug path free of MinIO side+// effects and preserves the current memory behaviour (raw bytes discarded, not+// held until a persist stage). An empty kb_id only occurs in canvas debug+// (dry-run) mode; production ingestion always supplies a KB.+func TestImageUploadDecorator_DebugSkipsUpload(t *testing.T) {+ prev := ChunkImageUploader+ ChunkImageUploader = func(_ context.Context, _, _ string, _ []byte) (string, error) {+ t.Fatalf("ChunkImageUploader must not be called in a debug run")+ return "", nil+ }+ t.Cleanup(func() { ChunkImageUploader = prev })++ comp, err := NewOneChunker(nil)+ if err != nil {+ t.Fatalf("NewOneChunker: %v", err)+ }+ decorated := &imageUploadDecorator{inner: comp}++ inputs := map[string]any{+ "name": "doc.pdf",+ "kb_id": "", // debug mode: no KB -> no upload+ "doc_id": testDocID,+ "chunks": []map[string]any{+ {"content_with_weight": "a cropped figure", "image": "data:image/png;base64," + pngBase64},+ },+ }+ out, err := decorated.Invoke(context.Background(), nil, inputs)+ if err != nil {+ t.Fatalf("decorated Invoke: %v", err)+ }+ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) == 0 {+ t.Fatalf("expected chunks in output, got %#v", out)+ }+ ck := chunks[0]+ if _, stillHas := ck["image"]; stillHas {+ t.Errorf("raw image should be dropped in debug run, but ck[\"image\"] is still present")+ }+ if id, _ := ck["img_id"].(string); id != "" {+ t.Errorf("img_id should not be set in debug run, got %q", id)+ }+}diff --git a/internal/ingestion/component/chunker/pdfcrop_nocgo.go b/internal/ingestion/component/chunker/pdfcrop_nocgo.goindex 179e199c33..1034dc13e1 100644--- a/internal/ingestion/component/chunker/pdfcrop_nocgo.go+++ b/internal/ingestion/component/chunker/pdfcrop_nocgo.go@@ -12,6 +12,8 @@ import (deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/ingestion/component/schema"++ "gorm.io/gorm")func newPDFEngineFromUpstream(_ context.Context, _ *gorm.DB, _ schema.ChunkerFromUpstream) (deepdoctype.PDFEngine, error) {diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.goindex 6b716fa76c..3c28a24223 100644--- a/internal/ingestion/component/chunker/register.go+++ b/internal/ingestion/component/chunker/register.go@@ -82,8 +82,19 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs mck["id"] = common.ChunkID(docID, text)}- if err := uploadChunkImages(ctx, chunks, ChunkImageUploader, kbID); err != nil {- return nil, err+ // kb_id is empty only in canvas debug (dry-run) mode; production ingestion+ // always supplies a KB, so kb_id == "" never occurs in normal operation.+ // Image bytes are uploaded to MinIO only when a KB is present (i.e. a+ // persist run); in debug we drop the raw bytes so they are not held in+ // memory until a persist stage that will never run.+ if kbID != "" {+ if err := uploadChunkImages(ctx, chunks, ChunkImageUploader, kbID); err != nil {+ return nil, err+ }+ } else {+ for _, ck := range chunks {+ delete(ck, "image")+ }}return out, nil}diff --git a/internal/ingestion/component/file.go b/internal/ingestion/component/file.goindex bd037079cb..f6a803a3fc 100644--- a/internal/ingestion/component/file.go+++ b/internal/ingestion/component/file.go@@ -151,6 +151,14 @@ func (c *FileComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[striif in.fileDesc != nil {out["file"] = in.fileDesc}+ // Pass through in-memory bytes when present (debug / dataflow dry-run+ // where no doc row exists in storage). The downstream Parser reads+ // `binary` first and skips the doc_id → storage lookup, so a debug run+ // can parse the uploaded file without a persisted document. Persist+ // runs set no `binary` here, so they keep resolving from storage.+ if len(in.binary) > 0 {+ out["binary"] = in.binary+ }// Publish the resolved run-level metadata into the workflow-wide// CanvasState.Globals bag so downstream components (Tokenizer,// Chunker, ...) read it from ctx instead of relying on this output@@ -169,6 +177,7 @@ type fileInputs struct {bucket stringpath stringfileDesc map[string]any+ binary []byte}// parseFileInputs parses and validates the upstream input map.@@ -180,6 +189,19 @@ func parseFileInputs(ctx context.Context, inputs map[string]any) (fileInputs, er}out := fileInputs{}+ // Debug / dataflow dry-run fast path: in-memory bytes were supplied+ // via the graph input `binary` (no persisted document exists). Use+ // them directly and skip the doc_id → storage resolution so a debug+ // run parses the uploaded file without a DB/storage round-trip. The+ // executor only sets `binary` for the non-persist (debug) marker doc.+ if b, ok := inputs["binary"].([]byte); ok && len(b) > 0 {+ out.binary = b+ if n, ok := getString(inputs, "name"); ok && n != "" {+ out.name = n+ }+ return out, nil+ }+if v, ok := getString(inputs, "doc_id"); ok && v != "" {out.docID = vout.name = vdiff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex dacfc6713d..9fef42ab57 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -535,6 +535,12 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st}}+ // 3b. (pages-based page selection is handled upstream via the+ // component setup: ParserConfig[cpnID][filetype]["pages"] is a+ // list of page ranges delivered through override_params and+ // consumed by the deepdoc/pdf parser. No inputs-level handling+ // here.)+// 4. Build the page slice sequentially. Per-page parallelism now// lives in the parser backends (e.g. internal/deepdoc/parser/pdf// fans out one worker per page and assembles in page order), sodiff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.goindex 8eea5e131c..05b9f5b042 100644--- a/internal/ingestion/component/parser_dispatch.go+++ b/internal/ingestion/component/parser_dispatch.go@@ -320,6 +320,16 @@ func pythonFamilyName(raw string) string {return ""}+// ParserFileFamily normalises a free-form file-type/extension hint to the+// python-side family identifier used as the key into a Parser component's+// setups (e.g. "pdf", "docx", "slides", "text&code"). It is the exported+// entry point for callers outside this package (e.g. the ingestion task+// executor that injects the debug page cap into override_params) that need+// to build the canonical ParserConfig[cpnID][family]["pages"] shape.+func ParserFileFamily(ext string) string {+ return pythonFamilyName(ext)+}+// jsonItemsToPages reshapes a parsed JSON payload into the// schema.Page layout the chunker side consumes. Each JSON item// becomes one page carrying `text`, `doc_type_kwd`, and anydiff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex 8e8318b796..007d4a3d52 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -360,7 +360,12 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map"chunks": schema.ChunkDocsToMaps(chunks),}- if contains(c.param.SearchMethod, "embedding") {+ // Embedding requires a KB: the embedder (and its embd_id) is configured+ // on the knowledgebase, so without kb_id there is nothing to resolve+ // against. A canvas-debug (dry-run) run has kb_id == "" by construction,+ // so embedding is skipped there — debug only exercises parse+chunk and+ // must stay side-effect free.+ if shouldHaveEmbedding(c.param.SearchMethod, kbID) {chunks, tokenCount, err := c.embedChunks(ctx, tenantID, kbID, embeddingModel, name, chunks)if err != nil {return nil, err@@ -368,7 +373,7 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapout["embedding_token_consumption"] = tokenCountout["chunks"] = schema.ChunkDocsToMaps(chunks)}- if err := validateTokenizerOutputs(chunks, c.param.SearchMethod, c.param.Fields); err != nil {+ if err := validateTokenizerOutputs(chunks, c.param.SearchMethod, c.param.Fields, kbID); err != nil {return nil, err}@@ -776,9 +781,15 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {return b.String()}-func validateTokenizerOutputs(chunks []schema.ChunkDoc, searchMethods, fields []string) error {+// shouldHaveEmbedding reports whether the tokenizer must attach embedding+// vectors: the search method requests embedding AND a KB is present.+func shouldHaveEmbedding(searchMethods []string, kbID string) bool {+ return contains(searchMethods, "embedding") && kbID != ""+}++func validateTokenizerOutputs(chunks []schema.ChunkDoc, searchMethods, fields []string, kbID string) error {needFullText := contains(searchMethods, "full_text")- needEmbedding := contains(searchMethods, "embedding")+ needEmbedding := shouldHaveEmbedding(searchMethods, kbID)if !needFullText && !needEmbedding {return nil}diff --git a/internal/ingestion/component/tokenizer_nonpersist_test.go b/internal/ingestion/component/tokenizer_nonpersist_test.gonew file mode 100644index 0000000000..1d9a8c2457--- /dev/null+++ b/internal/ingestion/component/tokenizer_nonpersist_test.go@@ -0,0 +1,109 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Regression guard for the canvas-debug (dry-run) run contract: a debug run+// carries no KB (kb_id == ""), so the tokenizer must skip embedding rather than+// resolving an embedder. It lives in the canvas-debug branch (not the+// ingestion-fixes branch) so the ingestion-fixes branch stays compilable on its+// own.++package component++import (+ "context"+ "testing"+)++// TestTokenizerComponent_DebugSkipsEmbedding is a regression gate for the+// canvas-debug (dry-run) contract: a debug run has no KB (kb_id == ""), so+// embedding MUST be skipped — there is no embedder (and no embd_id) to resolve+// against, and a debug run must stay side-effect free. The tokenizer must+// return its chunks without error, without invoking the embedder, and without+// attaching embedding vectors. Conversely, when a kb_id is present the embedding+// path must run.+//+// It uses embedding-only search_method so it runs without the C++ RAGAnalyzer+// pool (plain `go test`, no -tags integration), keeping the regression gate+// executable in every environment.+func TestTokenizerComponent_DebugSkipsEmbedding(t *testing.T) {+ stub := newStubEmbedder(4)+ cIntf, err := NewTokenizerComponentWithResolver(+ map[string]any{"search_method": []any{"embedding"}, "fields": []any{"text"}},+ func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil },+ )+ if err != nil {+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)+ }+ c := cIntf.(*TokenizerComponent)++ run := func(kbID string) (int32, map[string]any) {+ inputs := map[string]any{+ "name": "doc.pdf",+ "output_format": "chunks",+ "chunks": []map[string]any{+ {"text": "alpha bravo"},+ {"text": "charlie delta"},+ },+ }+ if kbID != "" {+ inputs["kb_id"] = kbID+ }+ out, err := c.Invoke(context.Background(), nil, inputs)+ if err != nil {+ t.Fatalf("Invoke(kb_id=%q): %v", kbID, err)+ }+ return stub.calls.Load(), out+ }++ // Debug (kb_id == ""): embedding must be skipped entirely.+ stub.calls.Store(0)+ debugCalls, debugOut := run("")+ if debugCalls != 0 {+ t.Fatalf("embedder called under debug (kb_id=\"\"): %d calls; embedding must be skipped", debugCalls)+ }+ if got := debugOut["embedding_token_consumption"]; got != nil {+ t.Errorf("embedding_token_consumption = %v under debug; must be omitted when embedding is skipped", got)+ }+ got, ok := debugOut["chunks"].([]map[string]any)+ if !ok || len(got) != 2 {+ t.Fatalf("chunks malformed under debug: %v", debugOut["chunks"])+ }+ for i, ck := range got {+ if _, has := ck["q_4_vec"]; has {+ t.Errorf("chunk[%d] carries q_4_vec under debug; embedding must be skipped", i)+ }+ }++ // With a KB: embedding must run.+ stub.calls.Store(0)+ kbCalls, kbOut := run("kb-1")+ if kbCalls != 2 {+ t.Fatalf("embedder calls = %d with kb_id set; want 2", kbCalls)+ }+ if got := kbOut["embedding_token_consumption"]; got == nil {+ t.Error("embedding_token_consumption missing with kb_id set; embedding accounting must run")+ }+ kbChunks, ok := kbOut["chunks"].([]map[string]any)+ if !ok || len(kbChunks) != 2 {+ t.Fatalf("chunks malformed with kb_id: %v", kbOut["chunks"])+ }+ for i, ck := range kbChunks {+ vec, ok := ck["q_4_vec"].([]float64)+ if !ok || len(vec) != 4 {+ t.Fatalf("chunk[%d] missing q_4_vec with kb_id set: %v", i, ck["q_4_vec"])+ }+ }+}diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.goindex 5dd986e6d2..5ab725a6a1 100644--- a/internal/ingestion/component/tokenizer_test.go+++ b/internal/ingestion/component/tokenizer_test.go@@ -70,6 +70,7 @@ func TestTokenizerComponent_Invoke_HappyPath(t *testing.T) {out, err := c.Invoke(context.Background(), nil, map[string]any{"tenant_id": "t1","model_id": "embd-1",+ "kb_id": "kb-1","name": "doc.pdf","output_format": "chunks","chunks": chunks,@@ -222,6 +223,7 @@ func TestTokenizerComponent_Invoke_BatchedEmbedding(t *testing.T) {}if _, err := c.Invoke(context.Background(), nil, map[string]any{"name": "doc.txt",+ "kb_id": "kb-1","output_format": "chunks","chunks": chunks,}); err != nil {@@ -298,6 +300,7 @@ func TestTokenizerComponent_Invoke_FullTextAndEmbedding(t *testing.T) {c, _ := withStubEmbedder(t, 4)out, err := c.Invoke(context.Background(), nil, map[string]any{"name": "doc.pdf",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "alpha bravo"}},})@@ -327,6 +330,7 @@ func TestTokenizerComponent_Invoke_EmbedNoResolver(t *testing.T) {requireTokenizerPool(t)c, _ := NewTokenizerComponent(nil)_, err := c.Invoke(context.Background(), nil, map[string]any{+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "alpha"}},})@@ -346,6 +350,7 @@ func TestTokenizerComponent_Invoke_EmbedderError(t *testing.T) {stub.err = errors.New("simulated upstream error")_, err := c.Invoke(context.Background(), nil, map[string]any{+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "alpha"}},})@@ -372,6 +377,7 @@ func TestTokenizerComponent_Invoke_EncoderCountMismatch(t *testing.T) {c := cIntf.(*TokenizerComponent)_ = stub_, err = c.Invoke(context.Background(), nil, map[string]any{+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "a"}, {"text": "b"}, {"text": "c"}},})@@ -432,6 +438,7 @@ func TestTokenizerComponent_Smoke_EndToEnd(t *testing.T) {out, err := c.Invoke(context.Background(), nil, map[string]any{"tenant_id": "tenant-smoke","model_id": "embd-smoke",+ "kb_id": "kb-1","name": "smoke.pdf","output_format": "chunks","chunks": []map[string]any{{"text": chunkText}},@@ -479,6 +486,7 @@ func TestTokenizerComponent_Embedding_MergesTitleAndContentVectors(t *testing.T)}out, err := c.Invoke(context.Background(), nil, map[string]any{"name": "doc.pdf",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "alpha"}, {"text": "beta"}},})@@ -514,6 +522,7 @@ func TestTokenizerComponent_Embedding_UsesFilenameWeight(t *testing.T) {c := cIntf.(*TokenizerComponent)out, err := c.Invoke(context.Background(), nil, map[string]any{"name": "doc.pdf",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "alpha"}},})@@ -544,6 +553,7 @@ func TestTokenizerComponent_Embedding_EmptyNameWarnsAndUsesContentVector(t *testout, err := c.Invoke(context.Background(), nil, map[string]any{"name": " ",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "alpha"}},})@@ -577,6 +587,7 @@ func TestTokenizerComponent_Embedding_UsesRawNameNotTrimmed(t *testing.T) {if _, err := c.Invoke(context.Background(), nil, map[string]any{"name": " report.pdf ",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "alpha"}},}); err != nil {@@ -599,6 +610,7 @@ func TestTokenizerComponent_Embedding_TruncatesByMaxTokensMinus10(t *testing.T)longText := strings.Repeat("hello world ", 20)if _, err := c.Invoke(context.Background(), nil, map[string]any{"name": "doc.pdf",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": longText}},}); err != nil {@@ -620,6 +632,7 @@ func TestTokenizerComponent_Embedding_SkipsEmptyCleanedTextsButReturnsZeroWhenAlc, stub := withStubEmbedder(t, 2)out, err := c.Invoke(context.Background(), nil, map[string]any{"name": "doc.pdf",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "<table><tr><td></td></tr></table>"},@@ -651,6 +664,7 @@ func TestTokenizerComponent_Embedding_SetsTokenConsumptionIncludingTitleCall(t *out, err := c.Invoke(context.Background(), nil, map[string]any{"name": "doc.pdf",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "alpha"}, {"text": "beta"}},})@@ -669,6 +683,7 @@ func TestTokenizerComponent_Embedding_BatchesByConfiguredBatchSize(t *testing.T)if _, err := c.Invoke(context.Background(), nil, map[string]any{"name": "doc.pdf",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "one"},@@ -722,11 +737,11 @@ func TestTokenizerComponent_InstanceResolversDoNotLeakAcrossComponents(t *testincompA := compAIntf.(*TokenizerComponent)compB := compBIntf.(*TokenizerComponent)- outA, err := compA.Invoke(context.Background(), nil, map[string]any{"name": "docA", "output_format": "chunks", "chunks": []map[string]any{{"text": "alpha"}}})+ outA, err := compA.Invoke(context.Background(), nil, map[string]any{"kb_id": "kb-1", "name": "docA", "output_format": "chunks", "chunks": []map[string]any{{"text": "alpha"}}})if err != nil {t.Fatalf("Invoke A: %v", err)}- outB, err := compB.Invoke(context.Background(), nil, map[string]any{"name": "docB", "output_format": "chunks", "chunks": []map[string]any{{"text": "beta"}}})+ outB, err := compB.Invoke(context.Background(), nil, map[string]any{"kb_id": "kb-1", "name": "docB", "output_format": "chunks", "chunks": []map[string]any{{"text": "beta"}}})if err != nil {t.Fatalf("Invoke B: %v", err)}diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex 25dcc1f5f7..2bbd48c199 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -140,6 +140,7 @@ func TestTokenizerComponent_Invoke_EmptyChunks(t *testing.T) {_ = stubout, err := c.Invoke(context.Background(), nil, map[string]any{+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{},})@@ -190,6 +191,7 @@ func TestTokenizerComponent_Invoke_EmbeddingOnly(t *testing.T) {}out, err := cIntf.(*TokenizerComponent).Invoke(context.Background(), nil, map[string]any{"name": "doc.pdf",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{{"text": "alpha bravo"}},})@@ -217,6 +219,7 @@ func TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero(t *tesc, stub := withStubEmbedder(t, 2)out, err := c.Invoke(context.Background(), nil, map[string]any{"name": "doc.pdf",+ "kb_id": "kb-1","output_format": "chunks","chunks": []map[string]any{},})@@ -286,14 +289,14 @@ func TestTokenizerComponent_NewTokenizerComponent_BadParam(t *testing.T) {}func TestValidateTokenizerOutputs_FullTextMissingReturnsError(t *testing.T) {- err := validateTokenizerOutputs([]schema.ChunkDoc{{Text: "alpha"}}, []string{"full_text"}, []string{"text"})+ err := validateTokenizerOutputs([]schema.ChunkDoc{{Text: "alpha"}}, []string{"full_text"}, []string{"text"}, "")if err == nil || !strings.Contains(err.Error(), "missing full_text tokens") {t.Fatalf("err = %v, want missing full_text tokens", err)}}func TestValidateTokenizerOutputs_EmbeddingMissingReturnsError(t *testing.T) {- err := validateTokenizerOutputs([]schema.ChunkDoc{{Text: "alpha"}}, []string{"embedding"}, []string{"text"})+ err := validateTokenizerOutputs([]schema.ChunkDoc{{Text: "alpha"}}, []string{"embedding"}, []string{"text"}, "kb-1")if err == nil || !strings.Contains(err.Error(), "missing embedding vector") {t.Fatalf("err = %v, want missing embedding vector", err)}@@ -301,7 +304,7 @@ func TestValidateTokenizerOutputs_EmbeddingMissingReturnsError(t *testing.T) {func TestValidateTokenizerOutputs_BothModesFailWhenOneMissing(t *testing.T) {ck := schema.ChunkDoc{Text: "alpha", ContentLtks: "tok", ContentSmLtks: "sm"}- err := validateTokenizerOutputs([]schema.ChunkDoc{ck}, []string{"full_text", "embedding"}, []string{"text"})+ err := validateTokenizerOutputs([]schema.ChunkDoc{ck}, []string{"full_text", "embedding"}, []string{"text"}, "kb-1")if err == nil || !strings.Contains(err.Error(), "missing embedding vector") {t.Fatalf("err = %v, want missing embedding vector", err)}@@ -317,7 +320,7 @@ func TestValidateTokenizerOutputs_SymbolOnlyContentLtksIsEmptyFails(t *testing.TContentLtks: "",ContentSmLtks: "",}- err := validateTokenizerOutputs([]schema.ChunkDoc{ck}, []string{"full_text"}, []string{"text"})+ err := validateTokenizerOutputs([]schema.ChunkDoc{ck}, []string{"full_text"}, []string{"text"}, "")if err == nil || !strings.Contains(err.Error(), "missing full_text tokens") {t.Fatalf("err = %v, want missing full_text tokens", err)}diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.goindex 1a5be500e3..bee1d14632 100644--- a/internal/ingestion/pipeline/template_integration_test.go+++ b/internal/ingestion/pipeline/template_integration_test.go@@ -39,6 +39,7 @@ import ("ragflow/internal/common"componentpkg "ragflow/internal/ingestion/component"_ "ragflow/internal/ingestion/component/chunker"+ "ragflow/internal/ingestion/testutil""ragflow/internal/storage""github.com/signintech/gopdf"@@ -89,6 +90,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {attachFixedEmbedderFactory(t, pipe)out, err := pipe.Run(context.Background(), map[string]any{"doc_id": docID,+ "kb_id": "test-kb",}, nil)if err != nil {t.Fatalf("Run: %v", err)@@ -180,6 +182,7 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {attachFixedEmbedderFactory(t, pipe)out, err := pipe.Run(context.Background(), map[string]any{"doc_id": docID,+ "kb_id": "test-kb",}, nil)if err != nil {t.Fatalf("Run: %v", err)@@ -268,6 +271,7 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T)attachFixedEmbedderFactory(t, pipe)out, err := pipe.Run(context.Background(), map[string]any{"doc_id": docID,+ "kb_id": "test-kb",}, nil)if err != nil {t.Fatalf("Run: %v", err)@@ -366,6 +370,7 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {attachFixedEmbedderFactory(t, pipe)out, err := pipe.Run(context.Background(), map[string]any{"doc_id": docID,+ "kb_id": "test-kb",}, nil)if err != nil {t.Fatalf("Run: %v", err)@@ -463,6 +468,7 @@ func TestPipelineRun_TemplateLaws_RealComponents(t *testing.T) {attachFixedEmbedderFactory(t, pipe)out, err := pipe.Run(context.Background(), map[string]any{"doc_id": docID,+ "kb_id": "test-kb",}, nil)if err != nil {t.Fatalf("Run: %v", err)@@ -545,6 +551,7 @@ func TestPipelineRun_TemplatePaper_RealComponents(t *testing.T) {attachFixedEmbedderFactory(t, pipe)out, err := pipe.Run(context.Background(), map[string]any{"doc_id": docID,+ "kb_id": "test-kb",}, nil)if err != nil {t.Fatalf("Run: %v", err)@@ -625,6 +632,7 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) {attachFixedEmbedderFactory(t, pipe)out, err := pipe.Run(context.Background(), map[string]any{"doc_id": docID,+ "kb_id": "test-kb",}, nil)if err != nil {t.Fatalf("Run: %v", err)@@ -867,6 +875,15 @@ func withRealTemplateDeps(t *testing.T) storage.Storage {storage.GetStorageFactory().SetStorage(mem)t.Cleanup(func() { storage.GetStorageFactory().SetStorage(origStorage) })+ // The runtime invokes every component with the package-level dao.DB+ // (see node_body.go). These "real components" tests exercise the+ // Parser/Extractor model-resolution paths, so wire an in-memory sqlite DB+ // via the shared testutil helper. It has no tenant/model rows, so+ // resolution gracefully skips — but the nil-db panic from a headless run+ // (db never initialized) is avoided without touching production code.+ db := testutil.SetupTestDB(t)+ t.Cleanup(testutil.ReplaceDBForTest(t, db))+refs := map[string]componentpkg.DocumentStorageRef{}componentpkg.ResolveDocumentStorageOverride = func(docID string) (*componentpkg.DocumentStorageRef, error) {ref, ok := refs[docID]diff --git a/internal/ingestion/task/constants.go b/internal/ingestion/task/constants.goindex 7d9eefc6af..e0450e56f2 100644--- a/internal/ingestion/task/constants.go+++ b/internal/ingestion/task/constants.go@@ -18,9 +18,6 @@ package task// Special doc_id values used in task messages (mirrors Python constants).const (- // CANVAS_DEBUG_DOC_ID marks a canvas debug pipeline task — no persistence.- CANVAS_DEBUG_DOC_ID = "dataflow_x"-// GRAPH_RAPTOR_FAKE_DOC_ID is the fake doc_id used for RAPTOR-generated chunks.GRAPH_RAPTOR_FAKE_DOC_ID = "graph_raptor_fake_doc"diff --git a/internal/ingestion/task/debug.go b/internal/ingestion/task/debug.gonew file mode 100644index 0000000000..b1e6ab5cbc--- /dev/null+++ b/internal/ingestion/task/debug.go@@ -0,0 +1,92 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package task++import (+ "strings"++ "github.com/google/uuid"+ "ragflow/internal/entity"+)++// NewDebugTaskContext builds an in-memory TaskContext for a canvas-debug+// (dataflow dry-run) request. The returned context is intentionally+// side-effect free:+//+// - It carries no knowledgebase: KB.ID and Doc.KbID are empty. A canvas+// debug run has no KB, so kb_id == "" — this is the single debug signal+// used throughout the ingestion pipeline (the tokenizer skips embedding,+// the chunker skips image upload, the executor skips the persist stage).+// kb_id == "" occurs ONLY in debug mode; production ingestion always+// supplies a KB, so this branch is never reached in normal operation.+// - Doc.ID is a fresh uuid (a throwaway marker, not a persisted row); it+// only needs to be non-empty for the executor's input validation.+// - The parser page cap (debug preview only parses the first few pages)+// is injected at run time by the executor via Run's override_params+// channel, keyed by the Parser component's cpnID and the file's family+// (see injectDebugPageCap). It deliberately does NOT live in+// ParserConfig as a flat key, because flat keys are dropped by+// override_params merging and never reach the parser.+//+// fileData is the raw bytes of the uploaded debug document (may be nil for a+// file-less dry-run). It is stored on TaskContext.File and threaded into the+// pipeline run as the "file" / "binary" input.+//+// The caller-supplied kb_id is deliberately ignored: debug never resolves an+// embedder or writes to a KB, so a debug context has none.+func NewDebugTaskContext(tenantID, canvasID, fileName string, fileData []byte) *TaskContext {+ doc := entity.Document{+ ID: uuid.New().String(),+ KbID: "",+ Name: &fileName,+ }+ if suffix, docType := deriveDocSuffixAndType(fileName); suffix != "" {+ doc.Suffix = suffix+ doc.Type = docType+ }++ return &TaskContext{+ Doc: doc,+ KB: entity.Knowledgebase{ID: ""},+ Tenant: entity.Tenant{ID: tenantID},+ PipelineID: canvasID,+ File: fileData,+ }+}++// IsDebug reports whether this context is a canvas-debug (dry-run) context.+// A debug context carries no knowledgebase (KB.ID == "", forced by+// NewDebugTaskContext) — the single debug signal used across the ingestion+// pipeline. Production ingestion always supplies a KB (LoadFromIngestionTask+// follows doc -> kb), so KB.ID == "" occurs ONLY in debug mode. A debug run+// must produce no persistent side effect: the executor skips the persist+// stage, the tokenizer skips embedding, and the chunker skips image upload.+func (t *TaskContext) IsDebug() bool {+ return t.KB.ID == ""+}++// deriveDocSuffixAndType extracts the file extension (e.g. ".pdf") and a+// lower-cased document type from a file name. It is best-effort: when the+// name has no usable extension both return "".+func deriveDocSuffixAndType(fileName string) (suffix, docType string) {+ dot := strings.LastIndex(fileName, ".")+ if dot < 0 || dot == len(fileName)-1 {+ return "", ""+ }+ ext := strings.ToLower(fileName[dot+1:])+ return "." + ext, ext+}diff --git a/internal/ingestion/task/debug_log_sink.go b/internal/ingestion/task/debug_log_sink.gonew file mode 100644index 0000000000..ce55ae406a--- /dev/null+++ b/internal/ingestion/task/debug_log_sink.go@@ -0,0 +1,286 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package task++import (+ "context"+ "encoding/json"+ "sync"+ "time"++ "ragflow/internal/ingestion/pipeline"+)++// DebugLogTTL is the Redis expiry for a debug-run log. It mirrors the Python+// pipeline's 30-minute update TTL (rag/flow/pipeline.py callback: set_obj(...,+// 60*30)); the Go side writes the log once at the end of the synchronous run.+const DebugLogTTL = 30 * time.Minute++// DebugLogStore is the minimal Redis surface the debug log sink needs. In+// production it is satisfied by *redis.Client (which exposes Set(key, value,+// ttl)); tests pass a capturing closure. Keeping it to a single method keeps the+// sink free of any redis-package import.+type DebugLogStore interface {+ Set(key, value string, ttl time.Duration) bool+}++// funcStore adapts a plain function to DebugLogStore so tests (and callers that+// already hold a go-redis client) can wire a sink without a named type.+type funcStore func(key, value string, ttl time.Duration) bool++// Set implements DebugLogStore.+func (f funcStore) Set(key, value string, ttl time.Duration) bool { return f(key, value, ttl) }++// component phases mirror runtime.ProgressPhase (Enter=0, Exit=1, Error=2).+// They are copied locally so the sink stays decoupled from the agent runtime+// package; the integer values are part of the pipeline's stable event contract.+const (+ phaseEnter = 0+ phaseExit = 1+ phaseError = 2+)++// Debug-log size guards. A debug run may emit an unbounded number of component+// lifecycle events (e.g. a misbehaving component looping), so the sink caps the+// number of components, the traces per component, and the per-message length.+// This stops a single run from exhausting memory or writing an oversized entry+// into the shared Redis. The bounds are far above any legitimate canvas and only+// catch pathological runs.+const (+ maxLogEntries = 1000 // distinct components recorded per run+ maxTracePerEntry = 200 // progress events kept per component+ maxMessageRunes = 1024 // rune length cap for a single trace message+ maxPayloadBytes = 1 << 20 // hard ceiling on the JSON written to Redis (1 MiB)+)++// debugLogEntry is one component's row in the front-end Log box: a component id+// plus an ordered list of trace lines. It matches the Python pipeline callback+// shape exactly (rag/flow/pipeline.py) so GetAgentLogs can return it verbatim+// and the front-end (useFetchMessageTrace) can read it unchanged.+type debugLogEntry struct {+ ComponentID string `json:"component_id"`+ Trace []debugTrace `json:"trace"`+}++type debugTrace struct {+ Progress float64 `json:"progress"`+ Message string `json:"message"`+ Datetime string `json:"datetime"`+ Timestamp float64 `json:"timestamp"`+ ElapsedTime float64 `json:"elapsed_time"`+ // Dsl carries the debug-run result (per-component outputs) on the END+ // marker's single trace entry, so the front-end "View result" page can+ // render parsed chunks. It mirrors Python's END marker `dsl`+ // (rag/flow/pipeline.py:98). Nil/empty when unset, in which case+ // encoding/json omits it (json:"omitempty").+ Dsl json.RawMessage `json:"dsl,omitempty"`+}++// DebugLogSink implements pipeline.ProgressSink by accumulating component+// lifecycle events into the [{component_id, trace}] array the front-end polls+// for. It deliberately owns no Redis I/O until Flush: all events are buffered+// in memory, so the sink is fully testable with a fake DebugLogStore and the+// executor can drive it without a live Redis. A run always ends with an END+// marker (appended in Flush) carrying a non-empty message, which is the exact+// signal the front-end uses to consider the run finished.+type DebugLogSink struct {+ canvasID string+ messageID string+ store DebugLogStore+ ttl time.Duration++ mu sync.Mutex+ entries []debugLogEntry+ lastTS float64+ // resultDSL is the debug-run result built by BuildDebugResultDSL and handed+ // to the sink via SetResult (the ResultSink capability). It is written onto+ // the END marker's trace entry in Flush. Empty until SetResult is called.+ resultDSL json.RawMessage+}++// NewDebugLogSink builds a sink that writes to "{canvasID}-{messageID}-logs".+func NewDebugLogSink(canvasID, messageID string, store DebugLogStore) *DebugLogSink {+ return &DebugLogSink{+ canvasID: canvasID,+ messageID: messageID,+ store: store,+ ttl: DebugLogTTL,+ }+}++// OnComponentTotal is a no-op for the debug log: the array shape does not use+// the component-count denominator (it is only consumed by the DB-backed sink+// for progress aggregation).+func (s *DebugLogSink) OnComponentTotal(_ context.Context, _ string, _ int) {}++// SetResult receives the debug-run result DSL (built by BuildDebugResultDSL)+// and stores it for the END marker. It implements the optional ResultSink+// capability the executor probes for, so the sink stays decoupled from how the+// DSL is produced. Safe to call at most once per run; a later call replaces the+// previous value.+func (s *DebugLogSink) SetResult(dsl map[string]any) {+ b, err := json.Marshal(dsl)+ if err != nil {+ return+ }+ s.mu.Lock()+ defer s.mu.Unlock()+ s.resultDSL = b+}++// OnComponentProgress maps one component lifecycle event to a trace line and+// merges it into the running array. Consecutive events for the same component+// append to that component's trace (matching the Python callback's merge+// behaviour); a new component opens a fresh array element.+func (s *DebugLogSink) OnComponentProgress(_ context.Context, ev pipeline.ProgressEvent) {+ now := time.Now()+ ts := float64(now.UnixMicro())+ progress := 0.0+ switch ev.Phase {+ case phaseExit:+ progress = 1.0+ case phaseError:+ progress = -1.0+ }++ message := ev.Message+ if ev.Phase == phaseError {+ // Prefix so the front-end timeline renders the line red+ // (web/src/pages/agent/pipeline-log-sheet/dataflow-timeline.tsx).+ message = "[ERROR] " + message+ }+ // Clamp the message so a runaway component cannot blow up the stored entry.+ message = truncateRunes(message, maxMessageRunes)++ entry := debugTrace{+ Progress: progress,+ Message: message,+ Datetime: now.Format("15:04:05"),+ Timestamp: float64(now.UnixNano()) / 1e9,+ ElapsedTime: 0,+ }++ s.mu.Lock()+ defer s.mu.Unlock()+ if s.lastTS != 0 {+ // ts and s.lastTS are UnixMicro(); report the delta in SECONDS so the+ // front-end's verbatim "<value>s" rendering (dataflow-timeline.tsx)+ // and the "< 0.000001" gates in the other timeline views stay correct.+ entry.ElapsedTime = (ts - s.lastTS) / 1e6+ }+ s.lastTS = ts++ if n := len(s.entries); n > 0 && s.entries[n-1].ComponentID == ev.Component {+ if len(s.entries[n-1].Trace) >= maxTracePerEntry {+ // This component already has its full quota of trace lines; drop the+ // rest so the buffer stays bounded.+ return+ }+ s.entries[n-1].Trace = append(s.entries[n-1].Trace, entry)+ return+ }+ if len(s.entries) >= maxLogEntries {+ // Component quota reached; ignore further components.+ return+ }+ s.entries = append(s.entries, debugLogEntry{+ ComponentID: ev.Component,+ Trace: []debugTrace{entry},+ })+}++// Flush serialises the accumulated array (plus a terminal END marker) and+// persists it to the store. finalErr, when non-nil, is surfaced in the END+// message so a failed run is still reported and detectable as complete. All+// Redis I/O is best-effort: a store failure is swallowed (the run must not+// abort because the log could not be written).+func (s *DebugLogSink) Flush(ctx context.Context, finalErr error) {+ s.mu.Lock()+ entries := append([]debugLogEntry(nil), s.entries...)++ now := time.Now()+ endTS := float64(now.UnixMicro())+ elapsed := 0.0+ if s.lastTS != 0 {+ // endTS and s.lastTS are UnixMicro(); report the delta in SECONDS+ // (see OnComponentProgress for the matching conversion).+ elapsed = (endTS - s.lastTS) / 1e6+ }+ endMsg := "Debug run completed"+ if finalErr != nil {+ endMsg = "[ERROR] " + finalErr.Error()+ }+ entries = append(entries, debugLogEntry{+ ComponentID: "END",+ Trace: []debugTrace{{+ Progress: 1.0,+ Message: endMsg,+ Datetime: now.Format("15:04:05"),+ Timestamp: float64(now.UnixNano()) / 1e9,+ ElapsedTime: elapsed,+ Dsl: s.resultDSL,+ }},+ })+ s.mu.Unlock()++ payload, err := json.Marshal(entries)+ if err != nil {+ return+ }+ // Hard ceiling: if a buggy/abusive run still exceeds it, drop the middle+ // entries (keeping the first few and the END marker) until under budget.+ if len(payload) > maxPayloadBytes {+ entries = trimToPayloadBudget(entries, maxPayloadBytes)+ if payload, err = json.Marshal(entries); err != nil {+ return+ }+ }+ key := s.canvasID + "-" + s.messageID + "-logs"+ s.store.Set(key, string(payload), s.ttl)+}++// truncateRunes returns s truncated to at most max runes, preserving the original+// bytes when shorter. Mirrors internal/service/agent_sessions.go's truncateRunes.+func truncateRunes(s string, max int) string {+ if max <= 0 {+ return ""+ }+ r := []rune(s)+ if len(r) <= max {+ return s+ }+ return string(r[:max])+}++// trimToPayloadBudget collapses the middle of the log (keeping the first few+// entries plus the END marker at the tail) until the marshaled size fits the+// byte budget. The END marker is always the last element, so it is preserved.+// The loop terminates because each pass shrinks the slice; once only the head+// and END remain the payload is tiny.+func trimToPayloadBudget(entries []debugLogEntry, budget int) []debugLogEntry {+ for len(entries) > 2 {+ if b, err := json.Marshal(entries); err == nil && len(b) <= budget {+ return entries+ }+ keep := len(entries) / 4+ if keep < 1 {+ keep = 1+ }+ entries = append(append([]debugLogEntry{}, entries[:keep]...), entries[len(entries)-1])+ }+ return entries+}diff --git a/internal/ingestion/task/debug_log_sink_test.go b/internal/ingestion/task/debug_log_sink_test.gonew file mode 100644index 0000000000..97388a33d6--- /dev/null+++ b/internal/ingestion/task/debug_log_sink_test.go@@ -0,0 +1,766 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package task++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "math"+ "strings"+ "sync"+ "testing"+ "time"+ "unicode/utf8"++ "gorm.io/gorm"+ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/pipeline"+)++// capturedStore is a fake DebugLogStore that records every write so tests can+// assert the exact JSON the sink would persist to Redis — no live Redis needed.+type capturedStore struct {+ mu sync.Mutex+ data map[string]string+}++func (c *capturedStore) Set(key, value string, ttl time.Duration) bool {+ c.mu.Lock()+ defer c.mu.Unlock()+ if c.data == nil {+ c.data = map[string]string{}+ }+ c.data[key] = value+ return true+}++func (c *capturedStore) get(key string) string {+ c.mu.Lock()+ defer c.mu.Unlock()+ return c.data[key]+}++// clientConsidersComplete mirrors the front-end completion predicate in+// web/src/pages/agent/hooks/use-fetch-pipeline-log.ts: the run is done only+// when the LAST array element has component_id == "END" and its first trace+// message is non-empty.+func clientConsidersComplete(arr []map[string]any) bool {+ if len(arr) == 0 {+ return false+ }+ last := arr[len(arr)-1]+ if last["component_id"] != "END" {+ return false+ }+ trace, ok := last["trace"].([]any)+ if !ok || len(trace) == 0 {+ return false+ }+ first, ok := trace[0].(map[string]any)+ if !ok {+ return false+ }+ msg, _ := first["message"].(string)+ return strings.TrimSpace(msg) != ""+}++func loadArray(t *testing.T, raw string) []map[string]any {+ t.Helper()+ var arr []map[string]any+ if err := json.Unmarshal([]byte(raw), &arr); err != nil {+ t.Fatalf("stored value is not a JSON array: %v body=%s", err, raw)+ }+ return arr+}++// TestDebugLogSink_RecordsTraceAndEndMarker covers the core shape contract:+// same-component events merge into one element's trace, a distinct component+// opens a new element, and Flush appends an END marker so the front-end can+// detect completion.+func TestDebugLogSink_RecordsTraceAndEndMarker(t *testing.T) {+ store := &capturedStore{}+ sink := NewDebugLogSink("c1", "m1", store)++ sink.OnComponentProgress(context.Background(), pipeline.ProgressEvent{+ Component: "File", Message: "File Started", Phase: phaseEnter,+ })+ sink.OnComponentProgress(context.Background(), pipeline.ProgressEvent{+ Component: "File", Message: "File Done", Phase: phaseExit,+ })+ sink.OnComponentProgress(context.Background(), pipeline.ProgressEvent{+ Component: "Chunker", Message: "Chunker Done", Phase: phaseExit,+ })++ sink.Flush(context.Background(), nil)++ raw := store.get("c1-m1-logs")+ if raw == "" {+ t.Fatalf("expected key c1-m1-logs to be written")+ }+ arr := loadArray(t, raw)+ // File (merged, 2 trace entries) + Chunker + END = 3 elements.+ if len(arr) != 3 {+ t.Fatalf("want 3 elements, got %d: %s", len(arr), raw)+ }+ if arr[0]["component_id"] != "File" {+ t.Errorf("elem0.component_id=%v want File", arr[0]["component_id"])+ }+ fileTrace, _ := arr[0]["trace"].([]any)+ if len(fileTrace) != 2 {+ t.Errorf("File trace should merge 2 entries, got %d: %s", len(fileTrace), raw)+ }+ if arr[1]["component_id"] != "Chunker" {+ t.Errorf("elem1.component_id=%v want Chunker", arr[1]["component_id"])+ }+ if arr[2]["component_id"] != "END" {+ t.Errorf("last component_id=%v want END", arr[2]["component_id"])+ }+ endTrace, _ := arr[2]["trace"].([]any)+ endFirst, _ := endTrace[0].(map[string]any)+ if strings.TrimSpace(endFirst["message"].(string)) == "" {+ t.Errorf("END message must be non-empty for completion signal: %s", raw)+ }+ if !clientConsidersComplete(arr) {+ t.Errorf("client would NOT consider run complete: %s", raw)+ }+}++// TestDebugLogSink_ErrorPrefix verifies that error-phase events get a [ERROR]+// prefix (so the front-end timeline renders them red) and that Flush with a+// non-nil error yields an END marker carrying the error, keeping the run+// detectable as finished.+func TestDebugLogSink_ErrorPrefix(t *testing.T) {+ store := &capturedStore{}+ sink := NewDebugLogSink("c1", "m1", store)++ sink.OnComponentProgress(context.Background(), pipeline.ProgressEvent{+ Component: "Tokenizer", Message: "Tokenizer: boom", Phase: phaseError,+ })+ sink.Flush(context.Background(), errors.New("boom"))++ raw := store.get("c1-m1-logs")+ arr := loadArray(t, raw)++ // First element is the errored component; its message must carry [ERROR].+ erroredTrace, _ := arr[0]["trace"].([]any)+ erroredFirst, _ := erroredTrace[0].(map[string]any)+ if !strings.HasPrefix(erroredFirst["message"].(string), "[ERROR] ") {+ t.Errorf("error trace message should be prefixed [ERROR] , got %q", erroredFirst["message"])+ }++ // Last element is END with the error surfaced, so the run still completes.+ lastTrace, _ := arr[len(arr)-1]["trace"].([]any)+ lastFirst, _ := lastTrace[0].(map[string]any)+ if !strings.HasPrefix(lastFirst["message"].(string), "[ERROR] ") {+ t.Errorf("END message on failure should be prefixed [ERROR] , got %q", lastFirst["message"])+ }+ if !clientConsidersComplete(arr) {+ t.Errorf("failed run must still be detectable as complete: %s", raw)+ }+}++// TestDebugLogSink_OnComponentTotalIsNoOp ensures the denominator callback+// (used by the DB-backed sink for progress aggregation) is a safe no-op here:+// the debug log array shape does not use it.+func TestDebugLogSink_OnComponentTotalIsNoOp(t *testing.T) {+ store := &capturedStore{}+ sink := NewDebugLogSink("c1", "m1", store)+ // Must not panic and must not write.+ sink.OnComponentTotal(context.Background(), "task-1", 5)+ if len(store.data) != 0 {+ t.Errorf("OnComponentTotal must not write to the store, wrote %v", store.data)+ }+}++// TestPipelineExecutor_DebugRunWritesLogViaSink drives the real+// PipelineExecutor debug branch (KB.ID == "") with an injected fake pipeline+// that emits progress through the sink. It proves the end-to-end wiring inside+// the executor: a ProgressSink attached via WithProgressSink is forwarded to the+// pipeline and its events land in the persisted debug-log array (with the END+// marker), without touching the DB/index persist path.+func TestPipelineExecutor_DebugRunWritesLogViaSink(t *testing.T) {+ ctx := context.Background()+ taskCtx := NewDebugTaskContext("tenant-1", "c1", "doc.pdf", nil)+ exec, err := NewPipelineExecutor(taskCtx, "c1", 0)+ if err != nil {+ t.Fatalf("NewPipelineExecutor: %v", err)+ }++ store := &capturedStore{}+ sink := NewDebugLogSink("c1", "m1", store)+ exec.WithProgressSink(sink)++ // Fake the canvas/DSL boundary: loadDSLFunc returns a dummy DSL and+ // runPipelineFunc emits progress through the (now attached) sink instead of+ // running a real pipeline.+ exec.loadDSLFunc = func(_ context.Context, _ string) (string, string, error) {+ return "dummy-dsl", "", nil+ }+ exec.runPipelineFunc = func(ctx context.Context, _ string) (map[string]any, string, error) {+ exec.progressSink.OnComponentProgress(ctx, pipeline.ProgressEvent{+ Component: "File", Message: "File Done", Phase: phaseExit,+ })+ exec.progressSink.OnComponentProgress(ctx, pipeline.ProgressEvent{+ Component: "Chunker", Message: "Chunker Done", Phase: phaseExit,+ })+ return map[string]any{}, "", nil+ }++ if _, err := exec.Execute(ctx); err != nil {+ t.Fatalf("Execute: %v", err)+ }+ // Mirrors runCanvasPipelineDebug: flush (incl. END marker) after the run.+ sink.Flush(ctx, nil)++ raw := store.get("c1-m1-logs")+ if raw == "" {+ t.Fatalf("expected debug log written to c1-m1-logs")+ }+ arr := loadArray(t, raw)+ if arr[len(arr)-1]["component_id"] != "END" {+ t.Errorf("last element must be END marker, got %s", raw)+ }+ if !clientConsidersComplete(arr) {+ t.Errorf("client would not consider run complete: %s", raw)+ }+}++// traceStubComponent is a no-op runtime.Component used to exercise the REAL+// pipeline → TrackProgress → DebugLogSink wiring (the enter→exit pairing) without+// a live DB or any real ingestion work.+type traceStubComponent struct{}++func (traceStubComponent) Invoke(_ context.Context, _ *gorm.DB, _ map[string]any) (map[string]any, error) {+ return map[string]any{"ok": true}, nil+}++// traceChunkComponent emits a real `chunks` output so the end-to-end wiring can+// be asserted to populate the END-marker DSL's params.outputs — the exact+// contract the front-end "View result" needs and the one the nested-state bug+// broke (empty Result tabs).+type traceChunkComponent struct{}++func (traceChunkComponent) Invoke(_ context.Context, _ *gorm.DB, _ map[string]any) (map[string]any, error) {+ // Use the real component output shape ([]map[string]any as produced by+ // ChunkDocsToMaps) so the end-to-end strip/recurse path for []map[string]any+ // is actually exercised — a []any stub hides the deepCopyStrip type gap.+ return map[string]any{+ "chunks": []map[string]any{+ {"text": "real chunk", "vector": []float64{0.5}},+ },+ }, nil+}++// TestDebugLogSink_RealPipeline_EachComponentTraceHasStartedAndDone drives a+// REAL pipeline (not the fake runPipelineFunc used elsewhere) through the same+// wiring a dataflow debug run uses: NewPipelineFromDSL + WithProgressSink+// (DebugLogSink) + Run. The canvas framework wraps each component in+// realComponentBody → runtime.TrackProgress, which MUST emit BOTH PhaseEnter+// ("<comp> Started", progress 0) and PhaseExit ("<comp> Done", progress 1) on+// success.+//+// This locks the contract the front-end depends on: every component ends with a+// Done line, not just a Started line. It is the regression guard for the+// observation that a debug run's polled log appeared to stay at progress 0 —+// per the code, each component's trace must contain [Started, Done].+func TestDebugLogSink_RealPipeline_EachComponentTraceHasStartedAndDone(t *testing.T) {+ const (+ compA = "trace.RealStubA"+ compB = "trace.RealStubB"+ )+ runtime.MustRegister(compA, runtime.CategoryIngestion,+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },+ runtime.Metadata{Version: "1.0.0"})+ runtime.MustRegister(compB, runtime.CategoryIngestion,+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },+ runtime.Metadata{Version: "1.0.0"})++ store := &capturedStore{}+ sink := NewDebugLogSink("c-trace", "m-trace", store)++ dsl := []byte(`{"dsl":{"components":{+ "begin":{"obj":{"component_name":"Begin","params":{}},"downstream":["a"]},+ "a":{"obj":{"component_name":"` + compA + `","params":{}},"upstream":["begin"],"downstream":["b"]},+ "b":{"obj":{"component_name":"` + compB + `","params":{}},"upstream":["a"]}+ },"path":["begin","a","b"],"graph":{"nodes":[]}}}`)++ pipe, err := pipeline.NewPipelineFromDSL(dsl, "task-trace",+ pipeline.WithProgressSink(sink), pipeline.WithDocumentID("doc-trace"))+ if err != nil {+ t.Fatalf("NewPipelineFromDSL: %v", err)+ }+ if _, err := pipe.Run(context.Background(), map[string]any{"name": "doc-trace"}, nil); err != nil {+ t.Fatalf("Run: %v", err)+ }+ sink.Flush(context.Background(), nil)++ raw := store.get("c-trace-m-trace-logs")+ if raw == "" {+ t.Fatalf("expected debug log written to c-trace-m-trace-logs")+ }+ arr := loadArray(t, raw)++ // Completion predicate must hold: END is last with a non-empty message.+ if arr[len(arr)-1]["component_id"] != "END" {+ t.Fatalf("last element must be END marker: %s", raw)+ }+ if !clientConsidersComplete(arr) {+ t.Fatalf("client would not consider run complete: %s", raw)+ }++ // Every real component (a, b) must show BOTH Started (progress 0) and Done+ // (progress 1) within its own trace — proving TrackProgress emitted the full+ // enter→exit lifecycle, not just enter.+ for _, comp := range []string{"a", "b"} {+ var gotStart, gotDone bool+ for _, el := range arr {+ if el["component_id"] != comp {+ continue+ }+ traces, _ := el["trace"].([]any)+ for _, tr := range traces {+ tm, _ := tr.(map[string]any)+ if tm == nil {+ continue+ }+ prog, _ := tm["progress"].(float64)+ msg, _ := tm["message"].(string)+ switch {+ case prog == 0 && msg == comp+" Started":+ gotStart = true+ case prog == 1 && msg == comp+" Done":+ gotDone = true+ }+ }+ }+ if !gotStart || !gotDone {+ t.Fatalf("component %q missing Started/Done in trace: %s", comp, raw)+ }+ }+}++// TestDebugLogSink_RealPipeline_EndMarkerCarriesDSL locks the end-to-end+// contract the front-end "View result" depends on: the debug log's END marker+// trace[0] MUST carry a `dsl` (built by BuildDebugResultDSL) whose `components`+// map mirrors the canvas. The front-end `useFetchPipelineResult` picks+// `latest.trace.at(0)` where `latest.component_id == "END"` and reads+// `dsl.components[<id>].obj` to render parsed chunks (parser.tsx:41). Without+// this, "View result" shows only the file preview — the bug this change fixes.+//+// It drives a REAL pipeline (same wiring as a dataflow debug run) so the `dsl`+// is built from an actual run output map, not a hand-built one.+func TestDebugLogSink_RealPipeline_EndMarkerCarriesDSL(t *testing.T) {+ const (+ compC = "trace.RealStubC"+ compD = "trace.RealStubD"+ )+ runtime.MustRegister(compC, runtime.CategoryIngestion,+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },+ runtime.Metadata{Version: "1.0.0"})+ runtime.MustRegister(compD, runtime.CategoryIngestion,+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },+ runtime.Metadata{Version: "1.0.0"})++ store := &capturedStore{}+ sink := NewDebugLogSink("c-dsl", "m-dsl", store)++ dsl := `{"dsl":{"components":{+ "begin":{"obj":{"component_name":"Begin","params":{}},"downstream":["c"]},+ "c":{"obj":{"component_name":"` + compC + `","params":{}},"upstream":["begin"],"downstream":["d"]},+ "d":{"obj":{"component_name":"` + compD + `","params":{}},"upstream":["c"]}+ },"path":["begin","c","d"],"graph":{"nodes":[{"id":"begin","data":{"name":"开始"}},{"id":"c","data":{"name":"解析"}},{"id":"d","data":{"name":"分词"}}]}}}`++ pipe, err := pipeline.NewPipelineFromDSL([]byte(dsl), "task-dsl",+ pipeline.WithProgressSink(sink), pipeline.WithDocumentID("doc-dsl"))+ if err != nil {+ t.Fatalf("NewPipelineFromDSL: %v", err)+ }+ output, err := pipe.Run(context.Background(), map[string]any{"name": "doc-dsl"}, nil)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ // The executor would call this after Run; here we invoke it directly.+ resultDSL, err := BuildDebugResultDSL(dsl, output)+ if err != nil {+ t.Fatalf("BuildDebugResultDSL: %v", err)+ }+ sink.SetResult(resultDSL)+ sink.Flush(context.Background(), nil)++ raw := store.get("c-dsl-m-dsl-logs")+ if raw == "" {+ t.Fatalf("expected debug log written to c-dsl-m-dsl-logs")+ }+ arr := loadArray(t, raw)++ // END must be the last element.+ if arr[len(arr)-1]["component_id"] != "END" {+ t.Fatalf("last element must be END marker: %s", raw)+ }+ endTrace, _ := arr[len(arr)-1]["trace"].([]any)+ if len(endTrace) == 0 {+ t.Fatalf("END trace is empty: %s", raw)+ }+ endFirst, _ := endTrace[0].(map[string]any)+ dslVal := endFirst["dsl"]+ if dslVal == nil {+ t.Fatalf("END marker trace[0] must carry a non-empty dsl: %s", raw)+ }++ // json.RawMessage decodes into a generic map as map[string]any (not a+ // string), so accept either form.+ var dslDoc map[string]any+ switch v := dslVal.(type) {+ case string:+ if err := json.Unmarshal([]byte(v), &dslDoc); err != nil {+ t.Fatalf("END dsl is not valid JSON: %v body=%s", err, v)+ }+ case map[string]any:+ dslDoc = v+ default:+ t.Fatalf("END dsl unexpected type %T: %s", v, raw)+ }+ components, ok := dslDoc["components"].(map[string]any)+ if !ok || len(components) == 0 {+ t.Fatalf("END dsl.components missing or empty: %s", raw)+ }+ // The dsl must mirror every component in the canvas (begin, c, d).+ for _, id := range []string{"begin", "c", "d"} {+ if _, exists := components[id]; !exists {+ t.Errorf("END dsl.components missing component %q: %s", id, raw)+ }+ }+}++// TestDebugLogSink_RealPipeline_EndMarkerDSLShowsChunks is the strongest+// regression guard for the nested-state bug: it drives a REAL pipeline whose+// component "c" emits actual `chunks`, then asserts the END-marker DSL carries+// a non-empty params.outputs for that component. With the bug (reading+// output[id] at the top level instead of output["state"][id]), the run output+// is nested and params.outputs stays empty — the front-end "View result" shows+// blank tabs. The companion TestDebugLogSink_RealPipeline_EndMarkerCarriesDSL+// only checks component keys exist (its stubs return {"ok":true}, no chunks),+// so it would not catch this. This test closes that end-to-end gap.+//+// It also pins the inverse: component "d" (stub returning {"ok":true}) has NO+// recognized output key, so its params.outputs must be absent — the safe-empty+// contract the front-end renders as a blank step.+func TestDebugLogSink_RealPipeline_EndMarkerDSLShowsChunks(t *testing.T) {+ const (+ compC = "trace.RealStubChunks"+ compD = "trace.RealStubD2"+ )+ runtime.MustRegister(compC, runtime.CategoryIngestion,+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceChunkComponent{}, nil },+ runtime.Metadata{Version: "1.0.0"})+ runtime.MustRegister(compD, runtime.CategoryIngestion,+ func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },+ runtime.Metadata{Version: "1.0.0"})++ store := &capturedStore{}+ sink := NewDebugLogSink("c-chunk", "m-chunk", store)++ dsl := `{"dsl":{"components":{+ "begin":{"obj":{"component_name":"Begin","params":{}},"downstream":["c"]},+ "c":{"obj":{"component_name":"` + compC + `","params":{"setups":{"pdf":{"parse_method":"general"}}}},"upstream":["begin"],"downstream":["d"]},+ "d":{"obj":{"component_name":"` + compD + `","params":{}},"upstream":["c"]}+ },"path":["begin","c","d"],"graph":{"nodes":[{"id":"begin","data":{"name":"开始"}},{"id":"c","data":{"name":"解析"}},{"id":"d","data":{"name":"分词"}}]}}}`++ pipe, err := pipeline.NewPipelineFromDSL([]byte(dsl), "task-chunk",+ pipeline.WithProgressSink(sink), pipeline.WithDocumentID("doc-chunk"))+ if err != nil {+ t.Fatalf("NewPipelineFromDSL: %v", err)+ }+ output, err := pipe.Run(context.Background(), map[string]any{"name": "doc-chunk"}, nil)+ if err != nil {+ t.Fatalf("Run: %v", err)+ }++ resultDSL, err := BuildDebugResultDSL(dsl, output)+ if err != nil {+ t.Fatalf("BuildDebugResultDSL: %v", err)+ }+ sink.SetResult(resultDSL)+ sink.Flush(context.Background(), nil)++ raw := store.get("c-chunk-m-chunk-logs")+ if raw == "" {+ t.Fatalf("expected debug log written")+ }+ arr := loadArray(t, raw)+ if arr[len(arr)-1]["component_id"] != "END" {+ t.Fatalf("last element must be END marker: %s", raw)+ }+ endTrace, _ := arr[len(arr)-1]["trace"].([]any)+ endFirst, _ := endTrace[0].(map[string]any)++ var dslDoc map[string]any+ switch v := endFirst["dsl"].(type) {+ case string:+ if err := json.Unmarshal([]byte(v), &dslDoc); err != nil {+ t.Fatalf("END dsl not valid JSON: %v", err)+ }+ case map[string]any:+ dslDoc = v+ default:+ t.Fatalf("END dsl unexpected type %T", v)+ }+ components, _ := dslDoc["components"].(map[string]any)++ // Component "c" emits chunks -> params.outputs MUST be populated.+ cParams := components["c"].(map[string]any)["obj"].(map[string]any)["params"].(map[string]any)+ cOutputs, ok := cParams["outputs"].(map[string]any)+ if !ok {+ t.Fatalf("REGRESSION: chunk-emitting component c has empty params.outputs; "++ "the front-end 'View result' would render blank tabs. params=%#v", cParams)+ }+ if of, _ := cOutputs["output_format"].(map[string]any); of["value"] != "chunks" {+ t.Errorf("c output_format=%#v want {value:\"chunks\"}", cOutputs["output_format"])+ }+ chunksVal := cOutputs["chunks"].(map[string]any)["value"].([]any)+ if len(chunksVal) != 1 {+ t.Fatalf("c chunks.value len=%d want 1", len(chunksVal))+ }+ chunk0 := chunksVal[0].(map[string]any)+ if chunk0["text"] != "real chunk" {+ t.Errorf("chunk text=%v want 'real chunk'", chunk0["text"])+ }+ if _, exists := chunk0["vector"]; exists {+ t.Errorf("vector must be stripped from chunk payload")+ }+ if _, ok := cParams["setups"].(map[string]any); !ok {+ t.Errorf("c params.setups must be carried from DSL: %#v", cParams)+ }++ // Component "d" emits {"ok":true} -> no recognized format -> outputs absent.+ dParams := components["d"].(map[string]any)["obj"].(map[string]any)["params"].(map[string]any)+ if _, exists := dParams["outputs"]; exists {+ t.Errorf("d has no recognized output, outputs must be absent: %#v", dParams)+ }+}++// TestDebugLogSink_ElapsedTimeIsInSeconds locks the unit contract for+// `elapsed_time`: it MUST be expressed in SECONDS, not the raw UnixMicro delta.+// This is the regression guard for the dataflow timeline showing values like+// "19951s" — the sink stored microseconds while the front-end renders the number+// verbatim with an "s" suffix (dataflow-timeline.tsx) and the rest of the app+// (webhook/workflow timelines) gates on `elapsed_time < 0.000001`, i.e. seconds.+//+// The test drives two progress events ~20ms apart and asserts the recorded+// elapsed_time is close to the measured wall-clock gap in seconds (not 1e6x+// larger, which is what an unfixed microsecond delta would produce).+func TestDebugLogSink_ElapsedTimeIsInSeconds(t *testing.T) {+ store := &capturedStore{}+ sink := NewDebugLogSink("c-elapsed", "m-elapsed", store)+ ctx := context.Background()++ t0 := time.Now()+ sink.OnComponentProgress(ctx, pipeline.ProgressEvent{+ Component: "A", Message: "A Started", Phase: phaseEnter,+ })+ time.Sleep(20 * time.Millisecond)+ t1 := time.Now()+ sink.OnComponentProgress(ctx, pipeline.ProgressEvent{+ Component: "A", Message: "A Done", Phase: phaseExit,+ })+ sink.Flush(ctx, nil)++ raw := store.get("c-elapsed-m-elapsed-logs")+ if raw == "" {+ t.Fatalf("expected debug log written to c-elapsed-m-elapsed-logs")+ }+ arr := loadArray(t, raw)+ if len(arr) < 2 {+ t.Fatalf("expected at least A + END elements, got %d: %s", len(arr), raw)+ }+ aTrace, _ := arr[0]["trace"].([]any)+ if len(aTrace) < 2 {+ t.Fatalf("A trace should have [Started, Done], got %d entries: %s", len(aTrace), raw)+ }+ doneTrace, _ := aTrace[1].(map[string]any)+ elapsed, ok := doneTrace["elapsed_time"].(float64)+ if !ok {+ t.Fatalf("elapsed_time missing or wrong type: %s", raw)+ }++ gapSeconds := t1.Sub(t0).Seconds()+ // Tolerance absorbs call-overhead jitter; the key is that elapsed is in the+ // same SECONDS scale as gapSeconds, not 1e6x larger (microseconds).+ // With an unfixed microsecond delta, elapsed would be ~gapSeconds*1e6 and+ // this assertion fails.+ if math.Abs(elapsed-gapSeconds) > math.Max(gapSeconds*0.5, 0.01) {+ t.Fatalf("elapsed_time=%v want ~%v seconds (sink must divide the UnixMicro delta by 1e6)", elapsed, gapSeconds)+ }+}++// TestDebugLogSink_TimestampIsInSeconds locks the unit contract for the+// `timestamp` field: it MUST be a Unix epoch in SECONDS, not the raw UnixMicro+// value. Although the dataflow timeline currently renders the `datetime` string+// rather than `timestamp`, the field is part of the stable log contract the+// front-end can rely on; emitting microseconds would put it 1e6 off from any+// consumer expecting a Unix timestamp. This matches the SECONDS convention+// already established by elapsed_time and the other (webhook/workflow) timeline+// views.+func TestDebugLogSink_TimestampIsInSeconds(t *testing.T) {+ store := &capturedStore{}+ sink := NewDebugLogSink("c-ts", "m-ts", store)+ ctx := context.Background()++ before := time.Now().Unix()+ sink.OnComponentProgress(ctx, pipeline.ProgressEvent{+ Component: "A", Message: "A Started", Phase: phaseEnter,+ })+ sink.Flush(ctx, nil)++ raw := store.get("c-ts-m-ts-logs")+ if raw == "" {+ t.Fatalf("expected debug log written to c-ts-m-ts-logs")+ }+ arr := loadArray(t, raw)+ aTrace, _ := arr[0]["trace"].([]any)+ if len(aTrace) == 0 {+ t.Fatalf("A trace is empty: %s", raw)+ }+ firstTrace, _ := aTrace[0].(map[string]any)+ ts, ok := firstTrace["timestamp"].(float64)+ if !ok {+ t.Fatalf("timestamp missing or wrong type: %s", raw)+ }++ // If the sink emitted UnixMicro, ts would be ~1.7e15; a Unix-seconds value+ // is ~1.7e9 and must be within a few seconds of `before`.+ diff := math.Abs(ts - float64(before))+ if diff > 5 {+ t.Fatalf("timestamp=%v want ~%d (Unix seconds, not UnixMicro)", ts, before)+ }+}++// TestDebugLogSink_RespectsCaps pins the size guards: a pathological run that+// emits far more components / trace lines / over-long messages than any+// legitimate canvas ever would must be clamped, while the END completion marker+// and the front-end's completion predicate stay intact. This is the regression+// guard for the unbounded-memory / oversized-Redis-entry DoS surface.+//+// Counts are deliberately bounded so the test stays fast: we only need to exceed+// each cap by a small margin to prove clamping, not replay millions of events.+func TestDebugLogSink_RespectsCaps(t *testing.T) {+ ctx := context.Background()++ // Component cap: more distinct components than maxLogEntries (each one trace,+ // one over-long message that must be truncated to the rune cap).+ storeA := &capturedStore{}+ sinkA := NewDebugLogSink("c1", "mA", storeA)+ for i := 0; i < maxLogEntries+200; i++ {+ sinkA.OnComponentProgress(ctx, pipeline.ProgressEvent{+ Component: fmt.Sprintf("Comp-%d", i),+ Message: strings.Repeat("x", 5000), // truncated to maxMessageRunes+ Phase: phaseExit,+ })+ }+ sinkA.Flush(ctx, nil)++ arrA := loadArray(t, storeA.get("c1-mA-logs"))+ // Flush always appends the terminal END marker, so count only real components.+ var compCountA int+ for _, el := range arrA {+ if el["component_id"] == "END" {+ continue+ }+ compCountA+++ }+ if compCountA > maxLogEntries {+ t.Fatalf("components=%d want <= %d", compCountA, maxLogEntries)+ }+ for _, el := range arrA {+ traces, _ := el["trace"].([]any)+ for _, tr := range traces {+ tm, _ := tr.(map[string]any)+ if tm == nil {+ continue+ }+ msg, _ := tm["message"].(string)+ if utf8.RuneCountInString(msg) > maxMessageRunes {+ t.Fatalf("message rune len=%d want <= %d", utf8.RuneCountInString(msg), maxMessageRunes)+ }+ }+ }+ if !clientConsidersComplete(arrA) {+ t.Errorf("clamped log must still satisfy completion predicate; raw=%s", storeA.get("c1-mA-logs"))+ }++ // Trace cap: a single component with more traces than maxTracePerEntry.+ storeB := &capturedStore{}+ sinkB := NewDebugLogSink("c1", "mB", storeB)+ for j := 0; j < maxTracePerEntry+50; j++ {+ sinkB.OnComponentProgress(ctx, pipeline.ProgressEvent{+ Component: "Busy",+ Message: "ok",+ Phase: phaseExit,+ })+ }+ sinkB.Flush(ctx, nil)++ arrB := loadArray(t, storeB.get("c1-mB-logs"))+ // Busy (the only real component) + the END completion marker = 2 elements.+ if len(arrB) != 2 {+ t.Fatalf("Busy + END expected, got %d elements: %s", len(arrB), storeB.get("c1-mB-logs"))+ }+ busyTrace, _ := arrB[0]["trace"].([]any)+ if len(busyTrace) > maxTracePerEntry {+ t.Fatalf("Busy trace len=%d want <= %d", len(busyTrace), maxTracePerEntry)+ }+ if !clientConsidersComplete(arrB) {+ t.Errorf("trace-capped log must still satisfy completion predicate; raw=%s", storeB.get("c1-mB-logs"))+ }+}++// TestDebugLogSink_PayloadHardCap pins the absolute byte ceiling on the JSON+// written to Redis. It fills to the per-entry caps (maxLogEntries components x+// maxTracePerEntry traces) — which pre-trim is ~16 MiB — and asserts the stored+// value never exceeds maxPayloadBytes, and still ends with the END marker.+func TestDebugLogSink_PayloadHardCap(t *testing.T) {+ store := &capturedStore{}+ sink := NewDebugLogSink("c1", "m1", store)+ ctx := context.Background()++ for i := 0; i < maxLogEntries; i++ {+ comp := fmt.Sprintf("Comp-%d", i)+ for j := 0; j < maxTracePerEntry; j++ {+ sink.OnComponentProgress(ctx, pipeline.ProgressEvent{+ Component: comp,+ Message: "short",+ Phase: phaseExit,+ })+ }+ }+ sink.Flush(ctx, nil)++ raw := store.get("c1-m1-logs")+ if raw == "" {+ t.Fatalf("expected debug log written to c1-m1-logs")+ }+ if len(raw) > maxPayloadBytes {+ t.Fatalf("stored payload bytes=%d want <= %d", len(raw), maxPayloadBytes)+ }+ arr := loadArray(t, raw)+ if !clientConsidersComplete(arr) {+ t.Errorf("hard-capped log must still satisfy completion predicate; arr=%s", raw)+ }+}diff --git a/internal/ingestion/task/debug_pages_integration_test.go b/internal/ingestion/task/debug_pages_integration_test.gonew file mode 100644index 0000000000..6d38d2dbc2--- /dev/null+++ b/internal/ingestion/task/debug_pages_integration_test.go@@ -0,0 +1,213 @@+//go:build integration++//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package task++import (+ "context"+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/component"+ "ragflow/internal/ingestion/pipeline"+ "ragflow/internal/storage"+)++// TestExecute_DebugViaEntry_HonorsPagesCap_Integration is the end-to-end+// proof that the canvas-debug page cap travels through Run's override_params+// channel (NOT pipeline inputs) and is honoured by the deepdoc/pdf parser.+//+// It parses a multi-page PDF twice through a real debug pipeline (real DSL,+// real pdfium parser, in-memory sqlite + storage):+//+// - Uncapped baseline: an explicit cpnID+family page cap of [[1, 1000000]]+// is supplied in ParserConfig. injectDebugPageCap must RESPECT it, so the+// parser reads every page.+// - Capped: no page cap is supplied, so injectDebugPageCap injects the+// debug default [[1, debugPageCapPages]], and the parser reads only the+// leading pages.+//+// The capped run must therefore yield strictly fewer chunk characters than+// the uncapped run. This exercises the full path the production code uses+// (Run's 3rd-arg override_params → applyOverrideParams → c.Setups →+// ConfigureFromSetup → deepdoc pdf Config.Pages).+//+// This is gated by //go:build integration because it depends on the native+// pdfium library, the 03_multipage.pdf fixture, and (optionally) the+// tokenizer resource dict.+func TestExecute_DebugViaEntry_HonorsPagesCap_Integration(t *testing.T) {+ requireTokenizerPool(t)+ mustLoadTaskTestConfig(t)++ origDB := dao.DB+ realDB := mustOpenTaskTestDB(t)+ dao.DB = realDB+ t.Cleanup(func() { dao.DB = origDB })++ realStorage := storage.NewMemoryStorage()+ origStorage := storage.GetStorageFactory().GetStorage()+ storage.GetStorageFactory().SetStorage(realStorage)+ t.Cleanup(func() { storage.GetStorageFactory().SetStorage(origStorage) })++ templatePath := filepath.Join(taskRepoRoot(t), "internal", "ingestion", "pipeline", "template", "ingestion_pipeline_general.json")+ templateBytes, err := os.ReadFile(templatePath)+ if err != nil {+ t.Fatalf("read template: %v", err)+ }+ templateBytes = disableTokenizerEmbeddingForTaskTemplate(t, templateBytes)++ // envelopeDSL is stored verbatim on the canvas (this is what production+ // persists), so loadDSLFromCanvas marshals the envelope and Run receives+ // it. injectDebugPageCap must unwrap it to find the Parser cpnID.+ var envelope struct {+ DSL json.RawMessage `json:"dsl"`+ }+ if err := json.Unmarshal(templateBytes, &envelope); err != nil {+ t.Fatalf("unmarshal template envelope: %v", err)+ }+ var canvasDSL entity.JSONMap+ if err := json.Unmarshal(templateBytes, &canvasDSL); err != nil {+ t.Fatalf("unmarshal template into canvas dsl: %v", err)+ }++ // Discover the Parser cpnID from the inner DSL.+ schemas, err := pipeline.ExtractAllComponentParams(envelope.DSL)+ if err != nil {+ t.Fatalf("ExtractAllComponentParams: %v", err)+ }+ var parserCpnID string+ for _, s := range schemas {+ if s.ComponentName == component.ComponentNameParser {+ parserCpnID = s.CpnID+ }+ }+ if parserCpnID == "" {+ t.Fatal("template has no Parser component")+ }++ pdfPath := filepath.Join(taskRepoRoot(t), "internal", "deepdoc", "parser", "pdf", "testdata", "pdfs", "03_multipage.pdf")+ pdfBytes, err := os.ReadFile(pdfPath)+ if err != nil {+ t.Skipf("read pdf fixture %s: %v", pdfPath, err)+ return+ }++ suffix := fmt.Sprintf("%d", time.Now().UnixNano())+ tenantID := taskLimit32("it_tenant_" + suffix)+ canvasID := taskLimit32("it_canvas_" + suffix)++ if err := realDB.Create(&entity.UserCanvas{+ ID: canvasID,+ UserID: tenantID,+ Permission: "me",+ CanvasCategory: "agent_canvas",+ DSL: canvasDSL,+ }).Error; err != nil {+ t.Fatalf("create user canvas: %v", err)+ }+ t.Cleanup(func() { _ = realDB.Where("id = ?", canvasID).Delete(&entity.UserCanvas{}).Error })++ // Explicit "parse all pages" cap (JSON-decoded []any form, the shape the+ // parser actually consumes). injectDebugPageCap must respect it and leave+ // it untouched, so the parser reads every page of the PDF.+ allPages := map[string]any{+ parserCpnID: map[string]any{+ "pdf": map[string]any{+ "pages": []any{[]any{1, 1000000}},+ },+ },+ }++ newDebugCtx := func(parserConfig map[string]any) *TaskContext {+ // A canvas-debug (dry-run) context carries no KB: KB.ID == "" is the+ // single debug signal. The executor then skips the persist stage and+ // injects the debug page cap (see injectDebugPageCap, gated on+ // KB.ID == "").+ return &TaskContext{+ Doc: entity.Document{+ ID: "debug-run-1",+ KbID: "",+ Name: taskStrPtr("03_multipage.pdf"),+ Type: "pdf",+ ParserConfig: parserConfig,+ },+ KB: entity.Knowledgebase{+ ID: "",+ TenantID: tenantID,+ EmbdID: "embd-1",+ },+ Tenant: entity.Tenant{ID: tenantID},+ File: pdfBytes,+ PipelineID: canvasID,+ }+ }++ // Uncapped baseline: explicit pages=all → executor respects override.+ uncappedExec, err := NewPipelineExecutor(newDebugCtx(allPages), canvasID, 0)+ if err != nil {+ t.Fatalf("NewPipelineExecutor (uncapped): %v", err)+ }+ uncapped, err := uncappedExec.Execute(context.Background())+ if err != nil {+ t.Fatalf("Execute (uncapped): %v", err)+ }++ // Capped: no pages override → executor injects [[1, debugPageCapPages]].+ cappedExec, err := NewPipelineExecutor(newDebugCtx(nil), canvasID, 0)+ if err != nil {+ t.Fatalf("NewPipelineExecutor (capped): %v", err)+ }+ capped, err := cappedExec.Execute(context.Background())+ if err != nil {+ t.Fatalf("Execute (capped): %v", err)+ }++ uncappedLen := len(joinedChunks(uncapped.Chunks))+ cappedLen := len(joinedChunks(capped.Chunks))+ if uncappedLen == 0 {+ t.Fatal("uncapped baseline produced no chunks; fixture/pipeline misconfigured")+ }+ if cappedLen == 0 {+ t.Fatal("pages cap produced no chunks; pages wiring broken")+ }+ if cappedLen >= uncappedLen {+ t.Errorf("pages cap had no effect: capped text len %d >= uncapped %d (pages not honoured via override_params)", cappedLen, uncappedLen)+ }+}++// joinedChunks concatenates the text of every chunk into a single string so+// the page-cap assertion does not depend on how the chunker splits pages.+func joinedChunks(chunks []map[string]any) string {+ var b strings.Builder+ for _, ck := range chunks {+ for _, key := range []string{"content_with_weight", "content", "text"} {+ if v, ok := ck[key].(string); ok && v != "" {+ b.WriteString(v)+ b.WriteString("\n")+ break+ }+ }+ }+ return b.String()+}diff --git a/internal/ingestion/task/debug_result_dsl.go b/internal/ingestion/task/debug_result_dsl.gonew file mode 100644index 0000000000..284bec3f34--- /dev/null+++ b/internal/ingestion/task/debug_result_dsl.go@@ -0,0 +1,253 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//+// This file is the Go analogue of Python's `Graph.__str__`+// (agent/canvas.py:119) used by the dataflow debug "View result" page.+// Python attaches `dsl = json.loads(str(self))` to the END debug-log marker+// (rag/flow/pipeline.py:98); the front-end reads `dsl.components[<id>].obj`+// to render each component's parsed output. Go has no per-component `__str__`+// serialization, so BuildDebugResultDSL combines the STATIC DSL structure+// (component_name / downstream / params / graph.nodes) with the RUN output map+// (state.go:284 `out[<componentID>] = <outputs>`) to emit the identical JSON+// shape. Result is a strict subset (UI-relevant fields only) of Python's, so+// the front-end renders chunks with parity and a smaller payload.++package task++import (+ "encoding/json"+ "fmt"+ "strings"+)++// ResultSink is an OPTIONAL capability a ProgressSink may implement to receive+// the debug-run result DSL. The pipeline executor probes for it via a type+// assertion, so the ProgressSink contract stays unchanged and non-debug+// (DB-backed) sinks simply ignore it — keeping the coupling one-directional.+type ResultSink interface {+ SetResult(dsl map[string]any)+}++// outputFormats is the priority order used to pick a component's payload key,+// mirroring NormalizeChunks (chunk_utils.go:27).+var outputFormats = []string{"chunks", "json", "text", "html", "markdown"}++// vectorKeys are dropped from payloads while copying. The front-end never+// renders raw vectors and Python's serialized component obj excludes them;+// stripping keeps the Redis-stored debug log at Python-scale size.+var vectorKeys = map[string]struct{}{+ "vector": {},+ "embedding": {},+ "q_vec": {},+ "feature": {},+}++// isVectorKey reports whether k is a raw embedding-vector key that must be+// stripped from debug payloads. It matches the fixed legacy keys+// (vector/embedding/feature) AND the dimension-scoped pattern q_<dim>_vec that+// the tokenizer actually emits (see hasEmbeddingVector, tokenizer.go:828, e.g.+// q_4_vec, q_1024_vec). The literal "q_vec" entry never matches those, so a+// bare map lookup would let real vectors leak into the Redis log.+func isVectorKey(k string) bool {+ if _, ok := vectorKeys[k]; ok {+ return true+ }+ return strings.HasPrefix(k, "q_") && strings.HasSuffix(k, "_vec")+}++// BuildDebugResultDSL builds the `dsl` object the debug-log END marker carries+// so the front-end "View result" page can render each component's output.+//+// dsl is the raw canvas DSL JSON (optionally wrapped as {"dsl": {...}}). output+// is the pipeline run output keyed by component id (output[<id>] is that+// component's outputs map, which may carry chunks/json/text/html/markdown).+func BuildDebugResultDSL(dsl string, output map[string]any) (map[string]any, error) {+ var tpl map[string]any+ if err := json.Unmarshal([]byte(dsl), &tpl); err != nil {+ return nil, fmt.Errorf("BuildDebugResultDSL: unmarshal dsl: %w", err)+ }+ root := tpl+ if nested, ok := tpl["dsl"].(map[string]any); ok {+ root = nested+ }++ components, ok := root["components"].(map[string]any)+ if !ok {+ return nil, fmt.Errorf("BuildDebugResultDSL: dsl missing components map")+ }++ built := make(map[string]any, len(components))+ for id, raw := range components {+ comp, _ := raw.(map[string]any)+ if comp == nil {+ comp = map[string]any{}+ }++ // component_name: prefer the nested obj.component_name (Python shape),+ // fall back to a top-level component_name.+ name := ""+ var staticParams map[string]any+ if obj, _ := comp["obj"].(map[string]any); obj != nil {+ name, _ = obj["component_name"].(string)+ if p, _ := obj["params"].(map[string]any); p != nil {+ staticParams = p+ }+ }+ if name == "" {+ name, _ = comp["component_name"].(string)+ }++ down := comp["downstream"] // preserve as-is (string or []any)++ // Build obj.params: start from a deep copy of the static DSL params+ // (setups/field_name/...), then inject the runtime outputs wrapper.+ mergedParams := map[string]any{}+ for k, v := range staticParams {+ mergedParams[k] = deepCopy(v)+ }+ if format, payload := detectFormat(lookupComponentOutput(output, id)); format != "" {+ mergedParams["outputs"] = map[string]any{+ format: map[string]any{+ "value": deepCopyStrip(payload),+ "type": "",+ },+ "output_format": map[string]any{"value": format},+ }+ }++ built[id] = map[string]any{+ "obj": map[string]any{+ "component_name": name,+ "params": mergedParams,+ },+ "downstream": down,+ "component_name": name,+ }+ }++ result := map[string]any{+ "components": built,+ "graph": deepCopy(root["graph"]),+ }+ return result, nil+}++// lookupComponentOutput resolves a single component's runtime output map from+// the pipeline run result.+//+// The production `pipe.Run` return value nests every component's outputs under+// output["state"][<componentID>] — finalizeResult (pipeline.go:636) attaches+// runState.Snapshot() (state.go:296, map[string]map[string]any keyed by cpn+// id), and statePost (scheduler.go:229) writes each component's top-level+// output keys there via SetVar(cpnID, k, v). So the canonical lookup is+// output["state"][id].+//+// A flat keyed-by-id shape (output[id] directly) is accepted as a fallback so+// this builder stays usable for hand-built outputs in tests and any+// non-Snapshot callers; it is never produced by the real pipeline.+func lookupComponentOutput(output map[string]any, id string) any {+ // The production run output nests each component under+ // output["state"][<id>] (finalizeResult → runState.Snapshot(), state.go:296).+ // Snapshot returns map[string]map[string]any, but some callers build a+ // map[string]any-shaped state, so accept BOTH concrete types — a+ // single-type assertion would silently fail the real shape and fall through+ // to the (usually empty) top-level lookup.+ switch state := output["state"].(type) {+ case map[string]map[string]any:+ if v, ok := state[id]; ok {+ return v+ }+ case map[string]any:+ if v, ok := state[id]; ok {+ return v+ }+ }+ // Fallback: flat shape (tests / non-Snapshot producers).+ return output[id]+}++// detectFormat returns the output key (chunks/json/text/html/markdown) present+// in a component's output map, by priority, plus the raw payload under it.+// Returns ("", nil) when the component produced no recognized output — the+// front-end then renders that step empty (matching Python's empty obj).+func detectFormat(out any) (string, any) {+ m, ok := out.(map[string]any)+ if !ok || m == nil {+ return "", nil+ }+ for _, f := range outputFormats {+ if v, exists := m[f]; exists && v != nil {+ return f, v+ }+ }+ return "", nil+}++// deepCopy returns a JSON-compatible deep copy of v (maps/slices/primitives),+// preserving structure but sharing nothing mutable with the source.+func deepCopy(v any) any {+ switch val := v.(type) {+ case map[string]any:+ cp := make(map[string]any, len(val))+ for k, vv := range val {+ cp[k] = deepCopy(vv)+ }+ return cp+ case []map[string]any:+ cp := make([]any, len(val))+ for i, vv := range val {+ cp[i] = deepCopy(vv)+ }+ return cp+ case []any:+ cp := make([]any, len(val))+ for i, vv := range val {+ cp[i] = deepCopy(vv)+ }+ return cp+ default:+ return v+ }+}++// deepCopyStrip is deepCopy plus dropping vectorKeys from every map it visits.+// Used for component payloads so raw embedding vectors never reach the log.+func deepCopyStrip(v any) any {+ switch val := v.(type) {+ case map[string]any:+ cp := make(map[string]any, len(val))+ for k, vv := range val {+ if isVectorKey(k) {+ continue+ }+ cp[k] = deepCopyStrip(vv)+ }+ return cp+ case []map[string]any:+ cp := make([]any, len(val))+ for i, vv := range val {+ cp[i] = deepCopyStrip(vv)+ }+ return cp+ case []any:+ cp := make([]any, len(val))+ for i, vv := range val {+ cp[i] = deepCopyStrip(vv)+ }+ return cp+ default:+ return v+ }+}diff --git a/internal/ingestion/task/debug_result_dsl_test.go b/internal/ingestion/task/debug_result_dsl_test.gonew file mode 100644index 0000000000..f443fba339--- /dev/null+++ b/internal/ingestion/task/debug_result_dsl_test.go@@ -0,0 +1,430 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package task++import (+ "testing"+)++// TestBuildDebugResultDSL locks the contract that the Go debug log's END-marker+// `dsl` must satisfy so the front-end "View result" page renders parsed chunks.+//+// The shape is the faithful Go analogue of Python's `Graph.__str__`+// (agent/canvas.py:119) + the END-marker `dsl` attachment (rag/flow/pipeline.py:98):+// for every component in the DSL we emit `dsl.components[<id>].obj` carrying+// `component_name` and `params.outputs[<format>].value`, plus `downstream` and+// `graph.nodes[].data.name`. The front-end reads exactly these keys+// (web/src/pages/dataflow-result/parser.tsx:41, hooks.ts:215).+func TestBuildDebugResultDSL(t *testing.T) {+ const (+ compA = "my.Parser"+ compB = "my.Tokenizer"+ )+ dsl := `{+ "dsl": {+ "components": {+ "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},+ "a": {"obj": {"component_name": "` + compA + `", "params": {"setups": {"pdf": {"parse_method": "general"}}}}, "upstream": ["begin"], "downstream": ["b"]},+ "b": {"obj": {"component_name": "` + compB + `", "params": {"field_name": "content"}}, "upstream": ["a"]}+ },+ "graph": {"nodes": [+ {"id": "begin", "data": {"name": "开始"}},+ {"id": "a", "data": {"name": "解析"}},+ {"id": "b", "data": {"name": "分词"}}+ ]}+ }+ }`++ // Run output: keyed by component id (state.go:284 `out[ref]=v`). `a` emits+ // chunks, `b` emits plain text, `begin` has no recognized output key.+ output := map[string]any{+ "a": map[string]any{+ "chunks": []any{+ map[string]any{"text": "hello", "vector": []float64{0.1, 0.2}},+ },+ },+ "b": map[string]any{"text": "plain"},+ }++ result, err := BuildDebugResultDSL(dsl, output)+ if err != nil {+ t.Fatalf("BuildDebugResultDSL: %v", err)+ }++ components, ok := result["components"].(map[string]any)+ if !ok {+ t.Fatalf("result.components missing or wrong type: %#v", result["components"])+ }+ if len(components) != 3 {+ t.Fatalf("want 3 components (begin,a,b), got %d: %#v", len(components), components)+ }++ // --- component "a": must carry chunks under params.outputs ---+ a, ok := components["a"].(map[string]any)+ if !ok {+ t.Fatalf("components.a wrong type: %#v", components["a"])+ }+ aObj, ok := a["obj"].(map[string]any)+ if !ok {+ t.Fatalf("components.a.obj wrong type: %#v", a["obj"])+ }+ if name, _ := aObj["component_name"].(string); name != compA {+ t.Errorf("components.a.obj.component_name=%v want %q", name, compA)+ }+ // downstream preserved+ if down, _ := a["downstream"].(string); down != "b" {+ // accept []any form too+ if dl, _ := a["downstream"].([]any); len(dl) != 1 || dl[0] != "b" {+ t.Errorf("components.a.downstream=%v want [b]", a["downstream"])+ }+ }+ aParams, ok := aObj["params"].(map[string]any)+ if !ok {+ t.Fatalf("components.a.obj.params wrong type: %#v", aObj["params"])+ }+ aOutputs, ok := aParams["outputs"].(map[string]any)+ if !ok {+ t.Fatalf("components.a.obj.params.outputs wrong type: %#v", aParams["outputs"])+ }+ // output_format selects "chunks"+ of, ok := aOutputs["output_format"].(map[string]any)+ if !ok || of["value"] != "chunks" {+ t.Errorf("components.a output_format=%#v want {value:\"chunks\"}", aOutputs["output_format"])+ }+ chunksOut, ok := aOutputs["chunks"].(map[string]any)+ if !ok {+ t.Fatalf("components.a outputs.chunks wrong type: %#v", aOutputs["chunks"])+ }+ chunksVal, _ := chunksOut["value"].([]any)+ if len(chunksVal) != 1 {+ t.Fatalf("components.a outputs.chunks.value len=%d want 1", len(chunksVal))+ }+ chunk0, _ := chunksVal[0].(map[string]any)+ if chunk0["text"] != "hello" {+ t.Errorf("chunk text=%v want hello", chunk0["text"])+ }+ // VECTOR STRIPPING: Python's serialized obj excludes raw vectors; verify gone.+ if _, exists := chunk0["vector"]; exists {+ t.Errorf("vector key must be stripped from chunk payload, but it is present: %#v", chunk0)+ }+ // static params (setups) carried through, matching Python's cpn["params"] copy.+ if setups, _ := aParams["setups"].(map[string]any); setups == nil {+ t.Errorf("static params.setups must be carried from DSL, got %#v", aParams)+ }++ // --- component "b": plain text format ---+ b, _ := components["b"].(map[string]any)+ bObj, _ := b["obj"].(map[string]any)+ bParams, _ := bObj["params"].(map[string]any)+ bOutputs, _ := bParams["outputs"].(map[string]any)+ if bOutputs["output_format"].(map[string]any)["value"] != "text" {+ t.Errorf("components.b output_format=%#v want {value:\"text\"}", bOutputs["output_format"])+ }+ // static params.field_name carried (used by front-end chunk text key).+ if fn, _ := bParams["field_name"].(string); fn != "content" {+ t.Errorf("components.b params.field_name=%v want content", fn)+ }++ // --- component "begin": no recognized output -> outputs empty, safe ---+ begin, _ := components["begin"].(map[string]any)+ beginObj, _ := begin["obj"].(map[string]any)+ beginParams, _ := beginObj["params"].(map[string]any)+ if _, exists := beginParams["outputs"]; exists {+ t.Errorf("begin has no run output, outputs must be absent: %#v", beginParams)+ }++ // --- graph.nodes copied through ---+ graph, ok := result["graph"].(map[string]any)+ if !ok {+ t.Fatalf("result.graph missing: %#v", result["graph"])+ }+ nodes, _ := graph["nodes"].([]any)+ if len(nodes) != 3 {+ t.Fatalf("graph.nodes len=%d want 3", len(nodes))+ }+}++// TestBuildDebugResultDSL_NestedState locks the REAL pipeline output shape.+//+// Unlike TestBuildDebugResultDSL (which uses a flat keyed-by-id map for+// readability), the production `pipe.Run` return value nests each component's+// outputs under output["state"][<componentID>] — see finalizeResult+// (pipeline.go:636, `merged["state"] = runState.Snapshot()`) and+// CanvasState.Snapshot() (state.go:296), which returns+// map[string]map[string]any keyed by cpn id. statePost (scheduler.go:229)+// writes each component's top-level output keys there via SetVar(cpnID, k, v).+//+// A prior bug read output[id] at the TOP level, which is always nil for the+// real run, so every component's params.outputs stayed empty and the front-end+// "View result" rendered blank tabs. This test pins the nested lookup so that+// regression cannot recur.+func TestBuildDebugResultDSL_NestedState(t *testing.T) {+ const (+ compA = "my.Parser"+ compB = "my.Tokenizer"+ )+ dsl := `{+ "dsl": {+ "components": {+ "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},+ "a": {"obj": {"component_name": "` + compA + `", "params": {"setups": {"pdf": {"parse_method": "general"}}}}, "upstream": ["begin"], "downstream": ["b"]},+ "b": {"obj": {"component_name": "` + compB + `", "params": {"field_name": "content"}}, "upstream": ["a"]}+ },+ "graph": {"nodes": [+ {"id": "begin", "data": {"name": "开始"}},+ {"id": "a", "data": {"name": "解析"}},+ {"id": "b", "data": {"name": "分词"}}+ ]}+ }+ }`++ // REAL shape: Snapshot() returns map[string]map[string]any (state.go:296),+ // so state is typed map[string]map[string]any at runtime — NOT+ // map[string]any. Using the real type here is what makes this test a true+ // regression guard: a single-type assertion in lookupComponentOutput would+ // pass a map[string]any-shaped test yet fail the production output.+ output := map[string]any{+ "state": map[string]map[string]any{+ "a": {+ "chunks": []any{+ map[string]any{"text": "hello", "vector": []float64{0.1, 0.2}},+ },+ },+ "b": {"text": "plain"},+ },+ }++ result, err := BuildDebugResultDSL(dsl, output)+ if err != nil {+ t.Fatalf("BuildDebugResultDSL: %v", err)+ }+ components, ok := result["components"].(map[string]any)+ if !ok {+ t.Fatalf("result.components missing: %#v", result["components"])+ }++ // "a" must carry chunks under params.outputs even though the run output is+ // nested under output["state"]["a"].+ aObj := components["a"].(map[string]any)["obj"].(map[string]any)+ aParams := aObj["params"].(map[string]any)+ aOutputs, ok := aParams["outputs"].(map[string]any)+ if !ok {+ t.Fatalf("NESTED-SHAPE REGRESSION: components.a params.outputs missing; "++ "BuildDebugResultDSL must read output[\"state\"][id], got params=%#v", aParams)+ }+ if of, _ := aOutputs["output_format"].(map[string]any); of["value"] != "chunks" {+ t.Errorf("components.a output_format=%#v want {value:\"chunks\"}", aOutputs["output_format"])+ }+ chunksVal := aOutputs["chunks"].(map[string]any)["value"].([]any)+ if len(chunksVal) != 1 {+ t.Fatalf("components.a outputs.chunks.value len=%d want 1", len(chunksVal))+ }+ chunk0 := chunksVal[0].(map[string]any)+ if chunk0["text"] != "hello" {+ t.Errorf("chunk text=%v want hello", chunk0["text"])+ }+ if _, exists := chunk0["vector"]; exists {+ t.Errorf("vector key must be stripped, but present: %#v", chunk0)+ }++ // "b": plain text nested under output["state"]["b"].+ bParams := components["b"].(map[string]any)["obj"].(map[string]any)["params"].(map[string]any)+ bOutputs, ok := bParams["outputs"].(map[string]any)+ if !ok {+ t.Fatalf("NESTED-SHAPE REGRESSION: components.b params.outputs missing; "++ "got params=%#v", bParams)+ }+ if of, _ := bOutputs["output_format"].(map[string]any); of["value"] != "text" {+ t.Errorf("components.b output_format=%#v want {value:\"text\"}", bOutputs["output_format"])+ }+}++// TestLookupComponentOutput locks the resolution rules of+// lookupComponentOutput: the production run output nests each component under+// output["state"][<id>] (finalizeResult + Snapshot), so that is the primary+// lookup; a flat keyed-by-id shape at the top level is the documented fallback+// for tests and any non-Snapshot caller. These branches are defensive and must+// keep behaving if the run-output envelope changes.+func TestLookupComponentOutput(t *testing.T) {+ const id = "a"++ // 1. nested present -> returns the nested value (primary path).+ out1 := map[string]any{"state": map[string]any{id: map[string]any{"chunks": "x"}}}+ if got := lookupComponentOutput(out1, id); got == nil {+ t.Errorf("case1: nested present, want non-nil, got nil")+ }++ // 2. nested present but NOT a map -> must fall back to top-level.+ out2 := map[string]any{"state": "not-a-map", id: map[string]any{"text": "y"}}+ if got := lookupComponentOutput(out2, id); got == nil {+ t.Errorf("case2: state wrong type, must fall back to top-level, got nil")+ } else if m, _ := got.(map[string]any); m["text"] != "y" {+ t.Errorf("case2: fallback got %#v want {text:\"y\"}", got)+ }++ // 3. nested present, id missing in state but present at top-level -> fallback.+ out3 := map[string]any{"state": map[string]any{"other": map[string]any{}}, id: map[string]any{"text": "z"}}+ if got := lookupComponentOutput(out3, id); got == nil {+ t.Errorf("case3: id missing in state, must fall back to top-level, got nil")+ } else if m, _ := got.(map[string]any); m["text"] != "z" {+ t.Errorf("case3: fallback got %#v want {text:\"z\"}", got)+ }++ // 4. nested present, id missing everywhere -> nil (safe empty).+ out4 := map[string]any{"state": map[string]any{"other": map[string]any{}}}+ if got := lookupComponentOutput(out4, id); got != nil {+ t.Errorf("case4: id missing everywhere, want nil, got %#v", got)+ }++ // 5. no "state" key at all -> flat top-level lookup.+ out5 := map[string]any{id: map[string]any{"json": "j"}}+ if got := lookupComponentOutput(out5, id); got == nil {+ t.Errorf("case5: flat-only, want non-nil, got nil")+ }+}++// TestDeepCopyStrip_StripsVectorsFromMapSlice locks the regression for review+// comment #5: real component chunk output is []map[string]any (as produced by+// ChunkDocsToMaps), which the type switch MUST recurse into AND strip vectorKeys+// from. Previously []map[string]any fell into the default branch — no recursion,+// no stripping — so embedding vectors leaked into the Redis debug log and the+// copy shared mutable state with the source.+func TestDeepCopyStrip_StripsVectorsFromMapSlice(t *testing.T) {+ src := []map[string]any{+ {+ "text": "real chunk",+ "vector": []float64{0.1, 0.2},+ "embedding": []float64{0.3},+ "q_1024_vec": []float64{0.7},+ "nested": map[string]any{"q_vec": []float64{0.9}, "q_4_vec": []float64{0.8}, "keep": "x"},+ },+ {"text": "second", "feature": []float64{0.4}},+ }++ got := deepCopyStrip(src)++ // Returned slice must be a deep copy (new []any holding new maps), not the+ // original slice/map identity.+ cp, ok := got.([]any)+ if !ok {+ t.Fatalf("deepCopyStrip returned %T, want []any", got)+ }+ if len(cp) != 2 {+ t.Fatalf("len=%d want 2", len(cp))+ }++ // First chunk: vector & embedding dropped, text kept, nested vector dropped.+ c0, ok := cp[0].(map[string]any)+ if !ok {+ t.Fatalf("element 0 type %T, want map[string]any", cp[0])+ }+ if _, exists := c0["vector"]; exists {+ t.Errorf("vector must be stripped, but present: %#v", c0)+ }+ if _, exists := c0["embedding"]; exists {+ t.Errorf("embedding must be stripped, but present: %#v", c0)+ }+ if _, exists := c0["q_1024_vec"]; exists {+ t.Errorf("q_1024_vec (dimension-scoped vector key) must be stripped, but present: %#v", c0)+ }+ if c0["text"] != "real chunk" {+ t.Errorf("text=%v want 'real chunk'", c0["text"])+ }+ nested, _ := c0["nested"].(map[string]any)+ if _, exists := nested["q_vec"]; exists {+ t.Errorf("nested q_vec must be stripped, but present: %#v", nested)+ }+ if _, exists := nested["q_4_vec"]; exists {+ t.Errorf("nested q_4_vec (dimension-scoped vector key) must be stripped, but present: %#v", nested)+ }+ if nested["keep"] != "x" {+ t.Errorf("nested.keep=%v want x", nested["keep"])+ }++ // Second chunk: feature (a vectorKey) stripped, text kept.+ c1, ok := cp[1].(map[string]any)+ if !ok {+ t.Fatalf("element 1 type %T, want map[string]any", cp[1])+ }+ if _, exists := c1["feature"]; exists {+ t.Errorf("feature must be stripped, but present: %#v", c1)+ }+ if c1["text"] != "second" {+ t.Errorf("text=%v want 'second'", c1["text"])+ }++ // Mutation isolation: mutating the copy must not touch the source.+ c0["text"] = "mutated"+ if src[0]["text"] != "real chunk" {+ t.Errorf("copy mutation leaked into source: %#v", src[0])+ }+}++// TestDeepCopy_MapSliceIsDeep locks that deepCopy also recurses []map[string]any+// (structure preservation, no vector-strip concern for plain deepCopy).+func TestDeepCopy_MapSliceIsDeep(t *testing.T) {+ src := []map[string]any{{"text": "a", "n": map[string]any{"v": 1}}}+ got := deepCopy(src)+ cp, ok := got.([]any)+ if !ok {+ t.Fatalf("deepCopy returned %T, want []any", got)+ }+ m, ok := cp[0].(map[string]any)+ if !ok {+ t.Fatalf("element type %T, want map[string]any", cp[0])+ }+ if m["text"] != "a" {+ t.Errorf("text=%v want a", m["text"])+ }+ if n, _ := m["n"].(map[string]any); n["v"] != 1 {+ t.Errorf("nested not copied: %#v", m["n"])+ }+ // isolate+ m["text"] = "mut"+ if src[0]["text"] != "a" {+ t.Errorf("deepCopy shares mutable state: %#v", src[0])+ }+}++// TestDetectFormat_Priority locks the format selection order+// (chunks > json > text > html > markdown) used to pick a component's payload+// key. A component that emits multiple recognized keys must surface the+// highest-priority one, matching NormalizeChunks and the front-end tab render.+func TestDetectFormat_Priority(t *testing.T) {+ cases := []struct {+ name string+ in map[string]any+ want string+ }{+ {"chunks beats text", map[string]any{"text": "t", "chunks": "c"}, "chunks"},+ {"json beats text", map[string]any{"text": "t", "json": "j"}, "json"},+ {"text beats html", map[string]any{"html": "h", "text": "t"}, "text"},+ {"html beats markdown", map[string]any{"markdown": "m", "html": "h"}, "html"},+ {"markdown alone", map[string]any{"markdown": "m"}, "markdown"},+ {"only unrecognized -> empty", map[string]any{"ok": true}, ""},+ {"empty map -> empty", map[string]any{}, ""},+ {"nil -> empty", nil, ""},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ got, _ := detectFormat(c.in)+ if got != c.want {+ t.Errorf("detectFormat(%#v) = %q, want %q", c.in, got, c.want)+ }+ })+ }+}diff --git a/internal/ingestion/task/debug_test.go b/internal/ingestion/task/debug_test.gonew file mode 100644index 0000000000..d39681f097--- /dev/null+++ b/internal/ingestion/task/debug_test.go@@ -0,0 +1,254 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package task++import (+ "context"+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "reflect"+ "testing"++ "gorm.io/gorm"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/component"+ "ragflow/internal/ingestion/pipeline"+)++// TestNewDebugTaskContext_InjectsDebugID asserts the debug context carries the+// side-effect-free markers: a fresh non-empty Doc.ID (a throwaway uuid, not a+// persisted row) and an empty KB (debug has no knowledgebase, so kb_id == "" is+// the debug signal used across the pipeline). The parser page cap is no longer+// stored as a flat ParserConfig key here — flat keys are dropped by the+// override_params merge and never reach the parser. It is injected at run time+// by injectDebugPageCap via Run's override_params channel (see+// TestInjectDebugPageCap).+func TestNewDebugTaskContext_InjectsDebugID(t *testing.T) {+ taskCtx := NewDebugTaskContext("t1", "canvas-1", "doc.pdf", []byte("page one\fpage two\fpage three"))++ if taskCtx.Doc.ID == "" {+ t.Errorf("Doc.ID = %q, want non-empty (uuid)", taskCtx.Doc.ID)+ }+ if taskCtx.Doc.ParserConfig != nil {+ t.Errorf("Doc.ParserConfig = %v, want nil (debug page cap is injected via override_params, not a flat ParserConfig key)", taskCtx.Doc.ParserConfig)+ }+ if taskCtx.Doc.KbID != "" {+ t.Errorf("Doc.KbID = %q, want empty (debug has no KB)", taskCtx.Doc.KbID)+ }+ if taskCtx.KB.ID != "" {+ t.Errorf("KB.ID = %q, want empty (debug has no KB)", taskCtx.KB.ID)+ }+ if taskCtx.Tenant.ID != "t1" {+ t.Errorf("Tenant.ID = %q, want t1", taskCtx.Tenant.ID)+ }+ if taskCtx.PipelineID != "canvas-1" {+ t.Errorf("PipelineID = %q, want canvas-1", taskCtx.PipelineID)+ }+}++// TestExecute_DebugViaEntry proves the entry-point constructor produces a+// valid debug TaskContext that routes through PipelineExecutor.Execute and+// returns the pipeline's chunks WITHOUT persisting (no index insert, no+// pipeline log).+func TestExecute_DebugViaEntry(t *testing.T) {+ taskCtx := NewDebugTaskContext("t1", "canvas-1", "doc.pdf", []byte("page one\fpage two\fpage three"))++ logCalled := false+ insertCalled := false++ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 0)+ if err != nil {+ t.Fatalf("NewPipelineExecutor: %v", err)+ }+ exec.+ WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {+ return "dsl", "canvas-1", nil+ }).+ WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {+ return map[string]any{+ "chunks": []map[string]any{+ {"text": "c1"},+ {"text": "c2"},+ {"text": "c3"},+ {"text": "c4"},+ },+ }, dsl, nil+ }).+ WithLogCreateFunc(func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {+ logCalled = true+ return nil+ }).+ WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {+ insertCalled = true+ return nil, nil+ })++ result, err := exec.Execute(context.Background())+ if err != nil {+ t.Fatalf("Execute: %v", err)+ }+ if result == nil {+ t.Fatal("result is nil")+ }+ if len(result.Chunks) != 4 {+ t.Errorf("len(result.Chunks) = %d, want 4", len(result.Chunks))+ }+ if logCalled {+ t.Error("pipeline log should NOT be created in a debug (kb_id == \"\") run")+ }+ if insertCalled {+ t.Error("chunk insert should NOT be called in a debug (kb_id == \"\") run")+ }+}++// TestInjectDebugPageCap verifies the canvas-debug page cap is delivered+// through Run's override_params channel (the existing ParserConfig shape),+// NOT through pipeline inputs. The cap must land at+// ParserConfig[cpnID][family]["pages"], expressed as the JSON-decoded+// []any{[]any{1, N}} form (a list of [from,to] pairs) — cpnID is the Parser+// component's instance id from the DSL and family is the document's filetype+// family. This is the exact shape NormalizeParserConfigPages produces after a+// storage JSON round-trip and the shape the deepdoc pdf parser consumes+// (NormalizePDFPages requires []any, not a Go [][]int).+//+// It also pins the regression: a flat top-level "pages" key would be dropped+// by the override_params merge and never reach the parser, so the cap must be+// nested under the cpnID.+func TestInjectDebugPageCap(t *testing.T) {+ templatePath := filepath.Join(taskRepoRoot(t), "internal", "ingestion", "pipeline", "template", "ingestion_pipeline_general.json")+ raw, err := os.ReadFile(templatePath)+ if err != nil {+ t.Fatalf("read template: %v", err)+ }+ var envelope struct {+ DSL json.RawMessage `json:"dsl"`+ }+ if err := json.Unmarshal(raw, &envelope); err != nil {+ t.Fatalf("unmarshal template envelope: %v", err)+ }+ dsl := string(envelope.DSL)++ // Discover the Parser cpnID the same way the executor does.+ schemas, err := pipeline.ExtractAllComponentParams(envelope.DSL)+ if err != nil {+ t.Fatalf("ExtractAllComponentParams: %v", err)+ }+ var parserCpnID string+ for _, s := range schemas {+ if s.ComponentName == component.ComponentNameParser {+ parserCpnID = s.CpnID+ }+ }+ if parserCpnID == "" {+ t.Fatal("template has no Parser component")+ }++ t.Run("pdf injects [1,2] under cpnID+family", func(t *testing.T) {+ parserConfig := map[string]any{}+ injectDebugPageCap(dsl, parserConfig, "pdf")+ famEntry, ok := parserConfig[parserCpnID].(map[string]any)+ if !ok {+ t.Fatalf("parserConfig[%q] = %T, want map[string]any", parserCpnID, parserConfig[parserCpnID])+ }+ pdf, ok := famEntry["pdf"].(map[string]any)+ if !ok {+ t.Fatalf("parserConfig[%q][\"pdf\"] = %T, want map[string]any", parserCpnID, famEntry["pdf"])+ }+ if !reflect.DeepEqual(pdf["pages"], []any{[]any{1, debugPageCapPages}}) {+ t.Errorf("parserConfig[%q][\"pdf\"][\"pages\"] = %v, want [[1, %d]]", parserCpnID, pdf["pages"], debugPageCapPages)+ }+ })++ t.Run("docx injects under docx family", func(t *testing.T) {+ parserConfig := map[string]any{}+ injectDebugPageCap(dsl, parserConfig, "docx")+ famEntry, ok := parserConfig[parserCpnID].(map[string]any)+ if !ok {+ t.Fatalf("parserConfig[%q] = %T, want map[string]any", parserCpnID, parserConfig[parserCpnID])+ }+ docx, ok := famEntry["docx"].(map[string]any)+ if !ok {+ t.Fatalf("parserConfig[%q][\"docx\"] = %T, want map[string]any", parserCpnID, famEntry["docx"])+ }+ if !reflect.DeepEqual(docx["pages"], []any{[]any{1, debugPageCapPages}}) {+ t.Errorf("parserConfig[%q][\"docx\"][\"pages\"] = %v, want [[1, %d]]", parserCpnID, docx["pages"], debugPageCapPages)+ }+ })++ t.Run("empty docType is a no-op", func(t *testing.T) {+ parserConfig := map[string]any{}+ injectDebugPageCap(dsl, parserConfig, "")+ if len(parserConfig) != 0 {+ t.Errorf("parserConfig = %v, want empty (no family derivable from empty docType)", parserConfig)+ }+ })++ t.Run("does not clobber existing parser params", func(t *testing.T) {+ parserConfig := map[string]any{+ parserCpnID: map[string]any{+ "pdf": map[string]any{"parse_method": "deepdoc"},+ },+ }+ injectDebugPageCap(dsl, parserConfig, "pdf")+ famEntry := parserConfig[parserCpnID].(map[string]any)+ pdf := famEntry["pdf"].(map[string]any)+ if pdf["parse_method"] != "deepdoc" {+ t.Errorf("parserConfig[%q][\"pdf\"][\"parse_method\"] = %v, want deepdoc (existing params must be preserved)", parserCpnID, pdf["parse_method"])+ }+ if !reflect.DeepEqual(pdf["pages"], []any{[]any{1, debugPageCapPages}}) {+ t.Errorf("parserConfig[%q][\"pdf\"][\"pages\"] = %v, want [[1, %d]]", parserCpnID, pdf["pages"], debugPageCapPages)+ }+ })++ t.Run("envelope dsl form (production shape)", func(t *testing.T) {+ // In production dsl is the canvas envelope {"dsl": {"components": ...}},+ // not the bare components map. injectDebugPageCap must still find the+ // Parser cpnID after unwrapping.+ wrapped := fmt.Sprintf(`{"dsl":%s}`, string(envelope.DSL))+ parserConfig := map[string]any{}+ injectDebugPageCap(wrapped, parserConfig, "pdf")+ famEntry, ok := parserConfig[parserCpnID].(map[string]any)+ if !ok {+ t.Fatalf("parserConfig[%q] = %T, want map[string]any (envelope dsl must unwrap)", parserCpnID, parserConfig[parserCpnID])+ }+ pdf, ok := famEntry["pdf"].(map[string]any)+ if !ok {+ t.Fatalf("parserConfig[%q][\"pdf\"] = %T, want map[string]any", parserCpnID, famEntry["pdf"])+ }+ if !reflect.DeepEqual(pdf["pages"], []any{[]any{1, debugPageCapPages}}) {+ t.Errorf("parserConfig[%q][\"pdf\"][\"pages\"] = %v, want [[1, %d]] (envelope dsl not unwrapped?)", parserCpnID, pdf["pages"], debugPageCapPages)+ }+ })++ t.Run("respects explicit caller-supplied cap", func(t *testing.T) {+ // When the document already carries an explicit cpnID+family page cap,+ // the debug default must NOT override it (so a wider/narrower cap wins).+ parserConfig := map[string]any{+ parserCpnID: map[string]any{+ "pdf": map[string]any{"pages": []any{[]any{1, 1000000}}},+ },+ }+ injectDebugPageCap(dsl, parserConfig, "pdf")+ famEntry := parserConfig[parserCpnID].(map[string]any)+ pdf := famEntry["pdf"].(map[string]any)+ if !reflect.DeepEqual(pdf["pages"], []any{[]any{1, 1000000}}) {+ t.Errorf("parserConfig[%q][\"pdf\"][\"pages\"] = %v, want [[1, 1000000]] (explicit cap must be respected, not overridden by debug default)", parserCpnID, pdf["pages"])+ }+ })+}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex dfd187a550..f8a3fbd5b5 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -28,6 +28,7 @@ import ("ragflow/internal/dao""ragflow/internal/engine""ragflow/internal/entity"+ "ragflow/internal/ingestion/component""ragflow/internal/ingestion/knowledge_compile"pipelinepkg "ragflow/internal/ingestion/pipeline"@@ -41,9 +42,14 @@ type PipelineResult struct {DocID stringKbID stringMetadata map[string]any+ Chunks []map[string]any // populated only in debug (dry-run) modeChunkCount intTokenConsumption intDuration float64 // pipeline wall-clock seconds+ // MessageID is the polling key for the debug-run log. The front-end reads+ // it from the run response and polls GET /agents/:id/logs/:message_id to+ // render progress; it is empty for non-debug (persist) runs.+ MessageID string}type PipelineExecutor struct {@@ -66,15 +72,14 @@ func validateTaskContext(taskCtx *TaskContext) error {if taskCtx.Doc.ID == "" {return fmt.Errorf("pipeline executor: empty document id")}- if taskCtx.Doc.KbID == "" {+ // A debug (dry-run) context carries no knowledgebase (see+ // TaskContext.IsDebug), so it must not be required to supply one.+ if !taskCtx.IsDebug() && taskCtx.Doc.KbID == "" {return fmt.Errorf("pipeline executor: empty document knowledgebase id")}if taskCtx.Doc.Name == nil || *taskCtx.Doc.Name == "" {return fmt.Errorf("pipeline executor: empty document name")}- if taskCtx.KB.ID == "" {- return fmt.Errorf("pipeline executor: empty knowledgebase id")- }if taskCtx.Tenant.ID == "" {return fmt.Errorf("pipeline executor: empty tenant id")}@@ -170,9 +175,10 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)return nil, err}- if s.taskCtx.Doc.ID == CANVAS_DEBUG_DOC_ID {- s.recordPipelineLog(ctx, dao.DB, s.taskCtx.Doc.ID, pipelineDSL, "done")- return nil, nil+ // A debug (dry-run) run produces no persistent side effect (no MinIO+ // image upload, no index insert, no pipeline log); see TaskContext.IsDebug.+ if s.taskCtx.IsDebug() {+ return s.collectDebugOutput(ctx, pipelineOutput, start)}result, err := s.processOutput(ctx, pipelineOutput, start)@@ -187,6 +193,22 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)return result, nil}+// collectDebugOutput builds a PipelineResult for a debug (dry-run) run.+// It surfaces the pipeline's chunks so a debug endpoint can render them, but+// performs no DB/index writes — the embedding vectors already computed by the+// pipeline run are left on the chunks. This keeps debug runs side-effect free.+func (s *PipelineExecutor) collectDebugOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {+ chunks := NormalizeChunks(pipelineOutput)+ return &PipelineResult{+ DocID: s.taskCtx.Doc.ID,+ KbID: s.taskCtx.Doc.KbID,+ Chunks: chunks,+ ChunkCount: countDistinctChunkIDs(chunks),+ TokenConsumption: GetEmbeddingTokenConsumption(pipelineOutput),+ Duration: time.Since(start).Seconds(),+ }, nil+}+func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {if pipelineOutput == nil {return nil, nil@@ -375,6 +397,12 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (}parserConfig := map[string]interface{}(s.taskCtx.Doc.ParserConfig)+ if parserConfig == nil {+ // Debug (dataflow dry-run) contexts intentionally carry no+ // ParserConfig; start from an empty map so the debug page cap can be+ // injected in place below without a nil-map assignment panic.+ parserConfig = map[string]interface{}{}+ }common.InjectExtractorLLMID(parserConfig, s.taskCtx.Tenant.LLMID)// When the dataset enables auto-metadata, ensure the Extractor node(s)// carry the enable_metadata mode + field schema so the LLM extraction fires@@ -404,15 +432,51 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (if s.taskCtx.Doc.ID != "" {inputs["doc_id"] = s.taskCtx.Doc.ID}- if s.taskCtx.File != nil {- inputs["file"] = s.taskCtx.File- }+ // Run-level metadata shared by both persist and debug (dataflow+ // dry-run) runs. In debug the KB is absent (NewDebugTaskContext forces+ // KB.ID == ""); in persist it is the document's own KB. Either way the+ // Tokenizer reads kb_id from CanvasState.Globals.inputs["tenant_id"] = s.taskCtx.Tenant.IDinputs["kb_id"] = s.taskCtx.KB.IDif s.taskCtx.KB.Language != nil {inputs["lang"] = *s.taskCtx.KB.Language}+ // File delivery and doc metadata differ between the two run modes.+ debug := s.taskCtx.IsDebug()+ if debug {+ // A debug (dry-run) run has no DB document row, so the parser+ // cannot resolve its bytes via doc_id → storage. Deliver the+ // uploaded bytes directly as `binary` (what the parser actually+ // reads) and surface the doc name/type so family detection works.+ if s.taskCtx.File != nil {+ inputs["file"] = s.taskCtx.File+ inputs["binary"] = s.taskCtx.File+ }+ if s.taskCtx.Doc.Name != nil && *s.taskCtx.Doc.Name != "" {+ inputs["name"] = *s.taskCtx.Doc.Name+ }+ if s.taskCtx.Doc.Type != "" {+ inputs["file_type"] = s.taskCtx.Doc.Type+ }+ } else {+ if s.taskCtx.File != nil {+ inputs["file"] = s.taskCtx.File+ }+ }++ // A canvas-debug (dataflow dry-run) must return a fast preview, so it+ // caps the parser to the first few pages. The cap is delivered through+ // override_params (Run's 3rd argument) — the SAME channel the+ // production ParserConfig uses — keyed by the Parser component's cpnID+ // and the document's filetype family. It is NOT passed through pipeline+ // inputs: the parser selects pages from ParserConfig[cpnID][family]+ // ["pages"] (a list of 1-indexed inclusive ranges), exactly mirroring+ // NormalizeParserConfigPages / pdf_pages_test.go. See injectDebugPageCap.+ if debug {+ injectDebugPageCap(dsl, parserConfig, s.taskCtx.Doc.Type)+ }+// Component params from Doc.ParserConfig — including the tenant LLM id// injected into Extractor components above — are passed to Run as// override_params, keyed by cpnID with override-wins. The DSL itself is@@ -421,9 +485,105 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (if err != nil {return nil, dsl, err}++ // Surface the debug-run result DSL to any sink that implements ResultSink+ // (the DebugLogSink used by canvas-debug runs). This mirrors Python's+ // END-marker `dsl` attachment (rag/flow/pipeline.py:98) so the front-end+ // "View result" page can render parsed chunks. The probe is an optional+ // capability: non-debug (DB-backed) sinks ignore it and the ProgressSink+ // contract is unchanged, keeping the coupling one-directional.+ if rs, ok := s.progressSink.(ResultSink); ok {+ if resultDSL, e := BuildDebugResultDSL(dsl, output); e == nil {+ rs.SetResult(resultDSL)+ }+ }+payload, err := pipelinepkg.ExtractPayload(dsl, output)if err != nil {return nil, dsl, err}return payload, dsl, nil}++// debugPageCapPages is the number of leading pages a canvas-debug+// (dataflow dry-run) parses. The debug preview must return fast, so we cap+// the parser to the first few pages. The cap is expressed as the 1-indexed+// inclusive range [1, debugPageCapPages], matching the production+// ParserConfig[cpnID][filetype]["pages"] shape (see NormalizeParserConfigPages).+const debugPageCapPages = 2++// injectDebugPageCap wires the canvas-debug page cap into parserConfig using+// the SAME channel the production ParserConfig travels through: Run's+// override_params (the 3rd argument), keyed by the Parser component's cpnID+// and the document's filetype family. It must NOT be passed as a pipeline+// input — the parser selects pages from ParserConfig[cpnID][family]["pages"],+// which the deepdoc/pdf parser consumes as a list of page ranges.+//+// dsl is the raw pipeline DSL (used only to discover the Parser component's+// cpnID); docType is the uploaded file's extension (e.g. "pdf"). When no+// Parser component can be found, or the docType yields no known family, the+// call is a no-op (the run parses everything, which is safe).+//+// dsl arrives as the canvas ENVELOPE ({ "dsl": { "components": ... } }) in+// production (loadDSLFromCanvas marshals canvas.DSL), so it is unwrapped the+// same way NewPipelineFromDSL does before ExtractAllComponentParams runs.+// An explicit page cap already present in parserConfig (keyed by cpnID ++// family) is respected and left untouched — the debug default is only a+// fallback, so a debug run can honour a narrower or wider caller-supplied cap.+func injectDebugPageCap(dsl string, parserConfig map[string]any, docType string) {+ // Unwrap the canvas envelope to the inner components map.+ var raw map[string]any+ if err := json.Unmarshal([]byte(dsl), &raw); err != nil {+ return+ }+ if env, ok := raw["dsl"].(map[string]any); ok && len(env) > 0 {+ raw = env+ }+ inner, err := json.Marshal(raw)+ if err != nil {+ return+ }+ schemas, err := pipelinepkg.ExtractAllComponentParams(inner)+ if err != nil {+ return+ }+ var parserCpnID string+ for _, s := range schemas {+ if s.ComponentName == component.ComponentNameParser {+ parserCpnID = s.CpnID+ break+ }+ }+ if parserCpnID == "" {+ return+ }+ family := component.ParserFileFamily(docType)+ if family == "" {+ return+ }+ // Respect an explicit page cap already present under cpnID + family.+ if cpnEntry, ok := parserConfig[parserCpnID].(map[string]any); ok {+ if famEntry, ok := cpnEntry[family].(map[string]any); ok {+ if _, has := famEntry["pages"]; has {+ return+ }+ }+ }+ cpnEntry, ok := parserConfig[parserCpnID].(map[string]any)+ if !ok {+ cpnEntry = map[string]any{}+ parserConfig[parserCpnID] = cpnEntry+ }+ famEntry, ok := cpnEntry[family].(map[string]any)+ if !ok {+ famEntry = map[string]any{}+ cpnEntry[family] = famEntry+ }+ // pages is delivered as a JSON-decoded map — the very shape a ParserConfig+ // arrives in from the API/storage JSON round-trip: a []any of [from,to]+ // pairs. The deepdoc/pdf parser's NormalizePDFPages requires this+ // []any-of-[]any form (not a Go [][]int), so it is built explicitly here.+ // The shallow override_params merge in applyOverrideParams preserves this+ // shape all the way to ConfigureFromSetup, so the cap is honoured.+ famEntry["pages"] = []any{[]any{1, debugPageCapPages}}+}diff --git a/internal/ingestion/task/pipeline_executor_persist_test.go b/internal/ingestion/task/pipeline_executor_persist_test.gonew file mode 100644index 0000000000..aeca8a05e4--- /dev/null+++ b/internal/ingestion/task/pipeline_executor_persist_test.go@@ -0,0 +1,151 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package task++import (+ "context"+ "testing"++ "gorm.io/gorm"+ "ragflow/internal/entity"+)++func strptr(s string) *string { return &s }++// TestExecute_DebugSkipsPipelineLog asserts that a debug run+// (KB.ID == "") returns before recording the pipeline operation+// log and before inserting chunks into the index. A debug run+// must produce no persistent side effects: no pipeline_log and no+// index insert.+func TestExecute_DebugSkipsPipelineLog(t *testing.T) {+ taskCtx := &TaskContext{+ Ctx: context.Background(),+ Doc: entity.Document{+ ID: "debug-run-1",+ KbID: "",+ Name: strptr("doc.pdf"),+ ParserID: "parser-1",+ Suffix: "pdf",+ Type: "pdf",+ SourceType: "local",+ },+ KB: entity.Knowledgebase{ID: ""},+ Tenant: entity.Tenant{ID: "tenant-1"},+ }++ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 10)+ if err != nil {+ t.Fatalf("NewPipelineExecutor: %v", err)+ }+ exec.WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {+ return `"dsl"`, "", nil+ })+ exec.WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {+ return map[string]any{"chunks": []map[string]any{}}, "dsl", nil+ })++ var logCalled bool+ exec.WithLogCreateFunc(func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {+ logCalled = true+ return nil+ })+ var insertCalled bool+ exec.WithInsertFunc(func(ctx context.Context, chunks []map[string]any, _, _ string) ([]string, error) {+ insertCalled = true+ return nil, nil+ })++ if _, err := exec.Execute(context.Background()); err != nil {+ t.Fatalf("Execute: %v", err)+ }+ if logCalled {+ t.Errorf("recordPipelineLog must not be called in a debug (kb_id == \"\") run")+ }+ if insertCalled {+ t.Errorf("index insert must not be called in a debug (kb_id == \"\") run")+ }+}++// TestExecute_DebugReturnsChunks asserts that a debug (kb_id == "") run+// returns the pipeline's chunks in the result instead of discarding them. The+// chunks are surfaced for a debug HTTP endpoint to render; no DB/index writes+// happen and the pipeline operation log is never recorded.+func TestExecute_DebugReturnsChunks(t *testing.T) {+ taskCtx := &TaskContext{+ Ctx: context.Background(),+ Doc: entity.Document{+ ID: "debug-run-1",+ KbID: "",+ Name: strptr("doc.pdf"),+ ParserID: "parser-1",+ Suffix: "pdf",+ Type: "pdf",+ SourceType: "local",+ },+ KB: entity.Knowledgebase{ID: ""},+ Tenant: entity.Tenant{ID: "tenant-1"},+ }++ exec, err := NewPipelineExecutor(taskCtx, "canvas-1", 10)+ if err != nil {+ t.Fatalf("NewPipelineExecutor: %v", err)+ }+ exec.WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {+ return "dsl", "canvas-1", nil+ })+ exec.WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {+ return map[string]any{+ "chunks": []map[string]any{+ {"id": "c1", "content": "hello", "text": "hello"},+ {"id": "c2", "content": "world", "text": "world"},+ {"id": "c3", "content": "debug", "text": "debug"},+ },+ "embedding_token_consumption": 12,+ }, "dsl", nil+ })++ var logCalled bool+ exec.WithLogCreateFunc(func(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {+ logCalled = true+ return nil+ })+ var insertCalled bool+ exec.WithInsertFunc(func(ctx context.Context, chunks []map[string]any, _, _ string) ([]string, error) {+ insertCalled = true+ return nil, nil+ })++ result, err := exec.Execute(context.Background())+ if err != nil {+ t.Fatalf("Execute: %v", err)+ }+ if result == nil {+ t.Fatalf("Execute returned nil result in debug (kb_id == \"\") mode; expected chunks")+ }+ if len(result.Chunks) != 3 {+ t.Errorf("expected 3 chunks in result, got %d", len(result.Chunks))+ }+ if result.ChunkCount != 3 {+ t.Errorf("expected ChunkCount 3, got %d", result.ChunkCount)+ }+ if logCalled {+ t.Errorf("recordPipelineLog must not be called in a debug (kb_id == \"\") run")+ }+ if insertCalled {+ t.Errorf("index insert must not be called in a debug (kb_id == \"\") run")+ }+}diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex 296063fc01..1fce515841 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -139,7 +139,6 @@ func TestNewPipelineExecutor_RejectsIncompleteTaskContext(t *testing.T) {{name: "missing doc id", mutate: func(ctx *TaskContext) { ctx.Doc.ID = "" }},{name: "missing kb id", mutate: func(ctx *TaskContext) { ctx.Doc.KbID = "" }},{name: "missing doc name", mutate: func(ctx *TaskContext) { ctx.Doc.Name = nil }},- {name: "missing knowledgebase id", mutate: func(ctx *TaskContext) { ctx.KB.ID = "" }},{name: "missing tenant id", mutate: func(ctx *TaskContext) { ctx.Tenant.ID = "" }},}@@ -155,6 +154,19 @@ func TestNewPipelineExecutor_RejectsIncompleteTaskContext(t *testing.T) {}}+// TestNewPipelineExecutor_AcceptsDebugTaskContext verifies the canvas-debug+// (dry-run) contract: a TaskContext with an empty KB.ID is valid because debug+// mode carries no knowledgebase. KB.ID == "" never occurs in production+// ingestion, which always supplies a KB.+func TestNewPipelineExecutor_AcceptsDebugTaskContext(t *testing.T) {+ ctx := makeTaskCtx()+ ctx.KB = entity.Knowledgebase{ID: ""}+ ctx.Doc.KbID = ""+ if _, err := NewPipelineExecutor(ctx, "flow-1", 0); err != nil {+ t.Fatalf("debug TaskContext rejected: %v", err)+ }+}+func TestNewPipelineExecutor_DocBulkSize(t *testing.T) {svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 128)if svc.docBulkSize != 128 {diff --git a/web/src/pages/agent/hooks/use-run-dataflow.ts b/web/src/pages/agent/hooks/use-run-dataflow.tsindex 6dac58acb9..da74fc8865 100644--- a/web/src/pages/agent/hooks/use-run-dataflow.ts+++ b/web/src/pages/agent/hooks/use-run-dataflow.ts@@ -44,6 +44,13 @@ export function useRunDataflow({return msgId;} else {+ // Even on a run failure the debug log (with the [ERROR] END marker) is+ // already written; surface its polling key so the log sheet can render+ // the failure timeline instead of leaving it stuck on an empty state.+ const msgId = get(res, 'data.data.message_id');+ if (msgId) {+ setMessageId(msgId);+ }message.error(get(res, 'data.message', ''));}},