Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(ingestion): stop persisting business data in pipeline operation-log DSL (#19631)

xugangqiange83c1963dec7
diff --git a/api/apps/restful_apis/agent_api.py b/api/apps/restful_apis/agent_api.py
index faf506872a..b1798010b4 100644
--- a/api/apps/restful_apis/agent_api.py
+++ b/api/apps/restful_apis/agent_api.py
@@ -39,7 +39,6 @@ from quart import Response, jsonify, request, make_response
from api.apps import AUTH_JWT, AUTH_API, AUTH_BETA, current_user, login_required
from api.apps.services.canvas_replica_service import CanvasReplicaService
from api.db import CanvasCategory
-from api.db.db_models import Task
from api.db.services.api_service import API4ConversationService
from api.db.services.canvas_service import (
CanvasTemplateService,
@@ -47,11 +46,9 @@ from api.db.services.canvas_service import (
completion as agent_completion,
completion_openai,
)
-from api.db.services.document_service import DocumentService
from api.db.services.file_service import FileService
from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.pipeline_operation_log_service import PipelineOperationLogService
-from api.db.services.task_service import CANVAS_DEBUG_DOC_ID, TaskService, queue_dataflow
+from api.db.services.task_service import CANVAS_DEBUG_DOC_ID, queue_dataflow
from api.db.services.user_service import TenantService, UserService
from api.db.services.user_canvas_version import UserCanvasVersionService
from api.utils.api_utils import (
@@ -1301,55 +1298,6 @@ async def reset_agent(agent_id, tenant_id):
return server_error_response(exc)
[email protected]("/agents/rerun", methods=["POST"]) # noqa: F821
-@validate_request("id", "dsl", "component_id")
-@login_required
-@add_tenant_id_to_kwargs
-async def rerun_agent(tenant_id):
- from rag.nlp import search
-
- req = await get_request_json()
- doc = PipelineOperationLogService.get_documents_info(req["id"])
- if not doc:
- return get_data_error_result(message="Document not found.")
- doc = doc[0]
- if not DocumentService.accessible(doc["id"], tenant_id):
- logging.warning(
- "rerun_agent denied: tenant_id=%s log_id=%s doc_id=%s",
- tenant_id,
- req["id"],
- doc["id"],
- )
- return get_data_error_result(message="Document not found.")
- if 0 < doc["progress"] < 1:
- return get_data_error_result(message=f"`{doc['name']}` is processing...")
-
- from rag.advanced_rag.knowlege_compile.dataset_nav import remove_dataset_nav_doc_sync
-
- remove_dataset_nav_doc_sync(tenant_id, doc["kb_id"], doc["id"])
- if settings.docStoreConn.index_exist(search.index_name(tenant_id), doc["kb_id"]):
- settings.docStoreConn.delete({"doc_id": doc["id"]}, search.index_name(tenant_id), doc["kb_id"])
- doc["progress_msg"] = ""
- doc["chunk_num"] = 0
- doc["token_num"] = 0
- DocumentService.clear_chunk_num_when_rerun(doc["id"])
- DocumentService.update_by_id(doc["id"], doc)
- TaskService.filter_delete([Task.doc_id == doc["id"]])
-
- dsl = req["dsl"]
- dsl["path"] = [req["component_id"]]
- PipelineOperationLogService.update_by_id(req["id"], {"dsl": dsl})
- queue_dataflow(
- tenant_id=tenant_id,
- flow_id=req["id"],
- task_id=get_uuid(),
- doc_id=doc["id"],
- priority=0,
- rerun=True,
- )
- return get_json_result(data=True)
-
-
@manager.route("/agents/test_db_connection", methods=["POST"]) # noqa: F821
@validate_request("db_type", "database", "username", "host", "port", "password")
@login_required
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index bbfa2479a3..e3fdd6a81d 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -1251,12 +1251,7 @@ func startServer(ctx context.Context) {
agentOpts.stateSerializer,
agentOpts.runTracker,
)
- // WithDocumentService wires the rerun dependency used by
- // POST /api/v1/agents/rerun (dataflow "re-run" in the pipeline
- // result viewer). RerunAgent fails closed without it, so this must
- // stay attached to NewAgentHandler.
- agentHandler := handler.NewAgentHandler(ctx, agentService, fileService).
- WithDocumentService(documentService)
+ agentHandler := handler.NewAgentHandler(ctx, agentService, fileService)
// Public chatbot/agentbot endpoints (api/v1/chatbots/...,
// api/v1/agentbots/...) and the agent attachment download.
diff --git a/internal/dao/pipeline_operation_log.go b/internal/dao/pipeline_operation_log.go
index ec13661f1f..a33b42329f 100644
--- a/internal/dao/pipeline_operation_log.go
+++ b/internal/dao/pipeline_operation_log.go
@@ -389,16 +389,6 @@ func (dao *PipelineOperationLogDAO) GetByID(ctx context.Context, db *gorm.DB, lo
return &log, nil
}
-// UpdateDSL replaces the DSL stored on a pipeline operation log. Used by the
-// dataflow rerun endpoint to persist the front-end's edited component
-// configuration plus the rerun entry point (dsl.path = [component_id]),
-// mirroring Python's PipelineOperationLogService.update_by_id(id, {"dsl": dsl}).
-func (dao *PipelineOperationLogDAO) UpdateDSL(ctx context.Context, db *gorm.DB, logID string, dsl entity.JSONMap) error {
- return db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
- Where("id = ?", logID).
- Update("dsl", dsl).Error
-}
-
// Create inserts a new pipeline operation log.
func (dao *PipelineOperationLogDAO) Create(ctx context.Context, db *gorm.DB, log *entity.PipelineOperationLog) error {
return db.WithContext(ctx).Create(log).Error
diff --git a/internal/handler/agent.go b/internal/handler/agent.go
index 575369cf3e..4e8398d9a2 100644
--- a/internal/handler/agent.go
+++ b/internal/handler/agent.go
@@ -39,7 +39,6 @@ import (
pipelinepkg "ragflow/internal/ingestion/pipeline"
"ragflow/internal/ingestion/task"
"ragflow/internal/service"
- "ragflow/internal/service/document"
"ragflow/internal/service/file"
"ragflow/internal/utility"
@@ -72,33 +71,12 @@ type chatAgentService interface {
RunAgent(ctx context.Context, userID, canvasID, sessionID, version string, userInput any, files []map[string]interface{}) (<-chan canvas.RunEvent, error)
}
-// documentRerunService is the surface RerunAgent needs from the document
-// service: re-run the ingestion pipeline a pipeline operation log points
-// at, with document accessibility enforced inside the service. Defined as
-// an interface (instead of taking the concrete *document.DocumentService)
-// so handler tests can inject a stub without spinning up the full service
-// (DB DAOs, storage clients, …). The production *document.DocumentService
-// satisfies this interface because its RerunDocument signature matches.
-type documentRerunService interface {
- RerunDocument(ctx context.Context, userID, logID string, dsl map[string]interface{}, componentID string) error
-}
-
-// Compile-time proof that the production document service satisfies the
-// rerun surface, keeping the interface and the service in lockstep.
-var _ documentRerunService = (*document.DocumentService)(nil)
-
// AgentHandler agent handler
type AgentHandler struct {
agentService *service.AgentService
chatRunner chatAgentService
fileService agentFileService
loader canvasLoader
- // documentService is required by RerunAgent. Wired in
- // cmd/ragflow_server.go after NewAgentHandler (which doesn't take it
- // to preserve the existing test-friendly signature). RerunAgent fails
- // closed (500) when it is nil: skipping the service would bypass the
- // document ownership gate.
- documentService documentRerunService
// redisGet fetches a raw string from Redis. Defaults to the global
// client (redis.Get) so production behaviour is unchanged; tests inject
// a miniredis-backed getter to exercise Agent log endpoints without a
@@ -126,15 +104,6 @@ type debugExecutor interface {
Execute(ctx context.Context) (*task.PipelineResult, error)
}
-// WithDocumentService injects the document service used by
-// RerunAgent to resolve the pipeline operation log, enforce
-// DocumentService accessibility, and enqueue the rerun. Returns the
-// receiver for chaining in the server wiring.
-func (h *AgentHandler) WithDocumentService(s documentRerunService) *AgentHandler {
- h.documentService = s
- return h
-}
-
// NewAgentHandler create agent handler
func NewAgentHandler(ctx context.Context, agentService *service.AgentService, fileService *file.FileService) *AgentHandler {
@@ -1508,84 +1477,6 @@ func extractUserInputWithQuery(inputs map[string]interface{}, query string) map[
return values
}
-// RerunAgent POST /api/v1/agents/rerun — requires id, dsl, and
-// component_id. The front-end dataflow "view result" page sends the
-// pipeline operation LOG id plus the edited DSL (web
-// useRerunDataflow); the service resolves log -> document, enforces
-// ownership, persists the edited DSL, and re-enqueues the ingestion
-// run (see DocumentService.RerunDocument).
-//
-// DocumentService.RerunDocument enforces accessibility BEFORE the
-// rerun. The gate is REQUIRED: a nil documentService turns a wiring
-// miss into an auth bypass (any caller could rerun an arbitrary doc
-// id without an ownership check), so we fail closed with 500 instead
-// of accepting the request. On denial the service returns
-// "Document not found." so a caller cannot probe whether a document
-// exists in another tenant.
-func (h *AgentHandler) RerunAgent(c *gin.Context) {
- user, code, msg := GetUser(c)
- if code != common.CodeSuccess {
- common.ResponseWithCodeData(c, code, nil, msg)
- return
- }
- var body struct {
- ID string `json:"id"`
- DSL map[string]interface{} `json:"dsl"`
- ComponentID string `json:"component_id"`
- }
- if err := c.ShouldBindJSON(&body); err != nil && !errors.Is(err, io.EOF) {
- common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Invalid request: "+err.Error())
- return
- }
- missing := make([]string, 0, 3)
- if body.ID == "" {
- missing = append(missing, "id")
- }
- if body.DSL == nil {
- missing = append(missing, "dsl")
- }
- if body.ComponentID == "" {
- missing = append(missing, "component_id")
- }
- if len(missing) > 0 {
- common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "required argument are missing: "+strings.Join(missing, ",")+"; ")
- return
- }
- // Fail closed on missing dependency: a nil documentService
- // means the handler was wired without the rerun service, which
- // would let any caller rerun an arbitrary doc id without proving
- // ownership. Surface as a 500 so a missing dependency is loud,
- // not silent.
- if h.documentService == nil {
- zap.L().Error("RerunAgent: documentService is nil; refusing request to prevent auth bypass")
- common.ResponseWithCodeData(c, common.CodeServerError, nil, "server misconfiguration: document service not wired")
- return
- }
- if err := h.documentService.RerunDocument(c.Request.Context(), user.ID, body.ID, body.DSL, body.ComponentID); err != nil {
- // Domain failures (unknown log/document, access denial,
- // document mid-run) map to the data-error envelope (code 102)
- // with the service's caller-safe message. Everything else is
- // an internal failure: log it server-side and answer 500 with
- // a generic message so internals (DB errors, queue failures)
- // are neither leaked to the tenant nor mislabeled as a data
- // error.
- var processingErr *document.RerunDocumentProcessingError
- switch {
- case errors.Is(err, document.ErrRerunDocumentNotFound):
- common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
- case errors.As(err, &processingErr):
- common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
- default:
- zap.L().Error("RerunAgent: rerun failed",
- zap.String("log_id", body.ID),
- zap.Error(err))
- common.ResponseWithCodeData(c, common.CodeServerError, nil, "rerun failed, please try again later")
- }
- return
- }
- common.SuccessWithData(c, true, "success")
-}
-
// TestDBConnection POST /api/v1/agents/test_db_connection
func (h *AgentHandler) TestDBConnection(c *gin.Context) {
user, code, msg := GetUser(c)
@@ -1748,8 +1639,7 @@ func (h *AgentHandler) checkCanvasAccessForHandler(c *gin.Context, userID, canva
// agent through eino's compose.Workflow.Invoke, which is reconstructed
// from the DSL on each run, so the replica's read-side acceleration
// is unnecessary and its write-side adds an out-of-band DB/cache sync
-// for no benefit. UpdateAgent / CreateAgent / RerunAgent follow the
-// same convention — DSL write only, no Redis replica. See the
+// for no benefit. UpdateAgent / CreateAgent follow the same convention — DSL write only, no Redis replica. See the
// "canvas-replica-not-porting" project memory for the design rationale.
//
// The reset DSL is returned in the response body so the front-end
diff --git a/internal/handler/agent_test.go b/internal/handler/agent_test.go
index d6bbc00677..b3d6732022 100644
--- a/internal/handler/agent_test.go
+++ b/internal/handler/agent_test.go
@@ -36,7 +36,6 @@ import (
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/service"
- "ragflow/internal/service/document"
"ragflow/internal/tokenizer"
)
@@ -1601,79 +1600,6 @@ func decodeOpenAICompatStream(t *testing.T, body string) ([]map[string]interface
return chunks, done
}
-// TestRerunAgent_RequiresAllFields covers the 101 branch: missing
-// any of id / dsl / component_id -> "required argument are missing: ..."
-func TestRerunAgent_RequiresAllFields(t *testing.T) {
- gin.SetMode(gin.TestMode)
- cases := []struct {
- name string
- body string
- missing string
- }{
- {"empty", `{}`, "id,dsl,component_id"},
- {"only_id", `{"id":"x"}`, "dsl,component_id"},
- {"id_dsl", `{"id":"x","dsl":{}}`, "component_id"},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- w := httptest.NewRecorder()
- c, _ := gin.CreateTestContext(w)
- c.Request = httptest.NewRequest("POST", "/api/v1/agents/rerun",
- strings.NewReader(tc.body))
- c.Request.Header.Set("Content-Type", "application/json")
- c.Set("user", &entity.User{ID: "u1"})
- c.Set("user_id", "u1")
-
- ctx := t.Context()
- h := NewAgentHandler(ctx, service.NewAgentService(), nil)
- h.RerunAgent(c)
-
- var resp map[string]interface{}
- _ = json.Unmarshal(w.Body.Bytes(), &resp)
- if code, _ := resp["code"].(float64); code != float64(common.CodeArgumentError) {
- t.Errorf("code = %v, want 101", code)
- }
- if msg, _ := resp["message"].(string); !strings.Contains(msg, "required argument are missing") {
- t.Errorf("message = %q, want to contain 'required argument are missing'", msg)
- }
- })
- }
-}
-
-// TestRerunAgent_AcceptsCompleteRequest covers the happy path: all
-// three required fields present + documentService wired with a
-// rerun that succeeds -> 200 / code 0, and the stub receives the
-// log id / dsl / component_id from the request verbatim.
-func TestRerunAgent_AcceptsCompleteRequest(t *testing.T) {
- gin.SetMode(gin.TestMode)
- w := httptest.NewRecorder()
- c, _ := gin.CreateTestContext(w)
- c.Request = httptest.NewRequest("POST", "/api/v1/agents/rerun",
- strings.NewReader(`{"id":"log-1","dsl":{"path":[]},"component_id":"c1"}`))
- c.Request.Header.Set("Content-Type", "application/json")
- c.Set("user", &entity.User{ID: "u1"})
- c.Set("user_id", "u1")
-
- stub := &stubDocService{}
- ctx := t.Context()
- h := NewAgentHandler(ctx, service.NewAgentService(), nil).
- WithDocumentService(stub)
- h.RerunAgent(c)
-
- var resp map[string]interface{}
- _ = json.Unmarshal(w.Body.Bytes(), &resp)
- if code, _ := resp["code"].(float64); code != float64(common.CodeSuccess) {
- t.Errorf("code = %v, want 0 (msg=%v)", code, resp["message"])
- }
- if stub.userID != "u1" || stub.logID != "log-1" || stub.componentID != "c1" {
- t.Errorf("stub args = user %q log %q component %q, want u1/log-1/c1",
- stub.userID, stub.logID, stub.componentID)
- }
- if _, ok := stub.dsl["path"]; !ok {
- t.Errorf("stub dsl = %v, want the request dsl passed through", stub.dsl)
- }
-}
-
// TestPromptsReturnsHardcodedFields covers the contract: the data
// payload must contain the four authoring guideline keys.
func TestPromptsReturnsHardcodedFields(t *testing.T) {
@@ -1701,155 +1627,6 @@ func TestPromptsReturnsHardcodedFields(t *testing.T) {
}
}
-// TestRerunAgent_RejectsInaccessibleDocument: POST /api/v1/agents/rerun
-// gates on RerunDocument resolving the log and validating document access
-// before accepting the request. A denial from the service must surface as
-// CodeDataError + "Document not found." so a caller cannot probe whether a
-// document exists in another tenant.
-func TestRerunAgent_RejectsInaccessibleDocument(t *testing.T) {
- gin.SetMode(gin.TestMode)
- w := httptest.NewRecorder()
- c, _ := gin.CreateTestContext(w)
- c.Request = httptest.NewRequest("POST", "/api/v1/agents/rerun",
- strings.NewReader(`{"id":"doc-victim","dsl":{"path":[]},"component_id":"c1"}`))
- c.Request.Header.Set("Content-Type", "application/json")
- c.Set("user", &entity.User{ID: "u1"})
- c.Set("user_id", "u1")
-
- // The stub models the service's deny path: unknown log or an
- // inaccessible document both collapse to ErrRerunDocumentNotFound.
- stub := &stubDocService{err: document.ErrRerunDocumentNotFound}
- ctx := t.Context()
- h := NewAgentHandler(ctx, service.NewAgentService(), nil).
- WithDocumentService(stub)
- h.RerunAgent(c)
-
- var resp map[string]interface{}
- _ = json.Unmarshal(w.Body.Bytes(), &resp)
- if code, _ := resp["code"].(float64); code != float64(common.CodeDataError) {
- t.Errorf("deny-all stub: want code %d (Document not found), got %v (msg=%v)",
- common.CodeDataError, code, resp["message"])
- }
- if msg, _ := resp["message"].(string); !strings.Contains(msg, "Document not found") {
- t.Errorf("deny-all stub: want message to contain 'Document not found', got %q", msg)
- }
-}
-
-// TestRerunAgent_NoDocumentServiceFailsClosed pins the fail-closed
-// rule: a nil documentService is treated as a wiring misconfiguration
-// that would create an auth bypass, NOT a backward-compatible "skip
-// the gate" state. The handler must return 500 / "server
-// misconfiguration" so a missing dependency is loud and gets fixed,
-// instead of silently allowing any caller to rerun an arbitrary doc id.
-func TestRerunAgent_NoDocumentServiceFailsClosed(t *testing.T) {
- gin.SetMode(gin.TestMode)
- w := httptest.NewRecorder()
- c, _ := gin.CreateTestContext(w)
- c.Request = httptest.NewRequest("POST", "/api/v1/agents/rerun",
- strings.NewReader(`{"id":"doc-anything","dsl":{"path":[]},"component_id":"c1"}`))
- c.Request.Header.Set("Content-Type", "application/json")
- c.Set("user", &entity.User{ID: "u1"})
- c.Set("user_id", "u1")
-
- ctx := t.Context()
- h := NewAgentHandler(ctx, service.NewAgentService(), nil)
- // Note: no WithDocumentService call → documentService is nil.
- // The production wiring (cmd/ragflow_server.go) chains
- // WithDocumentService onto NewAgentHandler; a nil here means the
- // handler was constructed without its required dependency.
- h.RerunAgent(c)
-
- var resp map[string]interface{}
- _ = json.Unmarshal(w.Body.Bytes(), &resp)
- if code, _ := resp["code"].(float64); code != float64(common.CodeServerError) {
- t.Errorf("nil documentService: want code %d (fail closed), got %v (msg=%v)",
- common.CodeServerError, code, resp["message"])
- }
- if msg, _ := resp["message"].(string); !strings.Contains(msg, "server misconfiguration") {
- t.Errorf("nil documentService: want message to mention misconfiguration, got %q", msg)
- }
-}
-
-// TestRerunAgent_InternalErrorReturns500: a non-domain failure (DB down,
-// queue publish failed, ...) is neither a caller data error nor safe to
-// echo back. The handler must answer CodeServerError with a generic
-// message and keep the raw error text out of the response.
-func TestRerunAgent_InternalErrorReturns500(t *testing.T) {
- gin.SetMode(gin.TestMode)
- w := httptest.NewRecorder()
- c, _ := gin.CreateTestContext(w)
- c.Request = httptest.NewRequest("POST", "/api/v1/agents/rerun",
- strings.NewReader(`{"id":"log-1","dsl":{"path":[]},"component_id":"c1"}`))
- c.Request.Header.Set("Content-Type", "application/json")
- c.Set("user", &entity.User{ID: "u1"})
- c.Set("user_id", "u1")
-
- stub := &stubDocService{err: errors.New("update pipeline log dsl: connection refused")}
- ctx := t.Context()
- h := NewAgentHandler(ctx, service.NewAgentService(), nil).
- WithDocumentService(stub)
- h.RerunAgent(c)
-
- var resp map[string]interface{}
- _ = json.Unmarshal(w.Body.Bytes(), &resp)
- if code, _ := resp["code"].(float64); code != float64(common.CodeServerError) {
- t.Errorf("internal error: want code %d, got %v (msg=%v)",
- common.CodeServerError, code, resp["message"])
- }
- if msg, _ := resp["message"].(string); strings.Contains(msg, "connection refused") {
- t.Errorf("internal error: raw error leaked into the response message %q", msg)
- }
-}
-
-// TestRerunAgent_DocumentProcessingIsDataError: a mid-run document is a
-// caller-facing conflict, so the typed processing error keeps the 102
-// envelope with the service's message.
-func TestRerunAgent_DocumentProcessingIsDataError(t *testing.T) {
- gin.SetMode(gin.TestMode)
- w := httptest.NewRecorder()
- c, _ := gin.CreateTestContext(w)
- c.Request = httptest.NewRequest("POST", "/api/v1/agents/rerun",
- strings.NewReader(`{"id":"log-1","dsl":{"path":[]},"component_id":"c1"}`))
- c.Request.Header.Set("Content-Type", "application/json")
- c.Set("user", &entity.User{ID: "u1"})
- c.Set("user_id", "u1")
-
- stub := &stubDocService{err: &document.RerunDocumentProcessingError{DocumentName: "hlm.docx"}}
- ctx := t.Context()
- h := NewAgentHandler(ctx, service.NewAgentService(), nil).
- WithDocumentService(stub)
- h.RerunAgent(c)
-
- var resp map[string]interface{}
- _ = json.Unmarshal(w.Body.Bytes(), &resp)
- if code, _ := resp["code"].(float64); code != float64(common.CodeDataError) {
- t.Errorf("processing error: want code %d, got %v (msg=%v)",
- common.CodeDataError, code, resp["message"])
- }
- if msg, _ := resp["message"].(string); !strings.Contains(msg, "is processing") {
- t.Errorf("processing error: want message to contain 'is processing', got %q", msg)
- }
-}
-
-// stubDocService stubs the documentRerunService surface for handler
-// tests: it records the arguments and replays err (nil = success,
-// document.ErrRerunDocumentNotFound = deny).
-type stubDocService struct {
- err error
- userID string
- logID string
- dsl map[string]interface{}
- componentID string
-}
-
-func (s *stubDocService) RerunDocument(_ context.Context, userID, logID string, dsl map[string]interface{}, componentID string) error {
- s.userID = userID
- s.logID = logID
- s.dsl = dsl
- s.componentID = componentID
- return s.err
-}
-
// TestAgentChatCompletions_FilesDeserialized verifies that the
// dataflow-debug 2D `files` shape (`[[{...}]]`, web use-run-dataflow.ts)
// still deserializes: the agentFiles unmarshaler normalizes it by taking
diff --git a/internal/ingestion/task/debug_log_sink_test.go b/internal/ingestion/task/debug_log_sink_test.go
index 8b09733e8a..586ed8ea87 100644
--- a/internal/ingestion/task/debug_log_sink_test.go
+++ b/internal/ingestion/task/debug_log_sink_test.go
@@ -404,7 +404,7 @@ func TestDebugLogSink_RealPipeline_EndMarkerCarriesDSL(t *testing.T) {
}
// The executor would call this after Run; here we invoke it directly.
- resultDSL, err := BuildDebugResultDSL(dsl, output)
+ resultDSL, err := BuildDebugResultDSL(dsl, output, true)
if err != nil {
t.Fatalf("BuildDebugResultDSL: %v", err)
}
@@ -502,7 +502,7 @@ func TestDebugLogSink_RealPipeline_EndMarkerDSLShowsChunks(t *testing.T) {
t.Fatalf("Run: %v", err)
}
- resultDSL, err := BuildDebugResultDSL(dsl, output)
+ resultDSL, err := BuildDebugResultDSL(dsl, output, true)
if err != nil {
t.Fatalf("BuildDebugResultDSL: %v", err)
}
diff --git a/internal/ingestion/task/debug_result_dsl.go b/internal/ingestion/task/debug_result_dsl.go
index b921968907..168c0a88f6 100644
--- a/internal/ingestion/task/debug_result_dsl.go
+++ b/internal/ingestion/task/debug_result_dsl.go
@@ -87,7 +87,7 @@ func IsVectorKey(k string) bool {
// dsl is the raw canvas DSL JSON (optionally wrapped as {"dsl": {...}}). output
// is the pipeline run output keyed by component id (output[<id>] is that
// component's outputs map, which may carry chunks/json/text/html/markdown).
-func BuildDebugResultDSL(dsl string, output map[string]any) (map[string]any, error) {
+func BuildDebugResultDSL(dsl string, output map[string]any, includeOutputs bool) (map[string]any, error) {
var tpl map[string]any
if err := json.Unmarshal([]byte(dsl), &tpl); err != nil {
return nil, fmt.Errorf("BuildDebugResultDSL: unmarshal dsl: %w", err)
@@ -133,34 +133,49 @@ func BuildDebugResultDSL(dsl string, output map[string]any) (map[string]any, err
for k, v := range staticParams {
mergedParams[k] = deepCopy(v, false)
}
- runOut, _ := lookupComponentOutput(output, id).(map[string]any)
- outputsWrapper := map[string]any{}
- if format, payload := detectFormat(runOut); format != "" {
- value := deepCopy(payload, true)
- outputsWrapper[format] = map[string]any{
- "value": value,
- "type": pythonTypeName(value),
+ // Never forward a pre-existing outputs wrapper from the input DSL into
+ // the rebuilt params: the input can be an output-bearing copy (e.g. a
+ // stale pipeline_operation_log row loaded back as a canvas), and the
+ // persisted log must carry the DSL definition ONLY. The preview path
+ // re-attaches the fresh runtime outputs below when includeOutputs is
+ // true, so deleting here does not affect that branch.
+ delete(mergedParams, "outputs")
+ if includeOutputs {
+ // Build the runtime outputs wrapper (business data) only when the
+ // caller needs it — the dry-run live preview (ResultSink/Redis). The
+ // persisted pipeline operation-log DSL must carry the DSL definition
+ // ONLY (no chunks/json/text/... business data), so the real-parse
+ // persist path passes includeOutputs=false and this block is skipped
+ // entirely: the outputs are never even constructed, not stripped after.
+ runOut, _ := lookupComponentOutput(output, id).(map[string]any)
+ outputsWrapper := map[string]any{}
+ if format, payload := detectFormat(runOut); format != "" {
+ value := deepCopy(payload, true)
+ outputsWrapper[format] = map[string]any{
+ "value": value,
+ "type": pythonTypeName(value),
+ }
+ outputsWrapper["output_format"] = map[string]any{
+ "value": format,
+ "type": pythonTypeName(format),
+ }
}
- outputsWrapper["output_format"] = map[string]any{
- "value": format,
- "type": pythonTypeName(format),
+ // TrackElapsed stamps the bookkeeping pair into every component's run
+ // output (internal/agent/canvas/node_body.go). Carry them into the
+ // outputs wrapper as plain {value, type} entries so the front-end
+ // timeline renders per-node elapsed times — it reads exactly
+ // params.outputs._elapsed_time.value
+ // (web/src/pages/dataflow-result/hooks.ts; agent/component/base.py
+ // set_output's the same keys).
+ for _, k := range bookkeepingKeys {
+ if v, ok := runOut[k]; ok && v != nil {
+ outputsWrapper[k] = map[string]any{"value": v, "type": pythonTypeName(v)}
+ }
}
- }
- // TrackElapsed stamps the bookkeeping pair into every component's run
- // output (internal/agent/canvas/node_body.go). Carry them into the
- // outputs wrapper as plain {value, type} entries so the front-end
- // timeline renders per-node elapsed times — it reads exactly
- // params.outputs._elapsed_time.value
- // (web/src/pages/dataflow-result/hooks.ts; agent/component/base.py
- // set_output's the same keys).
- for _, k := range bookkeepingKeys {
- if v, ok := runOut[k]; ok && v != nil {
- outputsWrapper[k] = map[string]any{"value": v, "type": pythonTypeName(v)}
+ if len(outputsWrapper) > 0 {
+ mergedParams["outputs"] = outputsWrapper
}
}
- if len(outputsWrapper) > 0 {
- mergedParams["outputs"] = outputsWrapper
- }
built[id] = map[string]any{
"obj": map[string]any{
diff --git a/internal/ingestion/task/debug_result_dsl_test.go b/internal/ingestion/task/debug_result_dsl_test.go
index 7fecbe9b30..1b34d7bf87 100644
--- a/internal/ingestion/task/debug_result_dsl_test.go
+++ b/internal/ingestion/task/debug_result_dsl_test.go
@@ -68,7 +68,7 @@ func TestBuildDebugResultDSL(t *testing.T) {
"begin": map[string]any{"_elapsed_time": 0.01, "_created_time": 99.0},
}
- result, err := BuildDebugResultDSL(dsl, output)
+ result, err := BuildDebugResultDSL(dsl, output, true)
if err != nil {
t.Fatalf("BuildDebugResultDSL: %v", err)
}
@@ -263,7 +263,7 @@ func TestBuildDebugResultDSL_NestedState(t *testing.T) {
},
}
- result, err := BuildDebugResultDSL(dsl, output)
+ result, err := BuildDebugResultDSL(dsl, output, true)
if err != nil {
t.Fatalf("BuildDebugResultDSL: %v", err)
}
@@ -519,3 +519,167 @@ func TestPythonTypeName(t *testing.T) {
})
}
}
+
+// TestBuildDebugResultDSL_IncludeOutputs locks the core contract of the
+// "no business data in the persisted log" change via the includeOutputs flag
+// (the user-chosen approach: the persist path NEVER constructs business data,
+// rather than building-then-stripping):
+//
+// - includeOutputs=false -> the result DSL carries the DSL DEFINITION only
+// (component_name / static params / downstream / graph / path / …); no
+// component has obj.params.outputs. This is what the persisted
+// pipeline_operation_log DSL uses.
+// - includeOutputs=true -> each component's runtime outputs wrapper
+// (obj.params.outputs) is present with the business payload (chunks), so
+// the dry-run live preview (ResultSink / Redis END marker) renders parsed
+// chunks.
+func TestBuildDebugResultDSL_IncludeOutputs(t *testing.T) {
+ const (
+ compA = "my.Parser"
+ compB = "my.Tokenizer"
+ )
+ dsl := `{
+ "dsl": {
+ "components": {
+ "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
+ "a": {"obj": {"component_name": "` + compA + `", "params": {"setups": {"pdf": {"parse_method": "general"}}}}, "upstream": ["begin"], "downstream": ["b"]},
+ "b": {"obj": {"component_name": "` + compB + `", "params": {"field_name": "content"}}, "upstream": ["a"]}
+ },
+ "graph": {"nodes": [{"id": "a", "data": {"name": "解析"}}]},
+ "path": ["begin", "a", "b"],
+ "task_id": "task-42"
+ }
+ }`
+ output := map[string]any{
+ "a": map[string]any{
+ "chunks": []any{map[string]any{"text": "hello", "vector": []float64{0.1, 0.2}}},
+ "_elapsed_time": 0.35,
+ },
+ "b": map[string]any{"text": "plain", "_elapsed_time": 0.02},
+ }
+
+ // --- includeOutputs=false: DSL definition only, no business data. ---
+ noBiz, err := BuildDebugResultDSL(dsl, output, false)
+ if err != nil {
+ t.Fatalf("BuildDebugResultDSL(includeOutputs=false): %v", err)
+ }
+ comps, ok := noBiz["components"].(map[string]any)
+ if !ok {
+ t.Fatalf("components missing: %#v", noBiz["components"])
+ }
+ for id, raw := range comps {
+ comp, _ := raw.(map[string]any)
+ obj, _ := comp["obj"].(map[string]any)
+ params, _ := obj["params"].(map[string]any)
+ if _, exists := params["outputs"]; exists {
+ t.Errorf("includeOutputs=false must NOT attach obj.params.outputs on %q, got %#v", id, params["outputs"])
+ }
+ }
+ // Non-components top-level keys preserved verbatim (round-trips through
+ // the rerun flow).
+ if p, _ := noBiz["path"].([]any); len(p) != 3 {
+ t.Errorf("path dropped: %#v", noBiz["path"])
+ }
+ if noBiz["task_id"] != "task-42" {
+ t.Errorf("task_id dropped: %v", noBiz["task_id"])
+ }
+ if noBiz["graph"] == nil {
+ t.Error("graph dropped")
+ }
+ // Static params preserved.
+ if aParams, _ := comps["a"].(map[string]any)["obj"].(map[string]any)["params"].(map[string]any); aParams["setups"] == nil {
+ t.Error("static params.setups dropped for includeOutputs=false")
+ }
+
+ // --- includeOutputs=true: full business data present. ---
+ withBiz, err := BuildDebugResultDSL(dsl, output, true)
+ if err != nil {
+ t.Fatalf("BuildDebugResultDSL(includeOutputs=true): %v", err)
+ }
+ wcomps, ok := withBiz["components"].(map[string]any)
+ if !ok {
+ t.Fatalf("components missing: %#v", withBiz["components"])
+ }
+ aObj, _ := wcomps["a"].(map[string]any)["obj"].(map[string]any)
+ aParams, _ := aObj["params"].(map[string]any)
+ aOutputs, ok := aParams["outputs"].(map[string]any)
+ if !ok {
+ t.Fatalf("includeOutputs=true must attach obj.params.outputs on 'a', got %#v", aParams)
+ }
+ if aOutputs["output_format"].(map[string]any)["value"] != "chunks" {
+ t.Errorf("output_format=%#v want {value:\"chunks\"}", aOutputs["output_format"])
+ }
+ chunksVal := aOutputs["chunks"].(map[string]any)["value"].([]any)
+ if len(chunksVal) != 1 || chunksVal[0].(map[string]any)["text"] != "hello" {
+ t.Errorf("chunks payload wrong: %#v", aOutputs["chunks"])
+ }
+ // The static params still ride along in the preview (same mergedParams).
+ if aParams["setups"] == nil {
+ t.Error("static params.setups dropped in preview")
+ }
+ // Bookkeeping keys surface in the preview outputs wrapper.
+ if et, _ := aOutputs["_elapsed_time"].(map[string]any); et["value"] != 0.35 {
+ t.Errorf("preview outputs._elapsed_time=%#v want {value:0.35}", aOutputs["_elapsed_time"])
+ }
+}
+
+// TestBuildDebugResultDSL_StripsPreExistingOutputs locks the defense-in-depth
+// guard for the persisted (includeOutputs=false) path: if the INPUT canvas DSL
+// ALREADY carries runtime outputs under obj.params.outputs (e.g. a stale
+// pipeline_operation_log row loaded back as a canvas, or any output-bearing
+// copy), the persisted builder must NOT forward those outputs into the new log
+// DSL. The persist path never constructs outputs (includeOutputs=false), and it
+// must also drop any pre-existing outputs in the static params so business data
+// cannot round-trip back into the persisted row (CWE-200).
+func TestBuildDebugResultDSL_StripsPreExistingOutputs(t *testing.T) {
+ const compA = "my.Parser"
+ // The input params already embed business data under outputs — this is the
+ // dangerous shape the persist path must refuse to carry forward.
+ dsl := `{
+ "dsl": {
+ "components": {
+ "a": {"obj": {"component_name": "` + compA + `", "params": {
+ "setups": {"pdf": {"parse_method": "general"}},
+ "outputs": {"chunks": {"value": [{"text": "LEAKED-BUSINESS-DATA"}], "type": "list"}}
+ }}, "downstream": []}
+ },
+ "path": ["a"],
+ "task_id": "task-42"
+ }
+ }`
+ output := map[string]any{
+ "a": map[string]any{"chunks": []any{map[string]any{"text": "fresh"}}},
+ }
+
+ // Persisted path: includeOutputs=false — must produce the DSL DEFINITION
+ // only, with no obj.params.outputs at all.
+ noBiz, err := BuildDebugResultDSL(dsl, output, false)
+ if err != nil {
+ t.Fatalf("BuildDebugResultDSL(includeOutputs=false): %v", err)
+ }
+ comps, ok := noBiz["components"].(map[string]any)
+ if !ok {
+ t.Fatalf("components missing: %#v", noBiz["components"])
+ }
+ aObj, _ := comps["a"].(map[string]any)["obj"].(map[string]any)
+ aParams, _ := aObj["params"].(map[string]any)
+ if _, exists := aParams["outputs"]; exists {
+ t.Errorf("REGRESSION: pre-existing obj.params.outputs leaked into persisted DSL: %#v", aParams["outputs"])
+ }
+ // Static params must still survive (definition, not business data).
+ if aParams["setups"] == nil {
+ t.Error("static params.setups dropped by the outputs-strip guard")
+ }
+
+ // Sanity: the preview path (includeOutputs=true) still attaches the fresh
+ // runtime outputs — the guard only affects the persisted copy.
+ withBiz, err := BuildDebugResultDSL(dsl, output, true)
+ if err != nil {
+ t.Fatalf("BuildDebugResultDSL(includeOutputs=true): %v", err)
+ }
+ wcomps := withBiz["components"].(map[string]any)
+ waParams := wcomps["a"].(map[string]any)["obj"].(map[string]any)["params"].(map[string]any)
+ if _, ok := waParams["outputs"].(map[string]any); !ok {
+ t.Error("includeOutputs=true must still attach fresh runtime outputs")
+ }
+}
diff --git a/internal/ingestion/task/debug_test.go b/internal/ingestion/task/debug_test.go
index 972f016f61..0f5fc97654 100644
--- a/internal/ingestion/task/debug_test.go
+++ b/internal/ingestion/task/debug_test.go
@@ -401,7 +401,7 @@ func TestBuildDebugResultDSL_Envelope(t *testing.T) {
}
// Raw (non-enveloped) DSL.
- rawRes, err := BuildDebugResultDSL(rawDSL, output)
+ rawRes, err := BuildDebugResultDSL(rawDSL, output, true)
if err != nil {
t.Fatalf("raw DSL: %v", err)
}
@@ -415,7 +415,7 @@ func TestBuildDebugResultDSL_Envelope(t *testing.T) {
// Enveloped DSL (production shape) must unwrap to the same result.
envDSL := `{"dsl": ` + rawDSL + `}`
- envRes, err := BuildDebugResultDSL(envDSL, output)
+ envRes, err := BuildDebugResultDSL(envDSL, output, true)
if err != nil {
t.Fatalf("enveloped DSL: %v", err)
}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 819afccaf2..170a53bef1 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -1172,31 +1172,45 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (
return payload, logDSL, nil
}
-// buildLogDSL returns the DSL string recorded for a pipeline run: the
-// run-result DSL (static canvas structure + each component's runtime outputs
-// merged into obj.params.outputs) when it can be built and marshaled,
-// otherwise the static dsl unchanged — log recording must never fail a run.
+// buildLogDSL returns the DSL string recorded for a pipeline run. The PERSISTED
+// copy carries the DSL DEFINITION only (static canvas structure + static
+// params + downstream + graph + path + …) — business data is never constructed
+// for it (BuildDebugResultDSL(dsl, output, false)). Log recording must never
+// fail a run, so on any build error the static dsl is returned unchanged.
//
// BuildDebugResultDSL needs the full run output, which is in scope only inside
// runPipelineWithDSL: the extracted payload returned there no longer carries
-// output["state"]. Two consumers:
-// - canvas-debug runs hand the map to the DebugLogSink END marker via the
-// ResultSink capability (END-marker `dsl` attachment,
-// rag/flow/pipeline.py:98);
-// - persist (dataset parse) runs return its JSON as the log DSL so the
-// pipeline operation log carries every component's output
-// (dsl=str(pipeline), rag/svr/task_executor_refactor/dataflow_service.py)
-// — without it the dataset log "View result" page renders blank panels.
+// output["state"]. Two consumers, split by the includeOutputs flag:
+// - canvas-debug (dry-run) runs wire a DebugLogSink, which implements the
+// optional ResultSink capability. buildLogDSL hands it the FULL result DSL
+// (BuildDebugResultDSL(dsl, output, true)) so the front-end "View result"
+// page renders parsed chunks from the Redis END marker
+// (rag/flow/pipeline.py:98). This is the ONLY place business data leaves
+// the executor, and it targets Redis — never pipeline_operation_log.
+// - persist (dataset parse) runs use a DB-backed sink that is NOT a
+// ResultSink, so the preview branch is skipped. The returned logDSL (DSL
+// definition only) is what recordPipelineLog persists, matching the
+// "log stores the DSL definition, not business data" contract; the
+// dataset log "View result" page therefore renders from the dry-run
+// preview, not from the persisted row.
//
// The sink probe stays an optional capability: non-debug (DB-backed) sinks
// ignore it and the ProgressSink contract is unchanged.
func (s *PipelineExecutor) buildLogDSL(dsl string, output map[string]any) string {
+ // Start from the static dsl: log recording must never fail a run, so on any
+ // build error we fall back to the static dsl unchanged rather than a
+ // half-written payload. The input dsl is the canvas DEFINITION (no runtime
+ // outputs), so the fallback is not a business-data leak (CWE-200) — the
+ // persisted copy stays definition-only and the log row is preserved for
+ // observability.
logDSL := dsl
- if resultDSL, e := BuildDebugResultDSL(dsl, output); e == nil {
- if rs, ok := s.progressSink.(ResultSink); ok {
- rs.SetResult(resultDSL, output)
- }
- if raw, e := json.Marshal(resultDSL); e == nil {
+ // Persisted log DSL: DSL definition ONLY. Business data is never
+ // constructed for the persisted copy (includeOutputs=false). Real parses
+ // persist this string via recordPipelineLog; a dry-run also builds it here
+ // but discards it at the IsDebug() early-return before recordPipelineLog,
+ // so no business data is written to pipeline_operation_log.
+ if persisted, e := BuildDebugResultDSL(dsl, output, false); e == nil {
+ if raw, e := json.Marshal(persisted); e == nil {
logDSL = string(raw)
} else {
common.Warn(fmt.Sprintf("marshal run-result dsl for pipeline log: %v", e))
@@ -1204,6 +1218,16 @@ func (s *PipelineExecutor) buildLogDSL(dsl string, output map[string]any) string
} else {
common.Warn(fmt.Sprintf("build run-result dsl for pipeline log: %v", e))
}
+ // Dry-run live preview: the ResultSink (DebugLogSink, wired ONLY for
+ // canvas-debug runs) receives the FULL result DSL with business data
+ // (includeOutputs=true) so the front-end "View result" page renders parsed
+ // chunks from Redis. Real-parse sinks are NOT ResultSink, so this branch is
+ // skipped and no business data is handed anywhere on the persist path.
+ if rs, ok := s.progressSink.(ResultSink); ok {
+ if previewDSL, e := BuildDebugResultDSL(dsl, output, true); e == nil {
+ rs.SetResult(previewDSL, output)
+ }
+ }
return logDSL
}
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index d109033ee4..bc26128345 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"errors"
- "math"
"reflect"
"sync"
"testing"
@@ -1379,24 +1378,22 @@ func TestMergeCompiledVariants(t *testing.T) {
}
}
-// TestRunPipelineWithDSL_LogDSLCarriesOutputs locks the dataset-parse
-// "View result" contract: the DSL runPipelineWithDSL returns for the pipeline
-// operation log (Execute passes it straight to recordPipelineLog) must carry
-// each component's runtime outputs under obj.params.outputs, mirroring
-// Python's dsl=str(pipeline)
-// (rag/svr/task_executor_refactor/dataflow_service.py). Before this, the log
-// stored the raw static DSL, so the dataset log "View result" page rendered
-// blank panels and "0s" elapsed times even though the chunks were indexed —
-// the front-end renders those panels exclusively from
-// dsl.components[<id>].obj.params.outputs
-// (web/src/pages/dataflow-result/parser.tsx, hooks.ts).
+// TestRunPipelineWithDSL_LogDSLStripsOutputs locks the "no business data in
+// the pipeline operation log" contract: the DSL runPipelineWithDSL returns for
+// the log (Execute passes it straight to recordPipelineLog) must NOT carry any
+// component's runtime outputs under obj.params.outputs. The log keeps the DSL
+// DEFINITION only (component structure, static params, downstream, graph,
+// path) so a historical run can be reconstructed and the dataset "View result"
+// / rerun UI no longer renders chunk business data. Dry-run previews still
+// carry outputs via ResultSink; only the persisted log is kept output-free
+// (buildLogDSL passes includeOutputs=false before recordPipelineLog).
//
// It drives the REAL runPipelineWithDSL with stub ingestion components, so
// the log DSL is built from an actual run output (nested under
// output["state"][<id>] by finalizeResult), not a hand-built one, and the
// enveloped {"dsl": {...}} input is unwrapped to the front-end shape
// (top-level components).
-func TestRunPipelineWithDSL_LogDSLCarriesOutputs(t *testing.T) {
+func TestRunPipelineWithDSL_LogDSLStripsOutputs(t *testing.T) {
const (
compC = "logdsl.RealStubChunks"
compD = "logdsl.RealStubD"
@@ -1431,46 +1428,22 @@ func TestRunPipelineWithDSL_LogDSLCarriesOutputs(t *testing.T) {
t.Fatalf("log DSL must carry top-level components (unwrapped canvas envelope): %s", logDSL)
}
- // Chunk-emitting component: params.outputs.chunks + output_format.
+ // Chunk-emitting component: the persisted log must carry the DSL DEFINITION
+ // only — NO runtime outputs under obj.params.outputs (the "no business data
+ // in the log" contract, Req 1). The dry-run preview (ResultSink/Redis) still
+ // carries full outputs for the dataset "View result" page; the persisted row
+ // must not. Static params survive so the rerun/canvas flow can reconstruct
+ // the component.
cParams, ok := components["c"].(map[string]any)["obj"].(map[string]any)["params"].(map[string]any)
if !ok {
t.Fatalf("log DSL components.c.obj.params missing: %s", logDSL)
}
- cOutputs, ok := cParams["outputs"].(map[string]any)
- if !ok {
- t.Fatalf("REGRESSION: log DSL has no params.outputs for chunk component c; "+
- "the dataset log 'View result' page would render a blank panel. params=%#v", cParams)
- }
- if of, _ := cOutputs["output_format"].(map[string]any); of["value"] != "chunks" {
- t.Errorf("c output_format=%#v want {value:\"chunks\"}", cOutputs["output_format"])
- }
- chunksVal, _ := cOutputs["chunks"].(map[string]any)["value"].([]any)
- if len(chunksVal) != 1 {
- t.Fatalf("c chunks.value len=%d want 1", len(chunksVal))
+ if _, ok := cParams["outputs"]; ok {
+ t.Errorf("REGRESSION: persisted log DSL must NOT carry business data (obj.params.outputs) "+
+ "for chunk component c, got %#v", cParams["outputs"])
}
- // TrackElapsed bookkeeping must ride along so the timeline shows real
- // per-node elapsed times (hooks.ts reads outputs._elapsed_time.value).
- et, ok := cOutputs["_elapsed_time"].(map[string]any)
- if !ok {
- t.Fatalf("c outputs._elapsed_time missing: %#v", cOutputs)
- }
- if _, ok := et["value"].(float64); !ok {
- t.Errorf("c outputs._elapsed_time.value=%#v want float64", et["value"])
- }
- // TrackElapsed stamps _created_time as an RFC3339Nano wall-clock string;
- // the outputs wrapper carries it verbatim with its type string.
- if cct, ok := cOutputs["_created_time"].(map[string]any); ok {
- cs, ok := cct["value"].(string)
- if !ok || cs == "" {
- t.Errorf("c outputs._created_time.value=%#v want non-empty string", cct["value"])
- } else if _, err := time.Parse(time.RFC3339Nano, cs); err != nil {
- t.Errorf("c outputs._created_time.value %q is not RFC3339Nano: %v", cs, err)
- }
- if cct["type"] != "<class 'str'>" {
- t.Errorf("c outputs._created_time.type=%#v want <class 'str'>", cct["type"])
- }
- } else {
- t.Errorf("c outputs._created_time missing: %#v", cOutputs)
+ if _, ok := cParams["setups"]; !ok {
+ t.Errorf("persisted log DSL must keep static params.setups for c, got %#v", cParams)
}
// Non-components top-level keys are carried verbatim; this fixture's DSL
// declares "path" — the round-tripped log must keep it for rerun-flow
@@ -1481,25 +1454,125 @@ func TestRunPipelineWithDSL_LogDSLCarriesOutputs(t *testing.T) {
}
// TestBuildLogDSL_FallbackToStaticDSL pins the guarantee that log recording
-// never fails a run: when the run-result DSL cannot be built or cannot be
-// marshaled, buildLogDSL must return the static dsl unchanged rather than a
-// half-written payload.
+// never fails a run: when the run-result DSL cannot be built (a malformed
+// canvas), buildLogDSL must return the static dsl unchanged rather than a
+// half-written payload. The input dsl is the canvas definition (no runtime
+// outputs), so the fallback is not a business-data leak and the log row is
+// preserved for observability (recordPipelineLog still receives a valid,
+// definition-only DSL).
+//
+// Note: business-data payloads (e.g. NaN inside a chunk value) no longer reach
+// the persisted copy at all — the persist path passes includeOutputs=false, so
+// the outputs wrapper is never constructed and cannot break marshaling. The
+// realistic fallback trigger is therefore a malformed DSL, tested below.
func TestBuildLogDSL_FallbackToStaticDSL(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-logdsl-fallback", 0)
- // Marshal failure: NaN is a valid float64 payload, so the run-result DSL
- // builds fine but json.Marshal rejects it.
- dsl := `{"dsl":{"components":{"a":{"obj":{"component_name":"X","params":{}}}}}}`
- if got := svc.buildLogDSL(dsl, map[string]any{
- "a": map[string]any{"text": math.NaN()},
- }); got != dsl {
- t.Errorf("marshal failure: log DSL must fall back to the static dsl\n got: %s\nwant: %s", got, dsl)
- }
-
// Build failure: a DSL without a components map cannot produce a
- // run-result DSL at all.
+ // run-result DSL at all. The static dsl is returned unchanged.
badDSL := `{"dsl":{"path":["a"]}}`
if got := svc.buildLogDSL(badDSL, nil); got != badDSL {
t.Errorf("build failure: log DSL must fall back to the static dsl\n got: %s\nwant: %s", got, badDSL)
}
}
+
+// persistOnlySink simulates the real-parse (DB-backed) sink: it implements
+// pipeline.ProgressSink but NOT task.ResultSink, so buildLogDSL must NOT call
+// SetResult and the persisted DSL must carry no business data.
+type persistOnlySink struct{}
+
+func (persistOnlySink) OnComponentTotal(context.Context, string, int) {}
+func (persistOnlySink) OnComponentProgress(context.Context, pipelinepkg.ProgressEvent) {}
+
+// capturingSink simulates the dry-run (DebugLogSink) sink: it implements BOTH
+// ProgressSink and ResultSink, so buildLogDSL hands it the full result DSL
+// (business data) via SetResult for the Redis END marker.
+type capturingSink struct {
+ got map[string]any
+ gotOut map[string]any
+}
+
+func (c *capturingSink) OnComponentTotal(context.Context, string, int) {}
+func (c *capturingSink) OnComponentProgress(context.Context, pipelinepkg.ProgressEvent) {}
+func (c *capturingSink) SetResult(dsl map[string]any, output map[string]any) {
+ c.got = dsl
+ c.gotOut = output
+}
+
+// TestBuildLogDSL_PersistStripsOutputs locks the core contract of the change:
+// the DSL string buildLogDSL returns for a NON-ResultSink (real-parse) sink
+// carries the DSL definition only — no component has obj.params.outputs, while
+// static params / graph / path survive. This is exactly what recordPipelineLog
+// persists to pipeline_operation_log.
+func TestBuildLogDSL_PersistStripsOutputs(t *testing.T) {
+ const dsl = `{"components": {"a": {"obj": {"component_name": "my.Parser", "params": {"setups": {"pdf": {"parse_method": "general"}}}}, "downstream": ["b"]}}, "graph": {"nodes": [{"id": "a"}]}, "path": ["a"]}`
+ output := map[string]any{
+ "a": map[string]any{"chunks": []any{map[string]any{"text": "hello"}}, "_elapsed_time": 0.35},
+ }
+
+ exec := &PipelineExecutor{progressSink: persistOnlySink{}}
+ logDSL := exec.buildLogDSL(dsl, output)
+
+ var parsed map[string]any
+ if err := json.Unmarshal([]byte(logDSL), &parsed); err != nil {
+ t.Fatalf("persisted log DSL must be valid JSON: %v (raw=%s)", err, logDSL)
+ }
+ comps, _ := parsed["components"].(map[string]any)
+ a, _ := comps["a"].(map[string]any)
+ aObj, _ := a["obj"].(map[string]any)
+ aParams, _ := aObj["params"].(map[string]any)
+ if _, ok := aParams["outputs"]; ok {
+ t.Errorf("persisted log DSL must NOT carry business data (obj.params.outputs), got %#v", aParams["outputs"])
+ }
+ // DSL definition preserved.
+ if _, ok := aParams["setups"]; !ok {
+ t.Error("persisted log DSL must keep static params.setups")
+ }
+ if _, ok := parsed["graph"]; !ok {
+ t.Error("persisted log DSL must keep graph")
+ }
+ if _, ok := parsed["path"]; !ok {
+ t.Error("persisted log DSL must keep path")
+ }
+}
+
+// TestBuildLogDSL_PreviewKeepsOutputs locks the dry-run (ResultSink) branch:
+// buildLogDSL calls ResultSink.SetResult with the FULL result DSL (business
+// data in obj.params.outputs), while the DSL it returns for persistence still
+// carries no business data. Dry-run never persists (IsDebug() early-return),
+// so the outputs only travel to Redis via SetResult.
+func TestBuildLogDSL_PreviewKeepsOutputs(t *testing.T) {
+ const dsl = `{"components": {"a": {"obj": {"component_name": "my.Parser", "params": {}}, "downstream": ["b"]}}}`
+ output := map[string]any{
+ "a": map[string]any{"chunks": []any{map[string]any{"text": "hello"}}, "_elapsed_time": 0.35},
+ }
+
+ cap := &capturingSink{}
+ exec := &PipelineExecutor{progressSink: cap}
+ logDSL := exec.buildLogDSL(dsl, output)
+
+ // The ResultSink preview must carry the full outputs (business data).
+ if cap.got == nil {
+ t.Fatal("buildLogDSL must call ResultSink.SetResult for a dry-run (ResultSink) sink")
+ }
+ caps, _ := cap.got["components"].(map[string]any)
+ ca, _ := caps["a"].(map[string]any)
+ caObj, _ := ca["obj"].(map[string]any)
+ caParams, _ := caObj["params"].(map[string]any)
+ if _, ok := caParams["outputs"]; !ok {
+ t.Error("dry-run preview DSL must carry obj.params.outputs (business data)")
+ }
+
+ // Persisted copy returned by buildLogDSL still must NOT carry business data.
+ var parsed map[string]any
+ if err := json.Unmarshal([]byte(logDSL), &parsed); err != nil {
+ t.Fatalf("persisted log DSL must be valid JSON: %v", err)
+ }
+ pc, _ := parsed["components"].(map[string]any)
+ pa, _ := pc["a"].(map[string]any)
+ paObj, _ := pa["obj"].(map[string]any)
+ paParams, _ := paObj["params"].(map[string]any)
+ if _, ok := paParams["outputs"]; ok {
+ t.Error("persisted log DSL must NOT carry business data even when a ResultSink preview exists")
+ }
+}
diff --git a/internal/router/agent_routes.go b/internal/router/agent_routes.go
index ee43320f34..f37dbbb32b 100644
--- a/internal/router/agent_routes.go
+++ b/internal/router/agent_routes.go
@@ -94,7 +94,6 @@ func RegisterAgentRoutes(g *gin.RouterGroup, h *handler.AgentHandler) {
// (api/apps/restful_apis/agent_api.py) has since removed the route.
// See plan: .claude/plans/agent-api-gaps-go-port.md §Gap E.
g.POST("/chat/completions", h.AgentChatCompletions)
- g.POST("/rerun", h.RerunAgent)
g.POST("/test_db_connection", h.TestDBConnection)
}
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index 88b9a98094..2fb1a3d77c 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -17,11 +17,10 @@ import (
)
// Accessible reports whether docID belongs to a knowledge base
-// reachable by userID. Used by agent endpoints (e.g. RerunAgent,
-// PR #15145) to gate destructive / run-again actions on a document
-// the caller has access to. Returns false on any lookup failure or
-// empty inputs so callers can treat a denial as a 404-equivalent
-// and avoid leaking whether the document exists at all.
+// reachable by userID. Used to gate actions on a document the caller
+// has access to. Returns false on any lookup failure or empty inputs
+// so callers can treat a denial as a 404-equivalent and avoid leaking
+// whether the document exists at all.
func (s *DocumentService) Accessible(ctx context.Context, docID, userID string) bool {
if docID == "" || userID == "" {
return false
diff --git a/internal/service/document/document_rerun.go b/internal/service/document/document_rerun.go
deleted file mode 100644
index 74b7518f3b..0000000000
--- a/internal/service/document/document_rerun.go
+++ /dev/null
@@ -1,120 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package document
-
-import (
- "context"
- "errors"
- "fmt"
-
- "gorm.io/gorm"
-
- "ragflow/internal/dao"
- "ragflow/internal/entity"
-)
-
-// ErrRerunDocumentNotFound is the rerun endpoint's only "unknown or denied"
-// answer: an unknown log id, a missing document/dataset, and an access
-// denial all collapse to the same message so a caller cannot probe whether
-// a document exists in another tenant.
-var ErrRerunDocumentNotFound = errors.New("Document not found.")
-
-// RerunDocumentProcessingError reports a rerun attempt while the document
-// is mid-run (0 < progress < 1). The handler maps it to the data-error
-// envelope, so the message is written for the caller, not the server log.
-type RerunDocumentProcessingError struct {
- DocumentName string
-}
-
-func (e *RerunDocumentProcessingError) Error() string {
- return fmt.Sprintf("`%s` is processing...", e.DocumentName)
-}
-
-// RerunDocument re-runs the ingestion pipeline a pipeline operation log
-// belongs to. It backs POST /api/v1/agents/rerun, whose id is the
-// ingestion LOG id, resolved to its document before the accessibility
-// gate.
-//
-// Steps: resolve log -> document and enforce DocumentService
-// accessibility; refuse while the document is mid-run (0 < progress < 1);
-// persist the front-end's edited DSL with dsl.path = [component_id] on
-// the log row; clear the prior parse results (chunks, counters, terminal
-// tasks) and enqueue a fresh ingestion run (StartParseDocuments with
-// RerunWithDelete), which records a new pipeline operation log the
-// front-end can track.
-//
-// The Go pipeline has no partial-resume entry point (execution always
-// starts at the graph entry; see pipeline.Pipeline.Run), so the rerun
-// re-executes the whole pipeline instead of resuming from component_id.
-//
-// Known divergence from the Python endpoint: the persisted edited DSL is
-// an audit record only. The Go ingestion worker sources the parse DSL
-// from the canvas (doc.PipelineID -> user_canvas.dsl; see
-// loadDSLFromCanvas) and IngestionTask carries no DSL/log reference, so
-// the edited dsl/component_id currently has no effect on the Go rerun.
-// Plumbing the log DSL through the task to the worker is a follow-up.
-func (s *DocumentService) RerunDocument(ctx context.Context, userID, logID string, dsl map[string]interface{}, componentID string) error {
- // A missing row is the caller-facing not-found; any other DB error is
- // an internal failure and must not be collapsed into it.
- opLog, err := s.pipelineLogDAO.GetByID(ctx, dao.DB, logID)
- if errors.Is(err, gorm.ErrRecordNotFound) {
- return ErrRerunDocumentNotFound
- }
- if err != nil {
- return fmt.Errorf("get pipeline operation log %s: %w", logID, err)
- }
- doc, err := s.documentDAO.GetByID(ctx, dao.DB, opLog.DocumentID)
- if errors.Is(err, gorm.ErrRecordNotFound) {
- return ErrRerunDocumentNotFound
- }
- if err != nil {
- return fmt.Errorf("get document %s: %w", opLog.DocumentID, err)
- }
- kb, err := s.kbDAO.GetByID(ctx, dao.DB, doc.KbID)
- if errors.Is(err, gorm.ErrRecordNotFound) {
- return ErrRerunDocumentNotFound
- }
- if err != nil {
- return fmt.Errorf("get knowledgebase %s: %w", doc.KbID, err)
- }
- if !s.Accessible(ctx, doc.ID, userID) {
- return ErrRerunDocumentNotFound
- }
- if doc.Progress > 0 && doc.Progress < 1 {
- name := opLog.DocumentName
- if doc.Name != nil && *doc.Name != "" {
- name = *doc.Name
- }
- return &RerunDocumentProcessingError{DocumentName: name}
- }
-
- if dsl != nil {
- // Persist for any non-nil dsl, including an empty map. Copy
- // before writing so the caller's DSL map is not mutated in
- // place.
- persisted := make(map[string]interface{}, len(dsl)+1)
- for k, v := range dsl {
- persisted[k] = v
- }
- persisted["path"] = []interface{}{componentID}
- if err := s.pipelineLogDAO.UpdateDSL(ctx, dao.DB, logID, entity.JSONMap(persisted)); err != nil {
- return fmt.Errorf("update pipeline log dsl: %w", err)
- }
- }
-
- return s.StartParseDocuments(ctx, doc, kb, userID, StartParseOptions{RerunWithDelete: true})
-}
diff --git a/internal/service/document/document_rerun_e2e_test.go b/internal/service/document/document_rerun_e2e_test.go
deleted file mode 100644
index e0ea6b3367..0000000000
--- a/internal/service/document/document_rerun_e2e_test.go
+++ /dev/null
@@ -1,114 +0,0 @@
-//go:build e2e
-
-package document
-
-import (
- "context"
- "testing"
- "time"
-
- "ragflow/internal/common"
- "ragflow/internal/engine"
- "ragflow/internal/entity"
- "ragflow/internal/ingestion/testutil"
- servicepkg "ragflow/internal/service"
-)
-
-// TestRerunDocument_E2E_EnqueuesThroughRealMessageQueue drives the rerun
-// through the production MessageQueueTaskPublisher into a real NATS
-// JetStream stream and consumes the task message back. The unit tier's
-// recording publisher only proves the publish call happened; this tier
-// proves the enqueue actually lands on tasks.RAGFLOW with a task id that
-// resolves to a SCHEDULED ingestion task for the document.
-func TestRerunDocument_E2E_EnqueuesThroughRealMessageQueue(t *testing.T) {
- db := testutil.SetupTestDB(t,
- &entity.IngestionTask{}, &entity.IngestionTaskLog{}, &entity.Task{},
- &entity.Document{}, &entity.Knowledgebase{}, &entity.Tenant{},
- &entity.File{}, &entity.File2Document{}, &entity.PipelineOperationLog{},
- )
- defer testutil.ReplaceDBForTest(t, db)()
-
- mq := testutil.SetupNatsEngine(t)
- previous := engine.GetMessageQueueEngine()
- engine.SetMessageQueueEngine(mq)
- t.Cleanup(func() { engine.SetMessageQueueEngine(previous) })
- if err := mq.InitConsumer("tasks.>"); err != nil {
- t.Fatalf("InitConsumer: %v", err)
- }
-
- testutil.SeedTestData(t, db,
- testutil.WithTenantID("tenant-1"),
- testutil.WithKBID("kb-1"),
- testutil.WithDocID("doc-1"),
- )
- // SeedTestData leaves a RUNNING ingestion task behind, which
- // clearDocumentParseResults would (rightly) refuse to clear. The rerun
- // starts from a completed parse, so drop the seeded in-flight rows.
- if err := db.Where("id = ?", "task-1").Delete(&entity.IngestionTask{}).Error; err != nil {
- t.Fatalf("drop seeded ingestion task: %v", err)
- }
- if err := db.Where("id = ?", "task-1").Delete(&entity.Task{}).Error; err != nil {
- t.Fatalf("drop seeded task: %v", err)
- }
- // Prior parse results the rerun must clear, and a finished progress.
- if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").
- Updates(map[string]interface{}{"token_num": 9, "chunk_num": 4, "progress": 1}).Error; err != nil {
- t.Fatalf("set doc counters: %v", err)
- }
- insertTestPipelineLog(t, "log-1", "doc-1", "kb-1", "tenant-1",
- entity.JSONMap{"components": map[string]interface{}{}})
-
- svc := testDocumentService(t)
- svc.ingestionTaskSvc.SetTaskPublisher(servicepkg.NewMessageQueueTaskPublisher())
-
- dsl := map[string]interface{}{
- "components": map[string]interface{}{"c1": map[string]interface{}{"obj": map[string]interface{}{}}},
- }
- if err := svc.RerunDocument(t.Context(), "tenant-1", "log-1", dsl, "c1"); err != nil {
- t.Fatalf("RerunDocument: %v", err)
- }
-
- // Counters cleared for the rerun.
- doc, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1")
- if err != nil {
- t.Fatalf("reload doc: %v", err)
- }
- if doc.TokenNum != 0 || doc.ChunkNum != 0 {
- t.Fatalf("doc counters after rerun = token %d chunk %d, want zeros", doc.TokenNum, doc.ChunkNum)
- }
-
- // The edited DSL is persisted on the log row with the rerun entry point.
- updated, err := svc.pipelineLogDAO.GetByID(t.Context(), db, "log-1")
- if err != nil {
- t.Fatalf("reload log: %v", err)
- }
- path, _ := updated.DSL["path"].([]interface{})
- if len(path) != 1 || path[0] != "c1" {
- t.Fatalf("log dsl path = %v, want [c1]", updated.DSL["path"])
- }
-
- // The enqueue actually landed on tasks.RAGFLOW.
- pullCtx, cancel := context.WithTimeout(t.Context(), time.Second)
- defer cancel()
- handles, err := mq.PullMessages(pullCtx, 1)
- if err != nil {
- t.Fatalf("PullMessages: %v", err)
- }
- if len(handles) != 1 {
- t.Fatalf("expected 1 message on tasks.RAGFLOW, got %d", len(handles))
- }
- defer func() { _ = handles[0].Ack() }()
- taskMsg := handles[0].GetMessage()
- if taskMsg.TaskType != common.TaskTypeIngestionTask {
- t.Fatalf("message task type = %s, want %s", taskMsg.TaskType, common.TaskTypeIngestionTask)
- }
-
- // ...and the referenced task row exists, already SCHEDULED.
- task, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, taskMsg.TaskID)
- if err != nil {
- t.Fatalf("load enqueued ingestion task %s: %v", taskMsg.TaskID, err)
- }
- if task.DocumentID != "doc-1" || task.DatasetID != "kb-1" || task.Status != common.SCHEDULED {
- t.Fatalf("ingestion task = %+v", task)
- }
-}
diff --git a/internal/service/document/document_rerun_test.go b/internal/service/document/document_rerun_test.go
deleted file mode 100644
index 82bc344997..0000000000
--- a/internal/service/document/document_rerun_test.go
+++ /dev/null
@@ -1,284 +0,0 @@
-package document
-
-import (
- "errors"
- "strings"
- "testing"
-
- "ragflow/internal/common"
- "ragflow/internal/dao"
- "ragflow/internal/entity"
-)
-
-func insertTestPipelineLog(t *testing.T, id, docID, kbID, tenantID string, dsl entity.JSONMap) {
- t.Helper()
- opLog := &entity.PipelineOperationLog{
- ID: id,
- DocumentID: docID,
- TenantID: tenantID,
- KbID: kbID,
- ParserID: "pipeline",
- DocumentName: "hlm.docx",
- DocumentSuffix: ".docx",
- DocumentType: "docx",
- SourceFrom: "local",
- TaskType: "parse",
- OperationStatus: "done",
- DSL: dsl,
- }
- if err := dao.DB.Create(opLog).Error; err != nil {
- t.Fatalf("insert test pipeline log: %v", err)
- }
-}
-
-func rerunTestService(t *testing.T) (*DocumentService, *recordingTaskPublisher) {
- t.Helper()
- svc := testDocumentService(t)
- publisher := &recordingTaskPublisher{}
- svc.ingestionTaskSvc.SetTaskPublisher(publisher)
- return svc, publisher
-}
-
-func TestRerunDocument_UnknownLogID(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
-
- svc, _ := rerunTestService(t)
- err := svc.RerunDocument(t.Context(), "tenant-1", "missing-log", entity.JSONMap{"components": struct{}{}}, "c1")
- if err != ErrRerunDocumentNotFound {
- t.Fatalf("err = %v, want ErrRerunDocumentNotFound", err)
- }
-}
-
-func TestRerunDocument_InaccessibleDocument(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
-
- insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
- insertTestDoc(t, "doc-1", "kb-1", 9, 4)
- insertTestPipelineLog(t, "log-1", "doc-1", "kb-1", "tenant-1", nil)
-
- // No user_tenant row joins "stranger" to tenant-1 and the kb is owned
- // by tenant-1, so Accessible must deny.
- svc, _ := rerunTestService(t)
- err := svc.RerunDocument(t.Context(), "stranger", "log-1", entity.JSONMap{"components": struct{}{}}, "c1")
- if err != ErrRerunDocumentNotFound {
- t.Fatalf("err = %v, want ErrRerunDocumentNotFound", err)
- }
-}
-
-func TestRerunDocument_DocumentProcessing(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
-
- insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
- insertTestDoc(t, "doc-1", "kb-1", 9, 4)
- if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("progress", 0.5).Error; err != nil {
- t.Fatalf("set progress: %v", err)
- }
- insertTestPipelineLog(t, "log-1", "doc-1", "kb-1", "tenant-1", nil)
-
- svc, _ := rerunTestService(t)
- err := svc.RerunDocument(t.Context(), "tenant-1", "log-1", entity.JSONMap{"components": struct{}{}}, "c1")
- if err == nil || !strings.Contains(err.Error(), "is processing") {
- t.Fatalf("err = %v, want 'is processing' message", err)
- }
- // The handler classifies via errors.As, so pin the typed contract.
- var processingErr *RerunDocumentProcessingError
- if !errors.As(err, &processingErr) {
- t.Fatalf("err = %T, want RerunDocumentProcessingError", err)
- }
-}
-
-func TestRerunDocument_RerunsAndPersistsDSL(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
-
- insertTestKB(t, "kb-1", "tenant-1", 1, 9, 4)
- insertTestDoc(t, "doc-1", "kb-1", 9, 4)
- // GetDocumentStorageAddress requires a non-empty document location.
- if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {
- t.Fatalf("set location: %v", err)
- }
- insertTestPipelineLog(t, "log-1", "doc-1", "kb-1", "tenant-1", entity.JSONMap{"components": map[string]interface{}{}})
-
- svc, publisher := rerunTestService(t)
- dsl := map[string]interface{}{
- "components": map[string]interface{}{"c1": map[string]interface{}{"obj": map[string]interface{}{}}},
- }
- if err := svc.RerunDocument(t.Context(), "tenant-1", "log-1", dsl, "c1"); err != nil {
- t.Fatalf("RerunDocument: %v", err)
- }
-
- // The caller's DSL map is not mutated in place.
- if _, ok := dsl["path"]; ok {
- t.Fatalf("caller dsl was mutated in place: %v", dsl)
- }
-
- // The edited DSL is persisted on the log with the rerun entry point.
- updated, err := svc.pipelineLogDAO.GetByID(t.Context(), db, "log-1")
- if err != nil {
- t.Fatalf("reload log: %v", err)
- }
- path, _ := updated.DSL["path"].([]interface{})
- if len(path) != 1 || path[0] != "c1" {
- t.Fatalf("log dsl path = %v, want [c1]", updated.DSL["path"])
- }
- if _, ok := updated.DSL["components"]; !ok {
- t.Fatalf("log dsl lost its components: %v", updated.DSL)
- }
-
- // A fresh ingestion task is enqueued for the document.
- if len(publisher.messages) != 1 {
- t.Fatalf("expected 1 published message, got %d", len(publisher.messages))
- }
- task, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, publisher.messages[0].TaskID)
- if err != nil {
- t.Fatalf("load ingestion task: %v", err)
- }
- // CreateAndEnqueue marks the task SCHEDULED once its queue message has
- // been published (the publisher here records instead of failing), so the
- // persisted row is already past CREATED by the time we reload it.
- if task.DocumentID != "doc-1" || task.DatasetID != "kb-1" || task.Status != common.SCHEDULED {
- t.Fatalf("ingestion task = %+v", task)
- }
-
- // Prior counters are cleared for the rerun.
- doc, err := svc.documentDAO.GetByID(t.Context(), db, "doc-1")
- if err != nil {
- t.Fatalf("reload doc: %v", err)
- }
- if doc.TokenNum != 0 || doc.ChunkNum != 0 {
- t.Fatalf("doc counters after rerun = token %d chunk %d, want zeros", doc.TokenNum, doc.ChunkNum)
- }
-}
-
-func TestRerunDocument_ClearsQueuedTaskPreTerminalLog(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
-
- insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
- insertTestDoc(t, "doc-1", "kb-1", 0, 0)
- if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {
- t.Fatalf("set location: %v", err)
- }
- insertTestPipelineLog(t, "log-1", "doc-1", "kb-1", "tenant-1", entity.JSONMap{"components": map[string]interface{}{}})
- insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
- queuedMsg := "Task is queued..."
- if err := dao.DB.Create(&entity.PipelineOperationLog{
- ID: "queued-log",
- DocumentID: "doc-1",
- TenantID: "tenant-1",
- KbID: "kb-1",
- ParserID: "naive",
- TaskType: "Parse",
- OperationStatus: string(entity.TaskStatusSchedule),
- ProgressMsg: &queuedMsg,
- }).Error; err != nil {
- t.Fatalf("seed queued log: %v", err)
- }
- // The clearing run owns this row, exactly as createOpenLogBestEffort
- // binds it in production.
- if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
- Update("pipeline_log_id", "queued-log").Error; err != nil {
- t.Fatalf("bind queued log: %v", err)
- }
-
- svc, _ := rerunTestService(t)
- if err := svc.RerunDocument(t.Context(), "tenant-1", "log-1", nil, "c1"); err != nil {
- t.Fatalf("RerunDocument: %v", err)
- }
- var surviving entity.PipelineOperationLog
- err := db.First(&surviving, "id = ?", "queued-log").Error
- if err == nil {
- t.Fatalf("stale queued row survived the rerun; the new run must start fresh")
- }
- if !dao.IsNotFoundErr(err) {
- t.Fatalf("load queued log: %v", err)
- }
-}
-
-// TestRerunDocument_KeepsAnotherRunsPreTerminalLog locks the ownership boundary: the
-// rerun cleanup drops only the row the cleared task was bound to, so an open
-// row belonging to another run is left alone.
-func TestRerunDocument_KeepsAnotherRunsPreTerminalLog(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
-
- insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
- insertTestDoc(t, "doc-1", "kb-1", 0, 0)
- if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {
- t.Fatalf("set location: %v", err)
- }
- insertTestPipelineLog(t, "log-1", "doc-1", "kb-1", "tenant-1", entity.JSONMap{"components": map[string]interface{}{}})
- insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
-
- queuedMsg := "Task is queued..."
- seed := func(id, docID string) {
- t.Helper()
- if err := dao.DB.Create(&entity.PipelineOperationLog{
- ID: id,
- DocumentID: docID,
- TenantID: "tenant-1",
- KbID: "kb-1",
- ParserID: "naive",
- TaskType: "Parse",
- OperationStatus: string(entity.TaskStatusSchedule),
- ProgressMsg: &queuedMsg,
- }).Error; err != nil {
- t.Fatalf("seed %s: %v", id, err)
- }
- }
- seed("own-log", "doc-1")
- seed("other-run-log", "doc-2")
- if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
- Update("pipeline_log_id", "own-log").Error; err != nil {
- t.Fatalf("bind own log: %v", err)
- }
-
- svc, _ := rerunTestService(t)
- if err := svc.RerunDocument(t.Context(), "tenant-1", "log-1", nil, "c1"); err != nil {
- t.Fatalf("RerunDocument: %v", err)
- }
-
- if err := db.First(&entity.PipelineOperationLog{}, "id = ?", "own-log").Error; !dao.IsNotFoundErr(err) {
- t.Fatalf("cleared run's own row should be deleted, got err=%v", err)
- }
- if err := db.First(&entity.PipelineOperationLog{}, "id = ?", "other-run-log").Error; err != nil {
- t.Fatalf("another run's open row was deleted by this run's cleanup: %v", err)
- }
-}
-
-func TestRerunDocument_EmptyDSLPersistsEntryPath(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
-
- insertTestKB(t, "kb-1", "tenant-1", 1, 9, 4)
- insertTestDoc(t, "doc-1", "kb-1", 9, 4)
- // GetDocumentStorageAddress requires a non-empty document location.
- if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("location", "loc-1").Error; err != nil {
- t.Fatalf("set location: %v", err)
- }
- insertTestPipelineLog(t, "log-1", "doc-1", "kb-1", "tenant-1", entity.JSONMap{"components": map[string]interface{}{}})
-
- svc, publisher := rerunTestService(t)
- if err := svc.RerunDocument(t.Context(), "tenant-1", "log-1", entity.JSONMap{}, "c1"); err != nil {
- t.Fatalf("RerunDocument: %v", err)
- }
-
- // An empty but non-nil dsl is still persisted, with the rerun entry
- // point recorded on the log row.
- updated, err := svc.pipelineLogDAO.GetByID(t.Context(), db, "log-1")
- if err != nil {
- t.Fatalf("reload log: %v", err)
- }
- path, _ := updated.DSL["path"].([]interface{})
- if len(path) != 1 || path[0] != "c1" {
- t.Fatalf("log dsl path = %v, want [c1]", updated.DSL["path"])
- }
-
- // The rerun itself is still enqueued.
- if len(publisher.messages) != 1 {
- t.Fatalf("expected 1 published message, got %d", len(publisher.messages))
- }
-}
diff --git a/test/testcases/restful_api/test_agents.py b/test/testcases/restful_api/test_agents.py
index a5cbc004e9..0b920c8f87 100644
--- a/test/testcases/restful_api/test_agents.py
+++ b/test/testcases/restful_api/test_agents.py
@@ -18,7 +18,6 @@ import json
import pytest
-
MINIMAL_DSL = {
"components": {
"begin": {
@@ -348,12 +347,3 @@ def test_agent_db_connection_validates_required_fields(rest_client):
payload = res.json()
assert payload["code"] == 101, payload
assert "required argument are missing" in payload["message"], payload
-
-
-def test_agent_rerun_requires_required_fields(rest_client):
- res = rest_client.post("/agents/rerun", json={"id": "flow-1"})
- assert res.status_code == 200
- payload = res.json()
- assert payload["code"] == 101, payload
- assert "required argument are missing" in payload["message"], payload
diff --git a/test/unit_test/api/apps/restful_apis/test_rerun_agent_authorization.py b/test/unit_test/api/apps/restful_apis/test_rerun_agent_authorization.py
deleted file mode 100644
index 1d3e6ca1de..0000000000
--- a/test/unit_test/api/apps/restful_apis/test_rerun_agent_authorization.py
+++ /dev/null
@@ -1,240 +0,0 @@
-#
-# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-"""Regression tests for pipeline rerun authorization in agent_api.rerun_agent."""
-
-import asyncio
-import importlib.util
-import sys
-from pathlib import Path
-from types import ModuleType, SimpleNamespace
-
-import pytest
-
-_MODULE_NAME = "test_rerun_agent_agent_api"
-_REPO_ROOT = Path(__file__).resolve().parents[5]
-_AGENT_API_PATH = _REPO_ROOT / "api" / "apps" / "restful_apis" / "agent_api.py"
-
-
-class _PassthroughManager:
- def route(self, *_args, **_kwargs):
- return lambda func: func
-
-
-def _stub(monkeypatch, name, **attrs):
- mod = ModuleType(name)
- for key, value in attrs.items():
- setattr(mod, key, value)
- monkeypatch.setitem(sys.modules, name, mod)
- return mod
-
-
-def _load_agent_api_for_rerun(monkeypatch, *, documents_info, accessible):
- monkeypatch.delitem(sys.modules, _MODULE_NAME, raising=False)
-
- destructive_calls = {"clear": 0, "update": 0, "delete_tasks": 0, "queue": 0, "index_delete": 0}
-
- acc_fn = accessible if callable(accessible) else (lambda *_a, **_k: accessible)
-
- _stub(
- monkeypatch,
- "api.apps",
- AUTH_JWT="jwt",
- AUTH_API="api",
- AUTH_BETA="beta",
- QuartAuthUnauthorized=Exception,
- current_user=SimpleNamespace(id="user-owner", is_superuser=True),
- login_required=lambda func=None, **_kwargs: (lambda f: f) if func is None else func,
- )
- _stub(monkeypatch, "api.apps.services.canvas_replica_service", CanvasReplicaService=SimpleNamespace())
- _stub(monkeypatch, "api.db", CanvasCategory=SimpleNamespace())
-
- task_model = SimpleNamespace()
- task_model.doc_id = "doc_id_field"
- _stub(monkeypatch, "api.db.db_models", Task=task_model)
-
- _stub(
- monkeypatch,
- "api.db.services.api_service",
- API4ConversationService=SimpleNamespace(
- get_by_id=lambda *_a, **_k: (False, None),
- save=lambda **_kwargs: True,
- delete_by_id=lambda *_a, **_k: True,
- query=lambda **_kwargs: [],
- ),
- )
- _stub(
- monkeypatch,
- "api.db.services.canvas_service",
- CanvasTemplateService=SimpleNamespace(),
- UserCanvasService=SimpleNamespace(accessible=lambda *_a, **_k: True, query=lambda **_kwargs: []),
- completion=lambda *_a, **_k: None,
- completion_openai=lambda *_a, **_k: None,
- )
- _stub(
- monkeypatch,
- "api.db.services.document_service",
- DocumentService=SimpleNamespace(
- accessible=acc_fn,
- clear_chunk_num_when_rerun=lambda _doc_id: destructive_calls.__setitem__("clear", destructive_calls["clear"] + 1),
- update_by_id=lambda *_a, **_k: destructive_calls.__setitem__("update", destructive_calls["update"] + 1) or True,
- ),
- )
- _stub(monkeypatch, "api.db.services.file_service", FileService=SimpleNamespace())
- _stub(monkeypatch, "api.db.services.knowledgebase_service", KnowledgebaseService=SimpleNamespace())
-
- def _update_log(*_a, **_k):
- destructive_calls["update_log"] = True
-
- _stub(
- monkeypatch,
- "api.db.services.pipeline_operation_log_service",
- PipelineOperationLogService=SimpleNamespace(
- get_documents_info=lambda _log_id: documents_info,
- update_by_id=_update_log,
- ),
- )
-
- class _TaskService:
- @staticmethod
- def filter_delete(*_a, **_k):
- destructive_calls["delete_tasks"] += 1
-
- _stub(
- monkeypatch,
- "api.db.services.task_service",
- CANVAS_DEBUG_DOC_ID="",
- TaskService=_TaskService,
- queue_dataflow=lambda *_a, **_k: destructive_calls.__setitem__("queue", destructive_calls["queue"] + 1),
- )
- _stub(monkeypatch, "api.db.services.user_service", TenantService=SimpleNamespace(), UserService=SimpleNamespace(get_by_id=lambda *_a, **_k: (False, None)))
- _stub(monkeypatch, "api.db.services.user_canvas_version", UserCanvasVersionService=SimpleNamespace())
-
- request_body = {"id": "log-victim", "component_id": "Parser:0", "dsl": {"path": [], "components": {}}}
-
- _stub(
- monkeypatch,
- "api.utils.api_utils",
- add_tenant_id_to_kwargs=lambda func: func,
- check_duplicate_ids=lambda ids, _kind="item": (ids, []),
- get_data_error_result=lambda message="Sorry": {"code": 102, "message": message, "data": None},
- get_error_data_result=lambda message="Sorry", code=102: {"code": code, "message": message, "data": None},
- get_json_result=lambda code=0, message="", data=None: {"code": code, "message": message, "data": data},
- get_result=lambda **kwargs: kwargs,
- get_request_json=lambda: _awaitable(request_body),
- server_error_response=lambda exc: {"code": 500, "message": str(exc)},
- validate_request=lambda *_a, **_k: lambda func: func,
- )
-
- doc_store = SimpleNamespace(
- index_exist=lambda *_a, **_k: True,
- delete=lambda *_a, **_k: destructive_calls.__setitem__("index_delete", destructive_calls["index_delete"] + 1),
- )
- common_settings = _stub(
- monkeypatch,
- "common.settings",
- retriever=SimpleNamespace(),
- kg_retriever=SimpleNamespace(),
- docStoreConn=doc_store,
- )
- monkeypatch.setitem(sys.modules, "common", SimpleNamespace(settings=common_settings))
- _stub(monkeypatch, "common.ssrf_guard", assert_host_is_safe=lambda *_a, **_k: None)
- _stub(monkeypatch, "common.constants", RetCode=SimpleNamespace(OPERATING_ERROR=109))
- _stub(
- monkeypatch,
- "common.misc_utils",
- get_uuid=lambda: "task-uuid",
- thread_pool_exec=lambda fn, *a, **k: fn(*a, **k),
- )
-
- rag_nlp_mod = ModuleType("rag.nlp")
- rag_nlp_mod.search = SimpleNamespace(index_name=lambda _tenant_id: "idx")
- monkeypatch.setitem(sys.modules, "rag.nlp", rag_nlp_mod)
-
- spec = importlib.util.spec_from_file_location(_MODULE_NAME, _AGENT_API_PATH)
- module = importlib.util.module_from_spec(spec)
- module.manager = _PassthroughManager()
- monkeypatch.setitem(sys.modules, _MODULE_NAME, module)
- spec.loader.exec_module(module)
- module._destructive_calls = destructive_calls
- module._request_body = request_body
- return module
-
-
-def _awaitable(value):
- async def _co():
- return value
-
- return _co()
-
-
-class TestRerunAgentAuthorization:
- def test_cross_tenant_log_id_is_rejected(self, monkeypatch):
- victim_doc = {
- "id": "doc-victim",
- "name": "secret.pdf",
- "progress": 0,
- "kb_id": "kb-victim",
- }
- module = _load_agent_api_for_rerun(
- monkeypatch,
- documents_info=[victim_doc],
- accessible=lambda _doc_id, user_id: user_id == "user-owner",
- )
-
- result = asyncio.run(module.rerun_agent(tenant_id="user-attacker"))
-
- assert result == {"code": 102, "message": "Document not found.", "data": None}
- assert module._destructive_calls["clear"] == 0
- assert module._destructive_calls["queue"] == 0
- assert module._destructive_calls["index_delete"] == 0
-
- def test_missing_log_returns_same_message(self, monkeypatch):
- module = _load_agent_api_for_rerun(
- monkeypatch,
- documents_info=[],
- accessible=lambda *_a, **_k: True,
- )
-
- missing = asyncio.run(module.rerun_agent(tenant_id="user-owner"))
-
- module = _load_agent_api_for_rerun(
- monkeypatch,
- documents_info=[{"id": "doc-victim", "name": "x.pdf", "progress": 0, "kb_id": "kb-victim"}],
- accessible=lambda *_a, **_k: False,
- )
- unauthorized = asyncio.run(module.rerun_agent(tenant_id="user-owner"))
-
- assert missing["message"] == unauthorized["message"] == "Document not found."
-
- def test_authorized_rerun_proceeds(self, monkeypatch):
- victim_doc = {
- "id": "doc-owner",
- "name": "mine.pdf",
- "progress": 0,
- "kb_id": "kb-owner",
- }
- module = _load_agent_api_for_rerun(
- monkeypatch,
- documents_info=[victim_doc],
- accessible=lambda *_a, **_k: True,
- )
-
- result = asyncio.run(module.rerun_agent(tenant_id="user-owner"))
-
- assert result == {"code": 0, "message": "", "data": True}
- assert module._destructive_calls["clear"] == 1
- assert module._destructive_calls["queue"] == 1
diff --git a/web/src/pages/dataflow-result/chunker.tsx b/web/src/pages/dataflow-result/chunker.tsx
index c11498dda0..088e5d9918 100644
--- a/web/src/pages/dataflow-result/chunker.tsx
+++ b/web/src/pages/dataflow-result/chunker.tsx
@@ -16,7 +16,6 @@ import ChunkCard from './components/chunk-card';
import CreatingModal from './components/chunk-creating-modal';
import ChunkResultBar from './components/chunk-result-bar';
import CheckboxSets from './components/chunk-result-bar/checkbox-sets';
-import RerunButton from './components/rerun-button';
import {
useChangeChunkTextMode,
useDeleteChunkByIds,
@@ -30,7 +29,7 @@ interface IProps {
step?: TimelineNode;
}
const ChunkerContainer = (props: IProps) => {
- const { isChange, setIsChange, step } = props;
+ const { setIsChange } = props;
const [selectedChunkIds, setSelectedChunkIds] = useState<string[]>([]);
const { t } = useTranslation();
@@ -142,16 +141,8 @@ const ChunkerContainer = (props: IProps) => {
onChunkUpdatingOk(e);
};
- const handleReRunFunc = () => {
- setIsChange(false);
- };
return (
<div className="w-full h-full">
- {isChange && (
- <div className=" absolute top-2 right-6">
- <RerunButton step={step} onRerun={handleReRunFunc} />
- </div>
- )}
<div className={classNames('flex flex-col w-full')}>
<Spin spinning={loading} className={styles.spin} size="large">
<div className="h-[50px] flex flex-row justify-between items-end pb-[5px]">
diff --git a/web/src/pages/dataflow-result/hooks.ts b/web/src/pages/dataflow-result/hooks.ts
index e866c8d497..6b866048db 100644
--- a/web/src/pages/dataflow-result/hooks.ts
+++ b/web/src/pages/dataflow-result/hooks.ts
@@ -178,49 +178,6 @@ export const useUpdateChunk = () => {
};
};
-export const useRerunDataflow = ({
- data,
-}: {
- data: IPipelineFileLogDetail;
-}) => {
- const [isChange, setIsChange] = useState(false);
-
- const { mutateAsync: handleReRunFunc, isPending: loading } = useMutation({
- mutationKey: ['pipelineRerun', data],
- mutationFn: async (newData: { value: IDslComponent; key: string }) => {
- const newDsl = {
- ...data.dsl,
- components: {
- ...data.dsl.components,
- [newData.key]: newData.value,
- },
- };
-
- // this Data provided to the interface
- const params = {
- id: data.id,
- dsl: newDsl,
- component_id: newData.key,
- };
- const { data: result } = await kbService.pipelineRerun(params);
- if (result.code === 0) {
- message.success(t('message.operated'));
- // queryClient.invalidateQueries({
- // queryKey: [type],
- // });
- }
- return result;
- },
- });
-
- return {
- loading,
- isChange,
- setIsChange,
- handleReRunFunc,
- };
-};
-
export const useTimelineDataFlow = (data: IPipelineFileLogDetail) => {
const timelineNodes: TimelineNode[] = useMemo(() => {
const nodes: Array<
diff --git a/web/src/pages/dataflow-result/index.tsx b/web/src/pages/dataflow-result/index.tsx
index 1f8fd9eee0..eaf4c2886f 100644
--- a/web/src/pages/dataflow-result/index.tsx
+++ b/web/src/pages/dataflow-result/index.tsx
@@ -8,7 +8,6 @@ import {
useGetChunkHighlights,
useGetPipelineResultSearchParams,
useHandleChunkCardClick,
- useRerunDataflow,
useSummaryInfo,
useTimelineDataFlow,
} from './hooks';
@@ -20,7 +19,6 @@ import { TimelineNode } from '@/components/originui/timeline';
import { PageHeader } from '@/components/page-header';
import Spotlight from '@/components/spotlight';
import { Button } from '@/components/ui/button';
-import { Modal } from '@/components/ui/modal/modal';
import { AgentCategory, AgentQuery } from '@/constants/agent';
import { Images } from '@/constants/common';
import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
@@ -110,56 +108,8 @@ const DataflowResult = () => {
return 'unknown';
}, [documentExtension, documentInfo?.name, documentInfo?.type, isAgent]);
- const {
- handleReRunFunc,
- isChange,
- setIsChange,
- loading: reRunLoading,
- } = useRerunDataflow({
- data: dataset,
- });
-
- const handleStepChange = (id: number | string, step: TimelineNode) => {
- if (isChange) {
- Modal.show({
- visible: true,
- className: '!w-[560px]',
- title: t('dataflowParser.changeStepModalTitle'),
- children: (
- <div
- className="text-sm text-text-secondary"
- dangerouslySetInnerHTML={{
- __html: t('dataflowParser.changeStepModalContent', {
- step: step?.title,
- }),
- }}
- ></div>
- ),
- onVisibleChange: () => {
- Modal.destroy();
- },
- footer: (
- <div className="flex justify-end gap-2">
- <Button variant={'outline'} onClick={() => Modal.destroy()}>
- {t('dataflowParser.changeStepModalCancelText')}
- </Button>
- <Button
- variant={'secondary'}
- className="!bg-state-error text-text-primary"
- onClick={() => {
- Modal.destroy();
- setActiveStepId(id);
- setIsChange(false);
- }}
- >
- {t('dataflowParser.changeStepModalConfirmText')}
- </Button>
- </div>
- ),
- });
- } else {
- setActiveStepId(id);
- }
+ const handleStepChange = (id: number | string) => {
+ setActiveStepId(id);
};
const { type } = useGetKnowledgeSearchParams();
@@ -232,9 +182,6 @@ const DataflowResult = () => {
currentTimeNode?.type === TimelineNodeType.contextGenerator) && (
<ParserContainer
isReadonly={isReadOnly}
- isChange={isChange}
- reRunLoading={reRunLoading}
- setIsChange={setIsChange}
step={currentTimeNode as TimelineNode}
data={
currentTimeNode.detail as {
@@ -244,7 +191,6 @@ const DataflowResult = () => {
}
summaryInfo={summaryInfo}
clickChunk={handleChunkCardClick}
- reRunFunc={handleReRunFunc}
/>
)}
{/* )} */}
diff --git a/web/src/pages/dataflow-result/parser.tsx b/web/src/pages/dataflow-result/parser.tsx
index 4995ea9e0e..c401c161ca 100644
--- a/web/src/pages/dataflow-result/parser.tsx
+++ b/web/src/pages/dataflow-result/parser.tsx
@@ -4,40 +4,23 @@ import { cn } from '@/lib/utils';
import classNames from 'classnames';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
-import ChunkResultBar from './components/chunk-result-bar';
-import CheckboxSets from './components/chunk-result-bar/checkbox-sets';
import FormatPreserEditor from './components/parse-editer';
-import RerunButton from './components/rerun-button';
import { TimelineNodeType } from './constant';
import { useChangeChunkTextMode } from './hooks';
import { IChunk, IDslComponent } from './interface';
interface IProps {
isReadonly: boolean;
- isChange: boolean;
- setIsChange: (isChange: boolean) => void;
step?: TimelineNode;
data: { value: IDslComponent; key: string };
- reRunLoading: boolean;
clickChunk: (chunk: IChunk) => void;
summaryInfo: string;
- reRunFunc: (data: { value: IDslComponent; key: string }) => void;
}
const ParserContainer = (props: IProps) => {
- const {
- isChange,
- setIsChange,
- step,
- data,
- reRunFunc,
- reRunLoading,
- clickChunk,
- isReadonly,
- summaryInfo,
- } = props;
+ const { step, data, clickChunk, isReadonly, summaryInfo } = props;
const { t } = useTranslation();
const [selectedChunkIds, setSelectedChunkIds] = useState<string[]>([]);
const [newChunkIndex, setNewChunkIndex] = useState<number | undefined>();
- const { changeChunkTextMode, textMode } = useChangeChunkTextMode();
+ const { textMode } = useChangeChunkTextMode();
const initialValue = useMemo(() => {
const outputs = data?.value?.obj?.params?.outputs;
const key = outputs?.output_format?.value;
@@ -66,48 +49,11 @@ const ParserContainer = (props: IProps) => {
const handleSave = (newContent: any) => {
console.log('newContent-change-->', newContent, initialValue);
if (JSON.stringify(newContent) !== JSON.stringify(initialValue)) {
- setIsChange(true);
setInitialText(newContent);
- } else {
- setIsChange(false);
}
// Here, the API is called to send newContent to the backend
};
- const handleReRunFunc = useCallback(() => {
- const newData: { value: IDslComponent; key: string } = {
- ...data,
- value: {
- ...data.value,
- obj: {
- ...data.value.obj,
- params: {
- ...(data.value?.obj?.params || {}),
- outputs: {
- ...(data.value?.obj?.params?.outputs || {}),
- [initialText.key]: {
- type: initialText.type,
- value: initialText.value,
- },
- },
- },
- },
- },
- };
- reRunFunc(newData);
- setIsChange(false);
- }, [data, initialText, reRunFunc, setIsChange]);
-
- const handleRemoveChunk = useCallback(async () => {
- if (selectedChunkIds.length > 0) {
- initialText.value = initialText.value.filter(
- (_item: any, index: number) => !selectedChunkIds.includes(index + ''),
- );
- setIsChange(true);
- setSelectedChunkIds([]);
- }
- }, [selectedChunkIds, initialText, setIsChange]);
-
const handleCheckboxClick = useCallback(
(id: string | number, checked: boolean) => {
setSelectedChunkIds((prev) => {
@@ -121,31 +67,10 @@ const ParserContainer = (props: IProps) => {
[],
);
- const selectAllChunk = useCallback(
- (checked: boolean) => {
- setSelectedChunkIds(
- checked ? initialText.value.map((_x: any, index: number) => index) : [],
- );
- },
- [initialText.value],
- );
-
const isChunck =
step?.type === TimelineNodeType.tokenChunker ||
step?.type === TimelineNodeType.titleChunker;
- const handleCreateChunk = useCallback(
- (text: string) => {
- const newText = [...initialText.value, { text: text || ' ' }];
- setNewChunkIndex(newText.length - 1);
- setInitialText({
- ...initialText,
- value: newText as any,
- });
- },
- [initialText],
- );
-
useEffect(() => {
if (newChunkIndex === undefined) return;
const timer = setTimeout(() => setNewChunkIndex(undefined), 3000);
@@ -154,15 +79,6 @@ const ParserContainer = (props: IProps) => {
return (
<>
- {isChange && !isReadonly && (
- <div className=" absolute top-2 right-6">
- <RerunButton
- step={step}
- onRerun={handleReRunFunc}
- loading={reRunLoading}
- />
- </div>
- )}
<div className={classNames('flex flex-col w-full')}>
{/* <Spin spinning={false} className="" size="large"> */}
<div className="h-[50px] flex flex-col justify-end pb-[5px]">
@@ -187,24 +103,6 @@ const ParserContainer = (props: IProps) => {
)}
</div>
- {isChunck && (
- <div className="pt-[5px] pb-[5px] flex justify-between items-center">
- {!isReadonly && (
- <CheckboxSets
- selectAllChunk={selectAllChunk}
- removeChunk={handleRemoveChunk}
- checked={selectedChunkIds.length === initialText.value.length}
- selectedChunkIds={selectedChunkIds}
- />
- )}
- <ChunkResultBar
- isReadonly={isReadonly}
- changeChunkTextMode={changeChunkTextMode}
- createChunk={handleCreateChunk}
- />
- </div>
- )}
-
<div
className={cn(
' border rounded-lg p-[20px] box-border w-[calc(100%-20px)] overflow-auto scrollbar-auto',
diff --git a/web/src/pages/dataset/dataset-overview/overview-table.tsx b/web/src/pages/dataset/dataset-overview/overview-table.tsx
index 5e7f832d9e..d127a60dc9 100644
--- a/web/src/pages/dataset/dataset-overview/overview-table.tsx
+++ b/web/src/pages/dataset/dataset-overview/overview-table.tsx
@@ -24,10 +24,7 @@ import {
RunningStatusMap,
} from '@/constants/knowledge';
import { useTranslate } from '@/hooks/common-hooks';
-import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
import { cn } from '@/lib/utils';
-import { PipelineResultSearchParams } from '@/pages/dataflow-result/constant';
-import { NavigateToDataflowResultProps } from '@/pages/dataflow-result/interface';
import { useDataSourceInfo } from '@/pages/user-setting/data-source/constant';
import { IDataSourceInfoMap } from '@/pages/user-setting/data-source/interface';
import { formatDate, formatSecondsToHumanReadable } from '@/utils/date';
@@ -44,9 +41,8 @@ import {
useReactTable,
} from '@tanstack/react-table';
import { TFunction } from 'i18next';
-import { ArrowUpDown, ClipboardList, Eye, MonitorUp } from 'lucide-react';
+import { ArrowUpDown, Eye, MonitorUp } from 'lucide-react';
import { FC, useMemo, useState } from 'react';
-import { useParams } from 'react-router';
import { RunningStatus } from '../dataset/constant';
import ProcessLogModal, { ILogInfo } from '../process-log-modal';
import { LogTabs } from './dataset-common';
@@ -55,10 +51,6 @@ import { DocumentLog, FileLogsTableProps, IFileLogItem } from './interface';
export const getFileLogsTableColumns = (
t: TFunction<'translation', string>,
showLog: (row: Row<IFileLogItem & DocumentLog>, active: LogTabs) => void,
- knowledgeId: string,
- navigateToDataflowResult: (
- props: NavigateToDataflowResultProps,
- ) => () => void,
dataSourceInfo: IDataSourceInfoMap,
) => {
// const { t } = useTranslate('knowledgeDetails');
@@ -214,23 +206,6 @@ export const getFileLogsTableColumns = (
>
<Eye />
</Button>
- {row.original.pipeline_id && (
- <Button
- variant="ghost"
- size="icon-sm"
- onClick={navigateToDataflowResult({
- id: row.original.id,
- [PipelineResultSearchParams.KnowledgeId]:
- row.original.kb_id || knowledgeId,
- [PipelineResultSearchParams.DocumentId]:
- row.original.document_id,
- [PipelineResultSearchParams.IsReadOnly]: 'false',
- [PipelineResultSearchParams.Type]: 'dataflow',
- })}
- >
- <ClipboardList />
- </Button>
- )}
</div>
),
},
@@ -368,9 +343,7 @@ const FileLogsTable: FC<FileLogsTableProps> = ({
const { t } = useTranslate('knowledgeDetails');
const { t: tDatasetOverview } = useTranslate('datasetOverview');
const [isModalVisible, setIsModalVisible] = useState(false);
- const { navigateToDataflowResult } = useNavigatePage();
const [logInfo, setLogInfo] = useState<IFileLogItem>();
- const knowledgeId = useParams().id;
const showLog = (row: Row<IFileLogItem & DocumentLog>) => {
const logDetail = {
taskId: row.original?.dsl?.task_id,
@@ -391,13 +364,7 @@ const FileLogsTable: FC<FileLogsTableProps> = ({
const { dataSourceInfo } = useDataSourceInfo();
const columns = useMemo(() => {
return active === LogTabs.FILE_LOGS
- ? getFileLogsTableColumns(
- t,
- showLog,
- knowledgeId || '',
- navigateToDataflowResult,
- dataSourceInfo,
- )
+ ? getFileLogsTableColumns(t, showLog, dataSourceInfo)
: getDatasetLogsTableColumns(t, showLog);
}, [active, t]);
diff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.ts
index 2c6339e402..cb5593f61d 100644
--- a/web/src/services/knowledge-service.ts
+++ b/web/src/services/knowledge-service.ts
@@ -88,10 +88,6 @@ const methods = {
url: listPipelines,
method: 'get',
},
- pipelineRerun: {
- url: api.pipelineRerun,
- method: 'post',
- },
};
const baseKbService = registerServer<keyof typeof methods>(methods, request);
diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts
index 785c3e9a5a..d2c074cc7d 100644
--- a/web/src/utils/api.ts
+++ b/web/src/utils/api.ts
@@ -236,7 +236,6 @@ export default {
`${restAPIv1}/datasets/${datasetId}/compilation/status`,
unbindPipelineTask: (datasetId: string, indexType: string, wipe?: boolean) =>
`${restAPIv1}/datasets/${datasetId}/${indexType.toLowerCase()}${wipe === false ? '?wipe=false' : ''}`,
- pipelineRerun: `${restAPIv1}/agents/rerun`,
getMetaData: (datasetId: string) =>
`${restAPIv1}/datasets/${datasetId}/metadata/summary`,
updateDocumentsMetadata: (datasetId: string) =>