RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
package task
import (
"context"
"encoding/json"
"errors"
"reflect"
"sync"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"ragflow/internal/agent/runtime"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/ingestion"
pipelinepkg "ragflow/internal/ingestion/pipeline"
indexdoc "ragflow/internal/ingestion/task/indexdoc"
)
// =============================================================================
// Test helpers
// =============================================================================
func strPtr(s string) *string { return &s }
func TestProcessOutputPurgesRunStateOnSuccessfulNoOutput(t *testing.T) {
for _, tc := range []struct {
name string
output map[string]any
}{
{name: "nil output", output: nil},
{name: "empty output", output: map[string]any{}},
} {
t.Run(tc.name, func(t *testing.T) {
taskCtx := &TaskContext{
Doc: entity.Document{ID: "doc-1"},
IngestionTask: &entity.IngestionTask{ID: "task-1"},
}
var purgedTaskID string
exec := &PipelineExecutor{
taskCtx: taskCtx,
purgeTaskState: func(ctx context.Context, taskID string) error {
purgedTaskID = taskID
return nil
},
}
if _, err := exec.processOutput(context.Background(), tc.output, time.Now()); err != nil {
t.Fatalf("processOutput: %v", err)
}
if purgedTaskID != "task-1" {
t.Fatalf("purged task id = %q, want task-1", purgedTaskID)
}
})
}
}
// TestMarkCompiledProductsHidden verifies the pipeline caller stages ONLY the
// wiki variant's per-document rows as available_int=0 (they are intermediate
// state for the consumer's merged pages). Tree / structure (page_index) /
// mindmap rows are the final per-document products — Python writes them visible
// at compile time — so they keep the index default available_int=1, as do
// ordinary source chunks.
func TestMarkCompiledProductsHidden(t *testing.T) {
chunks := []map[string]any{
{"id": "src-1", "content_with_weight": "ordinary source chunk"},
{"id": "tree-1", "compile_kwd": "tree", "content_with_weight": "tree node"},
{"id": "struct-1", "compile_kwd": "page_index", "content_with_weight": "entity A"},
{"id": "wiki-1", "compile_kwd": "wiki_page", "content_with_weight": "page X"},
{"id": "wiki-2", "compile_kwd": "wiki_section", "content_with_weight": "section X"},
{"id": "src-2", "content_with_weight": "another source chunk"},
}
markCompiledProductsHidden(chunks)
if v, ok := chunks[0]["available_int"]; ok {
t.Fatalf("ordinary source chunk should keep default available_int, got %v", v)
}
if v, ok := chunks[1]["available_int"]; ok {
t.Fatalf("tree row should stay visible (Python parity), got available_int=%v", v)
}
if v, ok := chunks[2]["available_int"]; ok {
t.Fatalf("page_index row should stay visible (Python parity), got available_int=%v", v)
}
if chunks[3]["available_int"] != 0 {
t.Fatalf("wiki page staging row should be available_int=0, got %v", chunks[3]["available_int"])
}
if chunks[4]["available_int"] != 0 {
t.Fatalf("wiki section staging row should be available_int=0, got %v", chunks[4]["available_int"])
}
if v, ok := chunks[5]["available_int"]; ok {
t.Fatalf("source chunk without compile_kwd should keep default available_int, got %v", v)
}
}
func TestWikiActiveStatesDecodeCheckpointValues(t *testing.T) {
states, err := wikiActiveStates(map[string]any{
"wiki_active_map_states": []any{map[string]any{
"key": "state-1", "tenant_id": "tenant-1", "dataset_id": "kb-1", "document_id": "doc-1", "payload": `{"plan":[]}`,
}},
})
if err != nil {
t.Fatalf("wikiActiveStates failed: %v", err)
}
if len(states) != 1 || states[0].Key != "state-1" || string(states[0].Payload) != `{"plan":[]}` {
t.Fatalf("decoded states = %#v", states)
}
}
// TestApplyDocumentAvailability verifies disabled documents (status=0) force
// every row — source chunks AND compiled products — to available_int=0
// (matching Python's doc_id-scoped availability toggle). Wiki staging rows are
// already 0 from markCompiledProductsHidden; the stamp is a no-op for them.
func TestApplyDocumentAvailability(t *testing.T) {
chunks := []map[string]any{
{"id": "src-1", "content_with_weight": "ordinary source chunk"},
{"id": "tree-1", "compile_kwd": "tree", "content_with_weight": "tree node"},
{"id": "wiki-1", "compile_kwd": "wiki_page", "content_with_weight": "page X", "available_int": 0},
{"id": "src-2", "content_with_weight": "another source chunk"},
}
markCompiledProductsHidden(chunks)
applyDocumentAvailability(chunks, strPtr("0"))
if chunks[0]["available_int"] != 0 {
t.Fatalf("disabled doc source chunk should be available_int=0, got %v", chunks[0]["available_int"])
}
if chunks[1]["available_int"] != 0 {
t.Fatalf("disabled doc compiled row should be available_int=0, got %v", chunks[1]["available_int"])
}
if chunks[2]["available_int"] != 0 {
t.Fatalf("wiki staging row should stay available_int=0, got %v", chunks[2]["available_int"])
}
if chunks[3]["available_int"] != 0 {
t.Fatalf("disabled doc source chunk should be available_int=0, got %v", chunks[3]["available_int"])
}
enabled := []map[string]any{
{"id": "src-3", "content_with_weight": "enabled source"},
}
applyDocumentAvailability(enabled, strPtr("1"))
if v, ok := enabled[0]["available_int"]; ok {
t.Fatalf("enabled doc should keep default available_int, got %v", v)
}
applyDocumentAvailability(enabled, nil)
if v, ok := enabled[0]["available_int"]; ok {
t.Fatalf("nil status should keep default available_int, got %v", v)
}
}
func makeTaskCtx() *TaskContext {
return &TaskContext{
IngestionTask: &entity.IngestionTask{
ID: "task-1",
DocumentID: "doc-1",
},
Doc: entity.Document{
ID: "doc-1",
KbID: "kb-1",
Name: strPtr("test-doc.pdf"),
Suffix: ".pdf",
Type: "pdf",
},
KB: entity.Knowledgebase{
ID: "kb-1",
TenantID: "tenant-1",
EmbdID: "embd-1",
},
Tenant: entity.Tenant{
ID: "tenant-1",
},
}
}
func setupPipelineExecutorTestDB(t *testing.T) func() {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.AutoMigrate(&entity.UserCanvas{}, &entity.PipelineOperationLog{}, &entity.Document{}, &entity.IngestionTask{}); err != nil {
t.Fatalf("auto-migrate sqlite: %v", err)
}
origDB := dao.DB
dao.DB = db
return func() { dao.DB = origDB }
}
// testPipelineDeps builds the dependency bundle the package tests use: the
// swapped-in test database (see setupPipelineExecutorDB) and no engine —
// executor tests stub the index writer with WithInsertFunc, and the
// engine-touching paths guard a nil engine.
func testPipelineDeps() *ingestion.Deps {
return &ingestion.Deps{DB: dao.DB}
}
func mustNewPipelineExecutor(t *testing.T, taskCtx *TaskContext, canvasID string, docBulkSize int) *PipelineExecutor {
t.Helper()
svc, err := NewPipelineExecutor(taskCtx, canvasID, docBulkSize, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor: %v", err)
}
return svc
}
// =============================================================================
// NewPipelineExecutor — constructor
// =============================================================================
func TestNewPipelineExecutor_Basic(t *testing.T) {
svc, err := NewPipelineExecutor(makeTaskCtx(), "flow-1", 0, testPipelineDeps())
if err != nil {
t.Fatalf("NewPipelineExecutor: %v", err)
}
if svc == nil {
t.Fatal("NewPipelineExecutor returned nil")
}
if svc.taskCtx == nil {
t.Error("taskCtx should not be nil")
}
if svc.indexWriter == nil || svc.loadDSLFunc == nil || svc.runPipelineFunc == nil {
t.Fatal("expected production dependencies to be fully initialized")
}
}
func TestNewPipelineExecutor_RejectsNilTaskContext(t *testing.T) {
_, err := NewPipelineExecutor(nil, "flow-1", 0, testPipelineDeps())
if err == nil {
t.Fatal("expected error for nil task context")
}
}
func TestNewPipelineExecutor_RejectsEmptyCanvasID(t *testing.T) {
_, err := NewPipelineExecutor(makeTaskCtx(), "", 0, testPipelineDeps())
if err == nil {
t.Fatal("expected error for empty canvas id")
}
}
func TestNewPipelineExecutor_RejectsIncompleteTaskContext(t *testing.T) {
tests := []struct {
name string
mutate func(*TaskContext)
}{
{name: "missing doc id", mutate: func(ctx *TaskContext) { ctx.Doc.ID = "" }},
{name: "missing kb id", mutate: func(ctx *TaskContext) { ctx.Doc.KbID = "" }},
{name: "missing doc name", mutate: func(ctx *TaskContext) { ctx.Doc.Name = nil }},
{name: "missing tenant id", mutate: func(ctx *TaskContext) { ctx.Tenant.ID = "" }},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := makeTaskCtx()
tt.mutate(ctx)
_, err := NewPipelineExecutor(ctx, "flow-1", 0, testPipelineDeps())
if err == nil {
t.Fatal("expected validation error")
}
})
}
}
// TestNewPipelineExecutor_AcceptsDebugTaskContext verifies the canvas-debug
// (dry-run) contract: a TaskContext with an empty KB.ID is valid because debug
// mode carries no knowledgebase. KB.ID == "" never occurs in production
// ingestion, which always supplies a KB.
func TestNewPipelineExecutor_AcceptsDebugTaskContext(t *testing.T) {
ctx := makeTaskCtx()
ctx.KB = entity.Knowledgebase{ID: ""}
ctx.Doc.KbID = ""
if _, err := NewPipelineExecutor(ctx, "flow-1", 0, testPipelineDeps()); err != nil {
t.Fatalf("debug TaskContext rejected: %v", err)
}
}
func TestNewPipelineExecutor_DocBulkSize(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 128)
if svc.docBulkSize != 128 {
t.Errorf("docBulkSize = %d, want 128", svc.docBulkSize)
}
}
func TestNewPipelineExecutor_CanvasID(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "my-flow-id", 0)
if svc.canvasID != "my-flow-id" {
t.Errorf("canvasID = %q, want %q", svc.canvasID, "my-flow-id")
}
}
func TestKB_Doc_Tenant_Accessors(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0)
if svc.KB().ID != "kb-1" {
t.Errorf("KB().ID = %q, want \"kb-1\"", svc.KB().ID)
}
if svc.Doc().ID != "doc-1" {
t.Errorf("Doc().ID = %q, want \"doc-1\"", svc.Doc().ID)
}
if svc.Tenant().ID != "tenant-1" {
t.Errorf("Tenant().ID = %q, want \"tenant-1\"", svc.Tenant().ID)
}
}
// =============================================================================
// processChunks
// =============================================================================
func TestPipelineExecutor_ProcessChunks_WrapsProcessChunksForPipeline(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0)
chunks := []map[string]any{{"text": "hello world"}}
meta, err := indexdoc.ProcessChunksForPipeline(chunks, svc.taskCtx.Doc.ID, *svc.taskCtx.Doc.Name, time.Now())
if err != nil {
t.Fatalf("ProcessChunksForPipeline: %v", err)
}
// Verify the wrapper method works correctly and chunks are processed
if chunks[0]["doc_id"] != "doc-1" {
t.Errorf("doc_id = %q, want \"doc-1\"", chunks[0]["doc_id"])
}
if meta != nil {
// No need to verify the detailed content of meta as ProcessChunksForPipeline already has comprehensive tests
}
}
// =============================================================================
// insertChunks
// =============================================================================
func TestInsertChunks_EmptyChunks(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).WithInsertFunc(
func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
return nil, nil
},
)
ctx := t.Context()
err := svc.indexWriter.Write(ctx, nil)
if err != nil {
t.Errorf("expected no error for nil chunks, got %v", err)
}
}
func TestInsertChunks_BaseNameAndDatasetID(t *testing.T) {
var capturedBaseName, capturedDatasetID string
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).WithInsertFunc(
func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
capturedBaseName = baseName
capturedDatasetID = datasetID
return nil, nil
},
)
ctx := t.Context()
chunks := []map[string]any{{"text": "hello"}}
err := svc.indexWriter.Write(ctx, chunks)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if capturedBaseName != "ragflow_tenant-1" {
t.Errorf("baseName = %q, want \"ragflow_tenant-1\"", capturedBaseName)
}
if capturedDatasetID != "kb-1" {
t.Errorf("datasetID = %q, want \"kb-1\"", capturedDatasetID)
}
}
func TestRecordPipelineLog_SharedWriterTerminalWithoutDSL(t *testing.T) {
cleanup := setupPipelineExecutorTestDB(t)
defer cleanup()
if err := dao.DB.Create(&entity.PipelineOperationLog{
ID: "run-1",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
OperationStatus: "5",
}).Error; err != nil {
t.Fatalf("seed pipeline log: %v", err)
}
docName := "terminal.pdf"
if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
TenantID: "tenant-1",
KbID: "kb-1",
DocumentID: "doc-1",
Status: "3",
Document: entity.Document{
ID: "doc-1",
KbID: "kb-1",
ParserID: "naive",
ParserConfig: entity.JSONMap{},
SourceType: "local",
Type: "pdf",
Name: &docName,
Suffix: ".pdf",
},
PipelineLogID: "run-1",
}); err != nil {
t.Fatalf("RecordPipelineLog: %v", err)
}
var log entity.PipelineOperationLog
if err := dao.DB.First(&log, "document_id = ?", "doc-1").Error; err != nil {
t.Fatalf("load pipeline log: %v", err)
}
if log.OperationStatus != "3" {
t.Fatalf("OperationStatus = %q, want explicit terminal status", log.OperationStatus)
}
if len(log.DSL) != 0 {
t.Fatalf("DSL = %v, want empty object for terminal writer without DSL", log.DSL)
}
}
func TestRecordPipelineLogKeepsResolvedVersionWhenTerminalWriterHasNone(t *testing.T) {
cleanup := setupPipelineExecutorTestDB(t)
defer cleanup()
version := "ver-1"
if err := dao.DB.Create(&entity.PipelineOperationLog{
ID: "run-1",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
OperationStatus: string(entity.TaskStatusRunning),
PipelineVersionID: &version,
}).Error; err != nil {
t.Fatalf("seed pipeline log: %v", err)
}
docName := "terminal.pdf"
if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
TenantID: "tenant-1",
KbID: "kb-1",
DocumentID: "doc-1",
Status: string(entity.TaskStatusFail),
Document: entity.Document{
ID: "doc-1",
KbID: "kb-1",
ParserID: "naive",
ParserConfig: entity.JSONMap{},
SourceType: "local",
Type: "pdf",
Name: &docName,
Suffix: ".pdf",
},
PipelineLogID: "run-1",
}); err != nil {
t.Fatalf("RecordPipelineLog: %v", err)
}
var log entity.PipelineOperationLog
if err := dao.DB.First(&log, "id = ?", "run-1").Error; err != nil {
t.Fatalf("load pipeline log: %v", err)
}
if log.PipelineVersionID == nil || *log.PipelineVersionID != version {
t.Fatalf("PipelineVersionID = %v, want the mid-run version %q", log.PipelineVersionID, version)
}
}
func TestRecordPipelineLogRejectsMissingRunIdentity(t *testing.T) {
err := RecordPipelineLog(t.Context(), nil, PipelineLogInput{DocumentID: "doc-1", Status: "3"})
if !errors.Is(err, ErrMissingRunIdentity) {
t.Fatalf("RecordPipelineLog error = %v, want ErrMissingRunIdentity", err)
}
}
func TestRecordPipelineLogInternalWriterRejectsMissingRunIdentity(t *testing.T) {
err := recordPipelineLog(t.Context(), nil, PipelineLogInput{DocumentID: "doc-1", Status: "3"})
if !errors.Is(err, ErrMissingRunIdentity) {
t.Fatalf("recordPipelineLog error = %v, want ErrMissingRunIdentity", err)
}
}
func TestRecordPipelineLog_TerminalWithoutDSLResolvesCanvasTitle(t *testing.T) {
cleanup := setupPipelineExecutorTestDB(t)
defer cleanup()
if err := dao.DB.Create(&entity.UserCanvas{
ID: "canvas-1",
UserID: "tenant-1",
Title: strPtr("My Pipeline"),
Avatar: strPtr("a.png"),
}).Error; err != nil {
t.Fatalf("seed agent: %v", err)
}
if err := dao.DB.AutoMigrate(&entity.Knowledgebase{}); err != nil {
t.Fatalf("migrate knowledgebase: %v", err)
}
if err := dao.DB.Create(&entity.Knowledgebase{
ID: "kb-1",
TenantID: "tenant-1",
}).Error; err != nil {
t.Fatalf("seed knowledgebase: %v", err)
}
if err := dao.DB.Create(&entity.PipelineOperationLog{
ID: "run-1",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
OperationStatus: "5",
}).Error; err != nil {
t.Fatalf("seed pipeline log: %v", err)
}
docName := "sample.avi"
if err := dao.DB.Create(&entity.Document{
ID: "doc-1",
KbID: "kb-1",
PipelineID: strPtr("canvas-1"),
ParserID: "naive",
ParserConfig: entity.JSONMap{},
Name: &docName,
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
KbID: "kb-1",
DocumentID: "doc-1",
PipelineID: "canvas-1",
Status: "3",
PipelineLogID: "run-1",
PipelineBindingResolved: true,
}); err != nil {
t.Fatalf("RecordPipelineLog: %v", err)
}
var log entity.PipelineOperationLog
if err := dao.DB.First(&log, "document_id = ?", "doc-1").Error; err != nil {
t.Fatalf("load pipeline log: %v", err)
}
if log.OperationStatus != "3" {
t.Fatalf("OperationStatus = %q, want terminal status", log.OperationStatus)
}
if log.PipelineID == nil || *log.PipelineID != "canvas-1" {
t.Fatalf("PipelineID = %v, want \"canvas-1\"", log.PipelineID)
}
if log.PipelineTitle == nil || *log.PipelineTitle != "My Pipeline" {
t.Fatalf("PipelineTitle = %v, want \"My Pipeline\"", log.PipelineTitle)
}
if log.Avatar == nil || *log.Avatar != "a.png" {
t.Fatalf("Avatar = %v, want \"a.png\"", log.Avatar)
}
}
func TestRecordPipelineLog_ReusesOpenPreTerminalRow(t *testing.T) {
cleanup := setupPipelineExecutorTestDB(t)
defer cleanup()
queuedMsg := "Task is queued..."
openLog := &entity.PipelineOperationLog{
ID: "open-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
DocumentName: "test-doc.pdf",
DocumentSuffix: ".pdf",
DocumentType: "pdf",
SourceFrom: "local",
TaskType: "Parse",
OperationStatus: "5",
ProgressMsg: &queuedMsg,
}
if err := dao.DB.Create(openLog).Error; err != nil {
t.Fatalf("seed open log: %v", err)
}
progress := 1.0
finalMsg := "Parser Done"
if err := dao.DB.Create(&entity.Document{
ID: "doc-1",
KbID: "kb-1",
ParserID: "naive",
ParserConfig: entity.JSONMap{},
SourceType: "local",
Type: "pdf",
CreatedBy: "tenant-1",
Name: strPtr("test-doc.pdf"),
Suffix: ".pdf",
Progress: progress,
ProgressMsg: &finalMsg,
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
TenantID: "tenant-1",
KbID: "kb-1",
DocumentID: "doc-1",
Status: "3",
PipelineLogID: "open-log",
}); err != nil {
t.Fatalf("RecordPipelineLog: %v", err)
}
var count int64
if err := dao.DB.Model(&entity.PipelineOperationLog{}).Where("document_id = ?", "doc-1").Count(&count).Error; err != nil {
t.Fatalf("count pipeline logs: %v", err)
}
if count != 1 {
t.Fatalf("pipeline log rows = %d, want 1 (terminal must reuse the bound queued row)", count)
}
var log entity.PipelineOperationLog
if err := dao.DB.First(&log, "id = ?", "open-log").Error; err != nil {
t.Fatalf("load open log: %v", err)
}
if log.OperationStatus != "3" {
t.Fatalf("OperationStatus = %q, want terminal status", log.OperationStatus)
}
if log.Progress != progress {
t.Fatalf("Progress = %v, want %v", log.Progress, progress)
}
if log.ProgressMsg == nil || *log.ProgressMsg != queuedMsg {
t.Fatalf("ProgressMsg = %v, want unchanged queued message", log.ProgressMsg)
}
if len(log.DSL) != 0 {
t.Fatalf("DSL = %v, want empty object for terminal writer without DSL", log.DSL)
}
}
// TestRecordPipelineLog_KeepsEarlyTimestampWhenDocumentHasNone locks the
// timestamp handover: the terminal snapshot must not blank the start time the
// queued row was opened with when the document carries none (a run that never
// reached the progress sink).
func TestRecordPipelineLog_KeepsEarlyTimestampWhenDocumentHasNone(t *testing.T) {
cleanup := setupPipelineExecutorTestDB(t)
defer cleanup()
openedAt := time.Date(2026, 1, 2, 3, 4, 5, 0, time.Local)
queuedMsg := "Task is queued..."
if err := dao.DB.Create(&entity.PipelineOperationLog{
ID: "open-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: "Parse",
OperationStatus: string(entity.TaskStatusRunning),
ProgressMsg: &queuedMsg,
ProcessBeginAt: &openedAt,
}).Error; err != nil {
t.Fatalf("seed open log: %v", err)
}
if err := dao.DB.Create(&entity.Document{
ID: "doc-1",
KbID: "kb-1",
ParserID: "naive",
ParserConfig: entity.JSONMap{},
SourceType: "local",
Type: "pdf",
CreatedBy: "tenant-1",
Suffix: ".pdf",
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
TenantID: "tenant-1",
KbID: "kb-1",
DocumentID: "doc-1",
Status: "4",
PipelineLogID: "open-log",
}); err != nil {
t.Fatalf("RecordPipelineLog: %v", err)
}
var log entity.PipelineOperationLog
if err := dao.DB.First(&log, "id = ?", "open-log").Error; err != nil {
t.Fatalf("load open log: %v", err)
}
if log.OperationStatus != "4" {
t.Fatalf("OperationStatus = %q, want terminal status", log.OperationStatus)
}
if log.ProcessBeginAt == nil || !log.ProcessBeginAt.Equal(openedAt) {
t.Fatalf("ProcessBeginAt = %v, want the queued row's %v", log.ProcessBeginAt, openedAt)
}
if log.SourceFrom != "local" || log.DocumentSuffix != ".pdf" || log.DocumentType != "pdf" {
t.Fatalf("terminal write did not refresh document metadata: source_from=%q suffix=%q type=%q",
log.SourceFrom, log.DocumentSuffix, log.DocumentType)
}
}
// TestTerminalDurationAnchorsToProcessBeginAt pins the single duration anchor:
// a persisted run measures from the document's process_begin_at (the value
// PrepareValidatedRun stamped), not from executor start, so the document and
// the pipeline operation log record one number. A begin time in the future
// (DB/worker clock skew) clamps to zero like UpdateRunState does; a missing
// begin time falls back to the executor start.
func TestTerminalDurationAnchorsToProcessBeginAt(t *testing.T) {
begin := time.Now().Add(-2 * time.Second)
exec := &PipelineExecutor{taskCtx: &TaskContext{Doc: entity.Document{ProcessBeginAt: &begin}}}
start := time.Now()
if d := exec.terminalDuration(start); d < 2.0 || d > 3.0 {
t.Fatalf("terminalDuration = %v, want ~2s measured from process_begin_at", d)
}
future := time.Now().Add(time.Minute)
execFuture := &PipelineExecutor{taskCtx: &TaskContext{Doc: entity.Document{ProcessBeginAt: &future}}}
if d := execFuture.terminalDuration(start); d != 0 {
t.Fatalf("terminalDuration with future begin = %v, want 0", d)
}
execNoBegin := &PipelineExecutor{taskCtx: &TaskContext{}}
if d := execNoBegin.terminalDuration(start); d < 0 || d > 1.0 {
t.Fatalf("terminalDuration without begin = %v, want executor-start fallback", d)
}
}
// TestRecordPipelineLog_TerminalDurationOverridesDocumentCopy locks the
// unification handover: when the run passes its measured terminal duration,
// the operation log records exactly that value — not the reloaded document's
// stale mid-run progress-sink value — so the final ApplyDocCounts write to
// document.process_duration and the log row agree.
func TestRecordPipelineLog_TerminalDurationOverridesDocumentCopy(t *testing.T) {
cleanup := setupPipelineExecutorTestDB(t)
defer cleanup()
if err := dao.DB.Create(&entity.PipelineOperationLog{
ID: "open-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: "Parse",
OperationStatus: string(entity.TaskStatusRunning),
}).Error; err != nil {
t.Fatalf("seed open log: %v", err)
}
if err := dao.DB.Create(&entity.Document{
ID: "doc-1",
KbID: "kb-1",
ParserID: "naive",
ParserConfig: entity.JSONMap{},
SourceType: "local",
Type: "pdf",
CreatedBy: "tenant-1",
Suffix: ".pdf",
ProcessDuration: 0.719,
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
terminal := 0.627
if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
TenantID: "tenant-1",
KbID: "kb-1",
DocumentID: "doc-1",
Status: "3",
PipelineLogID: "open-log",
TerminalDuration: &terminal,
}); err != nil {
t.Fatalf("RecordPipelineLog: %v", err)
}
var log entity.PipelineOperationLog
if err := dao.DB.First(&log, "id = ?", "open-log").Error; err != nil {
t.Fatalf("load open log: %v", err)
}
if log.ProcessDuration != terminal {
t.Fatalf("ProcessDuration = %v, want the run's terminal value %v, not the document's mid-run 0.719",
log.ProcessDuration, terminal)
}
}
// TestRecordPipelineLog_DoesNotAdoptAnotherRunsRow locks the run-isolation
// contract: a terminal write is bound to the row its own run opened, so a late
// write from a superseded run (whose row was replaced) can neither finalize nor
// even touch an open row belonging to another run of the same document.
func TestRecordPipelineLog_DoesNotAdoptAnotherRunsRow(t *testing.T) {
cleanup := setupPipelineExecutorTestDB(t)
defer cleanup()
// An open row for the document that belongs to a different run: either the
// superseded run's row (cleanup failed) or the replacement run's fresh
// queued row. It must survive the write below untouched.
otherMsg := "Task is queued..."
other := &entity.PipelineOperationLog{
ID: "other-run-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: "Parse",
OperationStatus: "5",
ProgressMsg: &otherMsg,
}
if err := dao.DB.Create(other).Error; err != nil {
t.Fatalf("seed other run's log: %v", err)
}
finalMsg := "Parser Done"
if err := dao.DB.Create(&entity.Document{
ID: "doc-1",
KbID: "kb-1",
ParserID: "naive",
ParserConfig: entity.JSONMap{},
SourceType: "local",
Type: "pdf",
CreatedBy: "tenant-1",
Suffix: ".pdf",
ProgressMsg: &finalMsg,
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
// This run's own row was deleted along with the superseded task.
if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
TenantID: "tenant-1",
KbID: "kb-1",
DocumentID: "doc-1",
Status: "3",
PipelineLogID: "superseded-log",
}); err != nil {
t.Fatalf("RecordPipelineLog: %v", err)
}
var reloaded entity.PipelineOperationLog
if err := dao.DB.First(&reloaded, "id = ?", "other-run-log").Error; err != nil {
t.Fatalf("reload other run's log: %v", err)
}
if reloaded.OperationStatus != "5" {
t.Fatalf("other run's row OperationStatus = %q, want %q (must not be touched)", reloaded.OperationStatus, "5")
}
var count int64
if err := dao.DB.Model(&entity.PipelineOperationLog{}).Where("document_id = ?", "doc-1").Count(&count).Error; err != nil {
t.Fatalf("count pipeline logs: %v", err)
}
if count != 1 {
t.Fatalf("pipeline log rows = %d, want 1 (a bound missing row must not create a duplicate)", count)
}
}
// =============================================================================
// updateDocumentMetadata
// =============================================================================
func TestRunPipeline_NilOutput(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0)
ctx := t.Context()
_, err := svc.processOutput(ctx, nil, time.Now())
if err != nil {
t.Errorf("expected nil error for nil output, got %v", err)
}
}
func TestRunPipeline_EmptyOutput(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0)
ctx := t.Context()
_, err := svc.processOutput(ctx, map[string]any{}, time.Now())
if err != nil {
t.Errorf("expected nil error for empty output, got %v", err)
}
}
func TestRunPipeline_NormalizedEmpty(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0)
ctx := t.Context()
_, err := svc.processOutput(ctx, map[string]any{"markdown": ""}, time.Now())
if err != nil {
t.Errorf("expected nil error for empty normalized output, got %v", err)
}
}
func TestRunPipeline_FullFlow(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).
WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
return nil, nil
})
output := map[string]any{
"chunks": []map[string]any{
{"text": "hello"},
{"text": "world"},
},
}
ctx := t.Context()
_, err := svc.processOutput(ctx, output, time.Now())
if err != nil {
t.Errorf("unexpected error: %v", err)
}
}
// TestPipelineExecutor_ProcessOutputMaterializesParentChunk verifies the
// persistence boundary turns chunker-level mom text into one hidden parent row
// and links every child to it. Removing that materialization would leave only
// independently retrievable child rows, which is the parent-child regression.
func TestPipelineExecutor_ProcessOutputMaterializesParentChunk(t *testing.T) {
var inserted []map[string]any
writeCalls := 0
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).WithInsertFunc(
func(_ context.Context, chunks []map[string]any, _, _ string) ([]string, error) {
writeCalls++
inserted = append(inserted, chunks...)
return nil, nil
},
)
_, err := svc.processOutput(t.Context(), map[string]any{
"chunks": []map[string]any{
{"text": "child one", "mom": "full parent passage"},
{"text": "child two", "mom": "full parent passage"},
},
}, time.Now())
if err != nil {
t.Fatalf("processOutput: %v", err)
}
if len(inserted) != 3 {
t.Fatalf("persisted rows = %d, want two children and one parent", len(inserted))
}
if writeCalls != 1 {
t.Fatalf("write calls = %d, want children and parents in one write", writeCalls)
}
var parent map[string]any
children := make([]map[string]any, 0, 2)
for _, row := range inserted {
if row["available_int"] == 0 && row["content_with_weight"] == "full parent passage" {
parent = row
continue
}
children = append(children, row)
}
if parent == nil {
t.Fatal("hidden parent row was not persisted")
}
parentID, _ := parent["id"].(string)
if parentID == "" {
t.Fatalf("parent id = %#v, want deterministic non-empty id", parent["id"])
}
for _, child := range children {
if child["mom_id"] != parentID {
t.Errorf("child mom_id = %#v, want parent id %q", child["mom_id"], parentID)
}
if _, exists := child["mom"]; exists {
t.Errorf("child retained pipeline-only mom field: %#v", child)
}
}
}
func TestPipelineExecutorCompensatesAfterExhaustingCombinedWriteRetries(t *testing.T) {
var deleted map[string]any
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).
WithInsertFunc(func(context.Context, []map[string]any, string, string) ([]string, error) {
return nil, errors.New("index unavailable")
}).
WithDeleteChunksFunc(func(_ context.Context, condition map[string]any, _, _ string) (int64, error) {
deleted = condition
return 0, nil
})
_, err := svc.processOutput(t.Context(), map[string]any{
"chunks": []map[string]any{{"text": "child", "mom": "parent"}},
}, time.Now())
if err == nil {
t.Fatal("processOutput succeeded after exhausted writes")
}
ids, ok := deleted["id"].([]string)
if !ok || len(ids) != 2 {
t.Fatalf("compensation ids = %#v, want child and parent IDs", deleted)
}
}
func TestPipelineExecutorBoundsCompensationCleanup(t *testing.T) {
var cleanupCtx context.Context
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).
WithInsertFunc(func(context.Context, []map[string]any, string, string) ([]string, error) {
return nil, errors.New("index unavailable")
}).
WithDeleteChunksFunc(func(ctx context.Context, _ map[string]any, _, _ string) (int64, error) {
cleanupCtx = ctx
return 0, nil
})
_, err := svc.processOutput(t.Context(), map[string]any{
"chunks": []map[string]any{{"text": "child", "mom": "parent"}},
}, time.Now())
if err == nil {
t.Fatal("processOutput succeeded after exhausted writes")
}
deadline, ok := cleanupCtx.Deadline()
if !ok {
t.Fatal("compensation cleanup context has no deadline")
}
if remaining := time.Until(deadline); remaining <= 0 || remaining > 5*time.Second {
t.Fatalf("compensation cleanup deadline remaining = %s, want (0, 5s]", remaining)
}
}
func TestRunPipeline_AlreadyHasVectors(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).
WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
return nil, nil
})
output := map[string]any{
"chunks": []map[string]any{
{"text": "hello", "q_768_vec": []float64{0.1, 0.2}},
},
}
ctx := t.Context()
_, err := svc.processOutput(ctx, output, time.Now())
if err != nil {
t.Errorf("unexpected error: %v", err)
}
}
func TestRunPipeline_ContextCanceled(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0)
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := svc.processOutput(ctx, map[string]any{
"chunks": []map[string]any{{"text": "hello"}},
}, time.Now())
if err == nil {
t.Error("expected context canceled error")
}
}
func TestPipelineExecutor_Run_MainFlowWithStubs(t *testing.T) {
inserted := false
taskCtx := makeTaskCtx()
taskCtx.PipelineID = "flow-1"
svc := mustNewPipelineExecutor(t, taskCtx, "flow-1", 0).
WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
return `{"nodes":[{"id":"n1"}],"edges":[]}`, "flow-corrected", nil
}).
WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {
return map[string]any{
"chunks": []map[string]any{
{"text": "hello world"},
},
}, dsl, nil
}).
WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
inserted = true
return nil, nil
})
_, err := svc.Execute(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !inserted {
t.Fatal("expected insertChunks to be called")
}
}
func TestPipelineExecutor_Execute_DoesNotLogFailedRun(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).
WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil
}).
WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {
return nil, dsl, errors.New("pipeline failed")
})
if _, err := svc.Execute(context.Background()); err == nil {
t.Fatal("Execute error = nil, want failure")
}
}
func TestPipelineExecutor_Execute_DoesNotLogCanceledRun(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0).
WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil
}).
WithRunPipelineFunc(func(ctx context.Context, dsl string) (map[string]any, string, error) {
return nil, dsl, context.Canceled
})
if _, err := svc.Execute(context.Background()); !errors.Is(err, context.Canceled) {
t.Fatalf("Execute error = %v, want context.Canceled", err)
}
}
// TestPipelineExecutor_Execute_PropagatesContext verifies the ctx passed to
// Execute is the ctx received by runPipelineFunc - the task context must flow
// through to the pipeline run.
func TestPipelineExecutor_Execute_PropagatesContext(t *testing.T) {
const key string = "trace"
taskCtx := makeTaskCtx()
ctx := t.Context()
taskCtx.Ctx = context.WithValue(ctx, key, "task-ctx")
svc := mustNewPipelineExecutor(t, taskCtx, "flow-1", 0).
WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil
}).
WithRunPipelineFunc(func(runCtx context.Context, dsl string) (map[string]any, string, error) {
if got := runCtx.Value(key); got != "task-ctx" {
t.Fatalf("runCtx value = %v, want task-ctx", got)
}
return map[string]any{"chunks": []map[string]any{{"text": "hello world"}}}, dsl, nil
}).
WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
return nil, nil
})
if _, err := svc.Execute(taskCtx.Ctx); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestPipelineExecutor_Execute_RecordsDoneOperationStatus(t *testing.T) {
taskCtx := makeTaskCtx()
taskCtx.Ctx = t.Context()
svc := mustNewPipelineExecutor(t, taskCtx, "flow-1", 0).
WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil
}).
WithRunPipelineFunc(func(runCtx context.Context, dsl string) (map[string]any, string, error) {
return map[string]any{"chunks": []map[string]any{{"text": "hello world"}}}, dsl, nil
}).
WithInsertFunc(func(ctx context.Context, chunks []map[string]any, baseName, datasetID string) ([]string, error) {
return nil, nil
})
if _, err := svc.Execute(taskCtx.Ctx); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
// TestPipelineExecutor_Execute_EmptyOutputRecordsTerminalDuration pins the
// no-chunk success branch: such a run never writes a terminal duration to the
// document (processOutput returns before ApplyDocCounts), so Execute must
// recompute from the same process_begin_at anchor and hand that value to the
// operation log instead of letting the row keep the stale mid-run
// progress-sink duration.
func TestPipelineExecutor_Execute_EmptyOutputRecordsTerminalDuration(t *testing.T) {
cleanup := setupPipelineExecutorTestDB(t)
defer cleanup()
begin := time.Now().Add(-2 * time.Second)
if err := dao.DB.Create(&entity.PipelineOperationLog{
ID: "open-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: "Parse",
OperationStatus: string(entity.TaskStatusRunning),
}).Error; err != nil {
t.Fatalf("seed open log: %v", err)
}
if err := dao.DB.Create(&entity.Document{
ID: "doc-1",
KbID: "kb-1",
ParserID: "naive",
ParserConfig: entity.JSONMap{},
SourceType: "local",
Type: "pdf",
CreatedBy: "tenant-1",
Suffix: ".pdf",
ProcessBeginAt: &begin,
ProcessDuration: 0.719,
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
logID := "open-log"
taskCtx := makeTaskCtx()
taskCtx.Ctx = t.Context()
taskCtx.IngestionTask.PipelineLogID = &logID
taskCtx.Doc.ProcessBeginAt = &begin
svc := mustNewPipelineExecutor(t, taskCtx, "flow-1", 0).
WithLoadDSLFunc(func(ctx context.Context, canvasID string) (string, string, error) {
return `{"nodes":[{"id":"n1"}],"edges":[]}`, canvasID, nil
}).
WithRunPipelineFunc(func(runCtx context.Context, dsl string) (map[string]any, string, error) {
return map[string]any{"chunks": []map[string]any{}}, dsl, nil
})
if _, err := svc.Execute(taskCtx.Ctx); err != nil {
t.Fatalf("unexpected error: %v", err)
}
var log entity.PipelineOperationLog
if err := dao.DB.First(&log, "id = ?", "open-log").Error; err != nil {
t.Fatalf("load log row: %v", err)
}
if log.OperationStatus != string(entity.TaskStatusDone) {
t.Fatalf("OperationStatus = %q, want DONE", log.OperationStatus)
}
if log.ProcessDuration < 2.0 || log.ProcessDuration > 3.0 {
t.Fatalf("ProcessDuration = %v, want ~2s measured from process_begin_at, not the stale 0.719", log.ProcessDuration)
}
}
// =============================================================================
// Stub implementations for testing
// =============================================================================
// recordingProgressSink captures progress events for asserting the executor
// forwards its sink through runPipelineWithDSL into the pipeline.
type recordingProgressSink struct {
mu sync.Mutex
total int
totalSet bool
events []pipelinepkg.ProgressEvent
}
func (r *recordingProgressSink) OnComponentTotal(ctx context.Context, taskID string, total int) {
r.mu.Lock()
defer r.mu.Unlock()
r.total = total
r.totalSet = true
}
func (r *recordingProgressSink) OnComponentProgress(ctx context.Context, ev pipelinepkg.ProgressEvent) {
r.mu.Lock()
defer r.mu.Unlock()
r.events = append(r.events, ev)
}
type sinkPassthroughStage struct{}
func (sinkPassthroughStage) Invoke(_ context.Context, _ *gorm.DB, inputs map[string]any) (map[string]any, error) {
return inputs, nil
}
// TestPipelineExecutorRunPipelineWithDSLForwardsSink verifies the sink set via
// WithProgressSink is threaded through runPipelineWithDSL into the pipeline,
// which reports the component total and lifecycle events back to the sink.
func TestPipelineExecutorRunPipelineWithDSLForwardsSink(t *testing.T) {
const nameA = "task.SinkPassthroughA"
runtime.MustRegister(nameA, runtime.CategoryIngestion,
func(_ string, _ map[string]any) (runtime.Component, error) { return sinkPassthroughStage{}, nil },
runtime.Metadata{Version: "1.0.0"})
sink := &recordingProgressSink{}
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-1", 0)
svc.WithProgressSink(sink)
dsl := `{"dsl":{"components":{"begin":{"obj":{"component_name":"Begin","params":{}},"downstream":["a"]},"a":{"obj":{"component_name":"` + nameA + `","params":{}},"upstream":["begin"]}},"path":["begin","a"],"graph":{"nodes":[]}}}`
ctx := t.Context()
if _, _, err := svc.runPipelineWithDSL(ctx, dsl); err != nil {
t.Fatalf("runPipelineWithDSL: %v", err)
}
sink.mu.Lock()
defer sink.mu.Unlock()
if !sink.totalSet || sink.total != 2 {
t.Fatalf("OnComponentTotal = (%d, set=%v), want 2", sink.total, sink.totalSet)
}
if len(sink.events) == 0 {
t.Fatal("expected progress events forwarded to sink, got none")
}
for _, ev := range sink.events {
if ev.TaskID != "task-1" {
t.Fatalf("event TaskID = %q, want task-1", ev.TaskID)
}
if ev.DocumentID != "doc-1" {
t.Fatalf("event DocumentID = %q, want doc-1", ev.DocumentID)
}
}
}
func TestCountOriginalChunkIDs(t *testing.T) {
// Empty list
if n := countOriginalChunkIDs(nil); n != 0 {
t.Fatalf("nil chunks: got %d, want 0", n)
}
// All unique
chunks := []map[string]any{
{"id": "a"},
{"id": "b"},
{"id": "c"},
}
if n := countOriginalChunkIDs(chunks); n != 3 {
t.Fatalf("all unique: got %d, want 3", n)
}
// Duplicates present — this is the key case
chunks = []map[string]any{
{"id": "x"},
{"id": "y"},
{"id": "x"}, // duplicate of [0]
{"id": "z"},
{"id": "y"}, // duplicate of [1]
}
if n := countOriginalChunkIDs(chunks); n != 3 {
t.Fatalf("with duplicates: got %d, want 3", n)
}
// Missing id fields are skipped
chunks = []map[string]any{
{"id": "one"},
{"text": "no id"},
{"id": "two"},
}
if n := countOriginalChunkIDs(chunks); n != 2 {
t.Fatalf("mixed present/absent ids: got %d, want 2", n)
}
// Compiler products are stored as chunks too, but are derived artifacts.
chunks = []map[string]any{
{"id": "source-1"},
{"id": "source-2"},
{"id": "wiki-page-1", "compile_kwd": "wiki_page"},
{"id": "wiki-section-1", "compile_kwd": "wiki_section"},
}
if n := countOriginalChunkIDs(chunks); n != 2 {
t.Fatalf("compiler products: got %d, want 2", n)
}
}
func TestMergeCompiledVariants(t *testing.T) {
if got := mergeCompiledVariants(nil, nil); got != nil {
t.Fatalf("empty variants = %v, want nil", got)
}
got := mergeCompiledVariants([]string{"wiki", "structure"}, []string{"wiki", "tree"})
want := []string{"structure", "tree", "wiki"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("merged variants = %v, want %v", got, want)
}
}
// TestRunPipelineWithDSL_LogDSLStripsOutputs locks the "no business data in
// the pipeline operation log" contract: the DSL runPipelineWithDSL returns for
// the log (Execute passes it straight to recordPipelineLog) must NOT carry any
// component's runtime outputs under obj.params.outputs. The log keeps the DSL
// DEFINITION only (component structure, static params, downstream, graph,
// path) so a historical run can be reconstructed and the dataset "View result"
// / rerun UI no longer renders chunk business data. Dry-run previews still
// carry outputs via ResultSink; only the persisted log is kept output-free
// (buildLogDSL passes includeOutputs=false before recordPipelineLog).
//
// It drives the REAL runPipelineWithDSL with stub ingestion components, so
// the log DSL is built from an actual run output (nested under
// output["state"][<id>] by finalizeResult), not a hand-built one, and the
// enveloped {"dsl": {...}} input is unwrapped to the front-end shape
// (top-level components).
func TestRunPipelineWithDSL_LogDSLStripsOutputs(t *testing.T) {
const (
compC = "logdsl.RealStubChunks"
compD = "logdsl.RealStubD"
)
runtime.MustRegister(compC, runtime.CategoryIngestion,
func(_ string, _ map[string]any) (runtime.Component, error) { return traceChunkComponent{}, nil },
runtime.Metadata{Version: "1.0.0"})
runtime.MustRegister(compD, runtime.CategoryIngestion,
func(_ string, _ map[string]any) (runtime.Component, error) { return traceStubComponent{}, nil },
runtime.Metadata{Version: "1.0.0"})
dsl := `{"dsl":{"components":{
"begin":{"obj":{"component_name":"Begin","params":{}},"downstream":["c"]},
"c":{"obj":{"component_name":"` + compC + `","params":{"setups":{"pdf":{"parse_method":"general"}}}},"upstream":["begin"],"downstream":["d"]},
"d":{"obj":{"component_name":"` + compD + `","params":{}},"upstream":["c"]}
},"path":["begin","c","d"],"graph":{"nodes":[{"id":"begin","data":{"name":"开始"}},{"id":"c","data":{"name":"解析"}},{"id":"d","data":{"name":"分词"}}]}}}`
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-logdsl", 0)
_, logDSL, err := svc.runPipelineWithDSL(t.Context(), dsl)
if err != nil {
t.Fatalf("runPipelineWithDSL: %v", err)
}
var doc map[string]any
if err := json.Unmarshal([]byte(logDSL), &doc); err != nil {
t.Fatalf("log DSL is not valid JSON: %v body=%s", err, logDSL)
}
// The enveloped canvas DSL must come back unwrapped: the front-end reads
// dsl.components at the top level.
components, ok := doc["components"].(map[string]any)
if !ok {
t.Fatalf("log DSL must carry top-level components (unwrapped canvas envelope): %s", logDSL)
}
// Chunk-emitting component: the persisted log must carry the DSL DEFINITION
// only — NO runtime outputs under obj.params.outputs (the "no business data
// in the log" contract, Req 1). The dry-run preview (ResultSink/Redis) still
// carries full outputs for the dataset "View result" page; the persisted row
// must not. Static params survive so the rerun/canvas flow can reconstruct
// the component.
cParams, ok := components["c"].(map[string]any)["obj"].(map[string]any)["params"].(map[string]any)
if !ok {
t.Fatalf("log DSL components.c.obj.params missing: %s", logDSL)
}
if _, ok := cParams["outputs"]; ok {
t.Errorf("REGRESSION: persisted log DSL must NOT carry business data (obj.params.outputs) "+
"for chunk component c, got %#v", cParams["outputs"])
}
if _, ok := cParams["setups"]; !ok {
t.Errorf("persisted log DSL must keep static params.setups for c, got %#v", cParams)
}
// Non-components top-level keys are carried verbatim; this fixture's DSL
// declares "path" — the round-tripped log must keep it for rerun-flow
// consumers.
if p, _ := doc["path"].([]any); len(p) != 3 || p[0] != "begin" || p[2] != "d" {
t.Errorf("log DSL path=%#v want [begin c d]", doc["path"])
}
}
// TestBuildLogDSL_FallbackToStaticDSL pins the guarantee that log recording
// never fails a run: when the run-result DSL cannot be built (a malformed
// canvas), buildLogDSL must return the static dsl unchanged rather than a
// half-written payload. The input dsl is the canvas definition (no runtime
// outputs), so the fallback is not a business-data leak and the log row is
// preserved for observability (recordPipelineLog still receives a valid,
// definition-only DSL).
//
// Note: business-data payloads (e.g. NaN inside a chunk value) no longer reach
// the persisted copy at all — the persist path passes includeOutputs=false, so
// the outputs wrapper is never constructed and cannot break marshaling. The
// realistic fallback trigger is therefore a malformed DSL, tested below.
func TestBuildLogDSL_FallbackToStaticDSL(t *testing.T) {
svc := mustNewPipelineExecutor(t, makeTaskCtx(), "flow-logdsl-fallback", 0)
// Build failure: a DSL without a components map cannot produce a
// run-result DSL at all. The static dsl is returned unchanged.
badDSL := `{"dsl":{"path":["a"]}}`
if got := svc.buildLogDSL(badDSL, nil); got != badDSL {
t.Errorf("build failure: log DSL must fall back to the static dsl\n got: %s\nwant: %s", got, badDSL)
}
}
// persistOnlySink simulates the real-parse (DB-backed) sink: it implements
// pipeline.ProgressSink but NOT task.ResultSink, so buildLogDSL must NOT call
// SetResult and the persisted DSL must carry no business data.
type persistOnlySink struct{}
func (persistOnlySink) OnComponentTotal(context.Context, string, int) {}
func (persistOnlySink) OnComponentProgress(context.Context, pipelinepkg.ProgressEvent) {}
// capturingSink simulates the dry-run (DebugLogSink) sink: it implements BOTH
// ProgressSink and ResultSink, so buildLogDSL hands it the full result DSL
// (business data) via SetResult for the Redis END marker.
type capturingSink struct {
got map[string]any
gotOut map[string]any
}
func (c *capturingSink) OnComponentTotal(context.Context, string, int) {}
func (c *capturingSink) OnComponentProgress(context.Context, pipelinepkg.ProgressEvent) {}
func (c *capturingSink) SetResult(dsl map[string]any, output map[string]any) {
c.got = dsl
c.gotOut = output
}
// TestBuildLogDSL_PersistStripsOutputs locks the core contract of the change:
// the DSL string buildLogDSL returns for a NON-ResultSink (real-parse) sink
// carries the DSL definition only — no component has obj.params.outputs, while
// static params / graph / path survive. This is exactly what recordPipelineLog
// persists to pipeline_operation_log.
func TestBuildLogDSL_PersistStripsOutputs(t *testing.T) {
const dsl = `{"components": {"a": {"obj": {"component_name": "my.Parser", "params": {"setups": {"pdf": {"parse_method": "general"}}}}, "downstream": ["b"]}}, "graph": {"nodes": [{"id": "a"}]}, "path": ["a"]}`
output := map[string]any{
"a": map[string]any{"chunks": []any{map[string]any{"text": "hello"}}, "_elapsed_time": 0.35},
}
exec := &PipelineExecutor{progressSink: persistOnlySink{}}
logDSL := exec.buildLogDSL(dsl, output)
var parsed map[string]any
if err := json.Unmarshal([]byte(logDSL), &parsed); err != nil {
t.Fatalf("persisted log DSL must be valid JSON: %v (raw=%s)", err, logDSL)
}
comps, _ := parsed["components"].(map[string]any)
a, _ := comps["a"].(map[string]any)
aObj, _ := a["obj"].(map[string]any)
aParams, _ := aObj["params"].(map[string]any)
if _, ok := aParams["outputs"]; ok {
t.Errorf("persisted log DSL must NOT carry business data (obj.params.outputs), got %#v", aParams["outputs"])
}
// DSL definition preserved.
if _, ok := aParams["setups"]; !ok {
t.Error("persisted log DSL must keep static params.setups")
}
if _, ok := parsed["graph"]; !ok {
t.Error("persisted log DSL must keep graph")
}
if _, ok := parsed["path"]; !ok {
t.Error("persisted log DSL must keep path")
}
}
// TestBuildLogDSL_PreviewKeepsOutputs locks the dry-run (ResultSink) branch:
// buildLogDSL calls ResultSink.SetResult with the FULL result DSL (business
// data in obj.params.outputs), while the DSL it returns for persistence still
// carries no business data. Dry-run never persists (IsDebug() early-return),
// so the outputs only travel to Redis via SetResult.
func TestBuildLogDSL_PreviewKeepsOutputs(t *testing.T) {
const dsl = `{"components": {"a": {"obj": {"component_name": "my.Parser", "params": {}}, "downstream": ["b"]}}}`
output := map[string]any{
"a": map[string]any{"chunks": []any{map[string]any{"text": "hello"}}, "_elapsed_time": 0.35},
}
cap := &capturingSink{}
exec := &PipelineExecutor{progressSink: cap}
logDSL := exec.buildLogDSL(dsl, output)
// The ResultSink preview must carry the full outputs (business data).
if cap.got == nil {
t.Fatal("buildLogDSL must call ResultSink.SetResult for a dry-run (ResultSink) sink")
}
caps, _ := cap.got["components"].(map[string]any)
ca, _ := caps["a"].(map[string]any)
caObj, _ := ca["obj"].(map[string]any)
caParams, _ := caObj["params"].(map[string]any)
if _, ok := caParams["outputs"]; !ok {
t.Error("dry-run preview DSL must carry obj.params.outputs (business data)")
}
// Persisted copy returned by buildLogDSL still must NOT carry business data.
var parsed map[string]any
if err := json.Unmarshal([]byte(logDSL), &parsed); err != nil {
t.Fatalf("persisted log DSL must be valid JSON: %v", err)
}
pc, _ := parsed["components"].(map[string]any)
pa, _ := pc["a"].(map[string]any)
paObj, _ := pa["obj"].(map[string]any)
paParams, _ := paObj["params"].(map[string]any)
if _, ok := paParams["outputs"]; ok {
t.Error("persisted log DSL must NOT carry business data even when a ResultSink preview exists")
}
}
func TestExecutePersistsResolvedVersionBeforeRun(t *testing.T) {
restore := setupPipelineExecutorTestDB(t)
defer restore()
db := dao.DB
if err := db.AutoMigrate(&entity.UserCanvas{}, &entity.UserCanvasVersion{}); err != nil {
t.Fatalf("migrate canvas tables: %v", err)
}
release := true
if err := db.Create(&entity.UserCanvas{
ID: "canvas-1",
UserID: "user-1",
CanvasCategory: "dataflow_canvas",
}).Error; err != nil {
t.Fatalf("seed canvas: %v", err)
}
if err := db.Create(&entity.UserCanvasVersion{
ID: "ver-1",
UserCanvasID: "canvas-1",
DSL: entity.JSONMap{"nodes": []any{}, "edges": []any{}},
Release: release,
BaseModel: entity.BaseModel{CreateTime: int64PtrForTest(1000)},
}).Error; err != nil {
t.Fatalf("seed canvas version: %v", err)
}
logID := "run-1"
if err := db.Create(&entity.PipelineOperationLog{
ID: logID,
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
DocumentName: "test-doc.pdf",
DocumentSuffix: ".pdf",
DocumentType: "pdf",
SourceFrom: "local",
OperationStatus: string(entity.TaskStatusRunning),
TaskType: string(entity.PipelineTaskTypeParse),
}).Error; err != nil {
t.Fatalf("seed pipeline log: %v", err)
}
taskCtx := makeTaskCtx()
taskCtx.PipelineID = "canvas-1"
taskCtx.IngestionTask.PipelineLogID = &logID
exec := mustNewPipelineExecutor(t, taskCtx, "canvas-1", 0)
exec.WithRunPipelineFunc(func(context.Context, string) (map[string]any, string, error) {
return nil, "", errors.New("pipeline failed")
})
if _, err := exec.Execute(t.Context()); err == nil {
t.Fatal("expected the injected pipeline failure")
}
var run entity.PipelineOperationLog
if err := db.First(&run, "id = ?", logID).Error; err != nil {
t.Fatalf("load pipeline log: %v", err)
}
if run.PipelineID == nil || *run.PipelineID != "canvas-1" {
t.Fatalf("PipelineID = %v, want canvas-1 after a failed run", run.PipelineID)
}
if run.PipelineVersionID == nil || *run.PipelineVersionID != "ver-1" {
t.Fatalf("PipelineVersionID = %v, want the latest resolved version after a failed run", run.PipelineVersionID)
}
}
func int64PtrForTest(v int64) *int64 { return &v }
func TestExecuteClearsStaleBindingForBuiltinRun(t *testing.T) {
restore := setupPipelineExecutorTestDB(t)
defer restore()
db := dao.DB
oldPipelineID := "canvas-1"
oldVersionID := "ver-1"
logID := "run-1"
if err := db.Create(&entity.PipelineOperationLog{
ID: logID,
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
DocumentName: "test-doc.pdf",
DocumentSuffix: ".pdf",
DocumentType: "pdf",
SourceFrom: "local",
OperationStatus: string(entity.TaskStatusRunning),
TaskType: string(entity.PipelineTaskTypeParse),
PipelineID: &oldPipelineID,
PipelineVersionID: &oldVersionID,
}).Error; err != nil {
t.Fatalf("seed pipeline log: %v", err)
}
taskCtx := makeTaskCtx()
taskCtx.IngestionTask.PipelineLogID = &logID
exec := mustNewPipelineExecutor(t, taskCtx, "naive", 0)
exec.WithLoadDSLFunc(func(_ context.Context, canvasID string) (string, string, error) {
return `{"nodes":[],"edges":[]}`, canvasID, nil
})
exec.WithRunPipelineFunc(func(context.Context, string) (map[string]any, string, error) {
return nil, "", errors.New("pipeline failed")
})
if _, err := exec.Execute(t.Context()); err == nil {
t.Fatal("expected the injected pipeline failure")
}
var run entity.PipelineOperationLog
if err := db.First(&run, "id = ?", logID).Error; err != nil {
t.Fatalf("load pipeline log: %v", err)
}
if run.PipelineID != nil || run.PipelineVersionID != nil {
t.Fatalf("pipeline binding = (%v, %v), want both nil after a builtin run", run.PipelineID, run.PipelineVersionID)
}
}
func TestRecordPipelineLogClearsStaleBindingForResolvedVersionlessRun(t *testing.T) {
restore := setupPipelineExecutorTestDB(t)
defer restore()
db := dao.DB
oldPipelineID := "canvas-1"
oldVersionID := "ver-1"
if err := db.Create(&entity.PipelineOperationLog{
ID: "run-1",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
DocumentName: "test-doc.pdf",
DocumentSuffix: ".pdf",
DocumentType: "pdf",
SourceFrom: "local",
OperationStatus: string(entity.TaskStatusRunning),
TaskType: string(entity.PipelineTaskTypeParse),
PipelineID: &oldPipelineID,
PipelineVersionID: &oldVersionID,
}).Error; err != nil {
t.Fatalf("seed pipeline log: %v", err)
}
currentPipelineID := "canvas-2"
docName := "test-doc.pdf"
if err := db.Create(&entity.Document{
ID: "doc-1", KbID: "kb-1", ParserID: "naive", ParserConfig: entity.JSONMap{},
Name: &docName, Suffix: ".pdf", Type: "pdf", SourceType: "local", Status: strPtr("1"),
PipelineID: ¤tPipelineID,
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
if err := RecordPipelineLog(t.Context(), db, PipelineLogInput{
TenantID: "tenant-1",
KbID: "kb-1",
DocumentID: "doc-1",
Status: string(entity.TaskStatusDone),
PipelineLogID: "run-1",
PipelineBindingResolved: true,
}); err != nil {
t.Fatalf("RecordPipelineLog: %v", err)
}
var run entity.PipelineOperationLog
if err := db.First(&run, "id = ?", "run-1").Error; err != nil {
t.Fatalf("load pipeline log: %v", err)
}
if run.PipelineID != nil || run.PipelineVersionID != nil {
t.Fatalf("pipeline binding = (%v, %v), want both nil for a versionless run", run.PipelineID, run.PipelineVersionID)
}
}