Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

52.6 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package task

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"ragflow/internal/common"
	"ragflow/internal/dao"
	"ragflow/internal/engine"
	enginetypes "ragflow/internal/engine/types"
	"ragflow/internal/entity"
	"ragflow/internal/ingestion"
	"ragflow/internal/ingestion/chunkcache"
	"ragflow/internal/ingestion/component"
	"ragflow/internal/ingestion/component/globals"
	kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
	"ragflow/internal/ingestion/knowledge_compile"
	"ragflow/internal/ingestion/mediacache"
	pipelinepkg "ragflow/internal/ingestion/pipeline"
	indexdoc "ragflow/internal/ingestion/task/indexdoc"

	"go.uber.org/zap"
	"gorm.io/gorm"
)

// PipelineResult is the outcome of a pipeline run: chunks have been
// indexed, and these bookkeeping inputs remain for the caller to apply to
// document state (metadata merge + chunk/token counter bumps).
type PipelineResult struct {
	DocID            string
	KbID             string
	Metadata         map[string]any
	Chunks           []map[string]any // populated only in debug (dry-run) mode
	ChunkCount       int
	TokenConsumption int
	Duration         float64 // pipeline wall-clock seconds
	// DocName, BuiltInMetadataConfig and AutoMetadataEnabled carry what the
	// document-state finalizer needs to apply built-in metadata
	// (update_time / file_name), mirroring Python apply_built_in_metadata.
	DocName               string
	BuiltInMetadataConfig []any
	AutoMetadataEnabled   bool
	// CompiledVariants and CompiledTaskTypes identify the document-level
	// products that the dataset consumer must merge after the ingestion task
	// reaches its terminal COMPLETED state.
	CompiledVariants  []string
	CompiledTaskTypes []string
	// MessageID is the polling key for the debug-run log. The front-end reads
	// it from the run response and polls GET /agents/:id/logs/:message_id to
	// render progress; it is empty for non-debug (persist) runs.
	MessageID string
}

type PipelineExecutor struct {
	taskCtx     *TaskContext
	canvasID    string
	docBulkSize int
	deps        *ingestion.Deps

	resolvedVersionID string

	indexWriter      *chunkIndexWriter
	deleteChunksFunc DeleteChunksFunc
	loadDSLFunc      func(ctx context.Context, canvasID string) (string, string, error)
	runPipelineFunc  func(ctx context.Context, dsl string) (map[string]any, string, error)
	purgeTaskState   func(ctx context.Context, taskID string) error
	progressSink     pipelinepkg.ProgressSink
}

func validateTaskContext(taskCtx *TaskContext) error {
	if taskCtx == nil {
		return fmt.Errorf("pipeline executor: nil task context")
	}
	if taskCtx.Doc.ID == "" {
		return fmt.Errorf("pipeline executor: empty document id")
	}
	// A debug (dry-run) context carries no knowledgebase (see
	// TaskContext.IsDebug), so it must not be required to supply one.
	if !taskCtx.IsDebug() && taskCtx.Doc.KbID == "" {
		return fmt.Errorf("pipeline executor: empty document knowledgebase id")
	}
	if taskCtx.Doc.Name == nil || *taskCtx.Doc.Name == "" {
		return fmt.Errorf("pipeline executor: empty document name")
	}
	if taskCtx.Tenant.ID == "" {
		return fmt.Errorf("pipeline executor: empty tenant id")
	}
	return nil
}

// noRefreshChunkInserter is the narrower inserter the ingestion path prefers:
// an engine that implements it can write chunks without waiting for an index
// refresh.
type noRefreshChunkInserter interface {
	InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error)
}

// insertChunksForIngestion writes non-final chunks through the engine's
// no-refresh path when it offers one, falling back to the plain inserter
// otherwise.
//
// Refreshing every batch adds unnecessary latency. The writer uses the regular
// inserter for its final batch, so all preceding batches become searchable
// before the document task is acknowledged and its completion event is sent.
//
// The first write creates the chunk store, and on Infinity that fixes the
// fulltext analyzer, so the dataset language has to travel with it.
func insertChunksForIngestion(eng engine.DocEngine, language string) InsertFunc {
	return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {
		if bulk, ok := eng.(noRefreshChunkInserter); ok {
			return bulk.InsertChunksNoRefresh(ctx, chunks, baseName, datasetID, language)
		}
		return eng.InsertChunks(ctx, chunks, baseName, datasetID, language)
	}
}

// insertFinalChunksForIngestion waits for the index refresh before returning.
// The final write makes every preceding no-refresh batch searchable before the
// document task is acknowledged and its completion event is published.
func insertFinalChunksForIngestion(eng engine.DocEngine, language string) InsertFunc {
	return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {
		return eng.InsertChunks(ctx, chunks, baseName, datasetID, language)
	}
}

// NewPipelineExecutor builds the executor for one document run. deps carries
// the process-wide singletons (doc engine, database handle) resolved at the
// server assembly point; the executor never resolves them itself.
func NewPipelineExecutor(
	taskCtx *TaskContext,
	canvasID string,
	docBulkSize int,
	deps *ingestion.Deps,
) (*PipelineExecutor, error) {
	if err := validateTaskContext(taskCtx); err != nil {
		return nil, err
	}
	if strings.TrimSpace(canvasID) == "" {
		return nil, fmt.Errorf("pipeline executor: empty canvas id")
	}
	if deps == nil {
		return nil, fmt.Errorf("pipeline executor: nil dependency bundle")
	}
	svc := &PipelineExecutor{
		taskCtx:     taskCtx,
		canvasID:    canvasID,
		docBulkSize: docBulkSize,
		deps:        deps,
		indexWriter: newChunkIndexWriter(
			insertChunksForIngestion(deps.Engine, datasetLanguage(taskCtx)),
			fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID),
			taskCtx.Doc.KbID,
			docBulkSize,
		).withFinalInsertFunc(insertFinalChunksForIngestion(deps.Engine, datasetLanguage(taskCtx))),
		deleteChunksFunc: func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error) {
			return deps.Engine.DeleteChunks(ctx, condition, baseName, datasetID)
		},
	}
	svc.loadDSLFunc = svc.loadDSLFromCanvas
	svc.runPipelineFunc = svc.runPipelineWithDSL
	svc.purgeTaskState = func(ctx context.Context, taskID string) error {
		if deps.KV == nil {
			return nil
		}
		return errors.Join(
			chunkcache.PurgeTask(ctx, deps.KV, taskID),
			mediacache.PurgeTask(ctx, deps.KV, taskID),
		)
	}
	return svc, nil
}

// DeleteChunksFunc removes partially written rows after a failed index write.
type DeleteChunksFunc func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error)

func (s *PipelineExecutor) WithInsertFunc(f InsertFunc) *PipelineExecutor {
	s.indexWriter.insertFunc = f
	s.indexWriter.finalInsertFunc = f
	return s
}

// WithDeleteChunksFunc replaces index-write compensation. It is used by the
// in-memory pipeline tests; production uses the configured document engine.
func (s *PipelineExecutor) WithDeleteChunksFunc(f DeleteChunksFunc) *PipelineExecutor {
	s.deleteChunksFunc = f
	return s
}
func (s *PipelineExecutor) WithLoadDSLFunc(f func(ctx context.Context, canvasID string) (string, string, error)) *PipelineExecutor {
	s.loadDSLFunc = f
	return s
}

func (s *PipelineExecutor) WithRunPipelineFunc(f func(ctx context.Context, dsl string) (map[string]any, string, error)) *PipelineExecutor {
	s.runPipelineFunc = f
	return s
}

// WithProgressSink injects a sink that receives pipeline component progress
// events. The sink owns all document/ingestion_task_log persistence; when
// unset, the pipeline runs DB-independent (progress events are dropped).
func (s *PipelineExecutor) WithProgressSink(sink pipelinepkg.ProgressSink) *PipelineExecutor {
	s.progressSink = sink
	return s
}

func (s *PipelineExecutor) KB() *entity.Knowledgebase { return &s.taskCtx.KB }
func (s *PipelineExecutor) Doc() *entity.Document     { return &s.taskCtx.Doc }
func (s *PipelineExecutor) Tenant() *entity.Tenant    { return &s.taskCtx.Tenant }

func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error) {
	start := time.Now()
	if err := ctx.Err(); err != nil {
		return nil, err
	}

	dsl, correctedID, err := s.loadDSLFunc(ctx, s.canvasID)
	if err != nil {
		return nil, err
	}
	if correctedID != "" {
		s.canvasID = correctedID
	}
	if !s.taskCtx.IsDebug() {
		s.persistResolvedPipeline(context.WithoutCancel(ctx))
	}

	pipelineOutput, pipelineDSL, err := s.runPipelineFunc(ctx, dsl)
	if err != nil {
		return nil, err
	}

	// A debug (dry-run) run produces no persistent side effect (no MinIO
	// image upload, no index insert, no pipeline log); see TaskContext.IsDebug.
	if s.taskCtx.IsDebug() {
		return s.collectDebugOutput(ctx, pipelineOutput, start)
	}

	result, err := s.processOutput(ctx, pipelineOutput, start)
	if err != nil {
		return nil, err
	}

	if pipelineDSL != "" && s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.PipelineLogID != nil && *s.taskCtx.IngestionTask.PipelineLogID != "" {
		var terminalDuration *float64
		if result != nil {
			terminalDuration = &result.Duration
		} else {
			// A successful run with no chunks never wrote a terminal duration
			// to the document, so recompute from the same process_begin_at
			// anchor here instead of letting the log copy the stale mid-run
			// value.
			d := s.terminalDuration(start)
			terminalDuration = &d
		}
		s.recordPipelineLog(context.WithoutCancel(ctx), s.deps.DB, s.taskCtx.Doc.ID, pipelineDSL, string(entity.TaskStatusDone), terminalDuration)
	}

	return result, nil
}

// collectDebugOutput builds a PipelineResult for a debug (dry-run) run.
// It surfaces the pipeline's chunks so a debug endpoint can render them, but
// performs no DB/index writes — the embedding vectors already computed by the
// pipeline run are left on the chunks. Run-scoped cache state is purged before
// the preview is returned.
func (s *PipelineExecutor) collectDebugOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {
	s.purgeRunState(ctx)
	chunks := indexdoc.NormalizeChunks(pipelineOutput)
	return &PipelineResult{
		DocID:            s.taskCtx.Doc.ID,
		KbID:             s.taskCtx.Doc.KbID,
		Chunks:           chunks,
		ChunkCount:       countOriginalChunkIDs(chunks),
		TokenConsumption: indexdoc.GetEmbeddingTokenConsumption(pipelineOutput),
		Duration:         time.Since(start).Seconds(),
	}, nil
}

func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {
	if pipelineOutput == nil {
		s.purgeRunState(ctx)
		return nil, nil
	}
	if err := ctx.Err(); err != nil {
		return nil, err
	}

	chunks := indexdoc.NormalizeChunks(pipelineOutput)
	if len(chunks) == 0 {
		s.purgeRunState(ctx)
		return nil, nil
	}

	embeddingTokenConsumption := indexdoc.GetEmbeddingTokenConsumption(pipelineOutput)
	metadata, err := indexdoc.ProcessChunksForPipeline(
		chunks,
		s.taskCtx.Doc.ID,
		*s.taskCtx.Doc.Name,
		time.Now(),
	)
	if err != nil {
		return nil, err
	}
	parentChunks := indexdoc.MaterializeParentChunks(s.taskCtx.Doc.KbID, chunks)

	tableMeta := indexdoc.AggregateTableDocMetadata(chunks, map[string]interface{}(s.taskCtx.Doc.ParserConfig))
	if tableMeta != nil {
		if metadata == nil {
			metadata = make(map[string]any)
		}
		for k, v := range tableMeta {
			if _, exists := metadata[k]; !exists {
				metadata[k] = v
			}
		}
	}

	// Per-document wiki products are staged as available_int=0 until the
	// dataset-level consumer merges them into available_int=1 products.
	// Final per-document structures and ordinary source chunks are written with
	// available_int=1 unless the document itself is disabled (status=0).
	markCompiledProductsHidden(chunks)
	// Reload status immediately before write: LoadFromIngestionTask copied a
	// snapshot that may be stale if BatchUpdateDocumentStatus ran mid-pipeline.
	docStatus := s.taskCtx.Doc.Status
	if s.deps.DB != nil {
		if persisted, err := dao.NewDocumentDAO().GetByID(ctx, s.deps.DB, s.taskCtx.Doc.ID); err == nil && persisted != nil {
			docStatus = persisted.Status
			s.taskCtx.Doc.Status = persisted.Status
		} else if err != nil {
			common.Warn(fmt.Sprintf("failed to reload document %s status before availability stamp: %v", s.taskCtx.Doc.ID, err))
		}
	}
	applyDocumentAvailability(chunks, docStatus)

	oldCompiledProductIDs, oldCompiledVariants, oldCompiledTaskTypes, err := s.loadDocumentCompiledState(ctx)
	if err != nil {
		return nil, err
	}
	indexChunks := append(chunks, parentChunks...)
	indexdoc.StampDatasetPagerank(indexChunks, s.taskCtx.KB.Pagerank)
	if err := s.indexWriter.Write(ctx, indexChunks); err != nil {
		if cleanupErr := s.compensateFailedIndexWrite(ctx, indexChunks); cleanupErr != nil {
			return nil, fmt.Errorf("write chunks: %w; compensate partial index write: %v", err, cleanupErr)
		}
		return nil, fmt.Errorf("write chunks: %w", err)
	}
	if err := s.reconcileDocumentCompiledProducts(ctx, oldCompiledProductIDs, chunks); err != nil {
		return nil, err
	}
	activeStates, err := wikiActiveStates(pipelineOutput)
	if err != nil {
		return nil, err
	}
	if err := putWikiActiveStates(ctx, s.deps.Engine, activeStates); err != nil {
		return nil, fmt.Errorf("persist Wiki active MAP state: %w", err)
	}

	// All chunks are now persisted. Defer the dataset-level post-processing
	// notification until the enclosing ingestion task reaches COMPLETED. The
	// index writes intentionally do not wait for refresh, so publishing here lets
	// the consumer race the final task bookkeeping and read an incomplete view of
	// the document's products.
	eventVariants := mergeCompiledVariants(oldCompiledVariants, compiledVariants(chunks))
	eventTaskTypes := mergeTaskTypes(oldCompiledTaskTypes, compiledTaskTypes(chunks))

	// Compilation products are derived artifacts and must not inflate the
	// document's source chunk counter shown by the document list API.
	chunkCount := countOriginalChunkIDs(chunks)

	builtInMetadata, autoMetaEnabled := builtInMetadataFromParserConfig(
		s.taskCtx.Doc.ParserConfig,
	)

	// Persist is now fully durable: every failure-capable step above
	// (index write, compiled-product reconcile, Wiki active-MAP state) has
	// succeeded. Only now drop the run-scoped cache entries this task produced,
	// so a failure in any of those steps leaves chunk results and parser media
	// intact for retry.
	s.purgeRunState(ctx)

	return &PipelineResult{
		DocID:                 s.taskCtx.Doc.ID,
		KbID:                  s.taskCtx.Doc.KbID,
		Metadata:              metadata,
		ChunkCount:            chunkCount,
		TokenConsumption:      embeddingTokenConsumption,
		Duration:              s.terminalDuration(start),
		DocName:               docNameValue(s.taskCtx.Doc.Name),
		BuiltInMetadataConfig: builtInMetadata,
		AutoMetadataEnabled:   autoMetaEnabled,
		CompiledVariants:      eventVariants,
		CompiledTaskTypes:     eventTaskTypes,
	}, nil
}

func (s *PipelineExecutor) purgeRunState(ctx context.Context) {
	if s == nil || s.taskCtx == nil || s.purgeTaskState == nil {
		return
	}
	taskID := ""
	if s.taskCtx.IngestionTask != nil {
		taskID = s.taskCtx.IngestionTask.ID
	}
	if taskID == "" && s.taskCtx.IsDebug() {
		taskID = "pipeline_" + s.taskCtx.Doc.ID
	}
	if taskID == "" {
		return
	}
	cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), runStateCleanupTimeout)
	defer cancel()
	if err := s.purgeTaskState(cleanupCtx, taskID); err != nil {
		common.Warn("failed to purge task run state",
			zap.String("task_id", taskID),
			zap.Error(err),
		)
	}
}

func (s *PipelineExecutor) compensateFailedIndexWrite(ctx context.Context, chunks []map[string]any) error {
	if s.deleteChunksFunc == nil {
		return nil
	}
	ids := make([]string, 0, len(chunks))
	seen := make(map[string]struct{}, len(chunks))
	for _, chunk := range chunks {
		id, _ := chunk["id"].(string)
		if id == "" {
			continue
		}
		if _, exists := seen[id]; exists {
			continue
		}
		seen[id] = struct{}{}
		ids = append(ids, id)
	}
	if len(ids) == 0 {
		return nil
	}
	cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
	defer cancel()
	_, err := s.deleteChunksFunc(
		cleanupCtx,
		map[string]any{"id": ids, "kb_id": s.taskCtx.Doc.KbID},
		s.indexWriter.baseName,
		s.taskCtx.Doc.KbID,
	)
	return err
}

// terminalDuration measures the run from the document's process_begin_at —
// the anchor PrepareValidatedRun stamped before the run and the one every
// mid-run progress-sink duration write uses — so the terminal value applied
// to document.process_duration and the value recorded in the pipeline
// operation log are the same number. Runs whose document carries no begin
// time fall back to the executor start.
func (s *PipelineExecutor) terminalDuration(start time.Time) float64 {
	if begin := s.taskCtx.Doc.ProcessBeginAt; begin != nil {
		duration := time.Since(*begin).Seconds()
		if duration < 0 {
			duration = 0
		}
		return duration
	}
	return time.Since(start).Seconds()
}

// builtInMetadataFromParserConfig extracts the built-in metadata config
// (update_time / file_name) and whether auto-metadata is enabled from the
// component-scoped Extractor node's modular metadata config. Legacy flat
// fields (enable_metadata / metadata_config / built_in_metadata at either the
// top level or on the node) are intentionally not supported.
// builtInMetadataFromParserConfig extracts the built-in metadata config
// (update_time / file_name) and whether auto-metadata is enabled from the
// component-scoped Extractor node's modular metadata config. Legacy flat
// fields (enable_metadata / metadata_config / built_in_metadata at either the
// top level or on the node) are intentionally not supported.
//
// The config lookup and the field-list normalisation live in common (see
// ExtractorMetadataConfig / MetadataRawFieldList) so the ingestion pipeline and the agentic
// metadata catalog read the very same shape.
func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool) {
	metaObj, ok := common.ExtractorMetadataConfig(parserConfig)
	if !ok {
		return nil, false
	}
	return common.MetadataRawFieldList(metaObj["built_in_metadata"]), common.ParserConfigBool(metaObj["enabled"])
}

func docNameValue(name *string) string {
	if name == nil {
		return ""
	}
	return *name
}

// countOriginalChunkIDs returns the number of distinct source chunk IDs in the
// slice. Knowledge-compiler products are also emitted as index chunks, but
// carry compile_kwd and must not be included in the document's chunk_count.
// After ProcessChunksForPipeline, every chunk carries an "id" field computed
// from xxhash(text+docID). Chunks with identical text share the same id and
// the search engine treats them as upserts, so the effective stored chunk count
// is the number of unique ids — not len(chunks). Mirrors the index-side
// deduplication that happens at write time.
func countOriginalChunkIDs(chunks []map[string]any) int {
	seen := make(map[string]struct{}, len(chunks))
	for _, ck := range chunks {
		if _, compiled := ck["compile_kwd"]; compiled {
			continue
		}
		id, _ := ck["id"].(string)
		if id == "" {
			continue
		}
		seen[id] = struct{}{}
	}
	return len(seen)
}

// markCompiledProductsHidden stages wiki per-document page rows as
// available_int=0. Only the wiki variant is hidden: its per-document page rows
// are intermediate state for the consumer's merged pages (available_int=1) and
// every wiki read path selects merged rows only.
//
// Tree / structure (page_index) / mindmap rows are the FINAL per-document
// products — Python writes them visible at compile time
// (_struct_to_doc_storage_doc sets no available_int, index default 1) and its
// claim recall / hybrid retrieval filter on available_int=1, so they stay
// searchable here too. Merged dataset-level products are written by the
// consumer, never here, so they are never double-marked.
func markCompiledProductsHidden(chunks []map[string]any) {
	for _, ck := range chunks {
		kwd := asCompiledKwd(ck)
		if kwd == "" {
			continue
		}
		if v, err := knowledge_compile.KwdToVariant(kwd); err == nil && v == kccommon.VariantWiki {
			ck["available_int"] = 0
		}
	}
}

// loadDocumentCompiledState snapshots the previous successful document
// compiler generation before the new pipeline output is written. Reading first
// avoids relying on immediate search visibility after a bulk index write.
func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]string, []string, []string, error) {
	if s == nil || s.taskCtx == nil || s.deps == nil || s.deps.Engine == nil {
		return nil, nil, nil, nil
	}
	docEngine := s.deps.Engine
	const pageSize = 1000
	indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)
	oldIDs := make([]string, 0)
	oldProducts := make([]map[string]any, 0)
	for offset := 0; ; offset += pageSize {
		result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{
			IndexNames:   []string{indexName},
			KbIDs:        []string{s.taskCtx.Doc.KbID},
			Offset:       offset,
			Limit:        pageSize,
			SelectFields: []string{"id", "compile_kwd", "compilation_template_kind_kwd", "type_kwd"},
			Filter:       map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},
		})
		if err != nil {
			return nil, nil, nil, fmt.Errorf("load document compiler products: %w", err)
		}
		if result == nil || len(result.Chunks) == 0 {
			break
		}
		for _, row := range result.Chunks {
			if strings.TrimSpace(asCompiledKwd(row)) == "" || enginetypes.IsNavigationRow(row) {
				continue
			}
			oldProducts = append(oldProducts, row)
			if id := strings.TrimSpace(anyString(row["id"])); id != "" {
				oldIDs = append(oldIDs, id)
			}
		}
		if int64(offset+len(result.Chunks)) >= result.Total {
			break
		}
	}
	return oldIDs, compiledVariants(oldProducts), compiledTaskTypes(oldProducts), nil
}

// reconcileDocumentCompiledProducts advances the document-level compiler
// generation only after the new pipeline output has been persisted. Reparse
// preparation keeps the previous compiled rows so a failed run does not erase
// the last usable document Wiki; this method removes rows absent from the new
// successful generation immediately before the dataset completion event.
func (s *PipelineExecutor) reconcileDocumentCompiledProducts(ctx context.Context, oldIDs []string, chunks []map[string]any) error {
	if s == nil || s.taskCtx == nil || s.deps == nil || s.deps.Engine == nil {
		return nil
	}
	docEngine := s.deps.Engine
	newIDs := make(map[string]struct{})
	for _, chunk := range chunks {
		if strings.TrimSpace(asCompiledKwd(chunk)) == "" {
			continue
		}
		if id := strings.TrimSpace(anyString(chunk["id"])); id != "" {
			newIDs[id] = struct{}{}
		}
	}
	staleIDs := make([]string, 0, len(oldIDs))
	for _, id := range oldIDs {
		if _, keep := newIDs[id]; !keep {
			staleIDs = append(staleIDs, id)
		}
	}
	const pageSize = 1000
	indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)
	for start := 0; start < len(staleIDs); start += pageSize {
		end := min(start+pageSize, len(staleIDs))
		if _, err := docEngine.DeleteChunks(ctx, map[string]any{
			"id":    staleIDs[start:end],
			"kb_id": s.taskCtx.Doc.KbID,
		}, indexName, s.taskCtx.Doc.KbID); err != nil {
			return fmt.Errorf("delete stale document compiler products: %w", err)
		}
	}
	return nil
}

// applyDocumentAvailability stamps every chunk — ordinary source chunks and
// compiled products alike — with available_int=0 when Document.status is "0"
// (disabled). Disabling before any chunks exist only updates MySQL; without
// this, later parsing would still write searchable available_int=1 rows.
// Wiki staging rows are already available_int=0 from markCompiledProductsHidden,
// so the stamp is a no-op for them; tree / structure rows become hidden
// together with their document, matching Python's doc_id-scoped availability
// toggle.
func applyDocumentAvailability(chunks []map[string]any, status *string) {
	if status == nil || *status != "0" {
		return
	}
	for _, ck := range chunks {
		ck["available_int"] = 0
	}
}

// compiledVariants returns the sorted, de-duplicated execution variants needed
// to route document completion to the dataset consumer.
func compiledVariants(chunks []map[string]any) []string {
	seen := map[string]struct{}{}
	for _, ck := range chunks {
		if _, ok := ck["compile_kwd"]; !ok || enginetypes.IsNavigationRow(ck) {
			continue
		}
		var v kccommon.Variant
		if kind := enginetypes.CompilationKind(ck); kind != "" {
			mapped, err := kccommon.KindToVariant(kind)
			if err != nil {
				continue // unknown template kind (O2a): skip, do not mis-route
			}
			v = mapped
		} else {
			// Fallback on compile_kwd via the shared knowledge_compile mapping
			// (which folds the inferred structure kinds list/set/hypergraph into
			// VariantStructure before the KindToVariant whitelist lookup).
			mapped, err := knowledge_compile.KwdToVariant(asCompiledKwd(ck))
			if err != nil {
				continue
			}
			v = mapped
		}
		if _, dup := seen[string(v)]; dup {
			continue
		}
		seen[string(v)] = struct{}{}
	}
	if len(seen) == 0 {
		return nil
	}
	out := make([]string, 0, len(seen))
	for k := range seen {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

// compiledTaskTypes returns the sorted, de-duplicated frontend categories of
// the compiled products in chunks. The raw template kind is preferred because
// several kinds intentionally share the structure execution variant.
func compiledTaskTypes(chunks []map[string]any) []string {
	seen := map[string]struct{}{}
	for _, ck := range chunks {
		if _, ok := ck["compile_kwd"]; !ok || enginetypes.IsNavigationRow(ck) {
			continue
		}
		taskType := ""
		if kind := enginetypes.CompilationKind(ck); kind != "" {
			if mapped, err := kccommon.KindToTaskType(kind); err == nil {
				taskType = mapped
			}
		}
		if taskType == "" {
			if mapped, err := knowledge_compile.KwdToVariant(asCompiledKwd(ck)); err == nil {
				taskType = kccommon.VariantToTaskType(mapped)
			}
		}
		if taskType != "" {
			seen[taskType] = struct{}{}
		}
	}
	return sortedTaskTypes(seen)
}

func mergeTaskTypes(groups ...[]string) []string {
	seen := map[string]struct{}{}
	for _, group := range groups {
		for _, taskType := range group {
			if taskType = strings.TrimSpace(taskType); taskType != "" {
				seen[taskType] = struct{}{}
			}
		}
	}
	return sortedTaskTypes(seen)
}

func sortedTaskTypes(seen map[string]struct{}) []string {
	out := make([]string, 0, len(seen))
	for taskType := range seen {
		out = append(out, taskType)
	}
	sort.Strings(out)
	return out
}

// mergeCompiledVariants returns the union of the previous and current
// document-level compiler variants. A reparse that removes a compiler template
// still needs to notify the dataset consumer so it can retract stale merged
// products; a document that never had compiler products produces an empty set
// and does not wake the dataset consumer at all.
func mergeCompiledVariants(previous, current []string) []string {
	seen := make(map[string]struct{}, len(previous)+len(current))
	for _, variants := range [][]string{previous, current} {
		for _, variant := range variants {
			variant = strings.TrimSpace(variant)
			if variant != "" {
				seen[variant] = struct{}{}
			}
		}
	}
	if len(seen) == 0 {
		return nil
	}
	merged := make([]string, 0, len(seen))
	for variant := range seen {
		merged = append(merged, variant)
	}
	sort.Strings(merged)
	return merged
}

// asCompiledKwd extracts the compile_kwd keyword value from a chunk map,
// tolerating both a bare string and a list-wrapped keyword column from the
// engine. It returns "" when absent.
func asCompiledKwd(c map[string]any) string {
	switch v := c["compile_kwd"].(type) {
	case string:
		return v
	case []string:
		if len(v) > 0 {
			return v[0]
		}
	case []any:
		if len(v) > 0 {
			if s, ok := v[0].(string); ok {
				return s
			}
		}
	}
	return ""
}

func wikiActiveStates(output map[string]any) ([]kccommon.WikiMapActiveState, error) {
	if output == nil || output["wiki_active_map_states"] == nil {
		return nil, nil
	}
	appendState := func(states []kccommon.WikiMapActiveState, value map[string]any) ([]kccommon.WikiMapActiveState, error) {
		state := kccommon.WikiMapActiveState{
			Key:        strings.TrimSpace(anyString(value["key"])),
			TenantID:   strings.TrimSpace(anyString(value["tenant_id"])),
			DatasetID:  strings.TrimSpace(anyString(value["dataset_id"])),
			DocumentID: strings.TrimSpace(anyString(value["document_id"])),
		}
		switch payload := value["payload"].(type) {
		case string:
			state.Payload = []byte(payload)
		case []byte:
			state.Payload = append([]byte(nil), payload...)
		default:
			return nil, fmt.Errorf("decode Wiki active MAP state: unsupported payload type %T", value["payload"])
		}
		if state.Key == "" || state.TenantID == "" || state.DatasetID == "" || state.DocumentID == "" {
			return nil, fmt.Errorf("decode Wiki active MAP state: incomplete scope")
		}
		return append(states, state), nil
	}

	switch values := output["wiki_active_map_states"].(type) {
	case []kccommon.WikiMapActiveState:
		return append([]kccommon.WikiMapActiveState(nil), values...), nil
	case []map[string]any:
		states := make([]kccommon.WikiMapActiveState, 0, len(values))
		for _, value := range values {
			var err error
			states, err = appendState(states, value)
			if err != nil {
				return nil, err
			}
		}
		return states, nil
	case []any:
		states := make([]kccommon.WikiMapActiveState, 0, len(values))
		for _, raw := range values {
			value, ok := raw.(map[string]any)
			if !ok {
				return nil, fmt.Errorf("decode Wiki active MAP state: unsupported item type %T", raw)
			}
			var err error
			states, err = appendState(states, value)
			if err != nil {
				return nil, err
			}
		}
		return states, nil
	default:
		return nil, fmt.Errorf("decode Wiki active MAP states: unsupported type %T", output["wiki_active_map_states"])
	}
}

// PipelineLogInput contains the identifiers and optional snapshots needed to
// persist a pipeline operation log without constructing a PipelineExecutor.
type PipelineLogInput struct {
	TenantID   string
	KbID       string
	DocumentID string
	PipelineID string
	DSL        string
	Status     string
	Document   entity.Document
	// PipelineLogID is the id of the pipeline_operation_log row this run owns
	// (ingestion_task.pipeline_log_id). The public writer requires it and updates
	// exactly that row, never creating a second one — so a superseded run whose
	// row was deleted cannot adopt the replacement run's row.
	PipelineLogID           string
	PipelineVersionID       string
	PipelineBindingResolved bool
	// TerminalDuration, when set, is this run's final duration in seconds,
	// measured from the document's process_begin_at. updateOpenLogRow records
	// it instead of the reloaded document's last mid-run value, so
	// pipeline_operation_log and document.process_duration hold one number.
	// Writers without it (failure/cancel, or a run that produced no chunks)
	// copy the document's stored value — still the last thing any writer put
	// there, so the tables stay aligned on those paths too.
	TerminalDuration *float64
}

// ErrMissingRunIdentity reports an attempt to persist an ingestion terminal
// outcome without the run row that owns it.
var ErrMissingRunIdentity = errors.New("pipeline log id is required")

// ErrPipelineVersionMissing is the single multi-version Missing sentinel. It is
// defined once in the dao package (dao.ErrPipelineVersionMissing) and re-exported
// here so the service and task layers share one identity (review: merge the
// duplicate service/task sentinels into one implementation).
var ErrPipelineVersionMissing = dao.ErrPipelineVersionMissing

// RecordPipelineLog persists a pipeline operation log without requiring
// executor setup. Callers should pass the status in Status.
//
// When the run created a queued row (CREATED/SCHEDULED), the terminal write
// advances that same row so the dataset detail page shows one entry per run.
// A missing run identity is rejected before any database lookup or fallback.
func RecordPipelineLog(ctx context.Context, db *gorm.DB, input PipelineLogInput) error {
	if strings.TrimSpace(input.PipelineLogID) == "" {
		return ErrMissingRunIdentity
	}
	return recordPipelineLog(ctx, db, input)
}

func recordPipelineLog(
	ctx context.Context,
	db *gorm.DB,
	input PipelineLogInput,
) error {
	if strings.TrimSpace(input.PipelineLogID) == "" {
		return ErrMissingRunIdentity
	}
	if db == nil {
		return errors.New("pipeline log database is required")
	}
	var dslMap entity.JSONMap
	if strings.TrimSpace(input.DSL) == "" {
		dslMap = entity.JSONMap{}
	} else if err := json.Unmarshal([]byte(input.DSL), &dslMap); err != nil {
		dslMap = entity.JSONMap{"raw": input.DSL}
	}

	// The task context contains the document snapshot loaded when the task
	// started. Reload it here so the operation log reflects the final progress
	// state written by the progress sink, matching the Python operation-log
	// creation path.
	doc := input.Document
	if doc.ID == "" {
		doc.ID = input.DocumentID
		doc.KbID = input.KbID
	}
	if db != nil {
		if persisted, err := dao.NewDocumentDAO().GetByID(ctx, db, input.DocumentID); err == nil && persisted != nil {
			doc = *persisted
		} else if err != nil {
			common.Warn(fmt.Sprintf("failed to reload document %s for pipeline log: %v", input.DocumentID, err))
		}
	}
	if input.KbID == "" {
		input.KbID = doc.KbID
	}
	if input.TenantID == "" && db != nil && input.KbID != "" {
		if kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, input.KbID); err == nil && kb != nil {
			input.TenantID = kb.TenantID
		} else if err != nil {
			return fmt.Errorf("load knowledgebase %s for pipeline log: %w", input.KbID, err)
		}
	}

	// Pipeline identity for the log row. A document without a user pipeline
	// selection runs on a builtin registry pipeline: its canvasID is the
	// parser_id, not a canvas row, so the log is titled with the document's
	// parser_id, reuses the document thumbnail as avatar, and leaves
	// pipeline_id empty. The canvas lookup must not depend on the DSL: terminal
	// failure/cancel logs are recorded without a DSL, and their pipeline title
	// and avatar must still resolve from the canvas row (mirrors the Python
	// operation-log creation path).
	pipelineTitle := doc.ParserID
	pipelineAvatar := doc.Thumbnail
	var pipelineID *string
	if input.PipelineID != "" {
		pipelineID = &input.PipelineID
		if db != nil {
			if canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, db, input.PipelineID); err == nil && canvas != nil {
				if canvas.Title != nil {
					pipelineTitle = *canvas.Title
				}
				pipelineAvatar = canvas.Avatar
			} else if err != nil && !errors.Is(err, dao.ErrUserCanvasNotFound) {
				common.Warn(fmt.Sprintf("failed to reload pipeline %s for operation log: %v", input.PipelineID, err))
			}
		}
	}

	operationStatus := input.Status
	statusValue := "1"
	if doc.Status != nil && *doc.Status != "" {
		statusValue = *doc.Status
	}
	if err := updateOpenLogRow(ctx, db, input, operationStatus, statusValue, pipelineID, pipelineTitle, pipelineAvatar, dslMap, doc); err != nil {
		return fmt.Errorf("advance pipeline log for document %s: %w", input.DocumentID, err)
	}
	return nil
}

// updateOpenLogRow advances the row this run already owns to its terminal
// state, filling in the DSL, the pipeline identity, and the final progress
// snapshot.
//
// The row is targeted by PipelineLogID, so a superseded run whose row was
// deleted with the task can never reach into the replacement run's row.
// RowsAffected is not used to decide whether to create: once a target row is
// known, the write is final — a lost CAS means another writer already finalized
// this run or the row was dropped with a superseded run, and neither may
// produce a second entry.
func updateOpenLogRow(ctx context.Context, db *gorm.DB, input PipelineLogInput, operationStatus, statusValue string, pipelineID *string, pipelineTitle string, pipelineAvatar *string, dslMap entity.JSONMap, doc entity.Document) error {
	targetID := input.PipelineLogID
	updates := map[string]interface{}{
		"operation_status": operationStatus,
		"status":           statusValue,
		"dsl":              dslMap,
		"pipeline_title":   pipelineTitle,
		"avatar":           pipelineAvatar,
		"progress":         doc.Progress,
		"process_duration": doc.ProcessDuration,
		"parser_id":        doc.ParserID,
		"source_from":      strings.SplitN(doc.SourceType, "/", 2)[0],
		"document_suffix":  doc.Suffix,
		"document_type":    doc.Type,
	}
	if doc.Name != nil {
		updates["document_name"] = *doc.Name
	}
	if input.PipelineBindingResolved {
		var versionValue interface{}
		if pipelineID != nil && input.PipelineVersionID != "" {
			versionValue = input.PipelineVersionID
		}
		updates["pipeline_id"] = pipelineID
		updates["pipeline_version_id"] = versionValue
	}
	if input.TerminalDuration != nil {
		updates["process_duration"] = *input.TerminalDuration
	}
	// The open row was created with a timestamp. Keep it when the reloaded
	// document carries none (a run that never reached the progress sink), so
	// the queued entry does not lose its start time.
	if doc.ProcessBeginAt != nil {
		updates["process_begin_at"] = doc.ProcessBeginAt
	}
	result := db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
		Where("id = ? AND operation_status IN ?", targetID, dao.OpenPipelineOperationStatuses()).
		Updates(updates)
	if result.Error != nil {
		return result.Error
	}
	// No second entry is created for a known target, so a row that is gone or
	// already finalized means this run's outcome is not recorded anywhere. That
	// is the intended outcome for a run whose row was dropped with a superseded
	// task; log it so the missing entry is not silent.
	if result.RowsAffected == 0 {
		common.Warn(fmt.Sprintf("pipeline log %s for document %s is gone or already terminal; %s entry dropped", targetID, input.DocumentID, operationStatus))
	}
	return nil
}

func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string, terminalDuration *float64) {
	pipelineID := ""
	if s.taskCtx.PipelineID != "" {
		pipelineID = s.canvasID
	}
	pipelineLogID := ""
	if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.PipelineLogID != nil {
		pipelineLogID = *s.taskCtx.IngestionTask.PipelineLogID
	}
	if err := recordPipelineLog(ctx, db, PipelineLogInput{
		TenantID:                s.Tenant().ID,
		KbID:                    s.KB().ID,
		DocumentID:              docID,
		PipelineID:              pipelineID,
		DSL:                     dsl,
		Status:                  status,
		Document:                s.taskCtx.Doc,
		PipelineLogID:           pipelineLogID,
		PipelineVersionID:       s.resolvedVersionID,
		PipelineBindingResolved: true,
		TerminalDuration:        terminalDuration,
	}); err != nil {
		common.Warn(fmt.Sprintf("failed to record pipeline log: %v", err))
	}
}

func (s *PipelineExecutor) persistResolvedPipeline(ctx context.Context) {
	task := s.taskCtx.IngestionTask
	if task == nil || task.PipelineLogID == nil || *task.PipelineLogID == "" || s.deps == nil || s.deps.DB == nil {
		return
	}
	pipelineID := ""
	if s.taskCtx.PipelineID != "" {
		pipelineID = s.canvasID
	}
	if err := dao.NewPipelineOperationLogDAO().SetResolvedPipeline(ctx, s.deps.DB, *task.PipelineLogID, pipelineID, s.resolvedVersionID); err != nil {
		common.Warn(fmt.Sprintf("persist pipeline %s version %s for run %s: %v", pipelineID, s.resolvedVersionID, *task.PipelineLogID, err))
	}
}

func (s *PipelineExecutor) loadDSLFromCanvas(ctx context.Context, canvasID string) (string, string, error) {
	if s == nil || s.taskCtx == nil {
		return "", "", fmt.Errorf("pipeline executor: nil task context")
	}
	if canvasID == "" {
		return "", "", fmt.Errorf("pipeline executor: empty canvas id")
	}
	s.resolvedVersionID = ""
	canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, s.deps.DB, canvasID)
	if err != nil {
		return "", "", fmt.Errorf("load canvas %s: %w", canvasID, err)
	}

	canvasTitle := ""
	if canvas.Title != nil {
		canvasTitle = *canvas.Title
	}
	common.Info(fmt.Sprintf("load canvas %s, name %s", canvasID, canvasTitle))

	// dataflow canvases are versioned (design section 6.2). An explicit pin
	// resolves that exact version; an empty pin resolves the latest version.
	// A missing or mis-owned version maps to ErrPipelineVersionMissing so the
	// parse task fails and the file enters an unparseable Missing state (C4).
	if canvas.CanvasCategory == "dataflow_canvas" {
		// Dataflow canvases are versioned (design section 6.2). An explicit pin
		// resolves that exact version; an empty pin resolves the latest published
		// version. A missing or mis-owned version returns ErrPipelineVersionMissing
		// so the parse task fails and the file enters an unparseable Missing state.
		raw, resolvedVersionID, verr := dao.LoadCanvasVersionDSL(ctx, s.deps.DB, canvasID, s.taskCtx.PipelineVersionID)
		if verr != nil {
			return "", "", verr
		}
		s.resolvedVersionID = resolvedVersionID
		return string(raw), canvasID, nil
	}

	// Agent / non-versioned canvases: read the live draft column (out of scope
	// for multi-version; mirrors the agent read path in the service layer).
	raw, err := json.Marshal(canvas.DSL)
	if err != nil {
		return "", "", fmt.Errorf("marshal canvas dsl %s: %w", canvasID, err)
	}
	return string(raw), canvasID, nil
}

// warnUnknownComponentParams logs a warning for any component id in the
// parserConfig whose id is absent from the pipeline DSL. The runtime merge
// (component params -> override_params) silently drops such entries, so we
// surface them here for operability. API-side validation
// already rejects unknown ids on write; this is purely a defensive guard
// for legacy/stale rows.
func warnUnknownComponentParams(dsl string, parserConfig map[string]any) {
	if len(parserConfig) == 0 {
		return
	}
	// dsl arrives as the canvas ENVELOPE ({ "dsl": { "components": ... } }) in
	// production, so it must be unwrapped before ExtractAllComponentParams
	// runs (that helper expects the inner DSL). The previous direct call
	// passed the enveloped DSL, whose "components" key is nested under "dsl",
	// so it silently returned an error and made this guard a no-op.
	inner, err := pipelinepkg.UnwrapCanvasDSL([]byte(dsl))
	if err != nil {
		common.Warn(fmt.Sprintf("warnUnknownComponentParams: cannot parse DSL to validate component params: %v", err))
		return
	}
	innerJSON, err := json.Marshal(inner)
	if err != nil {
		common.Warn(fmt.Sprintf("warnUnknownComponentParams: cannot re-encode DSL: %v", err))
		return
	}
	schemas, err := pipelinepkg.ExtractAllComponentParams(innerJSON)
	if err != nil {
		common.Warn(fmt.Sprintf("warnUnknownComponentParams: cannot parse DSL to validate component params: %v", err))
		return
	}
	dslCPNs := make(map[string]struct{}, len(schemas))
	for _, s := range schemas {
		dslCPNs[s.CpnID] = struct{}{}
	}
	for cpnID := range parserConfig {
		if _, ok := dslCPNs[cpnID]; !ok {
			common.Warn(fmt.Sprintf(
				"parser_config references cpnID %q not present in the pipeline DSL; it will be ignored at runtime", cpnID))
		}
	}
}

func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (map[string]any, string, error) {
	if s == nil || s.taskCtx == nil {
		return nil, dsl, fmt.Errorf("pipeline executor: nil task context")
	}

	parserConfig := map[string]interface{}(s.taskCtx.Doc.ParserConfig)
	if parserConfig == nil {
		// Debug (dataflow dry-run) contexts intentionally carry no
		// ParserConfig; start from an empty map so the debug page cap can be
		// injected in place below without a nil-map assignment panic.
		parserConfig = map[string]interface{}{}
	}
	// Legacy parser_config rows saved by the Python-era frontend nest the
	// Parser's setups under a "setups" key. Lift them BEFORE the runtime
	// merges this map over the DSL-baked params (override-wins on top-level
	// key): a nested entry merged as-is would lose to the DSL's flat families
	// when NewParserComponent applies its own flattening, and the debug page
	// cap below reads the flat shape too.
	parserConfig = pipelinepkg.NormalizeParserConfigSetups(parserConfig)

	// Surface component params whose cpnID is absent from the DSL. The
	// runtime merge (override_params) silently drops such entries;
	// API-side validation already rejects unknown ids on write, so this is a
	// defensive guard for legacy/stale rows.
	warnUnknownComponentParams(dsl, parserConfig)

	pipelineID := "pipeline_" + s.taskCtx.Doc.ID
	if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.ID != "" {
		pipelineID = s.taskCtx.IngestionTask.ID
	}
	pipe, err := pipelinepkg.NewPipelineFromDSL([]byte(dsl), pipelineID,
		pipelinepkg.WithProgressSink(s.progressSink),
		pipelinepkg.WithDocumentID(s.taskCtx.Doc.ID),
		pipelinepkg.WithKV(s.deps.KV))
	if err != nil {
		return nil, dsl, fmt.Errorf("compile pipeline dsl: %w", err)
	}
	inputs := map[string]any{}
	if s.taskCtx.Doc.ID != "" {
		inputs["doc_id"] = s.taskCtx.Doc.ID
	}
	// Run-level metadata shared by both persist and debug (dataflow
	// dry-run) runs. In debug the KB is absent (NewDebugTaskContext forces
	// KB.ID == ""); in persist it is the document's own KB. Either way the
	// Tokenizer reads kb_id from CanvasState.Globals.
	inputs["tenant_id"] = s.taskCtx.Tenant.ID
	inputs["kb_id"] = s.taskCtx.KB.ID
	if s.taskCtx.KB.Language != nil {
		inputs["lang"] = *s.taskCtx.KB.Language
	}

	// File delivery and doc metadata differ between the two run modes.
	debug := s.taskCtx.IsDebug()
	if debug {
		// A debug (dry-run) run has no DB document row, so the parser
		// cannot resolve its bytes via doc_id → storage. Deliver the
		// uploaded bytes directly as `binary` (what the parser actually
		// reads) and surface the doc name/type so family detection works.
		if s.taskCtx.File != nil {
			inputs["file"] = s.taskCtx.File
			inputs["binary"] = s.taskCtx.File
		}
		if s.taskCtx.Doc.Name != nil && *s.taskCtx.Doc.Name != "" {
			inputs["name"] = *s.taskCtx.Doc.Name
		}
		if s.taskCtx.Doc.Type != "" {
			inputs["file_type"] = s.taskCtx.Doc.Type
		}
		// A debug (dry-run) run with a chunker node keeps only the leading
		// N chunks for preview. The cap is delivered through pipeline inputs
		// (seeded into CanvasState.Globals by the pipeline run, read by the
		// chunker decorator via globals.DebugChunkCap) — the same run-level
		// channel as the other shared metadata, not override_params (the
		// decorator is built at compile time and cannot read run-time
		// override_params). An explicit caller-supplied cap is respected.
		inputs = injectDebugChunkCap(inputs)
	} else {
		if s.taskCtx.File != nil {
			inputs["file"] = s.taskCtx.File
		}
	}

	// A canvas-debug (dataflow dry-run) must return a fast preview, so it
	// caps the parser to the first few pages. The cap is delivered through
	// override_params (Run's 3rd argument) — the SAME channel the
	// production ParserConfig uses — keyed by the Parser component's cpnID
	// and the document's filetype family. It is NOT passed through pipeline
	// inputs: the parser selects pages from ParserConfig[cpnID][family]
	// ["pages"] (a list of 1-indexed inclusive ranges), exactly mirroring
	// NormalizeParserConfigPages / pdf_pages_test.go. The DSL/parser-family
	// knowledge now lives in pipeline.BuildParserPageCapOverride.
	if debug {
		parserConfig = pipelinepkg.BuildParserPageCapOverride(
			parserConfig, []byte(dsl), s.taskCtx.Doc.Type,
			debugPageCapPages, component.ComponentNameParser, component.ParserFileFamily)
	}

	// Component params from Doc.ParserConfig — including the tenant LLM id
	// injected into Extractor components above — are passed to Run as
	// override_params, keyed by cpnID with override-wins. The DSL itself is
	// compiled unchanged.
	output, err := pipe.Run(ctx, inputs, parserConfig)
	if err != nil {
		return nil, dsl, err
	}

	logDSL := s.buildLogDSL(dsl, output)

	payload, err := pipelinepkg.ExtractPayload(dsl, output)
	if err != nil {
		return nil, dsl, err
	}
	return payload, logDSL, nil
}

// buildLogDSL returns the DSL string recorded for a pipeline run. The PERSISTED
// copy carries the DSL DEFINITION only (static canvas structure + static
// params + downstream + graph + path + …) — business data is never constructed
// for it (BuildDebugResultDSL(dsl, output, false)). Log recording must never
// fail a run, so on any build error the static dsl is returned unchanged.
//
// BuildDebugResultDSL needs the full run output, which is in scope only inside
// runPipelineWithDSL: the extracted payload returned there no longer carries
// output["state"]. Two consumers, split by the includeOutputs flag:
//   - canvas-debug (dry-run) runs wire a DebugLogSink, which implements the
//     optional ResultSink capability. buildLogDSL hands it the FULL result DSL
//     (BuildDebugResultDSL(dsl, output, true)) so the front-end "View result"
//     page renders parsed chunks from the Redis END marker
//     (rag/flow/pipeline.py:98). This is the ONLY place business data leaves
//     the executor, and it targets Redis — never pipeline_operation_log.
//   - persist (dataset parse) runs use a DB-backed sink that is NOT a
//     ResultSink, so the preview branch is skipped. The returned logDSL (DSL
//     definition only) is what recordPipelineLog persists, matching the
//     "log stores the DSL definition, not business data" contract; the
//     dataset log "View result" page therefore renders from the dry-run
//     preview, not from the persisted row.
//
// The sink probe stays an optional capability: non-debug (DB-backed) sinks
// ignore it and the ProgressSink contract is unchanged.
func (s *PipelineExecutor) buildLogDSL(dsl string, output map[string]any) string {
	// Start from the static dsl: log recording must never fail a run, so on any
	// build error we fall back to the static dsl unchanged rather than a
	// half-written payload. The input dsl is the canvas DEFINITION (no runtime
	// outputs), so the fallback is not a business-data leak (CWE-200) — the
	// persisted copy stays definition-only and the log row is preserved for
	// observability.
	logDSL := dsl
	// Persisted log DSL: DSL definition ONLY. Business data is never
	// constructed for the persisted copy (includeOutputs=false). Real parses
	// persist this string via recordPipelineLog; a dry-run also builds it here
	// but discards it at the IsDebug() early-return before recordPipelineLog,
	// so no business data is written to pipeline_operation_log.
	if persisted, e := BuildDebugResultDSL(dsl, output, false); e == nil {
		if raw, e := json.Marshal(persisted); e == nil {
			logDSL = string(raw)
		} else {
			common.Warn(fmt.Sprintf("marshal run-result dsl for pipeline log: %v", e))
		}
	} else {
		common.Warn(fmt.Sprintf("build run-result dsl for pipeline log: %v", e))
	}
	// Dry-run live preview: the ResultSink (DebugLogSink, wired ONLY for
	// canvas-debug runs) receives the FULL result DSL with business data
	// (includeOutputs=true) so the front-end "View result" page renders parsed
	// chunks from Redis. Real-parse sinks are NOT ResultSink, so this branch is
	// skipped and no business data is handed anywhere on the persist path.
	if rs, ok := s.progressSink.(ResultSink); ok {
		if previewDSL, e := BuildDebugResultDSL(dsl, output, true); e == nil {
			rs.SetResult(previewDSL, output)
		}
	}
	return logDSL
}

// debugPageCapPages is the number of leading pages a canvas-debug
// (dataflow dry-run) parses. The debug preview must return fast, so we cap
// the parser to the first few pages. The cap is expressed as the 1-indexed
// inclusive range [1, debugPageCapPages], matching the production
// ParserConfig[cpnID][filetype]["pages"] shape (see NormalizeParserConfigPages).
const debugPageCapPages = 2

const runStateCleanupTimeout = 5 * time.Second

// injectDebugChunkCap sets the canvas-debug chunk cap on the run inputs when
// not already present. The cap is read by the chunker decorator (via
// CanvasState.Globals / globals.DebugChunkCap) and limits a debug (dry-run)
// preview to the leading N chunks. An existing value (a future caller-supplied
// override) is respected, mirroring BuildParserPageCapOverride's respect for an
// explicit page cap. A nil inputs map is initialized, so callers may pass a
// concrete or nil map.
func injectDebugChunkCap(inputs map[string]any) map[string]any {
	if inputs == nil {
		inputs = map[string]any{}
	}
	if _, ok := inputs[globals.DebugChunkCapKey]; !ok {
		inputs[globals.DebugChunkCapKey] = DebugChunkCapDefault
	}
	return inputs
}

// datasetLanguage is the language of the dataset being ingested, or "" when it
// is unset. It selects the fulltext analyzer on engines that fix it when the
// chunk store is created.
func datasetLanguage(taskCtx *TaskContext) string {
	if taskCtx == nil || taskCtx.KB.Language == nil {
		return ""
	}
	return *taskCtx.KB.Language
}