RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package task
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/engine"
enginetypes "ragflow/internal/engine/types"
"ragflow/internal/entity"
"ragflow/internal/ingestion"
"ragflow/internal/ingestion/chunkcache"
"ragflow/internal/ingestion/component"
"ragflow/internal/ingestion/component/globals"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/ingestion/knowledge_compile"
"ragflow/internal/ingestion/mediacache"
pipelinepkg "ragflow/internal/ingestion/pipeline"
indexdoc "ragflow/internal/ingestion/task/indexdoc"
"go.uber.org/zap"
"gorm.io/gorm"
)
// PipelineResult is the outcome of a pipeline run: chunks have been
// indexed, and these bookkeeping inputs remain for the caller to apply to
// document state (metadata merge + chunk/token counter bumps).
type PipelineResult struct {
DocID string
KbID string
Metadata map[string]any
Chunks []map[string]any // populated only in debug (dry-run) mode
ChunkCount int
TokenConsumption int
Duration float64 // pipeline wall-clock seconds
// DocName, BuiltInMetadataConfig and AutoMetadataEnabled carry what the
// document-state finalizer needs to apply built-in metadata
// (update_time / file_name), mirroring Python apply_built_in_metadata.
DocName string
BuiltInMetadataConfig []any
AutoMetadataEnabled bool
// CompiledVariants and CompiledTaskTypes identify the document-level
// products that the dataset consumer must merge after the ingestion task
// reaches its terminal COMPLETED state.
CompiledVariants []string
CompiledTaskTypes []string
// MessageID is the polling key for the debug-run log. The front-end reads
// it from the run response and polls GET /agents/:id/logs/:message_id to
// render progress; it is empty for non-debug (persist) runs.
MessageID string
}
type PipelineExecutor struct {
taskCtx *TaskContext
canvasID string
docBulkSize int
deps *ingestion.Deps
resolvedVersionID string
indexWriter *chunkIndexWriter
deleteChunksFunc DeleteChunksFunc
loadDSLFunc func(ctx context.Context, canvasID string) (string, string, error)
runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)
purgeTaskState func(ctx context.Context, taskID string) error
progressSink pipelinepkg.ProgressSink
}
func validateTaskContext(taskCtx *TaskContext) error {
if taskCtx == nil {
return fmt.Errorf("pipeline executor: nil task context")
}
if taskCtx.Doc.ID == "" {
return fmt.Errorf("pipeline executor: empty document id")
}
// A debug (dry-run) context carries no knowledgebase (see
// TaskContext.IsDebug), so it must not be required to supply one.
if !taskCtx.IsDebug() && taskCtx.Doc.KbID == "" {
return fmt.Errorf("pipeline executor: empty document knowledgebase id")
}
if taskCtx.Doc.Name == nil || *taskCtx.Doc.Name == "" {
return fmt.Errorf("pipeline executor: empty document name")
}
if taskCtx.Tenant.ID == "" {
return fmt.Errorf("pipeline executor: empty tenant id")
}
return nil
}
// noRefreshChunkInserter is the narrower inserter the ingestion path prefers:
// an engine that implements it can write chunks without waiting for an index
// refresh.
type noRefreshChunkInserter interface {
InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error)
}
// insertChunksForIngestion writes non-final chunks through the engine's
// no-refresh path when it offers one, falling back to the plain inserter
// otherwise.
//
// Refreshing every batch adds unnecessary latency. The writer uses the regular
// inserter for its final batch, so all preceding batches become searchable
// before the document task is acknowledged and its completion event is sent.
//
// The first write creates the chunk store, and on Infinity that fixes the
// fulltext analyzer, so the dataset language has to travel with it.
func insertChunksForIngestion(eng engine.DocEngine, language string) InsertFunc {
return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {
if bulk, ok := eng.(noRefreshChunkInserter); ok {
return bulk.InsertChunksNoRefresh(ctx, chunks, baseName, datasetID, language)
}
return eng.InsertChunks(ctx, chunks, baseName, datasetID, language)
}
}
// insertFinalChunksForIngestion waits for the index refresh before returning.
// The final write makes every preceding no-refresh batch searchable before the
// document task is acknowledged and its completion event is published.
func insertFinalChunksForIngestion(eng engine.DocEngine, language string) InsertFunc {
return func(ctx context.Context, chunks []map[string]any, baseName string, datasetID string) ([]string, error) {
return eng.InsertChunks(ctx, chunks, baseName, datasetID, language)
}
}
// NewPipelineExecutor builds the executor for one document run. deps carries
// the process-wide singletons (doc engine, database handle) resolved at the
// server assembly point; the executor never resolves them itself.
func NewPipelineExecutor(
taskCtx *TaskContext,
canvasID string,
docBulkSize int,
deps *ingestion.Deps,
) (*PipelineExecutor, error) {
if err := validateTaskContext(taskCtx); err != nil {
return nil, err
}
if strings.TrimSpace(canvasID) == "" {
return nil, fmt.Errorf("pipeline executor: empty canvas id")
}
if deps == nil {
return nil, fmt.Errorf("pipeline executor: nil dependency bundle")
}
svc := &PipelineExecutor{
taskCtx: taskCtx,
canvasID: canvasID,
docBulkSize: docBulkSize,
deps: deps,
indexWriter: newChunkIndexWriter(
insertChunksForIngestion(deps.Engine, datasetLanguage(taskCtx)),
fmt.Sprintf("ragflow_%s", taskCtx.Tenant.ID),
taskCtx.Doc.KbID,
docBulkSize,
).withFinalInsertFunc(insertFinalChunksForIngestion(deps.Engine, datasetLanguage(taskCtx))),
deleteChunksFunc: func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error) {
return deps.Engine.DeleteChunks(ctx, condition, baseName, datasetID)
},
}
svc.loadDSLFunc = svc.loadDSLFromCanvas
svc.runPipelineFunc = svc.runPipelineWithDSL
svc.purgeTaskState = func(ctx context.Context, taskID string) error {
if deps.KV == nil {
return nil
}
return errors.Join(
chunkcache.PurgeTask(ctx, deps.KV, taskID),
mediacache.PurgeTask(ctx, deps.KV, taskID),
)
}
return svc, nil
}
// DeleteChunksFunc removes partially written rows after a failed index write.
type DeleteChunksFunc func(ctx context.Context, condition map[string]any, baseName, datasetID string) (int64, error)
func (s *PipelineExecutor) WithInsertFunc(f InsertFunc) *PipelineExecutor {
s.indexWriter.insertFunc = f
s.indexWriter.finalInsertFunc = f
return s
}
// WithDeleteChunksFunc replaces index-write compensation. It is used by the
// in-memory pipeline tests; production uses the configured document engine.
func (s *PipelineExecutor) WithDeleteChunksFunc(f DeleteChunksFunc) *PipelineExecutor {
s.deleteChunksFunc = f
return s
}
func (s *PipelineExecutor) WithLoadDSLFunc(f func(ctx context.Context, canvasID string) (string, string, error)) *PipelineExecutor {
s.loadDSLFunc = f
return s
}
func (s *PipelineExecutor) WithRunPipelineFunc(f func(ctx context.Context, dsl string) (map[string]any, string, error)) *PipelineExecutor {
s.runPipelineFunc = f
return s
}
// WithProgressSink injects a sink that receives pipeline component progress
// events. The sink owns all document/ingestion_task_log persistence; when
// unset, the pipeline runs DB-independent (progress events are dropped).
func (s *PipelineExecutor) WithProgressSink(sink pipelinepkg.ProgressSink) *PipelineExecutor {
s.progressSink = sink
return s
}
func (s *PipelineExecutor) KB() *entity.Knowledgebase { return &s.taskCtx.KB }
func (s *PipelineExecutor) Doc() *entity.Document { return &s.taskCtx.Doc }
func (s *PipelineExecutor) Tenant() *entity.Tenant { return &s.taskCtx.Tenant }
func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error) {
start := time.Now()
if err := ctx.Err(); err != nil {
return nil, err
}
dsl, correctedID, err := s.loadDSLFunc(ctx, s.canvasID)
if err != nil {
return nil, err
}
if correctedID != "" {
s.canvasID = correctedID
}
if !s.taskCtx.IsDebug() {
s.persistResolvedPipeline(context.WithoutCancel(ctx))
}
pipelineOutput, pipelineDSL, err := s.runPipelineFunc(ctx, dsl)
if err != nil {
return nil, err
}
// A debug (dry-run) run produces no persistent side effect (no MinIO
// image upload, no index insert, no pipeline log); see TaskContext.IsDebug.
if s.taskCtx.IsDebug() {
return s.collectDebugOutput(ctx, pipelineOutput, start)
}
result, err := s.processOutput(ctx, pipelineOutput, start)
if err != nil {
return nil, err
}
if pipelineDSL != "" && s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.PipelineLogID != nil && *s.taskCtx.IngestionTask.PipelineLogID != "" {
var terminalDuration *float64
if result != nil {
terminalDuration = &result.Duration
} else {
// A successful run with no chunks never wrote a terminal duration
// to the document, so recompute from the same process_begin_at
// anchor here instead of letting the log copy the stale mid-run
// value.
d := s.terminalDuration(start)
terminalDuration = &d
}
s.recordPipelineLog(context.WithoutCancel(ctx), s.deps.DB, s.taskCtx.Doc.ID, pipelineDSL, string(entity.TaskStatusDone), terminalDuration)
}
return result, nil
}
// collectDebugOutput builds a PipelineResult for a debug (dry-run) run.
// It surfaces the pipeline's chunks so a debug endpoint can render them, but
// performs no DB/index writes — the embedding vectors already computed by the
// pipeline run are left on the chunks. Run-scoped cache state is purged before
// the preview is returned.
func (s *PipelineExecutor) collectDebugOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {
s.purgeRunState(ctx)
chunks := indexdoc.NormalizeChunks(pipelineOutput)
return &PipelineResult{
DocID: s.taskCtx.Doc.ID,
KbID: s.taskCtx.Doc.KbID,
Chunks: chunks,
ChunkCount: countOriginalChunkIDs(chunks),
TokenConsumption: indexdoc.GetEmbeddingTokenConsumption(pipelineOutput),
Duration: time.Since(start).Seconds(),
}, nil
}
func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map[string]any, start time.Time) (*PipelineResult, error) {
if pipelineOutput == nil {
s.purgeRunState(ctx)
return nil, nil
}
if err := ctx.Err(); err != nil {
return nil, err
}
chunks := indexdoc.NormalizeChunks(pipelineOutput)
if len(chunks) == 0 {
s.purgeRunState(ctx)
return nil, nil
}
embeddingTokenConsumption := indexdoc.GetEmbeddingTokenConsumption(pipelineOutput)
metadata, err := indexdoc.ProcessChunksForPipeline(
chunks,
s.taskCtx.Doc.ID,
*s.taskCtx.Doc.Name,
time.Now(),
)
if err != nil {
return nil, err
}
parentChunks := indexdoc.MaterializeParentChunks(s.taskCtx.Doc.KbID, chunks)
tableMeta := indexdoc.AggregateTableDocMetadata(chunks, map[string]interface{}(s.taskCtx.Doc.ParserConfig))
if tableMeta != nil {
if metadata == nil {
metadata = make(map[string]any)
}
for k, v := range tableMeta {
if _, exists := metadata[k]; !exists {
metadata[k] = v
}
}
}
// Per-document wiki products are staged as available_int=0 until the
// dataset-level consumer merges them into available_int=1 products.
// Final per-document structures and ordinary source chunks are written with
// available_int=1 unless the document itself is disabled (status=0).
markCompiledProductsHidden(chunks)
// Reload status immediately before write: LoadFromIngestionTask copied a
// snapshot that may be stale if BatchUpdateDocumentStatus ran mid-pipeline.
docStatus := s.taskCtx.Doc.Status
if s.deps.DB != nil {
if persisted, err := dao.NewDocumentDAO().GetByID(ctx, s.deps.DB, s.taskCtx.Doc.ID); err == nil && persisted != nil {
docStatus = persisted.Status
s.taskCtx.Doc.Status = persisted.Status
} else if err != nil {
common.Warn(fmt.Sprintf("failed to reload document %s status before availability stamp: %v", s.taskCtx.Doc.ID, err))
}
}
applyDocumentAvailability(chunks, docStatus)
oldCompiledProductIDs, oldCompiledVariants, oldCompiledTaskTypes, err := s.loadDocumentCompiledState(ctx)
if err != nil {
return nil, err
}
indexChunks := append(chunks, parentChunks...)
indexdoc.StampDatasetPagerank(indexChunks, s.taskCtx.KB.Pagerank)
if err := s.indexWriter.Write(ctx, indexChunks); err != nil {
if cleanupErr := s.compensateFailedIndexWrite(ctx, indexChunks); cleanupErr != nil {
return nil, fmt.Errorf("write chunks: %w; compensate partial index write: %v", err, cleanupErr)
}
return nil, fmt.Errorf("write chunks: %w", err)
}
if err := s.reconcileDocumentCompiledProducts(ctx, oldCompiledProductIDs, chunks); err != nil {
return nil, err
}
activeStates, err := wikiActiveStates(pipelineOutput)
if err != nil {
return nil, err
}
if err := putWikiActiveStates(ctx, s.deps.Engine, activeStates); err != nil {
return nil, fmt.Errorf("persist Wiki active MAP state: %w", err)
}
// All chunks are now persisted. Defer the dataset-level post-processing
// notification until the enclosing ingestion task reaches COMPLETED. The
// index writes intentionally do not wait for refresh, so publishing here lets
// the consumer race the final task bookkeeping and read an incomplete view of
// the document's products.
eventVariants := mergeCompiledVariants(oldCompiledVariants, compiledVariants(chunks))
eventTaskTypes := mergeTaskTypes(oldCompiledTaskTypes, compiledTaskTypes(chunks))
// Compilation products are derived artifacts and must not inflate the
// document's source chunk counter shown by the document list API.
chunkCount := countOriginalChunkIDs(chunks)
builtInMetadata, autoMetaEnabled := builtInMetadataFromParserConfig(
s.taskCtx.Doc.ParserConfig,
)
// Persist is now fully durable: every failure-capable step above
// (index write, compiled-product reconcile, Wiki active-MAP state) has
// succeeded. Only now drop the run-scoped cache entries this task produced,
// so a failure in any of those steps leaves chunk results and parser media
// intact for retry.
s.purgeRunState(ctx)
return &PipelineResult{
DocID: s.taskCtx.Doc.ID,
KbID: s.taskCtx.Doc.KbID,
Metadata: metadata,
ChunkCount: chunkCount,
TokenConsumption: embeddingTokenConsumption,
Duration: s.terminalDuration(start),
DocName: docNameValue(s.taskCtx.Doc.Name),
BuiltInMetadataConfig: builtInMetadata,
AutoMetadataEnabled: autoMetaEnabled,
CompiledVariants: eventVariants,
CompiledTaskTypes: eventTaskTypes,
}, nil
}
func (s *PipelineExecutor) purgeRunState(ctx context.Context) {
if s == nil || s.taskCtx == nil || s.purgeTaskState == nil {
return
}
taskID := ""
if s.taskCtx.IngestionTask != nil {
taskID = s.taskCtx.IngestionTask.ID
}
if taskID == "" && s.taskCtx.IsDebug() {
taskID = "pipeline_" + s.taskCtx.Doc.ID
}
if taskID == "" {
return
}
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), runStateCleanupTimeout)
defer cancel()
if err := s.purgeTaskState(cleanupCtx, taskID); err != nil {
common.Warn("failed to purge task run state",
zap.String("task_id", taskID),
zap.Error(err),
)
}
}
func (s *PipelineExecutor) compensateFailedIndexWrite(ctx context.Context, chunks []map[string]any) error {
if s.deleteChunksFunc == nil {
return nil
}
ids := make([]string, 0, len(chunks))
seen := make(map[string]struct{}, len(chunks))
for _, chunk := range chunks {
id, _ := chunk["id"].(string)
if id == "" {
continue
}
if _, exists := seen[id]; exists {
continue
}
seen[id] = struct{}{}
ids = append(ids, id)
}
if len(ids) == 0 {
return nil
}
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
_, err := s.deleteChunksFunc(
cleanupCtx,
map[string]any{"id": ids, "kb_id": s.taskCtx.Doc.KbID},
s.indexWriter.baseName,
s.taskCtx.Doc.KbID,
)
return err
}
// terminalDuration measures the run from the document's process_begin_at —
// the anchor PrepareValidatedRun stamped before the run and the one every
// mid-run progress-sink duration write uses — so the terminal value applied
// to document.process_duration and the value recorded in the pipeline
// operation log are the same number. Runs whose document carries no begin
// time fall back to the executor start.
func (s *PipelineExecutor) terminalDuration(start time.Time) float64 {
if begin := s.taskCtx.Doc.ProcessBeginAt; begin != nil {
duration := time.Since(*begin).Seconds()
if duration < 0 {
duration = 0
}
return duration
}
return time.Since(start).Seconds()
}
// builtInMetadataFromParserConfig extracts the built-in metadata config
// (update_time / file_name) and whether auto-metadata is enabled from the
// component-scoped Extractor node's modular metadata config. Legacy flat
// fields (enable_metadata / metadata_config / built_in_metadata at either the
// top level or on the node) are intentionally not supported.
// builtInMetadataFromParserConfig extracts the built-in metadata config
// (update_time / file_name) and whether auto-metadata is enabled from the
// component-scoped Extractor node's modular metadata config. Legacy flat
// fields (enable_metadata / metadata_config / built_in_metadata at either the
// top level or on the node) are intentionally not supported.
//
// The config lookup and the field-list normalisation live in common (see
// ExtractorMetadataConfig / MetadataRawFieldList) so the ingestion pipeline and the agentic
// metadata catalog read the very same shape.
func builtInMetadataFromParserConfig(parserConfig entity.JSONMap) ([]any, bool) {
metaObj, ok := common.ExtractorMetadataConfig(parserConfig)
if !ok {
return nil, false
}
return common.MetadataRawFieldList(metaObj["built_in_metadata"]), common.ParserConfigBool(metaObj["enabled"])
}
func docNameValue(name *string) string {
if name == nil {
return ""
}
return *name
}
// countOriginalChunkIDs returns the number of distinct source chunk IDs in the
// slice. Knowledge-compiler products are also emitted as index chunks, but
// carry compile_kwd and must not be included in the document's chunk_count.
// After ProcessChunksForPipeline, every chunk carries an "id" field computed
// from xxhash(text+docID). Chunks with identical text share the same id and
// the search engine treats them as upserts, so the effective stored chunk count
// is the number of unique ids — not len(chunks). Mirrors the index-side
// deduplication that happens at write time.
func countOriginalChunkIDs(chunks []map[string]any) int {
seen := make(map[string]struct{}, len(chunks))
for _, ck := range chunks {
if _, compiled := ck["compile_kwd"]; compiled {
continue
}
id, _ := ck["id"].(string)
if id == "" {
continue
}
seen[id] = struct{}{}
}
return len(seen)
}
// markCompiledProductsHidden stages wiki per-document page rows as
// available_int=0. Only the wiki variant is hidden: its per-document page rows
// are intermediate state for the consumer's merged pages (available_int=1) and
// every wiki read path selects merged rows only.
//
// Tree / structure (page_index) / mindmap rows are the FINAL per-document
// products — Python writes them visible at compile time
// (_struct_to_doc_storage_doc sets no available_int, index default 1) and its
// claim recall / hybrid retrieval filter on available_int=1, so they stay
// searchable here too. Merged dataset-level products are written by the
// consumer, never here, so they are never double-marked.
func markCompiledProductsHidden(chunks []map[string]any) {
for _, ck := range chunks {
kwd := asCompiledKwd(ck)
if kwd == "" {
continue
}
if v, err := knowledge_compile.KwdToVariant(kwd); err == nil && v == kccommon.VariantWiki {
ck["available_int"] = 0
}
}
}
// loadDocumentCompiledState snapshots the previous successful document
// compiler generation before the new pipeline output is written. Reading first
// avoids relying on immediate search visibility after a bulk index write.
func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]string, []string, []string, error) {
if s == nil || s.taskCtx == nil || s.deps == nil || s.deps.Engine == nil {
return nil, nil, nil, nil
}
docEngine := s.deps.Engine
const pageSize = 1000
indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)
oldIDs := make([]string, 0)
oldProducts := make([]map[string]any, 0)
for offset := 0; ; offset += pageSize {
result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{
IndexNames: []string{indexName},
KbIDs: []string{s.taskCtx.Doc.KbID},
Offset: offset,
Limit: pageSize,
SelectFields: []string{"id", "compile_kwd", "compilation_template_kind_kwd", "type_kwd"},
Filter: map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},
})
if err != nil {
return nil, nil, nil, fmt.Errorf("load document compiler products: %w", err)
}
if result == nil || len(result.Chunks) == 0 {
break
}
for _, row := range result.Chunks {
if strings.TrimSpace(asCompiledKwd(row)) == "" || enginetypes.IsNavigationRow(row) {
continue
}
oldProducts = append(oldProducts, row)
if id := strings.TrimSpace(anyString(row["id"])); id != "" {
oldIDs = append(oldIDs, id)
}
}
if int64(offset+len(result.Chunks)) >= result.Total {
break
}
}
return oldIDs, compiledVariants(oldProducts), compiledTaskTypes(oldProducts), nil
}
// reconcileDocumentCompiledProducts advances the document-level compiler
// generation only after the new pipeline output has been persisted. Reparse
// preparation keeps the previous compiled rows so a failed run does not erase
// the last usable document Wiki; this method removes rows absent from the new
// successful generation immediately before the dataset completion event.
func (s *PipelineExecutor) reconcileDocumentCompiledProducts(ctx context.Context, oldIDs []string, chunks []map[string]any) error {
if s == nil || s.taskCtx == nil || s.deps == nil || s.deps.Engine == nil {
return nil
}
docEngine := s.deps.Engine
newIDs := make(map[string]struct{})
for _, chunk := range chunks {
if strings.TrimSpace(asCompiledKwd(chunk)) == "" {
continue
}
if id := strings.TrimSpace(anyString(chunk["id"])); id != "" {
newIDs[id] = struct{}{}
}
}
staleIDs := make([]string, 0, len(oldIDs))
for _, id := range oldIDs {
if _, keep := newIDs[id]; !keep {
staleIDs = append(staleIDs, id)
}
}
const pageSize = 1000
indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)
for start := 0; start < len(staleIDs); start += pageSize {
end := min(start+pageSize, len(staleIDs))
if _, err := docEngine.DeleteChunks(ctx, map[string]any{
"id": staleIDs[start:end],
"kb_id": s.taskCtx.Doc.KbID,
}, indexName, s.taskCtx.Doc.KbID); err != nil {
return fmt.Errorf("delete stale document compiler products: %w", err)
}
}
return nil
}
// applyDocumentAvailability stamps every chunk — ordinary source chunks and
// compiled products alike — with available_int=0 when Document.status is "0"
// (disabled). Disabling before any chunks exist only updates MySQL; without
// this, later parsing would still write searchable available_int=1 rows.
// Wiki staging rows are already available_int=0 from markCompiledProductsHidden,
// so the stamp is a no-op for them; tree / structure rows become hidden
// together with their document, matching Python's doc_id-scoped availability
// toggle.
func applyDocumentAvailability(chunks []map[string]any, status *string) {
if status == nil || *status != "0" {
return
}
for _, ck := range chunks {
ck["available_int"] = 0
}
}
// compiledVariants returns the sorted, de-duplicated execution variants needed
// to route document completion to the dataset consumer.
func compiledVariants(chunks []map[string]any) []string {
seen := map[string]struct{}{}
for _, ck := range chunks {
if _, ok := ck["compile_kwd"]; !ok || enginetypes.IsNavigationRow(ck) {
continue
}
var v kccommon.Variant
if kind := enginetypes.CompilationKind(ck); kind != "" {
mapped, err := kccommon.KindToVariant(kind)
if err != nil {
continue // unknown template kind (O2a): skip, do not mis-route
}
v = mapped
} else {
// Fallback on compile_kwd via the shared knowledge_compile mapping
// (which folds the inferred structure kinds list/set/hypergraph into
// VariantStructure before the KindToVariant whitelist lookup).
mapped, err := knowledge_compile.KwdToVariant(asCompiledKwd(ck))
if err != nil {
continue
}
v = mapped
}
if _, dup := seen[string(v)]; dup {
continue
}
seen[string(v)] = struct{}{}
}
if len(seen) == 0 {
return nil
}
out := make([]string, 0, len(seen))
for k := range seen {
out = append(out, k)
}
sort.Strings(out)
return out
}
// compiledTaskTypes returns the sorted, de-duplicated frontend categories of
// the compiled products in chunks. The raw template kind is preferred because
// several kinds intentionally share the structure execution variant.
func compiledTaskTypes(chunks []map[string]any) []string {
seen := map[string]struct{}{}
for _, ck := range chunks {
if _, ok := ck["compile_kwd"]; !ok || enginetypes.IsNavigationRow(ck) {
continue
}
taskType := ""
if kind := enginetypes.CompilationKind(ck); kind != "" {
if mapped, err := kccommon.KindToTaskType(kind); err == nil {
taskType = mapped
}
}
if taskType == "" {
if mapped, err := knowledge_compile.KwdToVariant(asCompiledKwd(ck)); err == nil {
taskType = kccommon.VariantToTaskType(mapped)
}
}
if taskType != "" {
seen[taskType] = struct{}{}
}
}
return sortedTaskTypes(seen)
}
func mergeTaskTypes(groups ...[]string) []string {
seen := map[string]struct{}{}
for _, group := range groups {
for _, taskType := range group {
if taskType = strings.TrimSpace(taskType); taskType != "" {
seen[taskType] = struct{}{}
}
}
}
return sortedTaskTypes(seen)
}
func sortedTaskTypes(seen map[string]struct{}) []string {
out := make([]string, 0, len(seen))
for taskType := range seen {
out = append(out, taskType)
}
sort.Strings(out)
return out
}
// mergeCompiledVariants returns the union of the previous and current
// document-level compiler variants. A reparse that removes a compiler template
// still needs to notify the dataset consumer so it can retract stale merged
// products; a document that never had compiler products produces an empty set
// and does not wake the dataset consumer at all.
func mergeCompiledVariants(previous, current []string) []string {
seen := make(map[string]struct{}, len(previous)+len(current))
for _, variants := range [][]string{previous, current} {
for _, variant := range variants {
variant = strings.TrimSpace(variant)
if variant != "" {
seen[variant] = struct{}{}
}
}
}
if len(seen) == 0 {
return nil
}
merged := make([]string, 0, len(seen))
for variant := range seen {
merged = append(merged, variant)
}
sort.Strings(merged)
return merged
}
// asCompiledKwd extracts the compile_kwd keyword value from a chunk map,
// tolerating both a bare string and a list-wrapped keyword column from the
// engine. It returns "" when absent.
func asCompiledKwd(c map[string]any) string {
switch v := c["compile_kwd"].(type) {
case string:
return v
case []string:
if len(v) > 0 {
return v[0]
}
case []any:
if len(v) > 0 {
if s, ok := v[0].(string); ok {
return s
}
}
}
return ""
}
func wikiActiveStates(output map[string]any) ([]kccommon.WikiMapActiveState, error) {
if output == nil || output["wiki_active_map_states"] == nil {
return nil, nil
}
appendState := func(states []kccommon.WikiMapActiveState, value map[string]any) ([]kccommon.WikiMapActiveState, error) {
state := kccommon.WikiMapActiveState{
Key: strings.TrimSpace(anyString(value["key"])),
TenantID: strings.TrimSpace(anyString(value["tenant_id"])),
DatasetID: strings.TrimSpace(anyString(value["dataset_id"])),
DocumentID: strings.TrimSpace(anyString(value["document_id"])),
}
switch payload := value["payload"].(type) {
case string:
state.Payload = []byte(payload)
case []byte:
state.Payload = append([]byte(nil), payload...)
default:
return nil, fmt.Errorf("decode Wiki active MAP state: unsupported payload type %T", value["payload"])
}
if state.Key == "" || state.TenantID == "" || state.DatasetID == "" || state.DocumentID == "" {
return nil, fmt.Errorf("decode Wiki active MAP state: incomplete scope")
}
return append(states, state), nil
}
switch values := output["wiki_active_map_states"].(type) {
case []kccommon.WikiMapActiveState:
return append([]kccommon.WikiMapActiveState(nil), values...), nil
case []map[string]any:
states := make([]kccommon.WikiMapActiveState, 0, len(values))
for _, value := range values {
var err error
states, err = appendState(states, value)
if err != nil {
return nil, err
}
}
return states, nil
case []any:
states := make([]kccommon.WikiMapActiveState, 0, len(values))
for _, raw := range values {
value, ok := raw.(map[string]any)
if !ok {
return nil, fmt.Errorf("decode Wiki active MAP state: unsupported item type %T", raw)
}
var err error
states, err = appendState(states, value)
if err != nil {
return nil, err
}
}
return states, nil
default:
return nil, fmt.Errorf("decode Wiki active MAP states: unsupported type %T", output["wiki_active_map_states"])
}
}
// PipelineLogInput contains the identifiers and optional snapshots needed to
// persist a pipeline operation log without constructing a PipelineExecutor.
type PipelineLogInput struct {
TenantID string
KbID string
DocumentID string
PipelineID string
DSL string
Status string
Document entity.Document
// PipelineLogID is the id of the pipeline_operation_log row this run owns
// (ingestion_task.pipeline_log_id). The public writer requires it and updates
// exactly that row, never creating a second one — so a superseded run whose
// row was deleted cannot adopt the replacement run's row.
PipelineLogID string
PipelineVersionID string
PipelineBindingResolved bool
// TerminalDuration, when set, is this run's final duration in seconds,
// measured from the document's process_begin_at. updateOpenLogRow records
// it instead of the reloaded document's last mid-run value, so
// pipeline_operation_log and document.process_duration hold one number.
// Writers without it (failure/cancel, or a run that produced no chunks)
// copy the document's stored value — still the last thing any writer put
// there, so the tables stay aligned on those paths too.
TerminalDuration *float64
}
// ErrMissingRunIdentity reports an attempt to persist an ingestion terminal
// outcome without the run row that owns it.
var ErrMissingRunIdentity = errors.New("pipeline log id is required")
// ErrPipelineVersionMissing is the single multi-version Missing sentinel. It is
// defined once in the dao package (dao.ErrPipelineVersionMissing) and re-exported
// here so the service and task layers share one identity (review: merge the
// duplicate service/task sentinels into one implementation).
var ErrPipelineVersionMissing = dao.ErrPipelineVersionMissing
// RecordPipelineLog persists a pipeline operation log without requiring
// executor setup. Callers should pass the status in Status.
//
// When the run created a queued row (CREATED/SCHEDULED), the terminal write
// advances that same row so the dataset detail page shows one entry per run.
// A missing run identity is rejected before any database lookup or fallback.
func RecordPipelineLog(ctx context.Context, db *gorm.DB, input PipelineLogInput) error {
if strings.TrimSpace(input.PipelineLogID) == "" {
return ErrMissingRunIdentity
}
return recordPipelineLog(ctx, db, input)
}
func recordPipelineLog(
ctx context.Context,
db *gorm.DB,
input PipelineLogInput,
) error {
if strings.TrimSpace(input.PipelineLogID) == "" {
return ErrMissingRunIdentity
}
if db == nil {
return errors.New("pipeline log database is required")
}
var dslMap entity.JSONMap
if strings.TrimSpace(input.DSL) == "" {
dslMap = entity.JSONMap{}
} else if err := json.Unmarshal([]byte(input.DSL), &dslMap); err != nil {
dslMap = entity.JSONMap{"raw": input.DSL}
}
// The task context contains the document snapshot loaded when the task
// started. Reload it here so the operation log reflects the final progress
// state written by the progress sink, matching the Python operation-log
// creation path.
doc := input.Document
if doc.ID == "" {
doc.ID = input.DocumentID
doc.KbID = input.KbID
}
if db != nil {
if persisted, err := dao.NewDocumentDAO().GetByID(ctx, db, input.DocumentID); err == nil && persisted != nil {
doc = *persisted
} else if err != nil {
common.Warn(fmt.Sprintf("failed to reload document %s for pipeline log: %v", input.DocumentID, err))
}
}
if input.KbID == "" {
input.KbID = doc.KbID
}
if input.TenantID == "" && db != nil && input.KbID != "" {
if kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, input.KbID); err == nil && kb != nil {
input.TenantID = kb.TenantID
} else if err != nil {
return fmt.Errorf("load knowledgebase %s for pipeline log: %w", input.KbID, err)
}
}
// Pipeline identity for the log row. A document without a user pipeline
// selection runs on a builtin registry pipeline: its canvasID is the
// parser_id, not a canvas row, so the log is titled with the document's
// parser_id, reuses the document thumbnail as avatar, and leaves
// pipeline_id empty. The canvas lookup must not depend on the DSL: terminal
// failure/cancel logs are recorded without a DSL, and their pipeline title
// and avatar must still resolve from the canvas row (mirrors the Python
// operation-log creation path).
pipelineTitle := doc.ParserID
pipelineAvatar := doc.Thumbnail
var pipelineID *string
if input.PipelineID != "" {
pipelineID = &input.PipelineID
if db != nil {
if canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, db, input.PipelineID); err == nil && canvas != nil {
if canvas.Title != nil {
pipelineTitle = *canvas.Title
}
pipelineAvatar = canvas.Avatar
} else if err != nil && !errors.Is(err, dao.ErrUserCanvasNotFound) {
common.Warn(fmt.Sprintf("failed to reload pipeline %s for operation log: %v", input.PipelineID, err))
}
}
}
operationStatus := input.Status
statusValue := "1"
if doc.Status != nil && *doc.Status != "" {
statusValue = *doc.Status
}
if err := updateOpenLogRow(ctx, db, input, operationStatus, statusValue, pipelineID, pipelineTitle, pipelineAvatar, dslMap, doc); err != nil {
return fmt.Errorf("advance pipeline log for document %s: %w", input.DocumentID, err)
}
return nil
}
// updateOpenLogRow advances the row this run already owns to its terminal
// state, filling in the DSL, the pipeline identity, and the final progress
// snapshot.
//
// The row is targeted by PipelineLogID, so a superseded run whose row was
// deleted with the task can never reach into the replacement run's row.
// RowsAffected is not used to decide whether to create: once a target row is
// known, the write is final — a lost CAS means another writer already finalized
// this run or the row was dropped with a superseded run, and neither may
// produce a second entry.
func updateOpenLogRow(ctx context.Context, db *gorm.DB, input PipelineLogInput, operationStatus, statusValue string, pipelineID *string, pipelineTitle string, pipelineAvatar *string, dslMap entity.JSONMap, doc entity.Document) error {
targetID := input.PipelineLogID
updates := map[string]interface{}{
"operation_status": operationStatus,
"status": statusValue,
"dsl": dslMap,
"pipeline_title": pipelineTitle,
"avatar": pipelineAvatar,
"progress": doc.Progress,
"process_duration": doc.ProcessDuration,
"parser_id": doc.ParserID,
"source_from": strings.SplitN(doc.SourceType, "/", 2)[0],
"document_suffix": doc.Suffix,
"document_type": doc.Type,
}
if doc.Name != nil {
updates["document_name"] = *doc.Name
}
if input.PipelineBindingResolved {
var versionValue interface{}
if pipelineID != nil && input.PipelineVersionID != "" {
versionValue = input.PipelineVersionID
}
updates["pipeline_id"] = pipelineID
updates["pipeline_version_id"] = versionValue
}
if input.TerminalDuration != nil {
updates["process_duration"] = *input.TerminalDuration
}
// The open row was created with a timestamp. Keep it when the reloaded
// document carries none (a run that never reached the progress sink), so
// the queued entry does not lose its start time.
if doc.ProcessBeginAt != nil {
updates["process_begin_at"] = doc.ProcessBeginAt
}
result := db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
Where("id = ? AND operation_status IN ?", targetID, dao.OpenPipelineOperationStatuses()).
Updates(updates)
if result.Error != nil {
return result.Error
}
// No second entry is created for a known target, so a row that is gone or
// already finalized means this run's outcome is not recorded anywhere. That
// is the intended outcome for a run whose row was dropped with a superseded
// task; log it so the missing entry is not silent.
if result.RowsAffected == 0 {
common.Warn(fmt.Sprintf("pipeline log %s for document %s is gone or already terminal; %s entry dropped", targetID, input.DocumentID, operationStatus))
}
return nil
}
func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string, terminalDuration *float64) {
pipelineID := ""
if s.taskCtx.PipelineID != "" {
pipelineID = s.canvasID
}
pipelineLogID := ""
if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.PipelineLogID != nil {
pipelineLogID = *s.taskCtx.IngestionTask.PipelineLogID
}
if err := recordPipelineLog(ctx, db, PipelineLogInput{
TenantID: s.Tenant().ID,
KbID: s.KB().ID,
DocumentID: docID,
PipelineID: pipelineID,
DSL: dsl,
Status: status,
Document: s.taskCtx.Doc,
PipelineLogID: pipelineLogID,
PipelineVersionID: s.resolvedVersionID,
PipelineBindingResolved: true,
TerminalDuration: terminalDuration,
}); err != nil {
common.Warn(fmt.Sprintf("failed to record pipeline log: %v", err))
}
}
func (s *PipelineExecutor) persistResolvedPipeline(ctx context.Context) {
task := s.taskCtx.IngestionTask
if task == nil || task.PipelineLogID == nil || *task.PipelineLogID == "" || s.deps == nil || s.deps.DB == nil {
return
}
pipelineID := ""
if s.taskCtx.PipelineID != "" {
pipelineID = s.canvasID
}
if err := dao.NewPipelineOperationLogDAO().SetResolvedPipeline(ctx, s.deps.DB, *task.PipelineLogID, pipelineID, s.resolvedVersionID); err != nil {
common.Warn(fmt.Sprintf("persist pipeline %s version %s for run %s: %v", pipelineID, s.resolvedVersionID, *task.PipelineLogID, err))
}
}
func (s *PipelineExecutor) loadDSLFromCanvas(ctx context.Context, canvasID string) (string, string, error) {
if s == nil || s.taskCtx == nil {
return "", "", fmt.Errorf("pipeline executor: nil task context")
}
if canvasID == "" {
return "", "", fmt.Errorf("pipeline executor: empty canvas id")
}
s.resolvedVersionID = ""
canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, s.deps.DB, canvasID)
if err != nil {
return "", "", fmt.Errorf("load canvas %s: %w", canvasID, err)
}
canvasTitle := ""
if canvas.Title != nil {
canvasTitle = *canvas.Title
}
common.Info(fmt.Sprintf("load canvas %s, name %s", canvasID, canvasTitle))
// dataflow canvases are versioned (design section 6.2). An explicit pin
// resolves that exact version; an empty pin resolves the latest version.
// A missing or mis-owned version maps to ErrPipelineVersionMissing so the
// parse task fails and the file enters an unparseable Missing state (C4).
if canvas.CanvasCategory == "dataflow_canvas" {
// Dataflow canvases are versioned (design section 6.2). An explicit pin
// resolves that exact version; an empty pin resolves the latest published
// version. A missing or mis-owned version returns ErrPipelineVersionMissing
// so the parse task fails and the file enters an unparseable Missing state.
raw, resolvedVersionID, verr := dao.LoadCanvasVersionDSL(ctx, s.deps.DB, canvasID, s.taskCtx.PipelineVersionID)
if verr != nil {
return "", "", verr
}
s.resolvedVersionID = resolvedVersionID
return string(raw), canvasID, nil
}
// Agent / non-versioned canvases: read the live draft column (out of scope
// for multi-version; mirrors the agent read path in the service layer).
raw, err := json.Marshal(canvas.DSL)
if err != nil {
return "", "", fmt.Errorf("marshal canvas dsl %s: %w", canvasID, err)
}
return string(raw), canvasID, nil
}
// warnUnknownComponentParams logs a warning for any component id in the
// parserConfig whose id is absent from the pipeline DSL. The runtime merge
// (component params -> override_params) silently drops such entries, so we
// surface them here for operability. API-side validation
// already rejects unknown ids on write; this is purely a defensive guard
// for legacy/stale rows.
func warnUnknownComponentParams(dsl string, parserConfig map[string]any) {
if len(parserConfig) == 0 {
return
}
// dsl arrives as the canvas ENVELOPE ({ "dsl": { "components": ... } }) in
// production, so it must be unwrapped before ExtractAllComponentParams
// runs (that helper expects the inner DSL). The previous direct call
// passed the enveloped DSL, whose "components" key is nested under "dsl",
// so it silently returned an error and made this guard a no-op.
inner, err := pipelinepkg.UnwrapCanvasDSL([]byte(dsl))
if err != nil {
common.Warn(fmt.Sprintf("warnUnknownComponentParams: cannot parse DSL to validate component params: %v", err))
return
}
innerJSON, err := json.Marshal(inner)
if err != nil {
common.Warn(fmt.Sprintf("warnUnknownComponentParams: cannot re-encode DSL: %v", err))
return
}
schemas, err := pipelinepkg.ExtractAllComponentParams(innerJSON)
if err != nil {
common.Warn(fmt.Sprintf("warnUnknownComponentParams: cannot parse DSL to validate component params: %v", err))
return
}
dslCPNs := make(map[string]struct{}, len(schemas))
for _, s := range schemas {
dslCPNs[s.CpnID] = struct{}{}
}
for cpnID := range parserConfig {
if _, ok := dslCPNs[cpnID]; !ok {
common.Warn(fmt.Sprintf(
"parser_config references cpnID %q not present in the pipeline DSL; it will be ignored at runtime", cpnID))
}
}
}
func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (map[string]any, string, error) {
if s == nil || s.taskCtx == nil {
return nil, dsl, fmt.Errorf("pipeline executor: nil task context")
}
parserConfig := map[string]interface{}(s.taskCtx.Doc.ParserConfig)
if parserConfig == nil {
// Debug (dataflow dry-run) contexts intentionally carry no
// ParserConfig; start from an empty map so the debug page cap can be
// injected in place below without a nil-map assignment panic.
parserConfig = map[string]interface{}{}
}
// Legacy parser_config rows saved by the Python-era frontend nest the
// Parser's setups under a "setups" key. Lift them BEFORE the runtime
// merges this map over the DSL-baked params (override-wins on top-level
// key): a nested entry merged as-is would lose to the DSL's flat families
// when NewParserComponent applies its own flattening, and the debug page
// cap below reads the flat shape too.
parserConfig = pipelinepkg.NormalizeParserConfigSetups(parserConfig)
// Surface component params whose cpnID is absent from the DSL. The
// runtime merge (override_params) silently drops such entries;
// API-side validation already rejects unknown ids on write, so this is a
// defensive guard for legacy/stale rows.
warnUnknownComponentParams(dsl, parserConfig)
pipelineID := "pipeline_" + s.taskCtx.Doc.ID
if s.taskCtx.IngestionTask != nil && s.taskCtx.IngestionTask.ID != "" {
pipelineID = s.taskCtx.IngestionTask.ID
}
pipe, err := pipelinepkg.NewPipelineFromDSL([]byte(dsl), pipelineID,
pipelinepkg.WithProgressSink(s.progressSink),
pipelinepkg.WithDocumentID(s.taskCtx.Doc.ID),
pipelinepkg.WithKV(s.deps.KV))
if err != nil {
return nil, dsl, fmt.Errorf("compile pipeline dsl: %w", err)
}
inputs := map[string]any{}
if s.taskCtx.Doc.ID != "" {
inputs["doc_id"] = s.taskCtx.Doc.ID
}
// Run-level metadata shared by both persist and debug (dataflow
// dry-run) runs. In debug the KB is absent (NewDebugTaskContext forces
// KB.ID == ""); in persist it is the document's own KB. Either way the
// Tokenizer reads kb_id from CanvasState.Globals.
inputs["tenant_id"] = s.taskCtx.Tenant.ID
inputs["kb_id"] = s.taskCtx.KB.ID
if s.taskCtx.KB.Language != nil {
inputs["lang"] = *s.taskCtx.KB.Language
}
// File delivery and doc metadata differ between the two run modes.
debug := s.taskCtx.IsDebug()
if debug {
// A debug (dry-run) run has no DB document row, so the parser
// cannot resolve its bytes via doc_id → storage. Deliver the
// uploaded bytes directly as `binary` (what the parser actually
// reads) and surface the doc name/type so family detection works.
if s.taskCtx.File != nil {
inputs["file"] = s.taskCtx.File
inputs["binary"] = s.taskCtx.File
}
if s.taskCtx.Doc.Name != nil && *s.taskCtx.Doc.Name != "" {
inputs["name"] = *s.taskCtx.Doc.Name
}
if s.taskCtx.Doc.Type != "" {
inputs["file_type"] = s.taskCtx.Doc.Type
}
// A debug (dry-run) run with a chunker node keeps only the leading
// N chunks for preview. The cap is delivered through pipeline inputs
// (seeded into CanvasState.Globals by the pipeline run, read by the
// chunker decorator via globals.DebugChunkCap) — the same run-level
// channel as the other shared metadata, not override_params (the
// decorator is built at compile time and cannot read run-time
// override_params). An explicit caller-supplied cap is respected.
inputs = injectDebugChunkCap(inputs)
} else {
if s.taskCtx.File != nil {
inputs["file"] = s.taskCtx.File
}
}
// A canvas-debug (dataflow dry-run) must return a fast preview, so it
// caps the parser to the first few pages. The cap is delivered through
// override_params (Run's 3rd argument) — the SAME channel the
// production ParserConfig uses — keyed by the Parser component's cpnID
// and the document's filetype family. It is NOT passed through pipeline
// inputs: the parser selects pages from ParserConfig[cpnID][family]
// ["pages"] (a list of 1-indexed inclusive ranges), exactly mirroring
// NormalizeParserConfigPages / pdf_pages_test.go. The DSL/parser-family
// knowledge now lives in pipeline.BuildParserPageCapOverride.
if debug {
parserConfig = pipelinepkg.BuildParserPageCapOverride(
parserConfig, []byte(dsl), s.taskCtx.Doc.Type,
debugPageCapPages, component.ComponentNameParser, component.ParserFileFamily)
}
// Component params from Doc.ParserConfig — including the tenant LLM id
// injected into Extractor components above — are passed to Run as
// override_params, keyed by cpnID with override-wins. The DSL itself is
// compiled unchanged.
output, err := pipe.Run(ctx, inputs, parserConfig)
if err != nil {
return nil, dsl, err
}
logDSL := s.buildLogDSL(dsl, output)
payload, err := pipelinepkg.ExtractPayload(dsl, output)
if err != nil {
return nil, dsl, err
}
return payload, logDSL, nil
}
// buildLogDSL returns the DSL string recorded for a pipeline run. The PERSISTED
// copy carries the DSL DEFINITION only (static canvas structure + static
// params + downstream + graph + path + …) — business data is never constructed
// for it (BuildDebugResultDSL(dsl, output, false)). Log recording must never
// fail a run, so on any build error the static dsl is returned unchanged.
//
// BuildDebugResultDSL needs the full run output, which is in scope only inside
// runPipelineWithDSL: the extracted payload returned there no longer carries
// output["state"]. Two consumers, split by the includeOutputs flag:
// - canvas-debug (dry-run) runs wire a DebugLogSink, which implements the
// optional ResultSink capability. buildLogDSL hands it the FULL result DSL
// (BuildDebugResultDSL(dsl, output, true)) so the front-end "View result"
// page renders parsed chunks from the Redis END marker
// (rag/flow/pipeline.py:98). This is the ONLY place business data leaves
// the executor, and it targets Redis — never pipeline_operation_log.
// - persist (dataset parse) runs use a DB-backed sink that is NOT a
// ResultSink, so the preview branch is skipped. The returned logDSL (DSL
// definition only) is what recordPipelineLog persists, matching the
// "log stores the DSL definition, not business data" contract; the
// dataset log "View result" page therefore renders from the dry-run
// preview, not from the persisted row.
//
// The sink probe stays an optional capability: non-debug (DB-backed) sinks
// ignore it and the ProgressSink contract is unchanged.
func (s *PipelineExecutor) buildLogDSL(dsl string, output map[string]any) string {
// Start from the static dsl: log recording must never fail a run, so on any
// build error we fall back to the static dsl unchanged rather than a
// half-written payload. The input dsl is the canvas DEFINITION (no runtime
// outputs), so the fallback is not a business-data leak (CWE-200) — the
// persisted copy stays definition-only and the log row is preserved for
// observability.
logDSL := dsl
// Persisted log DSL: DSL definition ONLY. Business data is never
// constructed for the persisted copy (includeOutputs=false). Real parses
// persist this string via recordPipelineLog; a dry-run also builds it here
// but discards it at the IsDebug() early-return before recordPipelineLog,
// so no business data is written to pipeline_operation_log.
if persisted, e := BuildDebugResultDSL(dsl, output, false); e == nil {
if raw, e := json.Marshal(persisted); e == nil {
logDSL = string(raw)
} else {
common.Warn(fmt.Sprintf("marshal run-result dsl for pipeline log: %v", e))
}
} else {
common.Warn(fmt.Sprintf("build run-result dsl for pipeline log: %v", e))
}
// Dry-run live preview: the ResultSink (DebugLogSink, wired ONLY for
// canvas-debug runs) receives the FULL result DSL with business data
// (includeOutputs=true) so the front-end "View result" page renders parsed
// chunks from Redis. Real-parse sinks are NOT ResultSink, so this branch is
// skipped and no business data is handed anywhere on the persist path.
if rs, ok := s.progressSink.(ResultSink); ok {
if previewDSL, e := BuildDebugResultDSL(dsl, output, true); e == nil {
rs.SetResult(previewDSL, output)
}
}
return logDSL
}
// debugPageCapPages is the number of leading pages a canvas-debug
// (dataflow dry-run) parses. The debug preview must return fast, so we cap
// the parser to the first few pages. The cap is expressed as the 1-indexed
// inclusive range [1, debugPageCapPages], matching the production
// ParserConfig[cpnID][filetype]["pages"] shape (see NormalizeParserConfigPages).
const debugPageCapPages = 2
const runStateCleanupTimeout = 5 * time.Second
// injectDebugChunkCap sets the canvas-debug chunk cap on the run inputs when
// not already present. The cap is read by the chunker decorator (via
// CanvasState.Globals / globals.DebugChunkCap) and limits a debug (dry-run)
// preview to the leading N chunks. An existing value (a future caller-supplied
// override) is respected, mirroring BuildParserPageCapOverride's respect for an
// explicit page cap. A nil inputs map is initialized, so callers may pass a
// concrete or nil map.
func injectDebugChunkCap(inputs map[string]any) map[string]any {
if inputs == nil {
inputs = map[string]any{}
}
if _, ok := inputs[globals.DebugChunkCapKey]; !ok {
inputs[globals.DebugChunkCapKey] = DebugChunkCapDefault
}
return inputs
}
// datasetLanguage is the language of the dataset being ingested, or "" when it
// is unset. It selects the fulltext analyzer on engines that fix it when the
// chunk store is created.
func datasetLanguage(taskCtx *TaskContext) string {
if taskCtx == nil || taskCtx.KB.Language == nil {
return ""
}
return *taskCtx.KB.Language
}