Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(ingestion): record pipeline version on operation logs (#21916)

jay777215b54cb0dad44
diff --git a/docs/administrator/upgrade_and_migration/database_change_log.md b/docs/administrator/upgrade_and_migration/database_change_log.md
index 63b0bbbe74..9ed615c70b 100644
--- a/docs/administrator/upgrade_and_migration/database_change_log.md
+++ b/docs/administrator/upgrade_and_migration/database_change_log.md
@@ -334,7 +334,7 @@ These steps are not gated by the version marker. They use schema or data checks
| `ingestion_task` | Unique index on `document_id`. |
| `knowledgebase` | Adds the `VIRTUAL` generated column `name_ci` (`LOWER(name)` for `status='1'` rows, `NULL` otherwise) and the unique index `idx_kb_tenant_name_ci (tenant_id, name_ci)`, so soft-deleted rows never block name reuse. |
| `user_canvas` | Unique index `(user_id, canvas_category, title)`. |
-| `pipeline_operation_log` | Adds `run_count int NULL` and the index `idx_pipeline_operation_log_document_run`. |
+| `pipeline_operation_log` | Adds `run_count int NULL`, `pipeline_version_id varchar(32) NULL` (the concrete dataflow canvas version a run executed), and the index `idx_pipeline_operation_log_document_run`. |
| `ingestion_task` | Adds `pipeline_log_id varchar(32) NULL`. |
| `ingestion_task_log` | Adds `pipeline_log_id varchar(32) NULL`, `event_type tinyint NOT NULL DEFAULT 4`, and the index `idx_ingestion_task_log_pipeline_id`. |
| `dialog`, `tenant_llm`, `api_token`, `canvas_template`, `system_settings`, `knowledgebase` | Column types converged to the Peewee models, for example `dialog.top_k int NOT NULL DEFAULT 1024`, `tenant_llm.api_key text`, `api_token.dialog_id varchar(32)`, `canvas_template.title` / `description` nullable longtext, `system_settings.value text NOT NULL`, `knowledgebase.raptor_task_finish_at` / `mindmap_task_finish_at` datetime. |
diff --git a/internal/dao/database_test.go b/internal/dao/database_test.go
index c02dadf617..7ebefb198a 100644
--- a/internal/dao/database_test.go
+++ b/internal/dao/database_test.go
@@ -384,7 +384,7 @@ func TestMigrateIngestionLogRunIdentity(t *testing.T) {
t.Fatalf("migrateIngestionLogRunIdentity: %v", err)
}
migrator := db.Migrator()
- for _, column := range []string{"run_count"} {
+ for _, column := range []string{"run_count", "pipeline_version_id"} {
if !migrator.HasColumn(&entity.PipelineOperationLog{}, column) {
t.Fatalf("pipeline_operation_log missing %s", column)
}
@@ -408,6 +408,13 @@ func TestMigrateIngestionLogRunIdentity(t *testing.T) {
if legacyRunCount != nil {
t.Fatalf("legacy run_count = %d, want NULL", *legacyRunCount)
}
+ var legacyVersionID *string
+ if err := db.Raw(`SELECT pipeline_version_id FROM pipeline_operation_log WHERE id = 'legacy-log'`).Scan(&legacyVersionID).Error; err != nil {
+ t.Fatalf("read legacy pipeline version: %v", err)
+ }
+ if legacyVersionID != nil {
+ t.Fatalf("legacy pipeline_version_id = %q, want NULL", *legacyVersionID)
+ }
if err := db.Exec(`INSERT INTO pipeline_operation_log (id, document_id, run_count) VALUES ('run-1', 'doc-1', 1)`).Error; err != nil {
t.Fatalf("insert first numbered run: %v", err)
}
diff --git a/internal/dao/migration.go b/internal/dao/migration.go
index 52c87122c1..fffb4d0448 100644
--- a/internal/dao/migration.go
+++ b/internal/dao/migration.go
@@ -433,10 +433,9 @@ func migrateIngestionTaskPipelineLogID(ctx context.Context, db *gorm.DB) error {
return nil
}
-// migrateIngestionLogRunIdentity adds the columns and indexes that make each
-// ingestion event belong to one immutable pipeline-operation-log run. It is
-// deliberately explicit: the runtime startup path auto-migrates ingestion
-// tables and cannot be trusted to converge an existing schema safely.
+// migrateIngestionLogRunIdentity adds the run identity and pipeline version
+// columns. It is explicit because runtime AutoMigrate cannot be trusted to
+// converge an existing schema safely.
func migrateIngestionLogRunIdentity(ctx context.Context, db *gorm.DB) error {
migrator := db.WithContext(ctx).Migrator()
if migrator.HasTable(&entity.PipelineOperationLog{}) {
@@ -450,6 +449,13 @@ func migrateIngestionLogRunIdentity(ctx context.Context, db *gorm.DB) error {
if err := createIngestionLogIndexIfMissing(migrator, &entity.PipelineOperationLog{}, "idx_pipeline_operation_log_document_run", "add pipeline_operation_log document/run index"); err != nil {
return err
}
+ if !migrator.HasColumn(&entity.PipelineOperationLog{}, "pipeline_version_id") {
+ if err := db.WithContext(ctx).Exec(
+ "ALTER TABLE pipeline_operation_log ADD COLUMN pipeline_version_id varchar(32) NULL",
+ ).Error; err != nil && !isDuplicateColumnError(err) {
+ return fmt.Errorf("add pipeline_operation_log.pipeline_version_id: %w", err)
+ }
+ }
}
if migrator.HasTable(&entity.IngestionTaskLog{}) {
diff --git a/internal/dao/pipeline_operation_log.go b/internal/dao/pipeline_operation_log.go
index 8d85b155e1..9532a9fd27 100644
--- a/internal/dao/pipeline_operation_log.go
+++ b/internal/dao/pipeline_operation_log.go
@@ -193,19 +193,20 @@ func OpenPipelineOperationStatuses() []string {
// columns that are known before the pipeline finishes; the DSL and the final
// progress/counters stay empty until the terminal writer fills them in.
type OpenLogInput struct {
- DocumentID string
- KbID string
- TenantID string
- PipelineID string
- PipelineTitle string
- ParserID string
- DocumentName string
- DocumentSuffix string
- DocumentType string
- SourceFrom string
- Avatar *string
- OperationStatus string
- RunCount int
+ DocumentID string
+ KbID string
+ TenantID string
+ PipelineID string
+ PipelineTitle string
+ PipelineVersionID string
+ ParserID string
+ DocumentName string
+ DocumentSuffix string
+ DocumentType string
+ SourceFrom string
+ Avatar *string
+ OperationStatus string
+ RunCount int
}
// CreateOpenLog opens the pre-terminal row for a queued, numbered run. Its
@@ -246,6 +247,10 @@ func (dao *PipelineOperationLogDAO) CreateOpenLog(ctx context.Context, db *gorm.
title := input.PipelineTitle
log.PipelineTitle = &title
}
+ if input.PipelineVersionID != "" {
+ versionID := input.PipelineVersionID
+ log.PipelineVersionID = &versionID
+ }
if err := db.WithContext(ctx).Create(log).Error; err != nil {
return nil, err
}
@@ -288,6 +293,26 @@ func (dao *PipelineOperationLogDAO) AdvanceOpenLog(ctx context.Context, db *gorm
return result.RowsAffected == 1, nil
}
+// SetResolvedPipeline records the pipeline and version a run resolved.
+func (dao *PipelineOperationLogDAO) SetResolvedPipeline(ctx context.Context, db *gorm.DB, logID, pipelineID, versionID string) error {
+ if logID == "" {
+ return nil
+ }
+ var pipelineValue, versionValue interface{}
+ if pipelineID != "" {
+ pipelineValue = pipelineID
+ }
+ if versionID != "" {
+ versionValue = versionID
+ }
+ return db.WithContext(ctx).Model(&entity.PipelineOperationLog{}).
+ Where("id = ? AND operation_status IN ?", logID, OpenPipelineOperationStatuses()).
+ Updates(map[string]interface{}{
+ "pipeline_id": pipelineValue,
+ "pipeline_version_id": versionValue,
+ }).Error
+}
+
// GetByIDAndKBID fetches a single ingestion log scoped to its knowledge base.
func (dao *PipelineOperationLogDAO) GetByIDAndKBID(ctx context.Context, db *gorm.DB, logID, kbID string) (*entity.PipelineOperationLog, error) {
var log entity.PipelineOperationLog
diff --git a/internal/dao/pipeline_operation_log_test.go b/internal/dao/pipeline_operation_log_test.go
new file mode 100644
index 0000000000..2c6e8b5eec
--- /dev/null
+++ b/internal/dao/pipeline_operation_log_test.go
@@ -0,0 +1,158 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/entity"
+)
+
+func setupPipelineOperationLogTestDB(t *testing.T) *gorm.DB {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err := db.AutoMigrate(&entity.PipelineOperationLog{}); err != nil {
+ t.Fatalf("auto-migrate pipeline operation log: %v", err)
+ }
+ return db
+}
+
+func TestCreateOpenLogPersistsPipelineVersion(t *testing.T) {
+ db := setupPipelineOperationLogTestDB(t)
+ dao := NewPipelineOperationLogDAO()
+ ctx := context.Background()
+
+ run, err := dao.CreateOpenLog(ctx, db, OpenLogInput{
+ DocumentID: "doc-1",
+ KbID: "kb-1",
+ TenantID: "tenant-1",
+ ParserID: "naive",
+ DocumentName: "a.txt",
+ DocumentSuffix: ".txt",
+ DocumentType: "txt",
+ SourceFrom: "local",
+ OperationStatus: string(entity.TaskStatusUnstart),
+ RunCount: 1,
+ PipelineVersionID: "ver-1",
+ })
+ if err != nil {
+ t.Fatalf("CreateOpenLog: %v", err)
+ }
+ if run.PipelineVersionID == nil || *run.PipelineVersionID != "ver-1" {
+ t.Fatalf("PipelineVersionID = %v, want ver-1", run.PipelineVersionID)
+ }
+
+ unversioned, err := dao.CreateOpenLog(ctx, db, OpenLogInput{
+ DocumentID: "doc-2",
+ KbID: "kb-1",
+ TenantID: "tenant-1",
+ ParserID: "naive",
+ DocumentName: "b.txt",
+ DocumentSuffix: ".txt",
+ DocumentType: "txt",
+ SourceFrom: "local",
+ OperationStatus: string(entity.TaskStatusUnstart),
+ RunCount: 1,
+ })
+ if err != nil {
+ t.Fatalf("CreateOpenLog without version: %v", err)
+ }
+ if unversioned.PipelineVersionID != nil {
+ t.Fatalf("PipelineVersionID = %v, want nil", unversioned.PipelineVersionID)
+ }
+}
+
+func TestSetResolvedPipelineOnlyUpdatesOpenRuns(t *testing.T) {
+ db := setupPipelineOperationLogTestDB(t)
+ dao := NewPipelineOperationLogDAO()
+ ctx := context.Background()
+
+ open := &entity.PipelineOperationLog{
+ ID: "open-run",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ DocumentName: "a.txt",
+ DocumentSuffix: ".txt",
+ DocumentType: "txt",
+ SourceFrom: "local",
+ OperationStatus: string(entity.TaskStatusRunning),
+ }
+ if err := db.Create(open).Error; err != nil {
+ t.Fatalf("seed open run: %v", err)
+ }
+
+ if err := dao.SetResolvedPipeline(ctx, db, "open-run", "canvas-1", "ver-1"); err != nil {
+ t.Fatalf("SetResolvedPipeline: %v", err)
+ }
+ var openReloaded entity.PipelineOperationLog
+ if err := db.First(&openReloaded, "id = ?", "open-run").Error; err != nil {
+ t.Fatalf("load open run: %v", err)
+ }
+ if openReloaded.PipelineID == nil || *openReloaded.PipelineID != "canvas-1" {
+ t.Fatalf("open PipelineID = %v, want canvas-1", openReloaded.PipelineID)
+ }
+ if openReloaded.PipelineVersionID == nil || *openReloaded.PipelineVersionID != "ver-1" {
+ t.Fatalf("open PipelineVersionID = %v, want ver-1", openReloaded.PipelineVersionID)
+ }
+
+ if err := dao.SetResolvedPipeline(ctx, db, "open-run", "", ""); err != nil {
+ t.Fatalf("clear resolved pipeline: %v", err)
+ }
+ var openCleared entity.PipelineOperationLog
+ if err := db.First(&openCleared, "id = ?", "open-run").Error; err != nil {
+ t.Fatalf("load cleared open run: %v", err)
+ }
+ if openCleared.PipelineID != nil || openCleared.PipelineVersionID != nil {
+ t.Fatalf("cleared pipeline identity = (%v, %v), want both nil", openCleared.PipelineID, openCleared.PipelineVersionID)
+ }
+
+ terminal := &entity.PipelineOperationLog{
+ ID: "terminal-run",
+ DocumentID: "doc-2",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ DocumentName: "b.txt",
+ DocumentSuffix: ".txt",
+ DocumentType: "txt",
+ SourceFrom: "local",
+ OperationStatus: string(entity.TaskStatusDone),
+ }
+ if err := db.Create(terminal).Error; err != nil {
+ t.Fatalf("seed terminal run: %v", err)
+ }
+ if err := dao.SetResolvedPipeline(ctx, db, "terminal-run", "canvas-2", "ver-2"); err != nil {
+ t.Fatalf("SetResolvedPipeline on terminal run: %v", err)
+ }
+
+ var reloaded entity.PipelineOperationLog
+ if err := db.First(&reloaded, "id = ?", "terminal-run").Error; err != nil {
+ t.Fatalf("load terminal run: %v", err)
+ }
+ if reloaded.PipelineID != nil || reloaded.PipelineVersionID != nil {
+ t.Fatalf("terminal pipeline identity = (%v, %v), want both nil", reloaded.PipelineID, reloaded.PipelineVersionID)
+ }
+}
diff --git a/internal/dao/user_canvas_version.go b/internal/dao/user_canvas_version.go
index ebcc61a5b1..9babe72162 100644
--- a/internal/dao/user_canvas_version.go
+++ b/internal/dao/user_canvas_version.go
@@ -73,35 +73,46 @@ var ErrCanvasVersionLimitExceeded = errors.New("version limit reached (maximum 6
// ErrPipelineVersionMissing so callers can surface a Missing (unparseable) state.
// Both the service (component-param resolution) and task (runtime parse) layers
// delegate here, so the resolution logic and its error contract live in one place.
-func LoadCanvasVersionDSL(ctx context.Context, db *gorm.DB, canvasID, versionID string) ([]byte, error) {
+//
+// The second result is the resolved version id, used by callers that record
+// which version a run executed.
+func LoadCanvasVersionDSL(ctx context.Context, db *gorm.DB, canvasID, versionID string) ([]byte, string, error) {
versionDAO := NewUserCanvasVersionDAO()
if versionID != "" {
v, err := versionDAO.GetByID(ctx, db, versionID)
if err != nil {
if errors.Is(err, ErrUserCanvasVersionNotFound) {
- return nil, fmt.Errorf("%s (version %s): %w", userFacingVersionHint, versionID, ErrPipelineVersionMissing)
+ return nil, "", fmt.Errorf("%s (version %s): %w", userFacingVersionHint, versionID, ErrPipelineVersionMissing)
}
- return nil, fmt.Errorf("load pipeline version %s: %w", versionID, err)
+ return nil, "", fmt.Errorf("load pipeline version %s: %w", versionID, err)
}
if v.UserCanvasID != canvasID {
- return nil, fmt.Errorf("%s (version %s does not belong to canvas %s): %w", userFacingVersionHint, versionID, canvasID, ErrPipelineVersionMissing)
+ return nil, "", fmt.Errorf("%s (version %s does not belong to canvas %s): %w", userFacingVersionHint, versionID, canvasID, ErrPipelineVersionMissing)
}
if len(v.DSL) == 0 {
- return nil, fmt.Errorf("%s (version %s has no DSL): %w", userFacingVersionHint, versionID, ErrPipelineVersionMissing)
+ return nil, "", fmt.Errorf("%s (version %s has no DSL): %w", userFacingVersionHint, versionID, ErrPipelineVersionMissing)
+ }
+ raw, err := marshalCanvasVersionDSL(versionID, v.DSL)
+ if err != nil {
+ return nil, "", err
}
- return marshalCanvasVersionDSL(versionID, v.DSL)
+ return raw, versionID, nil
}
v, err := versionDAO.GetLatest(ctx, db, canvasID)
if err != nil {
if errors.Is(err, ErrUserCanvasVersionNotFound) {
- return nil, fmt.Errorf("%s (canvas %s has no published version): %w", userFacingVersionHint, canvasID, ErrPipelineVersionMissing)
+ return nil, "", fmt.Errorf("%s (canvas %s has no published version): %w", userFacingVersionHint, canvasID, ErrPipelineVersionMissing)
}
- return nil, fmt.Errorf("load latest version for canvas %s: %w", canvasID, err)
+ return nil, "", fmt.Errorf("load latest version for canvas %s: %w", canvasID, err)
}
if len(v.DSL) == 0 {
- return nil, fmt.Errorf("%s (latest version %s has no DSL): %w", userFacingVersionHint, v.ID, ErrPipelineVersionMissing)
+ return nil, "", fmt.Errorf("%s (latest version %s has no DSL): %w", userFacingVersionHint, v.ID, ErrPipelineVersionMissing)
+ }
+ raw, err := marshalCanvasVersionDSL(v.ID, v.DSL)
+ if err != nil {
+ return nil, "", err
}
- return marshalCanvasVersionDSL(v.ID, v.DSL)
+ return raw, v.ID, nil
}
func marshalCanvasVersionDSL(versionID string, dsl entity.JSONMap) ([]byte, error) {
diff --git a/internal/dao/user_canvas_version_test.go b/internal/dao/user_canvas_version_test.go
index d80021844f..1def7c2f29 100644
--- a/internal/dao/user_canvas_version_test.go
+++ b/internal/dao/user_canvas_version_test.go
@@ -188,3 +188,38 @@ func TestUserCanvasVersionDAO_SaveOrReplaceLatestTxStoresAlias(t *testing.T) {
}
func ptrStr(v string) *string { return &v }
+
+func TestLoadCanvasVersionDSLReturnsResolvedVersionID(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open db: %v", err)
+ }
+ if err := db.AutoMigrate(&entity.UserCanvasVersion{}); err != nil {
+ t.Fatalf("migrate: %v", err)
+ }
+ rows := []entity.UserCanvasVersion{
+ {
+ ID: "v-old", UserCanvasID: "canvas-1", DSL: entity.JSONMap{"version": "old"},
+ BaseModel: entity.BaseModel{CreateTime: int64Ptr(1000)},
+ },
+ {
+ ID: "v-new", UserCanvasID: "canvas-1", DSL: entity.JSONMap{"version": "new"},
+ BaseModel: entity.BaseModel{CreateTime: int64Ptr(2000)},
+ },
+ }
+ for i := range rows {
+ if err := db.Create(&rows[i]).Error; err != nil {
+ t.Fatalf("seed version %s: %v", rows[i].ID, err)
+ }
+ }
+
+ if _, resolved, err := LoadCanvasVersionDSL(t.Context(), db, "canvas-1", "v-old"); err != nil || resolved != "v-old" {
+ t.Fatalf("explicit pin resolved = %q, err = %v, want v-old", resolved, err)
+ }
+ if _, resolved, err := LoadCanvasVersionDSL(t.Context(), db, "canvas-1", ""); err != nil || resolved != "v-new" {
+ t.Fatalf("latest pin resolved = %q, err = %v, want v-new", resolved, err)
+ }
+ if _, resolved, err := LoadCanvasVersionDSL(t.Context(), db, "canvas-1", "missing"); err == nil || resolved != "" {
+ t.Fatalf("missing pin resolved = %q, err = %v, want empty + error", resolved, err)
+ }
+}
diff --git a/internal/entity/pipeline.go b/internal/entity/pipeline.go
index 409a82728d..2806a10b38 100644
--- a/internal/entity/pipeline.go
+++ b/internal/entity/pipeline.go
@@ -28,19 +28,20 @@ type PipelineOperationLog struct {
DocumentID string `gorm:"column:document_id;size:32;index;uniqueIndex:idx_pipeline_operation_log_document_run,priority:1" json:"document_id"`
// RunCount is the monotonically allocated display number for a numbered
// ingestion run.
- RunCount *int `gorm:"column:run_count;uniqueIndex:idx_pipeline_operation_log_document_run,priority:2" json:"run_count,omitempty"`
- TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"`
- KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"`
- PipelineID *string `gorm:"column:pipeline_id;size:32;index" json:"pipeline_id,omitempty"`
- PipelineTitle *string `gorm:"column:pipeline_title;size:32;index" json:"pipeline_title,omitempty"`
- ParserID string `gorm:"column:parser_id;size:32;not null;index" json:"parser_id"`
- DocumentName string `gorm:"column:document_name;size:255;not null" json:"document_name"`
- DocumentSuffix string `gorm:"column:document_suffix;size:255;not null" json:"document_suffix"`
- DocumentType string `gorm:"column:document_type;size:255;not null" json:"document_type"`
- SourceFrom string `gorm:"column:source_from;size:255;not null" json:"source_from"`
- Progress float64 `gorm:"column:progress;default:0;index" json:"progress"`
- ProgressMsg *string `gorm:"column:progress_msg;type:longtext" json:"progress_msg,omitempty"`
- ProcessBeginAt *time.Time `gorm:"column:process_begin_at;index" json:"process_begin_at,omitempty"`
+ RunCount *int `gorm:"column:run_count;uniqueIndex:idx_pipeline_operation_log_document_run,priority:2" json:"run_count,omitempty"`
+ TenantID string `gorm:"column:tenant_id;size:32;not null;index" json:"tenant_id"`
+ KbID string `gorm:"column:kb_id;size:32;not null;index" json:"kb_id"`
+ PipelineID *string `gorm:"column:pipeline_id;size:32;index" json:"pipeline_id,omitempty"`
+ PipelineTitle *string `gorm:"column:pipeline_title;size:32;index" json:"pipeline_title,omitempty"`
+ PipelineVersionID *string `gorm:"column:pipeline_version_id;size:32" json:"pipeline_version_id,omitempty"`
+ ParserID string `gorm:"column:parser_id;size:32;not null;index" json:"parser_id"`
+ DocumentName string `gorm:"column:document_name;size:255;not null" json:"document_name"`
+ DocumentSuffix string `gorm:"column:document_suffix;size:255;not null" json:"document_suffix"`
+ DocumentType string `gorm:"column:document_type;size:255;not null" json:"document_type"`
+ SourceFrom string `gorm:"column:source_from;size:255;not null" json:"source_from"`
+ Progress float64 `gorm:"column:progress;default:0;index" json:"progress"`
+ ProgressMsg *string `gorm:"column:progress_msg;type:longtext" json:"progress_msg,omitempty"`
+ ProcessBeginAt *time.Time `gorm:"column:process_begin_at;index" json:"process_begin_at,omitempty"`
// ProcessDuration is the persisted terminal run duration in seconds. The
// authoritative end time is recovered as ProcessBeginAt + ProcessDuration when
// needed, so no separate process_end_at column is required.
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index 339300d218..aa6d8b530e 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -1315,11 +1315,22 @@ func (e *Ingestor) recordTerminalPipelineLog(ctx context.Context, ingestionTask
common.Warn(fmt.Sprintf("record terminal pipeline log for task %s: run %s belongs to document %s/dataset %s, expected %s/%s", ingestionTask.ID, run.ID, run.DocumentID, run.KbID, ingestionTask.DocumentID, ingestionTask.DatasetID))
return
}
+ // Keep the run's own binding: the document may have been re-bound since.
+ pipelineID := ""
+ if run.PipelineID != nil {
+ pipelineID = strings.TrimSpace(*run.PipelineID)
+ }
+ versionID := ""
+ if pipelineID != "" && run.PipelineVersionID != nil {
+ versionID = strings.TrimSpace(*run.PipelineVersionID)
+ }
input := taskpkg.PipelineLogInput{
- KbID: ingestionTask.DatasetID,
- DocumentID: ingestionTask.DocumentID,
- Status: status,
- PipelineLogID: *ingestionTask.PipelineLogID,
+ KbID: ingestionTask.DatasetID,
+ DocumentID: ingestionTask.DocumentID,
+ PipelineID: pipelineID,
+ PipelineVersionID: versionID,
+ Status: status,
+ PipelineLogID: *ingestionTask.PipelineLogID,
}
if err := taskpkg.RecordPipelineLog(ctx, e.deps.DB, input); err != nil {
common.Warn(fmt.Sprintf("record terminal pipeline log for task %s document %s: %v", ingestionTask.ID, ingestionTask.DocumentID, err))
diff --git a/internal/ingestion/service/terminal_pipeline_log_test.go b/internal/ingestion/service/terminal_pipeline_log_test.go
new file mode 100644
index 0000000000..917202c56f
--- /dev/null
+++ b/internal/ingestion/service/terminal_pipeline_log_test.go
@@ -0,0 +1,74 @@
+package service
+
+import (
+ "testing"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/testutil"
+)
+
+func TestRecordTerminalPipelineLogPreservesRunIdentity(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ status := string(entity.StatusValid)
+ if err := db.Create(&entity.Knowledgebase{
+ ID: "kb-1", TenantID: "tenant-1", Name: "kb-1", EmbdID: "embd-1",
+ CreatedBy: "user-1", Permission: string(entity.TenantPermissionMe), Status: &status,
+ }).Error; err != nil {
+ t.Fatalf("seed knowledgebase: %v", err)
+ }
+
+ docName := "doc.txt"
+ newPipelineID := "new-canvas"
+ newVersionID := "new-ver"
+ if err := db.Create(&entity.Document{
+ ID: "doc-1", KbID: "kb-1", ParserID: "naive", ParserConfig: entity.JSONMap{},
+ Name: &docName, Suffix: ".txt", Type: "txt", Status: &status,
+ PipelineID: &newPipelineID, PipelineVersionID: &newVersionID,
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ runCount := 1
+ oldPipelineID := "old-canvas"
+ oldVersionID := "old-ver"
+ if err := db.Create(&entity.PipelineOperationLog{
+ ID: "run-1", DocumentID: "doc-1", TenantID: "tenant-1", KbID: "kb-1",
+ ParserID: "naive", DocumentName: docName, DocumentSuffix: ".txt",
+ DocumentType: "txt", SourceFrom: "local", TaskType: string(entity.PipelineTaskTypeParse),
+ OperationStatus: string(entity.TaskStatusRunning), RunCount: &runCount,
+ PipelineID: &oldPipelineID, PipelineVersionID: &oldVersionID,
+ }).Error; err != nil {
+ t.Fatalf("seed pipeline log: %v", err)
+ }
+
+ pipelineLogID := "run-1"
+ task := &entity.IngestionTask{
+ ID: "task-1", DocumentID: "doc-1", DatasetID: "kb-1", PipelineLogID: &pipelineLogID,
+ }
+ ingestor := newUnitIngestor("test", 1, nil)
+ ingestor.recordTerminalPipelineLog(t.Context(), task, string(entity.TaskStatusFail), "boom")
+
+ var run entity.PipelineOperationLog
+ if err := db.First(&run, "id = ?", "run-1").Error; err != nil {
+ t.Fatalf("load pipeline log: %v", err)
+ }
+ if run.OperationStatus != string(entity.TaskStatusFail) {
+ t.Fatalf("OperationStatus = %q, want FAIL", run.OperationStatus)
+ }
+ if run.PipelineID == nil || *run.PipelineID != oldPipelineID {
+ t.Fatalf("PipelineID = %v, want the run's %q", run.PipelineID, oldPipelineID)
+ }
+ if run.PipelineVersionID == nil || *run.PipelineVersionID != oldVersionID {
+ t.Fatalf("PipelineVersionID = %v, want the run's %q", run.PipelineVersionID, oldVersionID)
+ }
+
+ var terminal entity.IngestionTaskLog
+ if err := db.Where("pipeline_log_id = ? AND event_type = ?", "run-1", dao.EventTypeTerminal).
+ First(&terminal).Error; err != nil {
+ t.Fatalf("terminal event missing: %v", err)
+ }
+}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index d46f96a106..acfe9d98de 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -78,6 +78,8 @@ type PipelineExecutor struct {
docBulkSize int
deps *ingestion.Deps
+ resolvedVersionID string
+
indexWriter *chunkIndexWriter
deleteChunksFunc DeleteChunksFunc
loadDSLFunc func(ctx context.Context, canvasID string) (string, string, error)
@@ -239,6 +241,9 @@ func (s *PipelineExecutor) Execute(ctx context.Context) (*PipelineResult, error)
if correctedID != "" {
s.canvasID = correctedID
}
+ if !s.taskCtx.IsDebug() {
+ s.persistResolvedPipeline(context.WithoutCancel(ctx))
+ }
pipelineOutput, pipelineDSL, err := s.runPipelineFunc(ctx, dsl)
if err != nil {
@@ -863,7 +868,9 @@ type PipelineLogInput struct {
// (ingestion_task.pipeline_log_id). The public writer requires it and updates
// exactly that row, never creating a second one — so a superseded run whose
// row was deleted cannot adopt the replacement run's row.
- PipelineLogID string
+ PipelineLogID string
+ PipelineVersionID string
+ PipelineBindingResolved bool
// TerminalDuration, when set, is this run's final duration in seconds,
// measured from the document's process_begin_at. updateOpenLogRow records
// it instead of the reloaded document's last mid-run value, so
@@ -934,9 +941,6 @@ func recordPipelineLog(
if input.KbID == "" {
input.KbID = doc.KbID
}
- if input.PipelineID == "" && doc.PipelineID != nil {
- input.PipelineID = strings.TrimSpace(*doc.PipelineID)
- }
if input.TenantID == "" && db != nil && input.KbID != "" {
if kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, input.KbID); err == nil && kb != nil {
input.TenantID = kb.TenantID
@@ -997,7 +1001,6 @@ func updateOpenLogRow(ctx context.Context, db *gorm.DB, input PipelineLogInput,
"operation_status": operationStatus,
"status": statusValue,
"dsl": dslMap,
- "pipeline_id": pipelineID,
"pipeline_title": pipelineTitle,
"avatar": pipelineAvatar,
"progress": doc.Progress,
@@ -1010,6 +1013,14 @@ func updateOpenLogRow(ctx context.Context, db *gorm.DB, input PipelineLogInput,
if doc.Name != nil {
updates["document_name"] = *doc.Name
}
+ if input.PipelineBindingResolved {
+ var versionValue interface{}
+ if pipelineID != nil && input.PipelineVersionID != "" {
+ versionValue = input.PipelineVersionID
+ }
+ updates["pipeline_id"] = pipelineID
+ updates["pipeline_version_id"] = versionValue
+ }
if input.TerminalDuration != nil {
updates["process_duration"] = *input.TerminalDuration
}
@@ -1045,20 +1056,36 @@ func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, d
pipelineLogID = *s.taskCtx.IngestionTask.PipelineLogID
}
if err := recordPipelineLog(ctx, db, PipelineLogInput{
- TenantID: s.Tenant().ID,
- KbID: s.KB().ID,
- DocumentID: docID,
- PipelineID: pipelineID,
- DSL: dsl,
- Status: status,
- Document: s.taskCtx.Doc,
- PipelineLogID: pipelineLogID,
- TerminalDuration: terminalDuration,
+ TenantID: s.Tenant().ID,
+ KbID: s.KB().ID,
+ DocumentID: docID,
+ PipelineID: pipelineID,
+ DSL: dsl,
+ Status: status,
+ Document: s.taskCtx.Doc,
+ PipelineLogID: pipelineLogID,
+ PipelineVersionID: s.resolvedVersionID,
+ PipelineBindingResolved: true,
+ TerminalDuration: terminalDuration,
}); err != nil {
common.Warn(fmt.Sprintf("failed to record pipeline log: %v", err))
}
}
+func (s *PipelineExecutor) persistResolvedPipeline(ctx context.Context) {
+ task := s.taskCtx.IngestionTask
+ if task == nil || task.PipelineLogID == nil || *task.PipelineLogID == "" || s.deps == nil || s.deps.DB == nil {
+ return
+ }
+ pipelineID := ""
+ if s.taskCtx.PipelineID != "" {
+ pipelineID = s.canvasID
+ }
+ if err := dao.NewPipelineOperationLogDAO().SetResolvedPipeline(ctx, s.deps.DB, *task.PipelineLogID, pipelineID, s.resolvedVersionID); err != nil {
+ common.Warn(fmt.Sprintf("persist pipeline %s version %s for run %s: %v", pipelineID, s.resolvedVersionID, *task.PipelineLogID, err))
+ }
+}
+
func (s *PipelineExecutor) loadDSLFromCanvas(ctx context.Context, canvasID string) (string, string, error) {
if s == nil || s.taskCtx == nil {
return "", "", fmt.Errorf("pipeline executor: nil task context")
@@ -1066,6 +1093,7 @@ func (s *PipelineExecutor) loadDSLFromCanvas(ctx context.Context, canvasID strin
if canvasID == "" {
return "", "", fmt.Errorf("pipeline executor: empty canvas id")
}
+ s.resolvedVersionID = ""
canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, s.deps.DB, canvasID)
if err != nil {
return "", "", fmt.Errorf("load canvas %s: %w", canvasID, err)
@@ -1086,10 +1114,11 @@ func (s *PipelineExecutor) loadDSLFromCanvas(ctx context.Context, canvasID strin
// resolves that exact version; an empty pin resolves the latest published
// version. A missing or mis-owned version returns ErrPipelineVersionMissing
// so the parse task fails and the file enters an unparseable Missing state.
- raw, verr := dao.LoadCanvasVersionDSL(ctx, s.deps.DB, canvasID, s.taskCtx.PipelineVersionID)
+ raw, resolvedVersionID, verr := dao.LoadCanvasVersionDSL(ctx, s.deps.DB, canvasID, s.taskCtx.PipelineVersionID)
if verr != nil {
return "", "", verr
}
+ s.resolvedVersionID = resolvedVersionID
return string(raw), canvasID, nil
}
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index 1e1e6478ba..716c1677c7 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -408,6 +408,51 @@ func TestRecordPipelineLog_SharedWriterTerminalWithoutDSL(t *testing.T) {
}
}
+func TestRecordPipelineLogKeepsResolvedVersionWhenTerminalWriterHasNone(t *testing.T) {
+ cleanup := setupPipelineExecutorTestDB(t)
+ defer cleanup()
+ version := "ver-1"
+ if err := dao.DB.Create(&entity.PipelineOperationLog{
+ ID: "run-1",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ OperationStatus: string(entity.TaskStatusRunning),
+ PipelineVersionID: &version,
+ }).Error; err != nil {
+ t.Fatalf("seed pipeline log: %v", err)
+ }
+
+ docName := "terminal.pdf"
+ if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ DocumentID: "doc-1",
+ Status: string(entity.TaskStatusFail),
+ Document: entity.Document{
+ ID: "doc-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ SourceType: "local",
+ Type: "pdf",
+ Name: &docName,
+ Suffix: ".pdf",
+ },
+ PipelineLogID: "run-1",
+ }); err != nil {
+ t.Fatalf("RecordPipelineLog: %v", err)
+ }
+
+ var log entity.PipelineOperationLog
+ if err := dao.DB.First(&log, "id = ?", "run-1").Error; err != nil {
+ t.Fatalf("load pipeline log: %v", err)
+ }
+ if log.PipelineVersionID == nil || *log.PipelineVersionID != version {
+ t.Fatalf("PipelineVersionID = %v, want the mid-run version %q", log.PipelineVersionID, version)
+ }
+}
+
func TestRecordPipelineLogRejectsMissingRunIdentity(t *testing.T) {
err := RecordPipelineLog(t.Context(), nil, PipelineLogInput{DocumentID: "doc-1", Status: "3"})
if !errors.Is(err, ErrMissingRunIdentity) {
@@ -464,13 +509,13 @@ func TestRecordPipelineLog_TerminalWithoutDSLResolvesCanvasTitle(t *testing.T) {
t.Fatalf("seed document: %v", err)
}
- // Mirrors Ingestor.recordTerminalPipelineLog: only the terminal status is
- // known; pipeline_id and DSL are absent and must come from the document.
if err := RecordPipelineLog(t.Context(), dao.DB, PipelineLogInput{
- KbID: "kb-1",
- DocumentID: "doc-1",
- Status: "3",
- PipelineLogID: "run-1",
+ KbID: "kb-1",
+ DocumentID: "doc-1",
+ PipelineID: "canvas-1",
+ Status: "3",
+ PipelineLogID: "run-1",
+ PipelineBindingResolved: true,
}); err != nil {
t.Fatalf("RecordPipelineLog: %v", err)
}
@@ -1472,3 +1517,174 @@ func TestBuildLogDSL_PreviewKeepsOutputs(t *testing.T) {
t.Error("persisted log DSL must NOT carry business data even when a ResultSink preview exists")
}
}
+
+func TestExecutePersistsResolvedVersionBeforeRun(t *testing.T) {
+ restore := setupPipelineExecutorTestDB(t)
+ defer restore()
+ db := dao.DB
+ if err := db.AutoMigrate(&entity.UserCanvas{}, &entity.UserCanvasVersion{}); err != nil {
+ t.Fatalf("migrate canvas tables: %v", err)
+ }
+ release := true
+ if err := db.Create(&entity.UserCanvas{
+ ID: "canvas-1",
+ UserID: "user-1",
+ CanvasCategory: "dataflow_canvas",
+ }).Error; err != nil {
+ t.Fatalf("seed canvas: %v", err)
+ }
+ if err := db.Create(&entity.UserCanvasVersion{
+ ID: "ver-1",
+ UserCanvasID: "canvas-1",
+ DSL: entity.JSONMap{"nodes": []any{}, "edges": []any{}},
+ Release: release,
+ BaseModel: entity.BaseModel{CreateTime: int64PtrForTest(1000)},
+ }).Error; err != nil {
+ t.Fatalf("seed canvas version: %v", err)
+ }
+
+ logID := "run-1"
+ if err := db.Create(&entity.PipelineOperationLog{
+ ID: logID,
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ DocumentName: "test-doc.pdf",
+ DocumentSuffix: ".pdf",
+ DocumentType: "pdf",
+ SourceFrom: "local",
+ OperationStatus: string(entity.TaskStatusRunning),
+ TaskType: string(entity.PipelineTaskTypeParse),
+ }).Error; err != nil {
+ t.Fatalf("seed pipeline log: %v", err)
+ }
+
+ taskCtx := makeTaskCtx()
+ taskCtx.PipelineID = "canvas-1"
+ taskCtx.IngestionTask.PipelineLogID = &logID
+ exec := mustNewPipelineExecutor(t, taskCtx, "canvas-1", 0)
+ exec.WithRunPipelineFunc(func(context.Context, string) (map[string]any, string, error) {
+ return nil, "", errors.New("pipeline failed")
+ })
+
+ if _, err := exec.Execute(t.Context()); err == nil {
+ t.Fatal("expected the injected pipeline failure")
+ }
+
+ var run entity.PipelineOperationLog
+ if err := db.First(&run, "id = ?", logID).Error; err != nil {
+ t.Fatalf("load pipeline log: %v", err)
+ }
+ if run.PipelineID == nil || *run.PipelineID != "canvas-1" {
+ t.Fatalf("PipelineID = %v, want canvas-1 after a failed run", run.PipelineID)
+ }
+ if run.PipelineVersionID == nil || *run.PipelineVersionID != "ver-1" {
+ t.Fatalf("PipelineVersionID = %v, want the latest resolved version after a failed run", run.PipelineVersionID)
+ }
+}
+
+func int64PtrForTest(v int64) *int64 { return &v }
+
+func TestExecuteClearsStaleBindingForBuiltinRun(t *testing.T) {
+ restore := setupPipelineExecutorTestDB(t)
+ defer restore()
+ db := dao.DB
+
+ oldPipelineID := "canvas-1"
+ oldVersionID := "ver-1"
+ logID := "run-1"
+ if err := db.Create(&entity.PipelineOperationLog{
+ ID: logID,
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ DocumentName: "test-doc.pdf",
+ DocumentSuffix: ".pdf",
+ DocumentType: "pdf",
+ SourceFrom: "local",
+ OperationStatus: string(entity.TaskStatusRunning),
+ TaskType: string(entity.PipelineTaskTypeParse),
+ PipelineID: &oldPipelineID,
+ PipelineVersionID: &oldVersionID,
+ }).Error; err != nil {
+ t.Fatalf("seed pipeline log: %v", err)
+ }
+
+ taskCtx := makeTaskCtx()
+ taskCtx.IngestionTask.PipelineLogID = &logID
+ exec := mustNewPipelineExecutor(t, taskCtx, "naive", 0)
+ exec.WithLoadDSLFunc(func(_ context.Context, canvasID string) (string, string, error) {
+ return `{"nodes":[],"edges":[]}`, canvasID, nil
+ })
+ exec.WithRunPipelineFunc(func(context.Context, string) (map[string]any, string, error) {
+ return nil, "", errors.New("pipeline failed")
+ })
+
+ if _, err := exec.Execute(t.Context()); err == nil {
+ t.Fatal("expected the injected pipeline failure")
+ }
+
+ var run entity.PipelineOperationLog
+ if err := db.First(&run, "id = ?", logID).Error; err != nil {
+ t.Fatalf("load pipeline log: %v", err)
+ }
+ if run.PipelineID != nil || run.PipelineVersionID != nil {
+ t.Fatalf("pipeline binding = (%v, %v), want both nil after a builtin run", run.PipelineID, run.PipelineVersionID)
+ }
+}
+
+func TestRecordPipelineLogClearsStaleBindingForResolvedVersionlessRun(t *testing.T) {
+ restore := setupPipelineExecutorTestDB(t)
+ defer restore()
+ db := dao.DB
+
+ oldPipelineID := "canvas-1"
+ oldVersionID := "ver-1"
+ if err := db.Create(&entity.PipelineOperationLog{
+ ID: "run-1",
+ DocumentID: "doc-1",
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ DocumentName: "test-doc.pdf",
+ DocumentSuffix: ".pdf",
+ DocumentType: "pdf",
+ SourceFrom: "local",
+ OperationStatus: string(entity.TaskStatusRunning),
+ TaskType: string(entity.PipelineTaskTypeParse),
+ PipelineID: &oldPipelineID,
+ PipelineVersionID: &oldVersionID,
+ }).Error; err != nil {
+ t.Fatalf("seed pipeline log: %v", err)
+ }
+ currentPipelineID := "canvas-2"
+ docName := "test-doc.pdf"
+ if err := db.Create(&entity.Document{
+ ID: "doc-1", KbID: "kb-1", ParserID: "naive", ParserConfig: entity.JSONMap{},
+ Name: &docName, Suffix: ".pdf", Type: "pdf", SourceType: "local", Status: strPtr("1"),
+ PipelineID: &currentPipelineID,
+ }).Error; err != nil {
+ t.Fatalf("seed document: %v", err)
+ }
+
+ if err := RecordPipelineLog(t.Context(), db, PipelineLogInput{
+ TenantID: "tenant-1",
+ KbID: "kb-1",
+ DocumentID: "doc-1",
+ Status: string(entity.TaskStatusDone),
+ PipelineLogID: "run-1",
+ PipelineBindingResolved: true,
+ }); err != nil {
+ t.Fatalf("RecordPipelineLog: %v", err)
+ }
+
+ var run entity.PipelineOperationLog
+ if err := db.First(&run, "id = ?", "run-1").Error; err != nil {
+ t.Fatalf("load pipeline log: %v", err)
+ }
+ if run.PipelineID != nil || run.PipelineVersionID != nil {
+ t.Fatalf("pipeline binding = (%v, %v), want both nil for a versionless run", run.PipelineID, run.PipelineVersionID)
+ }
+}
diff --git a/internal/service/dataset/ingestion.go b/internal/service/dataset/ingestion.go
index 74bdba6a7e..00f4070a63 100644
--- a/internal/service/dataset/ingestion.go
+++ b/internal/service/dataset/ingestion.go
@@ -423,6 +423,7 @@ func fileIngestionLogToMap(log *entity.PipelineOperationLog, latestEvent *servic
"kb_id": log.KbID,
"pipeline_id": stringPointerValue(log.PipelineID),
"pipeline_title": stringPointerValue(log.PipelineTitle),
+ "pipeline_version_id": stringPointerValue(log.PipelineVersionID),
"parser_id": log.ParserID,
"document_name": log.DocumentName,
"document_suffix": log.DocumentSuffix,
diff --git a/internal/service/dataset/ingestion_messages_test.go b/internal/service/dataset/ingestion_messages_test.go
index 36aff250f2..f7ad1c6852 100644
--- a/internal/service/dataset/ingestion_messages_test.go
+++ b/internal/service/dataset/ingestion_messages_test.go
@@ -462,3 +462,43 @@ func TestListIngestionLogsRejectsUnknownLogType(t *testing.T) {
})
}
}
+
+func TestListIngestionLogsExposesPipelineVersionID(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.PipelineOperationLog{}); err != nil {
+ t.Fatalf("migrate pipeline operation log: %v", err)
+ }
+ insertCompilationOwnerKB(t, "kb-1", "user-1")
+ runCount := 1
+ versionID := "ver-1"
+ if err := db.Create(&entity.PipelineOperationLog{
+ ID: "run-1",
+ DocumentID: "doc-1",
+ TenantID: "user-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ DocumentName: "doc.txt",
+ DocumentSuffix: ".txt",
+ DocumentType: "text",
+ SourceFrom: "local",
+ TaskType: string(entity.PipelineTaskTypeParse),
+ OperationStatus: string(entity.TaskStatusDone),
+ RunCount: &runCount,
+ PipelineVersionID: &versionID,
+ }).Error; err != nil {
+ t.Fatalf("insert versioned run: %v", err)
+ }
+
+ result, code, err := NewDatasetService().ListIngestionLogs(withDatasetTestContext(t.Context(), "user-1", "user-1"), "kb-1", 1, 30, nil, nil, "", "", "file", "", "")
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("ListIngestionLogs = (%+v, %v, %v), want success", result, code, err)
+ }
+ logs, ok := result["logs"].([]map[string]interface{})
+ if !ok || len(logs) != 1 {
+ t.Fatalf("logs = %#v, want one row", result["logs"])
+ }
+ if got := logs[0]["pipeline_version_id"]; got != versionID {
+ t.Fatalf("pipeline_version_id = %v, want %q", got, versionID)
+ }
+}
diff --git a/internal/service/ingestion_task_service.go b/internal/service/ingestion_task_service.go
index c23d355053..0b5f10e2b6 100644
--- a/internal/service/ingestion_task_service.go
+++ b/internal/service/ingestion_task_service.go
@@ -1144,6 +1144,9 @@ func (s *IngestionTaskService) buildOpenLogInput(ctx context.Context, task *enti
if doc.PipelineID != nil {
if pipelineID := strings.TrimSpace(*doc.PipelineID); pipelineID != "" {
input.PipelineID = pipelineID
+ if doc.PipelineVersionID != nil {
+ input.PipelineVersionID = strings.TrimSpace(*doc.PipelineVersionID)
+ }
if canvas, err := s.userCanvasDAO.GetByID(ctx, dao.DB, pipelineID); err == nil && canvas != nil {
if canvas.Title != nil && *canvas.Title != "" {
input.PipelineTitle = *canvas.Title
diff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.go
index 823da6512a..9cc5b0ce8f 100644
--- a/internal/service/ingestion_task_service_test.go
+++ b/internal/service/ingestion_task_service_test.go
@@ -2112,6 +2112,53 @@ func TestIngestionTaskServiceRetryAfterTerminalOpensFreshPipelineLog(t *testing.
}
}
+func TestIngestionTaskServiceOpensRunWithDocumentPipelineVersion(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
+ pipelineID := "canvas-1"
+ if err := db.Create(&entity.UserCanvas{
+ ID: pipelineID,
+ UserID: "user-1",
+ CanvasCategory: "dataflow_canvas",
+ }).Error; err != nil {
+ t.Fatalf("insert pipeline: %v", err)
+ }
+ versionID := "ver-1"
+ if err := db.Create(&entity.Document{
+ ID: "doc-1",
+ KbID: "kb-1",
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{},
+ Suffix: ".txt",
+ Status: sptr("1"),
+ PipelineID: &pipelineID,
+ PipelineVersionID: &versionID,
+ }).Error; err != nil {
+ t.Fatalf("insert versioned document: %v", err)
+ }
+
+ svc := NewIngestionTaskService()
+ svc.taskPublisher = &recordingTaskPublisher{}
+ task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
+ DocumentID: "doc-1", UserID: "user-1", DatasetID: "kb-1", Status: common.CREATED,
+ })
+ if err != nil {
+ t.Fatalf("CreateAndEnqueue: %v", err)
+ }
+ if task.PipelineLogID == nil {
+ t.Fatal("enqueued task has no pipeline log id")
+ }
+
+ var run entity.PipelineOperationLog
+ if err := db.First(&run, "id = ?", *task.PipelineLogID).Error; err != nil {
+ t.Fatalf("load pipeline log: %v", err)
+ }
+ if run.PipelineVersionID == nil || *run.PipelineVersionID != versionID {
+ t.Fatalf("PipelineVersionID = %v, want %q", run.PipelineVersionID, versionID)
+ }
+}
+
func TestIngestionTaskServiceRetryAllocatesNextRunCount(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
diff --git a/internal/service/pipeline_params.go b/internal/service/pipeline_params.go
index 4558e34911..5c2e54c7e6 100644
--- a/internal/service/pipeline_params.go
+++ b/internal/service/pipeline_params.go
@@ -65,7 +65,8 @@ func loadCanvasDSLJSON(ctx context.Context, canvasID string, versionID *string)
if versionID != nil {
pin = strings.TrimSpace(*versionID)
}
- return dao.LoadCanvasVersionDSL(ctx, dao.DB, canvasID, pin)
+ dsl, _, err := dao.LoadCanvasVersionDSL(ctx, dao.DB, canvasID, pin)
+ return dsl, err
}
func marshalCanvasDSL(canvasID string, dsl entity.JSONMap) ([]byte, error) {