RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
package service
import (
"context"
"errors"
"reflect"
"strings"
"sync"
"testing"
"time"
"unicode/utf8"
"gorm.io/gorm"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/permission"
)
type recordingTaskPublisher struct {
subject string
messages []common.TaskMessage
err error
beforeReturn func(taskID string)
}
type failFirstTaskPublisher struct {
messages []common.TaskMessage
}
type duplicatePublishRecorder struct {
mu sync.Mutex
messages []common.TaskMessage
firstPublished chan struct{}
secondPublished chan struct{}
releaseFirst chan struct{}
releaseSecond chan struct{}
}
func (p *failFirstTaskPublisher) PublishTaskMessage(_ string, msg common.TaskMessage) error {
p.messages = append(p.messages, msg)
if len(p.messages) == 1 {
return errors.New("publish failed")
}
return nil
}
func (p *duplicatePublishRecorder) PublishTaskMessage(_ string, msg common.TaskMessage) error {
p.mu.Lock()
p.messages = append(p.messages, msg)
publishNumber := len(p.messages)
p.mu.Unlock()
switch publishNumber {
case 1:
close(p.firstPublished)
<-p.releaseFirst
case 2:
close(p.secondPublished)
<-p.releaseSecond
}
return nil
}
func (p *recordingTaskPublisher) PublishTaskMessage(subject string, msg common.TaskMessage) error {
p.subject = subject
p.messages = append(p.messages, msg)
if p.beforeReturn != nil {
p.beforeReturn(msg.TaskID)
}
return p.err
}
func TestIngestionTaskServiceCreateForDocumentsPublishesTaskMessages(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
publisher := &recordingTaskPublisher{}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
ctx := t.Context()
resp, err := svc.CreateForDocuments(permission.WithIdentity(ctx, "user-1", ""), "kb-1", []string{"doc-1"})
if err != nil {
t.Fatalf("CreateForDocuments failed: %v", err)
}
if len(resp) != 1 {
t.Fatalf("expected 1 response, got %d", len(resp))
}
if publisher.subject != "tasks.RAGFLOW" {
t.Fatalf("subject = %q, want %q", publisher.subject, "tasks.RAGFLOW")
}
if len(publisher.messages) != 1 {
t.Fatalf("expected 1 published message, got %d", len(publisher.messages))
}
msg := publisher.messages[0]
if msg.TaskType != common.TaskTypeIngestionTask {
t.Fatalf("task type = %q, want %q", msg.TaskType, common.TaskTypeIngestionTask)
}
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, msg.TaskID)
if err != nil {
t.Fatalf("load task: %v", err)
}
if task.DocumentID != "doc-1" || task.DatasetID != "kb-1" || task.UserID != "user-1" {
t.Fatalf("unexpected task: %+v", task)
}
}
func TestIngestionTaskServiceCreateForDocumentsRejectsDatasetMismatch(t *testing.T) {
tests := []struct {
name, docKB, taskStatus, wantError string
}{
{"foreign new task", "kb-2", "", gorm.ErrRecordNotFound.Error()},
{"foreign created task", "kb-2", common.CREATED, gorm.ErrRecordNotFound.Error()},
{"foreign failed task", "kb-2", common.FAILED, gorm.ErrRecordNotFound.Error()},
{"foreign stopped task", "kb-2", common.STOPPED, gorm.ErrRecordNotFound.Error()},
{"mismatched existing task", "kb-1", common.FAILED, "different dataset"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
insertTestKB(t, "kb-2", "tenant-2", 1, 0, 0)
insertTestDoc(t, "doc-1", tt.docKB, 0, 0)
if tt.taskStatus != "" {
insertTestIngestionTaskWithStatus(t, "task-1", "user-2", "doc-1", "kb-2", tt.taskStatus)
}
ctx := t.Context()
taskDAO := dao.NewIngestionTaskDAO()
before, err := taskDAO.GetByDocumentID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("load existing task: %v", err)
}
publisher := &recordingTaskPublisher{}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
responses, err := svc.CreateForDocuments(permission.WithIdentity(ctx, "user-1", ""), "kb-1", []string{"doc-1"})
if err != nil {
t.Fatalf("expected per-document failure: %v", err)
}
if len(responses) != 1 {
t.Fatalf("responses = %d, want 1", len(responses))
}
if responses[0].DocumentID != "doc-1" || !strings.Contains(responses[0].Result, tt.wantError) {
t.Fatalf("expected dataset rejection, got %+v", responses[0])
}
if len(publisher.messages) != 0 {
t.Fatalf("published rejected document: %+v", publisher.messages)
}
after, err := taskDAO.GetByDocumentID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("reload existing task: %v", err)
}
if !reflect.DeepEqual(before, after) {
t.Fatalf("task changed: before=%+v, after=%+v", before, after)
}
if got := countPipelineLogs(t, db, "doc-1"); got != 0 {
t.Fatalf("pipeline logs = %d, want 0", got)
}
})
}
}
func TestIngestionTaskServiceCreateForDocumentsRejectsMissingRunMetadata(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestDoc(t, "doc-1", "missing-kb", 0, 0)
publisher := &recordingTaskPublisher{}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
responses, err := svc.CreateForDocuments(permission.WithIdentity(t.Context(), "user-1", ""), "missing-kb", []string{"doc-1"})
if err != nil {
t.Fatalf("CreateForDocuments returns per-document failures: %v", err)
}
if len(responses) != 1 || !strings.Contains(responses[0].Result, "ensure run identity") {
t.Fatalf("unexpected responses: %+v", responses)
}
if len(publisher.messages) != 0 {
t.Fatalf("published messages = %d, want 0", len(publisher.messages))
}
task, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")
if err != nil {
t.Fatalf("load queued task: %v", err)
}
if task != nil {
t.Fatalf("task = %+v, want no task after identity setup fails", task)
}
if got := countPipelineLogs(t, db, "doc-1"); got != 0 {
t.Fatalf("pipeline log rows = %d, want 0 when metadata lookup fails", got)
}
}
func TestIngestionTaskServiceMarksTaskScheduledOnlyAfterPublish(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
statusDuringPublish := ""
publisher := &recordingTaskPublisher{
beforeReturn: func(taskID string) {
task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, taskID)
if err != nil {
t.Fatalf("load task during publish: %v", err)
}
statusDuringPublish = task.Status
},
}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
responses, err := svc.CreateForDocuments(permission.WithIdentity(t.Context(), "user-1", ""), "kb-1", []string{"doc-1"})
if err != nil {
t.Fatalf("CreateForDocuments failed: %v", err)
}
if statusDuringPublish != common.CREATED {
t.Fatalf("status during publish = %q, want %q", statusDuringPublish, common.CREATED)
}
if len(responses) != 1 || !strings.HasPrefix(responses[0].Result, "task_id: ") {
t.Fatalf("unexpected parse response: %+v", responses)
}
task, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")
if err != nil {
t.Fatalf("reload task: %v", err)
}
if task.Status != common.SCHEDULED {
t.Fatalf("status after publish = %q, want %q", task.Status, common.SCHEDULED)
}
}
func TestIngestionTaskServiceAcceptsConsumerWinningPublishRace(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
publisher := &recordingTaskPublisher{
beforeReturn: func(taskID string) {
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", taskID).
Update("status", common.RUNNING).Error; err != nil {
t.Fatalf("simulate consumer start: %v", err)
}
},
}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
responses, err := svc.CreateForDocuments(permission.WithIdentity(t.Context(), "user-1", ""), "kb-1", []string{"doc-1"})
if err != nil {
t.Fatalf("CreateForDocuments failed: %v", err)
}
if len(responses) != 1 || !strings.HasPrefix(responses[0].Result, "task_id: ") {
t.Fatalf("unexpected parse response: %+v", responses)
}
task, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")
if err != nil {
t.Fatalf("reload task: %v", err)
}
if task.Status != common.RUNNING {
t.Fatalf("status after consumer race = %q, want %q", task.Status, common.RUNNING)
}
}
func TestIngestionTaskServiceStartRunningTransitionsScheduledTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
task, err := NewIngestionTaskService().TransitionTaskToRunning(t.Context(), "task-1")
if err != nil {
t.Fatalf("StartRunning failed: %v", err)
}
if task.Status != common.RUNNING {
t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)
}
}
func TestIngestionTaskServiceStartRunningFromCreatedTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
task, err := NewIngestionTaskService().TransitionTaskToRunning(t.Context(), "task-1")
if err != nil {
t.Fatalf("StartRunning failed: %v", err)
}
if task.Status != common.RUNNING {
t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)
}
}
func TestIngestionTaskServiceTransitionFromRejectsConflict(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.STOPPED)
svc := NewIngestionTaskService()
ctx := t.Context()
_, err := svc.transitionFrom(ctx, "task-1", []string{common.CREATED, common.SCHEDULED}, common.RUNNING)
if err == nil {
t.Fatal("expected conflict error, got nil")
}
var conflictErr *TaskStatusConflictError
if !errors.As(err, &conflictErr) {
t.Fatalf("expected TaskStatusConflictError, got %T (%v)", err, err)
}
if conflictErr.ExpectedFrom != "CREATED/SCHEDULED" {
t.Fatalf("expected %q, got %q", "CREATED/SCHEDULED", conflictErr.ExpectedFrom)
}
if conflictErr.ActualCurrent != common.STOPPED {
t.Fatalf("actual current = %q, want %q", conflictErr.ActualCurrent, common.STOPPED)
}
}
func TestIngestionTaskServiceListByUserFiltersDataset(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
insertTestIngestionTask(t, "task-2", "user-1", "doc-2", "kb-2")
insertTestIngestionTask(t, "task-3", "user-2", "doc-3", "kb-1")
svc := NewIngestionTaskService()
datasetID := "kb-1"
ctx := t.Context()
tasks, err := svc.List(permission.WithIdentity(ctx, "user-1", ""), &datasetID, 0, 0)
if err != nil {
t.Fatalf("ListByUser failed: %v", err)
}
if len(tasks) != 1 {
t.Fatalf("expected 1 task, got %d", len(tasks))
}
if tasks[0].ID != "task-1" {
t.Fatalf("task ID = %q, want %q", tasks[0].ID, "task-1")
}
}
func TestIngestionTaskServiceRequestStopManyStopsOwnedTasks(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
ctx := t.Context()
userID := "user-1"
svc := NewIngestionTaskService()
tasks, err := svc.RequestStopMany(ctx, []string{"task-1"}, &userID)
if err != nil {
t.Fatalf("RequestStopMany failed: %v", err)
}
if len(tasks) != 1 {
t.Fatalf("expected 1 task response, got %d", len(tasks))
}
if tasks[0].Status != common.STOPPED {
t.Fatalf("status = %q, want %q", tasks[0].Status, common.STOPPED)
}
}
func TestIngestionTaskServiceRequestStopManyRejectsOtherUsersTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
ctx := t.Context()
userID := "user-2"
svc := NewIngestionTaskService()
if _, err := svc.RequestStopMany(ctx, []string{"task-1"}, &userID); err == nil {
t.Fatal("expected RequestStopMany to reject non-owner")
}
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if err != nil {
t.Fatalf("load task: %v", err)
}
if task.Status != common.CREATED {
t.Fatalf("status = %q, want %q", task.Status, common.CREATED)
}
}
func TestIngestionTaskServiceRequestStopManyAllowsAdmin(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
ctx := t.Context()
svc := NewIngestionTaskService()
tasks, err := svc.RequestStopMany(ctx, []string{"task-1"}, nil)
if err != nil {
t.Fatalf("RequestStopMany admin failed: %v", err)
}
if len(tasks) != 1 {
t.Fatalf("expected 1 task response, got %d", len(tasks))
}
if tasks[0].Status != common.STOPPED {
t.Fatalf("status = %q, want %q", tasks[0].Status, common.STOPPED)
}
}
func TestIngestionTaskServiceRemoveManyRemovesOwnedTasks(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
ctx := t.Context()
userID := "user-1"
svc := NewIngestionTaskService()
result, err := svc.RemoveMany(ctx, []string{"task-1"}, &userID)
if err != nil {
t.Fatalf("RemoveMany failed: %v", err)
}
if len(result) != 1 || result[0]["remove"] != "success" {
t.Fatalf("unexpected remove result: %+v", result)
}
if _, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1"); err == nil {
t.Fatal("task should be removed")
}
}
func TestIngestionTaskServiceStartRunningTransitionsCreatedTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
svc := NewIngestionTaskService()
ctx := t.Context()
task, err := svc.TransitionTaskToRunning(ctx, "task-1")
if err != nil {
t.Fatalf("StartRunning failed: %v", err)
}
if task.Status != common.RUNNING {
t.Fatalf("status = %q, want %q", task.Status, common.RUNNING)
}
}
// TestPrepareValidatedRunResetsDocumentProgress verifies validation precedes
// document initialization while preserving committed counts and progress_msg.
func TestPrepareValidatedRunResetsDocumentProgress(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 100, 10)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
// Seed the document as a partially-processed state that the start transition must reset.
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").
Updates(map[string]interface{}{
"progress": float64(0.5),
"progress_msg": "partial",
"process_duration": 0.719,
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
svc := NewIngestionTaskService()
ctx := t.Context()
task, err := svc.TransitionTaskToRunning(ctx, "task-1")
if err != nil {
t.Fatalf("TransitionTaskToRunning failed: %v", err)
}
svc.PrepareValidatedRun(ctx, task)
var doc entity.Document
if err := db.Where("id = ?", "doc-1").First(&doc).Error; err != nil {
t.Fatalf("reload document: %v", err)
}
if doc.Progress != 0 {
t.Fatalf("progress = %f, want 0", doc.Progress)
}
if doc.ChunkNum != 10 {
t.Fatalf("chunk_num = %d, want 10", doc.ChunkNum)
}
if doc.TokenNum != 100 {
t.Fatalf("token_num = %d, want 100", doc.TokenNum)
}
if doc.ProgressMsg == nil || *doc.ProgressMsg != "partial" {
t.Fatalf("progress_msg = %v, want partial", doc.ProgressMsg)
}
if doc.ProcessBeginAt == nil || doc.ProcessBeginAt.IsZero() {
t.Fatal("process_begin_at not set")
}
if doc.ProcessDuration != 0 {
t.Fatalf("process_duration = %f, want 0", doc.ProcessDuration)
}
}
// TestStartRunningLeavesTerminalDocumentUntouched locks in the no-resurrection
// invariant: a task already in a terminal status is returned as-is by
// StartRunning, and its document's finished run status/counters are not reset.
func TestStartRunningLeavesTerminalDocumentUntouched(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 100, 10)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").
Updates(map[string]interface{}{
"progress": float64(1.0),
"progress_msg": "done",
}).Error; err != nil {
t.Fatalf("seed document: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("status", common.COMPLETED).Error; err != nil {
t.Fatalf("set COMPLETED: %v", err)
}
svc := NewIngestionTaskService()
ctx := t.Context()
task, err := svc.TransitionTaskToRunning(ctx, "task-1")
if err != nil {
t.Fatalf("StartRunning failed: %v", err)
}
if task.Status != common.COMPLETED {
t.Fatalf("status = %q, want %q (terminal must be preserved)", task.Status, common.COMPLETED)
}
var doc entity.Document
if err := db.Where("id = ?", "doc-1").First(&doc).Error; err != nil {
t.Fatalf("reload document: %v", err)
}
if doc.Progress != 1.0 {
t.Fatalf("progress = %v, want 1.0", doc.Progress)
}
if doc.ChunkNum != 10 || doc.TokenNum != 100 {
t.Fatalf("counters changed: chunk_num=%d token_num=%d, want 10/100", doc.ChunkNum, doc.TokenNum)
}
}
// TestStartRunningFinalizesStoppingTask locks in the redelivery path: a
// STOPPING task (cancelled after being nacked, before any worker ran it) is
// moved to STOPPED by StartRunning, so the task reaches a terminal state and
// a later re-parse can transition it back to CREATED.
func TestStartRunningFinalizesStoppingTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("status", common.STOPPING).Error; err != nil {
t.Fatalf("set STOPPING: %v", err)
}
svc := NewIngestionTaskService()
ctx := t.Context()
task, err := svc.TransitionTaskToRunning(ctx, "task-1")
if err != nil {
t.Fatalf("StartRunning failed: %v", err)
}
if task.Status != common.STOPPED {
t.Fatalf("status = %q, want %q", task.Status, common.STOPPED)
}
}
func TestIngestionTaskServiceRequestStopTransitionsCreatedTaskToStopped(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
ctx := t.Context()
svc := NewIngestionTaskService()
task, err := svc.RequestStop(ctx, "task-1")
if err != nil {
t.Fatalf("RequestStop failed: %v", err)
}
if task.Status != common.STOPPED {
t.Fatalf("status = %q, want %q", task.Status, common.STOPPED)
}
}
func TestIngestionTaskServiceRequestStopTransitionsScheduledTaskToStopped(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("status", common.SCHEDULED).Error; err != nil {
t.Fatalf("set SCHEDULED: %v", err)
}
ctx := t.Context()
svc := NewIngestionTaskService()
task, err := svc.RequestStop(ctx, "task-1")
if err != nil {
t.Fatalf("RequestStop failed: %v", err)
}
if task.Status != common.STOPPED {
t.Fatalf("status = %q, want %q", task.Status, common.STOPPED)
}
}
func TestIngestionTaskServiceMarkCompletedRejectsNonRunningTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
ctx := t.Context()
svc := NewIngestionTaskService()
if err := svc.MarkCompleted(ctx, "task-1"); err == nil {
t.Fatal("expected MarkCompleted to reject non-running task")
}
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if err != nil {
t.Fatalf("load task: %v", err)
}
if task.Status != common.CREATED {
t.Fatalf("status = %q, want %q", task.Status, common.CREATED)
}
}
func TestIngestionTaskServiceMarkCompletedUpdatesTaskStatus(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", common.RUNNING).Error; err != nil {
t.Fatalf("set running status: %v", err)
}
ctx := t.Context()
svc := NewIngestionTaskService()
if err := svc.MarkCompleted(ctx, "task-1"); err != nil {
t.Fatalf("MarkCompleted failed: %v", err)
}
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if err != nil {
t.Fatalf("load task: %v", err)
}
if task.Status != common.COMPLETED {
t.Fatalf("status = %q, want %q", task.Status, common.COMPLETED)
}
}
func TestIngestionTaskServiceMarkFailedUpdatesTaskStatus(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", common.RUNNING).Error; err != nil {
t.Fatalf("set running status: %v", err)
}
ctx := t.Context()
svc := NewIngestionTaskService()
if err := svc.MarkFailed(ctx, "task-1", ""); err != nil {
t.Fatalf("MarkFailed failed: %v", err)
}
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if err != nil {
t.Fatalf("load task: %v", err)
}
if task.Status != common.FAILED {
t.Fatalf("status = %q, want %q", task.Status, common.FAILED)
}
}
func TestIngestionTaskServiceNewTaskStatusConflictErrorLoadsActualStatus(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", common.STOPPING).Error; err != nil {
t.Fatalf("set stopping status: %v", err)
}
ctx := t.Context()
svc := NewIngestionTaskService()
err := svc.newTaskStatusConflictError(ctx, "task-1", common.CREATED, common.RUNNING)
var conflictErr *TaskStatusConflictError
if !errors.As(err, &conflictErr) {
t.Fatalf("expected TaskStatusConflictError, got %T", err)
}
if conflictErr.TaskID != "task-1" || conflictErr.ExpectedFrom != common.CREATED || conflictErr.AttemptedTo != common.RUNNING || conflictErr.ActualCurrent != common.STOPPING {
t.Fatalf("unexpected conflict error: %+v", conflictErr)
}
}
func TestIngestionTaskServiceMarkCompletedReturnsTaskIDInTransitionError(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
ctx := t.Context()
svc := NewIngestionTaskService()
err := svc.MarkCompleted(ctx, "task-1")
var transitionErr *InvalidTaskTransitionError
if !errors.As(err, &transitionErr) {
t.Fatalf("expected InvalidTaskTransitionError, got %T", err)
}
if transitionErr.TaskID != "task-1" || transitionErr.From != common.CREATED || transitionErr.To != common.COMPLETED {
t.Fatalf("unexpected transition error: %+v", transitionErr)
}
}
func TestIngestionTaskServiceCreateAndEnqueueRetriesTerminalTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestDoc(t, "doc-2", "kb-1", 0, 0)
publisher := &recordingTaskPublisher{}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
testCases := []struct {
name string
status string
docID string
}{
{name: "failed", status: common.FAILED, docID: "doc-1"},
{name: "stopped", status: common.STOPPED, docID: "doc-2"},
}
ctx := t.Context()
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
publisher.subject = ""
publisher.messages = nil
if err := dao.DB.Where("id = ?", "task-1").Delete(&entity.IngestionTask{}).Error; err != nil {
t.Fatalf("clear task: %v", err)
}
insertTestIngestionTask(t, "task-1", "user-1", tc.docID, "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", tc.status).Error; err != nil {
t.Fatalf("set terminal status: %v", err)
}
task, err := svc.CreateAndEnqueue(ctx, &entity.IngestionTask{
DocumentID: tc.docID,
UserID: "user-1",
DatasetID: "kb-1",
Status: common.CREATED,
})
if err != nil {
t.Fatalf("CreateAndEnqueue failed: %v", err)
}
if task.ID != "task-1" {
t.Fatalf("task ID = %q, want task-1", task.ID)
}
if task.Status != common.SCHEDULED {
t.Fatalf("status = %q, want %q", task.Status, common.SCHEDULED)
}
if len(publisher.messages) != 1 || publisher.messages[0].TaskID != "task-1" {
t.Fatalf("unexpected published messages: %+v", publisher.messages)
}
reloaded, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if err != nil {
t.Fatalf("reload task: %v", err)
}
if reloaded.Status != common.SCHEDULED {
t.Fatalf("reloaded status = %q, want %q", reloaded.Status, common.SCHEDULED)
}
})
}
}
func TestIngestionTaskServiceCreateAndEnqueueSchedulesExistingCreatedTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "stale-task", "user-1", "doc-1", "kb-1", common.CREATED)
publisher := &recordingTaskPublisher{}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
DocumentID: "doc-1",
UserID: "user-1",
DatasetID: "kb-1",
Status: common.CREATED,
})
if err != nil {
t.Fatalf("CreateAndEnqueue failed: %v", err)
}
if task.ID != "stale-task" {
t.Fatalf("task ID = %q, want stale-task", task.ID)
}
if task.Status != common.SCHEDULED {
t.Fatalf("status = %q, want %q", task.Status, common.SCHEDULED)
}
if len(publisher.messages) != 1 || publisher.messages[0].TaskID != task.ID {
t.Fatalf("unexpected published messages: %+v", publisher.messages)
}
reloaded, err := dao.NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-1")
if err != nil {
t.Fatalf("load task after scheduling created task: %v", err)
}
if reloaded == nil || reloaded.ID != "stale-task" || reloaded.Status != common.SCHEDULED {
t.Fatalf("expected created task to be scheduled in place, task=%+v", reloaded)
}
}
func TestIngestionTaskServiceConcurrentCreatedTaskPublicationsConverge(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
publisher := &duplicatePublishRecorder{
firstPublished: make(chan struct{}),
secondPublished: make(chan struct{}),
releaseFirst: make(chan struct{}),
releaseSecond: make(chan struct{}),
}
firstService := NewIngestionTaskService()
firstService.taskPublisher = publisher
secondService := NewIngestionTaskService()
secondService.taskPublisher = publisher
type result struct {
task *entity.IngestionTask
err error
}
results := make(chan result, 2)
enqueue := func(svc *IngestionTaskService) {
task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
DocumentID: "doc-1",
UserID: "user-1",
DatasetID: "kb-1",
Status: common.CREATED,
})
results <- result{task: task, err: err}
}
go enqueue(firstService)
<-publisher.firstPublished
go enqueue(secondService)
<-publisher.secondPublished
close(publisher.releaseFirst)
first := <-results
if first.err != nil {
t.Fatalf("first CreateAndEnqueue: %v", first.err)
}
close(publisher.releaseSecond)
second := <-results
if second.err != nil {
t.Fatalf("second CreateAndEnqueue: %v", second.err)
}
if first.task.ID != "task-1" || second.task.ID != "task-1" {
t.Fatalf("concurrent requests returned task IDs %q and %q, want task-1", first.task.ID, second.task.ID)
}
publisher.mu.Lock()
messages := append([]common.TaskMessage(nil), publisher.messages...)
publisher.mu.Unlock()
if len(messages) != 2 || messages[0].TaskID != "task-1" || messages[1].TaskID != "task-1" {
t.Fatalf("published messages = %+v, want two messages for task-1", messages)
}
task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1")
if err != nil {
t.Fatalf("reload task: %v", err)
}
if task.Status != common.SCHEDULED {
t.Fatalf("task status after concurrent publication = %q, want %q", task.Status, common.SCHEDULED)
}
}
func TestIngestionTaskServiceScheduleCreatedTasksKeepsTaskCreatedOnPublishFailure(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
svc := NewIngestionTaskService()
svc.taskPublisher = &recordingTaskPublisher{err: errors.New("publish failed")}
if err := svc.ScheduleCreatedTasks(t.Context()); err == nil {
t.Fatal("expected ScheduleCreatedTasks to return publish error")
}
task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1")
if err != nil {
t.Fatalf("reload task: %v", err)
}
if task.Status != common.FAILED {
t.Fatalf("status after failed recovery publish = %q, want %q", task.Status, common.FAILED)
}
}
func TestIngestionTaskServiceScheduleCreatedTasksContinuesAfterPublishFailure(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestDoc(t, "doc-2", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
insertTestIngestionTaskWithStatus(t, "task-2", "user-1", "doc-2", "kb-1", common.CREATED)
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("create_time", 1).Error; err != nil {
t.Fatalf("set first task create time: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-2").Update("create_time", 2).Error; err != nil {
t.Fatalf("set second task create time: %v", err)
}
publisher := &failFirstTaskPublisher{}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
if err := svc.ScheduleCreatedTasks(t.Context()); err == nil {
t.Fatal("expected ScheduleCreatedTasks to return publish error")
}
if len(publisher.messages) != 2 {
t.Fatalf("published messages = %d, want 2", len(publisher.messages))
}
first, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1")
if err != nil {
t.Fatalf("reload first task: %v", err)
}
if first.Status != common.FAILED {
t.Fatalf("first task status = %q, want %q", first.Status, common.FAILED)
}
second, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-2")
if err != nil {
t.Fatalf("reload second task: %v", err)
}
if second.Status != common.SCHEDULED {
t.Fatalf("second task status = %q, want %q", second.Status, common.SCHEDULED)
}
}
func TestIngestionTaskServiceCreateAndEnqueueRejectsActiveExistingTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.SCHEDULED)
publisher := &recordingTaskPublisher{}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
ctx := t.Context()
_, err := svc.CreateAndEnqueue(ctx, &entity.IngestionTask{DocumentID: "doc-1", UserID: "user-1", DatasetID: "kb-1", Status: common.CREATED})
if err == nil {
t.Fatal("expected CreateAndEnqueue to reject existing created task")
}
if len(publisher.messages) != 0 {
t.Fatalf("expected no published messages, got %+v", publisher.messages)
}
}
func TestIngestionTaskServiceCreateAndEnqueueSettlesNewRunOnPublishFailure(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
publisher := &recordingTaskPublisher{err: errors.New("publish failed")}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
ctx := t.Context()
_, err := svc.CreateAndEnqueue(ctx, &entity.IngestionTask{
DocumentID: "doc-1",
UserID: "user-1",
DatasetID: "kb-1",
Status: common.CREATED,
})
if err == nil || err.Error() != "publish failed" {
t.Fatalf("expected publish failure, got %v", err)
}
task, getErr := dao.NewIngestionTaskDAO().GetByDocumentID(ctx, db, "doc-1")
if getErr != nil {
t.Fatalf("reload task by document id: %v", getErr)
}
if task == nil || task.Status != common.FAILED {
t.Fatalf("task = %+v, want a durably failed task", task)
}
var run entity.PipelineOperationLog
if err := db.Where("id = ?", *task.PipelineLogID).First(&run).Error; err != nil {
t.Fatalf("load numbered pipeline log: %v", err)
}
if run.RunCount == nil || *run.RunCount != 1 || run.OperationStatus != string(entity.TaskStatusFail) {
t.Fatalf("run = %+v, want numbered failed run", run)
}
}
func TestIngestionTaskServiceCreateAndEnqueueSettlesRetriedRunOnPublishFailure(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
// Seed the document and KB so buildOpenLogInput succeeds: without them no
// open row is created and the rollback cleanup below is never exercised.
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", common.FAILED).Error; err != nil {
t.Fatalf("set failed status: %v", err)
}
publisher := &recordingTaskPublisher{err: errors.New("publish failed")}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
ctx := t.Context()
_, err := svc.CreateAndEnqueue(ctx, &entity.IngestionTask{
DocumentID: "doc-1",
UserID: "user-1",
DatasetID: "kb-1",
Status: common.CREATED,
})
if err == nil || err.Error() != "publish failed" {
t.Fatalf("expected publish failure, got %v", err)
}
reloaded, getErr := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if getErr != nil {
t.Fatalf("reload task: %v", getErr)
}
if reloaded.Status != common.FAILED {
t.Fatalf("status = %q, want %q", reloaded.Status, common.FAILED)
}
if reloaded.PipelineLogID == nil {
t.Fatal("failed retry lost its run identity")
}
var run entity.PipelineOperationLog
if err := db.Where("id = ?", *reloaded.PipelineLogID).First(&run).Error; err != nil {
t.Fatalf("load failed retry run: %v", err)
}
if run.RunCount == nil || *run.RunCount != 1 || run.OperationStatus != string(entity.TaskStatusFail) {
t.Fatalf("run = %+v, want numbered failed retry", run)
}
}
func TestIngestionTaskServiceRemoveSettlesNumberedQueuedRun(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
runCount := 1
if err := db.Create(&entity.PipelineOperationLog{
ID: "numbered-open-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: string(entity.PipelineTaskTypeParse),
OperationStatus: string(entity.TaskStatusUnstart),
RunCount: &runCount,
}).Error; err != nil {
t.Fatalf("seed numbered pipeline log: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("pipeline_log_id", "numbered-open-log").Error; err != nil {
t.Fatalf("bind numbered pipeline log: %v", err)
}
if _, err := NewIngestionTaskService().Remove(t.Context(), "task-1", sptr("user-1")); err != nil {
t.Fatalf("Remove failed: %v", err)
}
var run entity.PipelineOperationLog
if err := db.First(&run, "id = ?", "numbered-open-log").Error; err != nil {
t.Fatalf("numbered run was deleted: %v", err)
}
if run.OperationStatus != string(entity.TaskStatusCancel) {
t.Fatalf("numbered run status = %q, want %q", run.OperationStatus, entity.TaskStatusCancel)
}
var terminal entity.IngestionTaskLog
if err := db.Where("pipeline_log_id = ? AND event_type = ?", "numbered-open-log", dao.EventTypeTerminal).
First(&terminal).Error; err != nil {
t.Fatalf("terminal event missing: %v", err)
}
if terminal.Message != "Task superseded by a new parse request." {
t.Fatalf("terminal message = %q, want supersede reason", terminal.Message)
}
}
func TestIngestionTaskServiceRemoveRetriesStoppedRunSettlement(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
runCount := 1
if err := db.Create(&entity.PipelineOperationLog{
ID: "remove-open-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: string(entity.PipelineTaskTypeParse),
OperationStatus: string(entity.TaskStatusUnstart),
RunCount: &runCount,
}).Error; err != nil {
t.Fatalf("seed numbered pipeline log: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("pipeline_log_id", "remove-open-log").Error; err != nil {
t.Fatalf("bind numbered pipeline log: %v", err)
}
settlementErr := errors.New("pipeline ledger unavailable")
const callbackName = "test:fail-remove-ledger-settlement"
if err := db.Callback().Update().Before("gorm:update").Register(callbackName, func(tx *gorm.DB) {
if tx.Statement.Table == (&entity.PipelineOperationLog{}).TableName() {
_ = tx.AddError(settlementErr)
}
}); err != nil {
t.Fatalf("register update failure: %v", err)
}
svc := NewIngestionTaskService()
if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); !errors.Is(err, settlementErr) {
t.Fatalf("first Remove error = %v, want settlement failure", err)
}
task, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-1")
if err != nil {
t.Fatalf("reload task after failed settlement: %v", err)
}
if task.Status != common.STOPPED {
t.Fatalf("task status = %q, want %q", task.Status, common.STOPPED)
}
var run entity.PipelineOperationLog
if err := db.First(&run, "id = ?", "remove-open-log").Error; err != nil {
t.Fatalf("load pipeline log after failed settlement: %v", err)
}
if run.OperationStatus != string(entity.TaskStatusUnstart) {
t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, entity.TaskStatusUnstart)
}
if err := db.Callback().Update().Remove(callbackName); err != nil {
t.Fatalf("remove update failure: %v", err)
}
if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); err != nil {
t.Fatalf("retry Remove: %v", err)
}
if _, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-1"); !errors.Is(err, gorm.ErrRecordNotFound) {
t.Fatalf("deleted task reload error = %v, want record not found", err)
}
if err := db.First(&run, "id = ?", "remove-open-log").Error; err != nil {
t.Fatalf("load pipeline log after retry: %v", err)
}
if run.OperationStatus != string(entity.TaskStatusCancel) {
t.Fatalf("pipeline run status after retry = %q, want %q", run.OperationStatus, entity.TaskStatusCancel)
}
var terminal entity.IngestionTaskLog
if err := db.Where("pipeline_log_id = ? AND event_type = ?", "remove-open-log", dao.EventTypeTerminal).
First(&terminal).Error; err != nil {
t.Fatalf("load terminal event after retry: %v", err)
}
if terminal.Message != "Task superseded by a new parse request." {
t.Fatalf("terminal message after retry = %q, want supersede reason", terminal.Message)
}
}
func TestIngestionTaskServiceRemoveSettlesTerminalTaskRun(t *testing.T) {
cases := []struct {
name string
taskStatus string
runStatus entity.TaskStatus
terminalText string
}{
{"failed", common.FAILED, entity.TaskStatusFail, taskTerminalMessageFailed},
{"completed", common.COMPLETED, entity.TaskStatusDone, taskTerminalMessageComplete},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", tc.taskStatus)
runCount := 1
if err := db.Create(&entity.PipelineOperationLog{
ID: "terminal-open-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: string(entity.PipelineTaskTypeParse),
OperationStatus: string(entity.TaskStatusUnstart),
RunCount: &runCount,
}).Error; err != nil {
t.Fatalf("seed numbered pipeline log: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("pipeline_log_id", "terminal-open-log").Error; err != nil {
t.Fatalf("bind numbered pipeline log: %v", err)
}
if _, err := NewIngestionTaskService().Remove(t.Context(), "task-1", sptr("user-1")); err != nil {
t.Fatalf("Remove failed: %v", err)
}
var run entity.PipelineOperationLog
if err := db.First(&run, "id = ?", "terminal-open-log").Error; err != nil {
t.Fatalf("numbered run was deleted: %v", err)
}
if run.OperationStatus != string(tc.runStatus) {
t.Fatalf("numbered run status = %q, want %q", run.OperationStatus, tc.runStatus)
}
var terminal entity.IngestionTaskLog
if err := db.Where("pipeline_log_id = ? AND event_type = ?", "terminal-open-log", dao.EventTypeTerminal).
First(&terminal).Error; err != nil {
t.Fatalf("terminal event missing: %v", err)
}
if terminal.Message != tc.terminalText {
t.Fatalf("terminal message = %q, want %q", terminal.Message, tc.terminalText)
}
})
}
}
func TestIngestionTaskServicePrepareDocumentTaskForCleanupRecordsCleanupReason(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
runCount := 1
if err := db.Create(&entity.PipelineOperationLog{
ID: "cleanup-open-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: string(entity.PipelineTaskTypeParse),
OperationStatus: string(entity.TaskStatusUnstart),
RunCount: &runCount,
}).Error; err != nil {
t.Fatalf("seed numbered pipeline log: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("pipeline_log_id", "cleanup-open-log").Error; err != nil {
t.Fatalf("bind numbered pipeline log: %v", err)
}
task, err := NewIngestionTaskService().PrepareDocumentTaskForCleanup(
t.Context(),
"doc-1",
"Task cancelled by document cleanup.",
)
if err != nil {
t.Fatalf("PrepareDocumentTaskForCleanup failed: %v", err)
}
if task == nil || task.Status != common.STOPPED {
t.Fatalf("cleanup task = %+v, want one STOPPED task", task)
}
var run entity.PipelineOperationLog
if err := db.First(&run, "id = ?", "cleanup-open-log").Error; err != nil {
t.Fatalf("load pipeline log: %v", err)
}
if run.OperationStatus != string(entity.TaskStatusCancel) {
t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, entity.TaskStatusCancel)
}
var terminal entity.IngestionTaskLog
if err := db.Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).
First(&terminal).Error; err != nil {
t.Fatalf("load terminal event: %v", err)
}
if terminal.Message != "Task cancelled by document cleanup." {
t.Fatalf("terminal message = %q, want cleanup reason", terminal.Message)
}
}
func TestIngestionTaskServicePrepareDocumentTaskForCleanupRetriesOpenLedgerSettlement(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
runCount := 1
if err := db.Create(&entity.PipelineOperationLog{
ID: "cleanup-open-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: string(entity.PipelineTaskTypeParse),
OperationStatus: string(entity.TaskStatusUnstart),
RunCount: &runCount,
}).Error; err != nil {
t.Fatalf("seed numbered pipeline log: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("pipeline_log_id", "cleanup-open-log").Error; err != nil {
t.Fatalf("bind numbered pipeline log: %v", err)
}
settlementErr := errors.New("pipeline ledger unavailable")
const callbackName = "test:fail-cleanup-ledger-settlement"
if err := db.Callback().Update().Before("gorm:update").Register(callbackName, func(tx *gorm.DB) {
if tx.Statement.Table == (&entity.PipelineOperationLog{}).TableName() {
_ = tx.AddError(settlementErr)
}
}); err != nil {
t.Fatalf("register update failure: %v", err)
}
svc := NewIngestionTaskService()
if _, err := svc.PrepareDocumentTaskForCleanup(
t.Context(),
"doc-1",
"Task cancelled by document cleanup.",
); !errors.Is(err, settlementErr) {
t.Fatalf("first cleanup error = %v, want settlement failure", err)
}
task, err := svc.ingestionTaskDAO.GetByID(t.Context(), db, "task-1")
if err != nil {
t.Fatalf("reload task after failed settlement: %v", err)
}
if task.Status != common.STOPPED {
t.Fatalf("task status = %q, want %q", task.Status, common.STOPPED)
}
var run entity.PipelineOperationLog
if err := db.First(&run, "id = ?", "cleanup-open-log").Error; err != nil {
t.Fatalf("load pipeline log after failed settlement: %v", err)
}
if run.OperationStatus != string(entity.TaskStatusUnstart) {
t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, entity.TaskStatusUnstart)
}
var terminalCount int64
if err := db.Model(&entity.IngestionTaskLog{}).
Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).
Count(&terminalCount).Error; err != nil {
t.Fatalf("count terminal events after failed settlement: %v", err)
}
if terminalCount != 0 {
t.Fatalf("terminal events after failed settlement = %d, want 0", terminalCount)
}
if err := db.Callback().Update().Remove(callbackName); err != nil {
t.Fatalf("remove update failure: %v", err)
}
task, err = svc.PrepareDocumentTaskForCleanup(
t.Context(),
"doc-1",
"Task cancelled by document cleanup.",
)
if err != nil {
t.Fatalf("retry PrepareDocumentTaskForCleanup: %v", err)
}
if task == nil || task.Status != common.STOPPED {
t.Fatalf("retried cleanup task = %+v, want one STOPPED task", task)
}
if err := db.First(&run, "id = ?", "cleanup-open-log").Error; err != nil {
t.Fatalf("load pipeline log after retry: %v", err)
}
if run.OperationStatus != string(entity.TaskStatusCancel) {
t.Fatalf("pipeline run status after retry = %q, want %q", run.OperationStatus, entity.TaskStatusCancel)
}
var terminal entity.IngestionTaskLog
if err := db.Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).
First(&terminal).Error; err != nil {
t.Fatalf("load terminal event after retry: %v", err)
}
if terminal.Message != "Task cancelled by document cleanup." {
t.Fatalf("terminal message after retry = %q, want cleanup reason", terminal.Message)
}
if err := db.Model(&entity.IngestionTaskLog{}).
Where("pipeline_log_id = ? AND event_type = ?", "cleanup-open-log", dao.EventTypeTerminal).
Count(&terminalCount).Error; err != nil {
t.Fatalf("count terminal events after retry: %v", err)
}
if terminalCount != 1 {
t.Fatalf("terminal events after retry = %d, want 1", terminalCount)
}
}
func TestIngestionTaskServicePrepareDocumentTaskForCleanupSettlesTerminalTaskLedger(t *testing.T) {
tests := []struct {
name string
taskStatus string
operationStatus entity.TaskStatus
wantMessage string
}{
{
name: "completed",
taskStatus: common.COMPLETED,
operationStatus: entity.TaskStatusDone,
wantMessage: "Task completed.",
},
{
name: "failed",
taskStatus: common.FAILED,
operationStatus: entity.TaskStatusFail,
wantMessage: "Task failed.",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", tc.taskStatus)
runCount := 1
run := &entity.PipelineOperationLog{
ID: "cleanup-terminal-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: string(entity.PipelineTaskTypeParse),
OperationStatus: string(entity.TaskStatusRunning),
RunCount: &runCount,
}
if err := db.Create(run).Error; err != nil {
t.Fatalf("seed numbered pipeline log: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("pipeline_log_id", run.ID).Error; err != nil {
t.Fatalf("bind numbered pipeline log: %v", err)
}
task, err := NewIngestionTaskService().PrepareDocumentTaskForCleanup(
t.Context(),
"doc-1",
"Task cancelled by document cleanup.",
)
if err != nil {
t.Fatalf("PrepareDocumentTaskForCleanup failed: %v", err)
}
if task == nil || task.Status != tc.taskStatus {
t.Fatalf("cleanup task = %+v, want status %q", task, tc.taskStatus)
}
if err := db.First(run, "id = ?", run.ID).Error; err != nil {
t.Fatalf("load pipeline log: %v", err)
}
if run.OperationStatus != string(tc.operationStatus) {
t.Fatalf("pipeline run status = %q, want %q", run.OperationStatus, tc.operationStatus)
}
var terminal entity.IngestionTaskLog
if err := db.Where("pipeline_log_id = ? AND event_type = ?", run.ID, dao.EventTypeTerminal).
First(&terminal).Error; err != nil {
t.Fatalf("load terminal event: %v", err)
}
if terminal.Message != tc.wantMessage {
t.Fatalf("terminal message = %q, want %q", terminal.Message, tc.wantMessage)
}
})
}
}
func TestIngestionTaskServicePrepareDocumentTaskForCleanupRetriesMissingTerminalEvent(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
runCount := 1
run := &entity.PipelineOperationLog{
ID: "cleanup-event-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: string(entity.PipelineTaskTypeParse),
OperationStatus: string(entity.TaskStatusDone),
RunCount: &runCount,
}
if err := db.Create(run).Error; err != nil {
t.Fatalf("seed numbered pipeline log: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("pipeline_log_id", run.ID).Error; err != nil {
t.Fatalf("bind numbered pipeline log: %v", err)
}
insertErr := errors.New("terminal event unavailable")
failInsert := true
const callbackName = "test:fail-cleanup-terminal-event"
if err := db.Callback().Create().Before("gorm:create").Register(callbackName, func(tx *gorm.DB) {
if !failInsert || tx.Statement.Table != (&entity.IngestionTaskLog{}).TableName() {
return
}
eventType, ok := tx.Statement.Dest.(map[string]interface{})["event_type"].(int)
if ok && eventType == dao.EventTypeTerminal {
_ = tx.AddError(insertErr)
}
}); err != nil {
t.Fatalf("register terminal insert failure: %v", err)
}
svc := NewIngestionTaskService()
if _, err := svc.PrepareDocumentTaskForCleanup(
t.Context(),
"doc-1",
"Task cancelled by document cleanup.",
); !errors.Is(err, insertErr) {
t.Fatalf("first cleanup error = %v, want terminal insert failure", err)
}
var terminalCount int64
if err := db.Model(&entity.IngestionTaskLog{}).
Where("pipeline_log_id = ? AND event_type = ?", run.ID, dao.EventTypeTerminal).
Count(&terminalCount).Error; err != nil {
t.Fatalf("count terminal events after failed insert: %v", err)
}
if terminalCount != 0 {
t.Fatalf("terminal events after failed insert = %d, want 0", terminalCount)
}
failInsert = false
if _, err := svc.PrepareDocumentTaskForCleanup(
t.Context(),
"doc-1",
"Task cancelled by document cleanup.",
); err != nil {
t.Fatalf("retry PrepareDocumentTaskForCleanup: %v", err)
}
if err := db.Model(&entity.IngestionTaskLog{}).
Where("pipeline_log_id = ? AND event_type = ?", run.ID, dao.EventTypeTerminal).
Count(&terminalCount).Error; err != nil {
t.Fatalf("count terminal events after retry: %v", err)
}
if terminalCount != 1 {
t.Fatalf("terminal events after retry = %d, want 1", terminalCount)
}
}
func TestIngestionTaskServiceRemovePurgesTaskState(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
purged := make(chan string, 1)
svc := NewIngestionTaskService()
svc.purgeTaskState = func(_ context.Context, taskID string) error {
purged <- taskID
return nil
}
if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); err != nil {
t.Fatalf("Remove failed: %v", err)
}
select {
case taskID := <-purged:
if taskID != "task-1" {
t.Fatalf("purged task = %q, want task-1", taskID)
}
case <-time.After(2 * time.Second):
t.Fatal("Remove did not schedule task-state purge")
}
}
func TestIngestionTaskServiceRemoveDoesNotWaitForTaskStatePurge(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-remove-async", "user-1", "doc-1", "kb-1", common.COMPLETED)
started := make(chan struct{})
release := make(chan struct{})
svc := NewIngestionTaskService()
svc.purgeTaskState = func(ctx context.Context, _ string) error {
close(started)
select {
case <-release:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
done := make(chan error, 1)
go func() {
_, err := svc.Remove(t.Context(), "task-remove-async", sptr("user-1"))
done <- err
}()
select {
case err := <-done:
if err != nil {
t.Fatalf("Remove failed: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("Remove blocked on task-state purge")
}
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("background task-state purge did not start")
}
close(release)
}
func TestIngestionTaskServiceRemoveKeepsDeleteWhenPurgeFails(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.COMPLETED)
svc := NewIngestionTaskService()
svc.purgeTaskState = func(_ context.Context, _ string) error {
return errors.New("kvrocks unavailable")
}
if _, err := svc.Remove(t.Context(), "task-1", sptr("user-1")); err != nil {
t.Fatalf("Remove failed after a purge error: %v", err)
}
if _, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1"); !errors.Is(err, gorm.ErrRecordNotFound) {
t.Fatalf("deleted task reload error = %v, want record not found", err)
}
}
func TestIngestionTaskServiceUpdateComponentTotalPersistsDenominator(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
ctx := t.Context()
svc := NewIngestionTaskService()
if err := svc.UpdateComponentTotal(ctx, "task-1", 4); err != nil {
t.Fatalf("UpdateComponentTotal failed: %v", err)
}
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if err != nil {
t.Fatalf("load task: %v", err)
}
if task.ComponentTotal != 4 {
t.Fatalf("component_total = %d, want 4", task.ComponentTotal)
}
}
func TestIngestionTaskServiceRecordLifecyclePersistsRunScopedLifecycleEvent(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
svc := NewIngestionTaskService()
if err := svc.RecordLifecycle(t.Context(), "run-1", "task-1", "Parser", 1, "Parser Done"); err != nil {
t.Fatalf("RecordLifecycle failed: %v", err)
}
logs, err := dao.NewIngestionTaskLogDAO().ListLogsByPipelineLogID(t.Context(), db, "run-1")
if err != nil {
t.Fatalf("list logs: %v", err)
}
if len(logs) != 1 {
t.Fatalf("event count = %d, want 1", len(logs))
}
event := logs[0]
if event.PipelineLogID == nil || *event.PipelineLogID != "run-1" {
t.Fatalf("pipeline_log_id = %v, want run-1", event.PipelineLogID)
}
if event.EventType != dao.EventTypeLifecycle || event.Component != "Parser" || event.Phase != 1 {
t.Fatalf("event = %+v, want lifecycle Parser/1", event)
}
}
func TestIngestionTaskServiceRecordMessageClearsLifecycleFields(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := NewIngestionTaskService().RecordMessage(t.Context(), "run-1", "task-1", "detail"); err != nil {
t.Fatalf("RecordMessage failed: %v", err)
}
var event entity.IngestionTaskLog
if err := db.Order("id DESC").First(&event).Error; err != nil {
t.Fatalf("load event: %v", err)
}
if event.EventType != dao.EventTypeMessage || event.Component != "" || event.Phase != 0 || event.Message != "detail" {
t.Fatalf("event = %+v, want message with empty component and phase", event)
}
}
func TestIngestionTaskServiceRecordEventRejectsInvalidIdentityAndPhase(t *testing.T) {
svc := NewIngestionTaskService()
cases := []struct {
name string
call func() error
}{
{name: "missing pipeline log id", call: func() error {
return svc.RecordMessage(t.Context(), "", "task-1", "detail")
}},
{name: "missing task id", call: func() error {
return svc.RecordMessage(t.Context(), "run-1", "", "detail")
}},
{name: "missing lifecycle component", call: func() error {
return svc.RecordLifecycle(t.Context(), "run-1", "task-1", "", 0, "started")
}},
{name: "invalid lifecycle phase", call: func() error {
return svc.RecordLifecycle(t.Context(), "run-1", "task-1", "Parser", 3, "bad")
}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if err := tc.call(); err == nil {
t.Fatal("expected event validation error")
}
})
}
}
func TestIngestionTaskServiceEventMessageIsBoundedByRunesAndBytes(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
message := strings.Repeat("界", 10_000)
if err := NewIngestionTaskService().RecordMessage(t.Context(), "run-1", "task-1", message); err != nil {
t.Fatalf("RecordMessage failed: %v", err)
}
var event entity.IngestionTaskLog
if err := db.Order("id DESC").First(&event).Error; err != nil {
t.Fatalf("load event: %v", err)
}
if got := len([]rune(event.Message)); got > 4_000 {
t.Fatalf("message rune length = %d, want <= 4000", got)
}
if got := len([]byte(event.Message)); got > 16_384 {
t.Fatalf("message byte length = %d, want <= 16384", got)
}
if !strings.Contains(event.Message, "… [truncated,") || !strings.HasSuffix(event.Message, " chars dropped]") {
t.Fatalf("message lacks truncation marker: %q", event.Message[len(event.Message)-64:])
}
if !utf8.ValidString(event.Message) {
t.Fatal("message is not valid UTF-8")
}
}
func TestIngestionTaskServiceUsesConfiguredEventMessageLimits(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
svc := NewIngestionTaskService()
if err := svc.SetIngestionLogSettings(IngestionLogSettings{
MaxRowsPerRun: 5,
MaxRowsPerDocument: 20,
MaxMessageChars: 8,
MaxMessageBytes: 16,
}); err != nil {
t.Fatalf("SetIngestionLogSettings failed: %v", err)
}
if err := svc.RecordMessage(t.Context(), "run-1", "task-1", "abcdefghijk"); err != nil {
t.Fatalf("RecordMessage failed: %v", err)
}
var event entity.IngestionTaskLog
if err := db.Order("id DESC").First(&event).Error; err != nil {
t.Fatalf("load event: %v", err)
}
if got := len([]rune(event.Message)); got > 8 || len([]byte(event.Message)) > 16 {
t.Fatalf("message limits = %d runes/%d bytes, want <= 8/16: %q", got, len([]byte(event.Message)), event.Message)
}
}
func TestIngestionTaskServiceAdvanceOpenLogLeavesLegacyMessageUntouched(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
legacy := "legacy value"
if err := db.Create(&entity.PipelineOperationLog{
ID: "run-1",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
TaskType: string(entity.PipelineTaskTypeParse),
OperationStatus: string(entity.TaskStatusUnstart),
ProgressMsg: &legacy,
}).Error; err != nil {
t.Fatalf("insert pipeline operation log: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("pipeline_log_id", "run-1").Error; err != nil {
t.Fatalf("bind pipeline operation log: %v", err)
}
task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, "task-1")
if err != nil {
t.Fatalf("load task: %v", err)
}
NewIngestionTaskService().advanceOpenLog(t.Context(), task, logFromUnstart, string(entity.TaskStatusSchedule))
var run entity.PipelineOperationLog
if err := db.First(&run, "id = ?", "run-1").Error; err != nil {
t.Fatalf("load run: %v", err)
}
if run.ProgressMsg == nil || *run.ProgressMsg != legacy {
t.Fatalf("progress_msg = %v, want unchanged legacy value %q", run.ProgressMsg, legacy)
}
}
func TestIngestionTaskServiceMarkStoppedTransitionsStoppingTask(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
// Override to STOPPING.
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("status", common.STOPPING).Error; err != nil {
t.Fatalf("set task STOPPING: %v", err)
}
ctx := t.Context()
svc := NewIngestionTaskService()
if err := svc.MarkStopped(ctx, "task-1"); err != nil {
t.Fatalf("MarkStopped failed: %v", err)
}
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if err != nil {
t.Fatalf("load task: %v", err)
}
if task.Status != common.STOPPED {
t.Fatalf("task status = %s, want STOPPED", task.Status)
}
}
func TestIngestionTaskServiceMarkStoppedIdempotentOnAlreadyStopped(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("status", common.STOPPED).Error; err != nil {
t.Fatalf("set task STOPPED: %v", err)
}
svc := NewIngestionTaskService()
ctx := t.Context()
if err := svc.MarkStopped(ctx, "task-1"); err != nil {
t.Fatalf("MarkStopped on already STOPPED task should be idempotent, got: %v", err)
}
}
// TestIngestionTaskServiceRequestStopNeverReopensTerminalTask locks the invariant
// that a terminal task (COMPLETED/STOPPED/FAILED) can never be moved back to
// STOPPING. A terminal task whose message was already acked must not regress to
// the in-flight STOPPING state (which has no settled worker and would be stuck
// forever). RequestStop is a no-op for terminal states.
func TestIngestionTaskServiceRequestStopNeverReopensTerminalTask(t *testing.T) {
for _, status := range []string{common.COMPLETED, common.STOPPED, common.FAILED} {
t.Run(status, func(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)
ctx := t.Context()
svc := NewIngestionTaskService()
task, err := svc.RequestStop(ctx, "task-1")
if err != nil {
t.Fatalf("RequestStop on %s task should be a no-op, got: %v", status, err)
}
if task.Status != status {
t.Fatalf("RequestStop moved %s task to %q, must stay %s", status, task.Status, status)
}
})
}
}
// TestValidateTransitionRejectsTerminalToStopping locks the state-machine
// invariant directly: STOPPING is only reachable from RUNNING, never from a
// terminal state (COMPLETED/STOPPED/FAILED) or from STOPPING itself.
func TestValidateTransitionRejectsTerminalToStopping(t *testing.T) {
for _, from := range []string{common.COMPLETED, common.STOPPED, common.FAILED, common.STOPPING, common.CREATED} {
if err := validateTransition(from, common.STOPPING); err == nil {
t.Errorf("validateTransition(%s -> STOPPING) = nil, want error", from)
}
}
if err := validateTransition(common.RUNNING, common.STOPPING); err != nil {
t.Errorf("validateTransition(RUNNING -> STOPPING) = %v, want nil", err)
}
}
func TestIngestionTaskServiceMarkFailedIdempotentOnAlreadyTerminal(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("status", common.COMPLETED).Error; err != nil {
t.Fatalf("set task COMPLETED: %v", err)
}
svc := NewIngestionTaskService()
ctx := t.Context()
if err := svc.MarkFailed(ctx, "task-1", ""); err != nil {
t.Fatalf("MarkFailed on already COMPLETED task should be idempotent, got: %v", err)
}
}
func TestIngestionTaskServiceMarkCompletedIdempotentOnAlreadyTerminal(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("status", common.FAILED).Error; err != nil {
t.Fatalf("set task FAILED: %v", err)
}
ctx := t.Context()
svc := NewIngestionTaskService()
if err := svc.MarkCompleted(ctx, "task-1"); err != nil {
t.Fatalf("MarkCompleted on already FAILED task should be idempotent, got: %v", err)
}
}
func loadOpenPipelineLog(t *testing.T, ctx context.Context, db *gorm.DB, documentID string) *entity.PipelineOperationLog {
t.Helper()
var open entity.PipelineOperationLog
if err := db.WithContext(ctx).
Where("document_id = ? AND operation_status IN ?", documentID, dao.OpenPipelineOperationStatuses()).
Order("create_time DESC").
Order("id DESC").
First(&open).Error; err != nil {
t.Fatalf("expected open pipeline log for document %s", documentID)
}
return &open
}
func countPipelineLogs(t *testing.T, db *gorm.DB, documentID string) int {
t.Helper()
var count int64
if err := db.Model(&entity.PipelineOperationLog{}).Where("document_id = ?", documentID).Count(&count).Error; err != nil {
t.Fatalf("count pipeline logs: %v", err)
}
return int(count)
}
func TestIngestionTaskServiceCreateForDocumentsOpensPreTerminalPipelineLog(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
svc := NewIngestionTaskService()
svc.taskPublisher = &recordingTaskPublisher{}
ctx := t.Context()
if _, err := svc.CreateForDocuments(permission.WithIdentity(ctx, "user-1", ""), "kb-1", []string{"doc-1"}); err != nil {
t.Fatalf("CreateForDocuments failed: %v", err)
}
open := loadOpenPipelineLog(t, ctx, db, "doc-1")
if open.OperationStatus != string(entity.TaskStatusSchedule) {
t.Fatalf("OperationStatus = %q, want %q (scheduled write wins over created)", open.OperationStatus, string(entity.TaskStatusSchedule))
}
if open.KbID != "kb-1" || open.TenantID != "tenant-1" {
t.Fatalf("unexpected kb/tenant scope: %+v", open)
}
if open.RunCount == nil || *open.RunCount != 1 {
t.Fatalf("run_count = %v, want 1", open.RunCount)
}
if open.ProgressMsg != nil {
t.Fatalf("ProgressMsg = %v, want legacy field untouched", open.ProgressMsg)
}
events, err := dao.NewIngestionTaskLogDAO().ListLogsByPipelineLogID(ctx, db, open.ID)
if err != nil {
t.Fatalf("list run events: %v", err)
}
var queuedMessage *entity.IngestionTaskLog
for _, event := range events {
if event.EventType == dao.EventTypeMessage && event.Message == "Task is queued..." {
queuedMessage = event
break
}
}
if queuedMessage == nil {
t.Fatalf("expected queued message event for run %s, events: %+v", open.ID, events)
}
if countPipelineLogs(t, db, "doc-1") != 1 {
t.Fatalf("expected exactly 1 pipeline log row for one run")
}
// The task must be bound to the row so the terminal writer knows which row
// belongs to this run.
task, err := dao.NewIngestionTaskDAO().GetByDocumentID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("reload task: %v", err)
}
if task.PipelineLogID == nil || *task.PipelineLogID != open.ID {
t.Fatalf("task PipelineLogID = %v, want %q (row bound to the owning run)", task.PipelineLogID, open.ID)
}
}
// loadTaskPipelineLogID returns the pipeline-operation-log id bound to a task.
func loadTaskPipelineLogID(t *testing.T, ctx context.Context, db *gorm.DB, taskID string) string {
t.Helper()
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, taskID)
if err != nil {
t.Fatalf("load task %s: %v", taskID, err)
}
if task.PipelineLogID == nil {
t.Fatalf("task %s has no bound pipeline log", taskID)
}
return *task.PipelineLogID
}
func TestIngestionTaskServiceStartRunningAdvancesPreTerminalPipelineLog(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
svc := NewIngestionTaskService()
svc.taskPublisher = &recordingTaskPublisher{}
ctx := t.Context()
resp, err := svc.CreateForDocuments(permission.WithIdentity(ctx, "user-1", ""), "kb-1", []string{"doc-1"})
if err != nil {
t.Fatalf("CreateForDocuments failed: %v", err)
}
if len(resp) != 1 {
t.Fatalf("expected 1 response, got %d", len(resp))
}
open := loadOpenPipelineLog(t, ctx, db, "doc-1")
if open.OperationStatus != string(entity.TaskStatusSchedule) {
t.Fatalf("OperationStatus after schedule = %q, want %q", open.OperationStatus, string(entity.TaskStatusSchedule))
}
task, err := dao.NewIngestionTaskDAO().GetByDocumentID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("reload task: %v", err)
}
task, err = svc.TransitionTaskToRunning(ctx, task.ID)
if err != nil {
t.Fatalf("TransitionTaskToRunning failed: %v", err)
}
svc.PrepareValidatedRun(ctx, task)
open = loadOpenPipelineLog(t, ctx, db, "doc-1")
if open.OperationStatus != string(entity.TaskStatusRunning) {
t.Fatalf("OperationStatus after start = %q, want %q", open.OperationStatus, string(entity.TaskStatusRunning))
}
if countPipelineLogs(t, db, "doc-1") != 1 {
t.Fatalf("expected running to reuse the queued row, not insert a second one")
}
if bound := loadTaskPipelineLogID(t, ctx, db, task.ID); bound != open.ID {
t.Fatalf("task bound to %q, want the same running row %q", bound, open.ID)
}
}
func TestIngestionTaskServiceRequestStopBeforeRunClosesPreTerminalPipelineLog(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
svc := NewIngestionTaskService()
svc.taskPublisher = &recordingTaskPublisher{}
ctx := t.Context()
if _, err := svc.CreateForDocuments(permission.WithIdentity(ctx, "user-1", ""), "kb-1", []string{"doc-1"}); err != nil {
t.Fatalf("CreateForDocuments failed: %v", err)
}
loadOpenPipelineLog(t, ctx, db, "doc-1")
task, err := dao.NewIngestionTaskDAO().GetByDocumentID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("reload task: %v", err)
}
stopped, err := svc.RequestStop(ctx, task.ID)
if err != nil {
t.Fatalf("RequestStop failed: %v", err)
}
if stopped.Status != common.STOPPED {
t.Fatalf("status = %q, want %q", stopped.Status, common.STOPPED)
}
var done entity.PipelineOperationLog
if err := db.Where("document_id = ?", "doc-1").First(&done).Error; err != nil {
t.Fatalf("load pipeline log: %v", err)
}
if done.OperationStatus != string(entity.TaskStatusCancel) {
t.Fatalf("OperationStatus = %q, want %q (stop without worker must close the row)", done.OperationStatus, string(entity.TaskStatusCancel))
}
if countPipelineLogs(t, db, "doc-1") != 1 {
t.Fatalf("expected stop to reuse the queued row, not insert a second one")
}
}
func TestIngestionTaskServiceRetryAfterTerminalOpensFreshPipelineLog(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", common.FAILED).Error; err != nil {
t.Fatalf("set failed status: %v", err)
}
svc := NewIngestionTaskService()
svc.taskPublisher = &recordingTaskPublisher{}
ctx := t.Context()
finished := &entity.PipelineOperationLog{
ID: "finished-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: "Parse",
OperationStatus: string(entity.TaskStatusFail),
}
if err := dao.DB.Create(finished).Error; err != nil {
t.Fatalf("seed finished log: %v", err)
}
if _, err := svc.CreateAndEnqueue(ctx, &entity.IngestionTask{
DocumentID: "doc-1",
UserID: "user-1",
DatasetID: "kb-1",
Status: common.CREATED,
}); err != nil {
t.Fatalf("CreateAndEnqueue retry failed: %v", err)
}
open := loadOpenPipelineLog(t, ctx, db, "doc-1")
if open.ID == "finished-log" {
t.Fatalf("retry reused the finished row; want a fresh queued row")
}
if countPipelineLogs(t, db, "doc-1") != 2 {
t.Fatalf("expected 2 pipeline log rows (finished + fresh queued)")
}
}
func TestIngestionTaskServiceOpensRunWithDocumentPipelineVersion(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 0, 0, 0)
pipelineID := "canvas-1"
if err := db.Create(&entity.UserCanvas{
ID: pipelineID,
UserID: "user-1",
CanvasCategory: "dataflow_canvas",
}).Error; err != nil {
t.Fatalf("insert pipeline: %v", err)
}
versionID := "ver-1"
if err := db.Create(&entity.Document{
ID: "doc-1",
KbID: "kb-1",
ParserID: "naive",
ParserConfig: entity.JSONMap{},
Suffix: ".txt",
Status: sptr("1"),
PipelineID: &pipelineID,
PipelineVersionID: &versionID,
}).Error; err != nil {
t.Fatalf("insert versioned document: %v", err)
}
svc := NewIngestionTaskService()
svc.taskPublisher = &recordingTaskPublisher{}
task, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
DocumentID: "doc-1", UserID: "user-1", DatasetID: "kb-1", Status: common.CREATED,
})
if err != nil {
t.Fatalf("CreateAndEnqueue: %v", err)
}
if task.PipelineLogID == nil {
t.Fatal("enqueued task has no pipeline log id")
}
var run entity.PipelineOperationLog
if err := db.First(&run, "id = ?", *task.PipelineLogID).Error; err != nil {
t.Fatalf("load pipeline log: %v", err)
}
if run.PipelineVersionID == nil || *run.PipelineVersionID != versionID {
t.Fatalf("PipelineVersionID = %v, want %q", run.PipelineVersionID, versionID)
}
}
func TestIngestionTaskServiceRetryAllocatesNextRunCount(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
svc := NewIngestionTaskService()
svc.taskPublisher = &recordingTaskPublisher{}
first, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
DocumentID: "doc-1", UserID: "user-1", DatasetID: "kb-1", Status: common.CREATED,
})
if err != nil {
t.Fatalf("create first run: %v", err)
}
if first.PipelineLogID == nil {
t.Fatal("first run has no pipeline log id")
}
svc.settlePublishFailure(t.Context(), first)
second, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
DocumentID: "doc-1", UserID: "user-1", DatasetID: "kb-1", Status: common.CREATED,
})
if err != nil {
t.Fatalf("retry: %v", err)
}
if second.PipelineLogID == nil || *second.PipelineLogID == *first.PipelineLogID {
t.Fatalf("second run id = %v, want a new run", second.PipelineLogID)
}
var secondLog entity.PipelineOperationLog
if err := db.First(&secondLog, "id = ?", *second.PipelineLogID).Error; err != nil {
t.Fatalf("load second run: %v", err)
}
if secondLog.RunCount == nil || *secondLog.RunCount != 2 {
t.Fatalf("second run_count = %v, want 2", secondLog.RunCount)
}
}
func TestIngestionTaskServiceReloadAndValidateRunIdentityRejectsInvalidBindings(t *testing.T) {
validRunCount := 1
invalidRunCount := 0
testCases := []struct {
name string
pipelineLogID *string
createRun bool
runDocumentID string
runDatasetID string
runCount *int
wantReason string
}{
{name: "missing pipeline log ID", wantReason: "missing_pipeline_log_id"},
{name: "pipeline log not found", pipelineLogID: sptr("missing-run"), wantReason: "pipeline_log_not_found"},
{name: "pipeline log document mismatch", pipelineLogID: sptr("run-1"), createRun: true, runDocumentID: "other-doc", runDatasetID: "kb-1", runCount: &validRunCount, wantReason: "pipeline_log_document_mismatch"},
{name: "pipeline log dataset mismatch", pipelineLogID: sptr("run-1"), createRun: true, runDocumentID: "doc-1", runDatasetID: "other-kb", runCount: &validRunCount, wantReason: "pipeline_log_dataset_mismatch"},
{name: "missing run count", pipelineLogID: sptr("run-1"), createRun: true, runDocumentID: "doc-1", runDatasetID: "kb-1", wantReason: "invalid_run_count"},
{name: "non-positive run count", pipelineLogID: sptr("run-1"), createRun: true, runDocumentID: "doc-1", runDatasetID: "kb-1", runCount: &invalidRunCount, wantReason: "invalid_run_count"},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if testCase.createRun {
if err := db.Create(&entity.PipelineOperationLog{
ID: *testCase.pipelineLogID,
DocumentID: testCase.runDocumentID,
KbID: testCase.runDatasetID,
TaskType: string(entity.PipelineTaskTypeParse),
OperationStatus: string(entity.TaskStatusRunning),
RunCount: testCase.runCount,
}).Error; err != nil {
t.Fatalf("create pipeline operation log: %v", err)
}
}
if testCase.pipelineLogID != nil {
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("pipeline_log_id", *testCase.pipelineLogID).Error; err != nil {
t.Fatalf("bind pipeline operation log: %v", err)
}
}
_, err := NewIngestionTaskService().ReloadAndValidateRunIdentity(t.Context(), "task-1")
var identityErr *InvalidRunIdentityError
if !errors.As(err, &identityErr) || identityErr.Reason != testCase.wantReason {
t.Fatalf("error = %v, want invalid run identity reason %q", err, testCase.wantReason)
}
})
}
}
// TestIngestionTaskServiceOpensPreTerminalLogBeforePublish locks the ordering that
// closes the orphan window: the run's row must exist before its message is
// published, so a worker that claims and finishes the task immediately still
// finds a row to terminalize.
func TestIngestionTaskServiceOpensPreTerminalLogBeforePublish(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
var logIDAtPublish string
publisher := &recordingTaskPublisher{
beforeReturn: func(taskID string) {
task, err := dao.NewIngestionTaskDAO().GetByID(t.Context(), db, taskID)
if err != nil {
t.Fatalf("load task at publish: %v", err)
}
if task.PipelineLogID == nil {
t.Fatal("task was published before its open log row was bound")
}
logIDAtPublish = *task.PipelineLogID
},
}
svc := NewIngestionTaskService()
svc.taskPublisher = publisher
if _, err := svc.CreateForDocuments(permission.WithIdentity(t.Context(), "user-1", ""), "kb-1", []string{"doc-1"}); err != nil {
t.Fatalf("CreateForDocuments failed: %v", err)
}
if logIDAtPublish == "" {
t.Fatal("publish did not observe a bound open log row")
}
open := loadOpenPipelineLog(t, t.Context(), db, "doc-1")
if open.ID != logIDAtPublish {
t.Fatalf("open row = %q, want the row bound before publish %q", open.ID, logIDAtPublish)
}
}
// TestIngestionTaskServiceStartRunningClosingStoppingTaskClosesPreTerminalLog locks
// the workerless STOPPING finalize: MQ redelivery of a STOPPING task stops it
// without ever running a terminal pipeline-log writer, so StartRunning must
// close the row itself.
func TestIngestionTaskServiceStartRunningClosingStoppingTaskClosesPreTerminalLog(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.STOPPING)
runningMsg := "Task is running..."
if err := dao.DB.Create(&entity.PipelineOperationLog{
ID: "running-log",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: "Parse",
OperationStatus: string(entity.TaskStatusRunning),
ProgressMsg: &runningMsg,
}).Error; err != nil {
t.Fatalf("seed running log: %v", err)
}
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("pipeline_log_id", "running-log").Error; err != nil {
t.Fatalf("bind running log: %v", err)
}
svc := NewIngestionTaskService()
task, err := svc.TransitionTaskToRunning(t.Context(), "task-1")
if err != nil {
t.Fatalf("StartRunning failed: %v", err)
}
if task.Status != common.STOPPED {
t.Fatalf("status = %q, want %q", task.Status, common.STOPPED)
}
var done entity.PipelineOperationLog
if err := db.First(&done, "id = ?", "running-log").Error; err != nil {
t.Fatalf("load running log: %v", err)
}
if done.OperationStatus != string(entity.TaskStatusCancel) {
t.Fatalf("OperationStatus = %q, want %q (workerless stop must close the row)", done.OperationStatus, string(entity.TaskStatusCancel))
}
}
// TestIngestionTaskServiceKeepsBoundRowOverUnrelatedOpenRow locks the ownership
// rule: a run reuses the row it is bound to, never an unrelated open row that
// happens to exist for the same document.
func TestIngestionTaskServiceKeepsBoundRowOverUnrelatedOpenRow(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
queuedMsg := "Task is queued..."
runCount := 1
seed := func(id, status string, runCount *int) {
t.Helper()
if err := dao.DB.Create(&entity.PipelineOperationLog{
ID: id,
DocumentID: "doc-1",
RunCount: runCount,
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: "Parse",
OperationStatus: status,
ProgressMsg: &queuedMsg,
}).Error; err != nil {
t.Fatalf("seed %s: %v", id, err)
}
}
seed("bound-log", string(entity.TaskStatusUnstart), &runCount)
seed("unrelated-log", string(entity.TaskStatusSchedule), nil)
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").
Update("pipeline_log_id", "bound-log").Error; err != nil {
t.Fatalf("bind: %v", err)
}
svc := NewIngestionTaskService()
svc.taskPublisher = &recordingTaskPublisher{}
if _, err := svc.CreateAndEnqueue(t.Context(), &entity.IngestionTask{
DocumentID: "doc-1",
UserID: "user-1",
DatasetID: "kb-1",
Status: common.CREATED,
}); err != nil {
t.Fatalf("CreateAndEnqueue failed: %v", err)
}
if got := countPipelineLogs(t, db, "doc-1"); got != 2 {
t.Fatalf("pipeline log rows = %d, want 2 (bound row advanced, no third row created)", got)
}
if bound := loadTaskPipelineLogID(t, t.Context(), db, "task-1"); bound != "bound-log" {
t.Fatalf("task bound to %q, want its own row bound-log", bound)
}
var bound entity.PipelineOperationLog
if err := db.First(&bound, "id = ?", "bound-log").Error; err != nil {
t.Fatalf("load bound log: %v", err)
}
if bound.OperationStatus != string(entity.TaskStatusSchedule) {
t.Fatalf("bound row status = %q, want %q", bound.OperationStatus, string(entity.TaskStatusSchedule))
}
}
func TestIngestionTaskServiceStaleSnapshotKeepsEstablishedOpenLogBinding(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
taskDAO := dao.NewIngestionTaskDAO()
firstSnapshot, err := taskDAO.GetByID(t.Context(), db, "task-1")
if err != nil {
t.Fatalf("load first task snapshot: %v", err)
}
staleSnapshot, err := taskDAO.GetByID(t.Context(), db, "task-1")
if err != nil {
t.Fatalf("load stale task snapshot: %v", err)
}
svc := NewIngestionTaskService()
if err := svc.ensureRunIdentity(t.Context(), firstSnapshot, nil); err != nil {
t.Fatalf("establish first identity: %v", err)
}
firstLogID := loadTaskPipelineLogID(t, t.Context(), db, "task-1")
if err := svc.ensureRunIdentity(t.Context(), staleSnapshot, nil); err != nil {
t.Fatalf("reuse established identity: %v", err)
}
if got := loadTaskPipelineLogID(t, t.Context(), db, "task-1"); got != firstLogID {
t.Fatalf("stale snapshot replaced binding %q with %q", firstLogID, got)
}
if got := countPipelineLogs(t, db, "doc-1"); got != 1 {
t.Fatalf("pipeline log rows = %d, want the original single row", got)
}
var firstLog entity.PipelineOperationLog
if err := db.First(&firstLog, "id = ?", firstLogID).Error; err != nil {
t.Fatalf("established open log was removed: %v", err)
}
}
// TestIngestionTaskServicePreTerminalLogAdvanceIsMonotonic locks the monotonic
// transition contract: a late queued write must not regress a row a concurrent
// worker already moved to running.
func TestIngestionTaskServicePreTerminalLogAdvanceIsMonotonic(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", common.CREATED)
openLog := &entity.PipelineOperationLog{
ID: "log-1",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: "Parse",
OperationStatus: string(entity.TaskStatusUnstart),
}
if err := dao.DB.Create(openLog).Error; err != nil {
t.Fatalf("seed open log: %v", err)
}
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("pipeline_log_id", "log-1").Error; err != nil {
t.Fatalf("bind task: %v", err)
}
svc := NewIngestionTaskService()
ctx := t.Context()
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if err != nil {
t.Fatalf("load task: %v", err)
}
svc.advanceOpenLog(ctx, task, logFromUnstart, string(entity.TaskStatusSchedule))
if got := loadOpenPipelineLog(t, ctx, db, "doc-1").OperationStatus; got != string(entity.TaskStatusSchedule) {
t.Fatalf("after schedule = %q, want %q", got, string(entity.TaskStatusSchedule))
}
svc.advanceOpenLog(ctx, task, logFromUnstartOrScheduled, string(entity.TaskStatusRunning))
if got := loadOpenPipelineLog(t, ctx, db, "doc-1").OperationStatus; got != string(entity.TaskStatusRunning) {
t.Fatalf("after start = %q, want %q", got, string(entity.TaskStatusRunning))
}
// The enqueue-time scheduled write lands late; it must not regress the row.
svc.advanceOpenLog(ctx, task, logFromUnstart, string(entity.TaskStatusSchedule))
open := loadOpenPipelineLog(t, ctx, db, "doc-1")
if open.OperationStatus != string(entity.TaskStatusRunning) {
t.Fatalf("late scheduled write regressed the row to %q, want %q", open.OperationStatus, string(entity.TaskStatusRunning))
}
if open.ID != "log-1" || countPipelineLogs(t, db, "doc-1") != 1 {
t.Fatalf("expected the same single row, got id=%q count=%d", open.ID, countPipelineLogs(t, db, "doc-1"))
}
}
// TestIngestionTaskServiceAdvanceRejectsMissingIdentity ensures a status
// advance never creates a run identity. Only publish-time EnsureRunIdentity
// may bind a task to a pipeline-operation-log row.
func TestIngestionTaskServiceAdvanceRejectsMissingIdentity(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-terminal", "kb-1", 0, 0)
insertTestDoc(t, "doc-live", "kb-1", 0, 0)
insertTestIngestionTaskWithStatus(t, "task-terminal", "user-1", "doc-terminal", "kb-1", common.STOPPED)
insertTestIngestionTaskWithStatus(t, "task-live", "user-1", "doc-live", "kb-1", common.CREATED)
svc := NewIngestionTaskService()
ctx := t.Context()
terminal, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-terminal")
if err != nil {
t.Fatalf("load terminal task: %v", err)
}
svc.advanceOpenLog(ctx, terminal, logFromUnstart, string(entity.TaskStatusSchedule))
if got := countPipelineLogs(t, db, "doc-terminal"); got != 0 {
t.Fatalf("terminal task resurrected %d queued row(s); want none", got)
}
live, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-live")
if err != nil {
t.Fatalf("load live task: %v", err)
}
svc.advanceOpenLog(ctx, live, logFromUnstart, string(entity.TaskStatusSchedule))
if got := countPipelineLogs(t, db, "doc-live"); got != 0 {
t.Fatalf("missing identity created %d pipeline log rows, want 0", got)
}
}
// A deleted ledger must not strand a terminal task during document cleanup.
// Active/queued tasks still require valid identity; runtime event writes remain strict.
func TestIngestionTaskServiceCleanupMissingRun(t *testing.T) {
for _, status := range []string{common.STOPPED, common.COMPLETED, common.FAILED, common.CREATED, common.SCHEDULED, common.RUNNING, common.STOPPING} {
t.Run(status, func(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("pipeline_log_id", "deleted-run").Error; err != nil {
t.Fatal(err)
}
svc := NewIngestionTaskService()
svc.purgeTaskState = func(context.Context, string) error { return nil }
terminal := common.IsTerminalTaskStatus(status)
prepared, err := svc.PrepareDocumentTaskForCleanup(t.Context(), "doc-1", "Document cleanup")
if terminal {
if err != nil || prepared == nil || prepared.ID != "task-1" {
t.Fatalf("terminal cleanup: task=%v err=%v", prepared, err)
}
} else if err == nil {
t.Fatal("nonterminal cleanup accepted missing run identity")
}
// Recreate the original state: preparing a queued task may freeze it.
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", status).Error; err != nil {
t.Fatal(err)
}
_, err = svc.Remove(t.Context(), "task-1", sptr("user-1"))
if terminal {
if err != nil {
t.Fatalf("Remove terminal task with deleted run: %v", err)
}
if _, err := svc.GetTask(t.Context(), "task-1"); !errors.Is(err, common.ErrTaskNotFound) {
t.Fatalf("task remains after removal: %v", err)
}
} else if err == nil {
t.Fatal("removed nonterminal task with missing run identity")
}
if err := svc.RecordTerminal(t.Context(), "deleted-run", "task-1", "runtime write"); !errors.Is(err, gorm.ErrRecordNotFound) {
t.Fatalf("runtime event error = %v, want missing run failure", err)
}
var events int64
if err := db.Model(&entity.IngestionTaskLog{}).Where("pipeline_log_id = ?", "deleted-run").Count(&events).Error; err != nil {
t.Fatal(err)
}
if events != 0 {
t.Fatalf("orphan terminal events = %d", events)
}
})
}
}
// recordingIngestLogEmitter captures emitted lifecycle events for assertions.
type recordingIngestLogEmitter struct {
mu sync.Mutex
logs []common.IngestionLog
}
func (e *recordingIngestLogEmitter) EmitIngestLog(_ context.Context, log common.IngestionLog) {
e.mu.Lock()
defer e.mu.Unlock()
e.logs = append(e.logs, log)
}
func (e *recordingIngestLogEmitter) last() common.IngestionLog {
e.mu.Lock()
defer e.mu.Unlock()
if len(e.logs) == 0 {
return common.IngestionLog{}
}
return e.logs[len(e.logs)-1]
}
// TestIngestionTaskServiceMarkFailedCarriesReason verifies that the failure
// reason supplied by the ingestor reaches the emitted lifecycle event's
// ErrorInfo, instead of being dropped (the previous gap: the run ledger got the
// message but the observability event carried an empty ErrorInfo).
func TestIngestionTaskServiceMarkFailedCarriesReason(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
if err := dao.DB.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("status", common.RUNNING).Error; err != nil {
t.Fatalf("set running status: %v", err)
}
ctx := t.Context()
emitter := &recordingIngestLogEmitter{}
svc := NewIngestionTaskService()
svc.SetEventEmitter(emitter)
const reason = "boom: nil deref"
if err := svc.MarkFailed(ctx, "task-1", reason); err != nil {
t.Fatalf("MarkFailed failed: %v", err)
}
got := emitter.last()
if got.ErrorInfo != reason {
t.Fatalf("emitted ErrorInfo = %q, want %q", got.ErrorInfo, reason)
}
if got.EventType != "failed" || got.Status != common.FAILED {
t.Fatalf("emitted event type/status = %q/%q, want failed/%q", got.EventType, got.Status, common.FAILED)
}
}
// TestPrepareValidatedRunEmitsRunningEventWithFreshClaimTime verifies that the
// running lifecycle event is emitted AFTER PrepareValidatedRun stamps the
// document's worker-claim time. Previously the running event was emitted in
// TransitionTaskToRunning, before the claim time was written, so its BeginTime
// (and therefore wait_duration_s) was either zero or the previous run's stale
// value.
func TestPrepareValidatedRunEmitsRunningEventWithFreshClaimTime(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestKB(t, "kb-1", "tenant-1", 1, 0, 0)
insertTestDoc(t, "doc-1", "kb-1", 0, 0)
insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
// Simulate a previous run that already stamped a claim time one hour ago.
stale := time.Now().Add(-time.Hour)
if err := db.Model(&entity.Document{}).Where("id = ?", "doc-1").Update("process_begin_at", stale).Error; err != nil {
t.Fatalf("seed stale claim time: %v", err)
}
// Bind a pipeline-operation-log row so advanceOpenLog/recordRunMessage succeed.
openLog := &entity.PipelineOperationLog{
ID: "log-1",
DocumentID: "doc-1",
TenantID: "tenant-1",
KbID: "kb-1",
ParserID: "naive",
TaskType: "Parse",
OperationStatus: string(entity.TaskStatusUnstart),
}
if err := db.Create(openLog).Error; err != nil {
t.Fatalf("seed open log: %v", err)
}
if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "task-1").Update("pipeline_log_id", "log-1").Error; err != nil {
t.Fatalf("bind task: %v", err)
}
ctx := t.Context()
emitter := &recordingIngestLogEmitter{}
svc := NewIngestionTaskService()
svc.SetEventEmitter(emitter)
task, err := dao.NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
if err != nil {
t.Fatalf("load task: %v", err)
}
svc.PrepareValidatedRun(ctx, task)
var running common.IngestionLog
found := false
for _, l := range emitter.logs {
if l.EventType == "running" {
running = l
found = true
}
}
if !found {
t.Fatalf("no running event emitted by PrepareValidatedRun")
}
if running.BeginTime.IsZero() {
t.Fatalf("running event BeginTime is zero - claim time was not stamped before emit")
}
if !running.BeginTime.After(stale) {
t.Fatalf("running event BeginTime = %v, want after stale claim time %v (must reflect current run)", running.BeginTime, stale)
}
if time.Since(running.BeginTime) > 30*time.Second {
t.Fatalf("running event BeginTime = %v, suspiciously old for a fresh claim", running.BeginTime)
}
}