Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

17.4 KB
Raw Blame
package dao

import (
	"errors"
	"testing"

	"ragflow/internal/common"
	"ragflow/internal/entity"

	"gorm.io/gorm"
)

func TestIngestionTaskDAOUpdateStatusIfCurrentSucceeds(t *testing.T) {
	db := setupTaskTestDB(t)
	orig := DB
	DB = db
	t.Cleanup(func() { DB = orig })

	task := &entity.IngestionTask{
		ID:         "task-1",
		UserID:     "user-1",
		DocumentID: "doc-1",
		DatasetID:  "kb-1",
		Status:     common.CREATED,
	}
	if err := db.Create(task).Error; err != nil {
		t.Fatalf("create task: %v", err)
	}

	ctx := t.Context()
	updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", []string{common.CREATED}, common.RUNNING)
	if err != nil {
		t.Fatalf("UpdateStatusIfCurrent failed: %v", err)
	}
	if !updated {
		t.Fatal("expected update to succeed")
	}

	reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
	if err != nil {
		t.Fatalf("reload task: %v", err)
	}
	if reloaded.Status != common.RUNNING {
		t.Fatalf("status = %q, want %q", reloaded.Status, common.RUNNING)
	}
}

func TestIngestionTaskDAOUpdateStatusIfCurrentMatchesMultipleStatuses(t *testing.T) {
	db := setupTaskTestDB(t)
	orig := DB
	DB = db
	t.Cleanup(func() { DB = orig })

	task := &entity.IngestionTask{
		ID:         "task-1",
		UserID:     "user-1",
		DocumentID: "doc-1",
		DatasetID:  "kb-1",
		Status:     common.SCHEDULED,
	}
	if err := db.Create(task).Error; err != nil {
		t.Fatalf("create task: %v", err)
	}

	ctx := t.Context()
	updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", []string{common.CREATED, common.SCHEDULED}, common.RUNNING)
	if err != nil {
		t.Fatalf("UpdateStatusIfCurrent failed: %v", err)
	}
	if !updated {
		t.Fatal("expected update to succeed")
	}

	reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
	if err != nil {
		t.Fatalf("reload task: %v", err)
	}
	if reloaded.Status != common.RUNNING {
		t.Fatalf("status = %q, want %q", reloaded.Status, common.RUNNING)
	}
}

func TestIngestionTaskDAOUpdateStatusIfCurrentEmptyStatusesReturnsFalse(t *testing.T) {
	db := setupTaskTestDB(t)
	orig := DB
	DB = db
	t.Cleanup(func() { DB = orig })

	task := &entity.IngestionTask{
		ID:         "task-1",
		UserID:     "user-1",
		DocumentID: "doc-1",
		DatasetID:  "kb-1",
		Status:     common.CREATED,
	}
	if err := db.Create(task).Error; err != nil {
		t.Fatalf("create task: %v", err)
	}

	ctx := t.Context()
	updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", nil, common.RUNNING)
	if err != nil {
		t.Fatalf("UpdateStatusIfCurrent failed: %v", err)
	}
	if updated {
		t.Fatal("expected update with empty fromStatuses to be rejected")
	}

	reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
	if err != nil {
		t.Fatalf("reload task: %v", err)
	}
	if reloaded.Status != common.CREATED {
		t.Fatalf("status = %q, want %q", reloaded.Status, common.CREATED)
	}
}

func TestIngestionTaskDAOCreateRejectsExistingTerminalTask(t *testing.T) {
	db := setupTaskTestDB(t)
	orig := DB
	DB = db
	t.Cleanup(func() { DB = orig })

	testCases := []struct {
		name   string
		status string
	}{
		{name: "failed", status: common.FAILED},
		{name: "stopped", status: common.STOPPED},
	}

	ctx := t.Context()
	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			if err := db.WithContext(ctx).Where("id = ?", "task-1").Delete(&entity.IngestionTask{}).Error; err != nil {
				t.Fatalf("clear task: %v", err)
			}
			task := &entity.IngestionTask{ID: "task-1", UserID: "user-1", DocumentID: "doc-1", DatasetID: "kb-1", Status: tc.status}
			if err := db.WithContext(ctx).Create(task).Error; err != nil {
				t.Fatalf("create task: %v", err)
			}
			_, err := NewIngestionTaskDAO().Create(ctx, db, &entity.IngestionTask{ID: "task-2", UserID: "user-1", DocumentID: "doc-1", DatasetID: "kb-1", Status: common.CREATED})
			if err == nil {
				t.Fatal("expected Create to reject duplicate document task")
			}
			reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
			if err != nil {
				t.Fatalf("reload task: %v", err)
			}
			if reloaded.Status != tc.status {
				t.Fatalf("status = %q, want %q", reloaded.Status, tc.status)
			}
		})
	}
}

func TestIngestionTaskDAODocumentIDIsUniqueAtDBLevel(t *testing.T) {
	db := setupTaskTestDB(t)
	orig := DB
	DB = db
	t.Cleanup(func() { DB = orig })

	first := &entity.IngestionTask{ID: "task-1", UserID: "user-1", DocumentID: "doc-1", DatasetID: "kb-1", Status: common.CREATED}
	if err := db.Create(first).Error; err != nil {
		t.Fatalf("create first task: %v", err)
	}

	second := &entity.IngestionTask{ID: "task-2", UserID: "user-1", DocumentID: "doc-1", DatasetID: "kb-1", Status: common.CREATED}
	err := db.Create(second).Error
	if !errors.Is(err, gorm.ErrDuplicatedKey) {
		t.Fatalf("expected duplicated key error, got %v", err)
	}
}

func TestIngestionTaskDAODeleteAllowsScheduledTask(t *testing.T) {
	db := setupTaskTestDB(t)
	orig := DB
	DB = db
	t.Cleanup(func() { DB = orig })

	task := &entity.IngestionTask{
		ID:         "task-scheduled",
		UserID:     "user-1",
		DocumentID: "doc-scheduled",
		DatasetID:  "kb-1",
		Status:     common.SCHEDULED,
	}
	if err := db.Create(task).Error; err != nil {
		t.Fatalf("create scheduled task: %v", err)
	}

	info, err := NewIngestionTaskDAO().Delete(t.Context(), db, task.ID, nil)
	if err != nil {
		t.Fatalf("delete scheduled task: %v", err)
	}
	if info == nil || info.TaskID != task.ID {
		t.Fatalf("unexpected task info: %+v", info)
	}
}

func TestIngestionTaskDAODeleteDoesNotRemoveTaskClaimedDuringDelete(t *testing.T) {
	db := setupTaskTestDB(t)
	task := &entity.IngestionTask{
		ID:         "task-scheduled",
		UserID:     "user-1",
		DocumentID: "doc-scheduled",
		DatasetID:  "kb-1",
		Status:     common.SCHEDULED,
	}
	if err := db.Create(task).Error; err != nil {
		t.Fatalf("create scheduled task: %v", err)
	}

	const callbackName = "test:claim-ingestion-task-before-delete"
	if err := db.Callback().Delete().Before("gorm:delete").Register(callbackName, func(tx *gorm.DB) {
		if tx.Statement.Table != task.TableName() {
			return
		}
		if err := tx.Session(&gorm.Session{NewDB: true}).Model(&entity.IngestionTask{}).Where("id = ?", task.ID).
			Update("status", common.RUNNING).Error; err != nil {
			tx.AddError(err)
		}
	}); err != nil {
		t.Fatalf("register delete callback: %v", err)
	}

	if _, err := NewIngestionTaskDAO().Delete(t.Context(), db, task.ID, nil); err == nil {
		t.Fatal("expected delete to reject task claimed during delete")
	}

	reloaded, err := NewIngestionTaskDAO().GetByID(t.Context(), db, task.ID)
	if err != nil {
		t.Fatalf("reload task after rejected delete: %v", err)
	}
	if reloaded.Status != common.SCHEDULED {
		t.Fatalf("status after rejected delete = %q, want %q", reloaded.Status, common.SCHEDULED)
	}
}

func TestIngestionTaskDAOListByStatus(t *testing.T) {
	db := setupTaskTestDB(t)
	orig := DB
	DB = db
	t.Cleanup(func() { DB = orig })

	for _, task := range []*entity.IngestionTask{
		{ID: "task-created", UserID: "user-1", DocumentID: "doc-created", DatasetID: "kb-1", Status: common.CREATED},
		{ID: "task-scheduled", UserID: "user-1", DocumentID: "doc-scheduled", DatasetID: "kb-1", Status: common.SCHEDULED},
	} {
		if err := db.Create(task).Error; err != nil {
			t.Fatalf("create task %s: %v", task.ID, err)
		}
	}

	tasks, err := NewIngestionTaskDAO().ListByStatus(t.Context(), db, common.CREATED)
	if err != nil {
		t.Fatalf("list CREATED tasks: %v", err)
	}
	if len(tasks) != 1 || tasks[0].ID != "task-created" {
		t.Fatalf("CREATED tasks = %+v, want only task-created", tasks)
	}
}

func TestIngestionTaskDAOListsTasks(t *testing.T) {
	db := setupTaskTestDB(t)
	orig := DB
	DB = db
	t.Cleanup(func() { DB = orig })

	for _, task := range []*entity.IngestionTask{
		{ID: "task-created", UserID: "user-1", DocumentID: "doc-created", DatasetID: "kb-1", Status: common.CREATED},
		{ID: "task-scheduled", UserID: "user-1", DocumentID: "doc-scheduled", DatasetID: "kb-1", Status: common.SCHEDULED},
	} {
		if err := db.Create(task).Error; err != nil {
			t.Fatalf("create task %s: %v", task.ID, err)
		}
	}

	d := NewIngestionTaskDAO()
	tasks, err := d.ListByUserID(t.Context(), db, "user-1", 0, 0)
	if err != nil {
		t.Fatalf("list tasks by user: %v", err)
	}
	if len(tasks) != 2 {
		t.Fatalf("listed tasks = %+v, want 2 tasks", tasks)
	}

	tasks, err = d.ListByUserIDAndDatasetID(t.Context(), db, "user-1", "kb-1", 0, 0)
	if err != nil {
		t.Fatalf("list tasks by user and dataset: %v", err)
	}
	if len(tasks) != 2 {
		t.Fatalf("listed tasks by dataset = %+v, want 2 tasks", tasks)
	}

	tasks, err = d.GetAllTasks(t.Context(), db, 0, 0)
	if err != nil {
		t.Fatalf("list all tasks: %v", err)
	}
	if len(tasks) != 2 {
		t.Fatalf("all listed tasks = %+v, want 2 tasks", tasks)
	}
}

func TestIngestionTaskDAOUpdateStatusIfCurrentRejectsMismatchedStatus(t *testing.T) {
	db := setupTaskTestDB(t)
	orig := DB
	DB = db
	t.Cleanup(func() { DB = orig })

	task := &entity.IngestionTask{
		ID:         "task-1",
		UserID:     "user-1",
		DocumentID: "doc-1",
		DatasetID:  "kb-1",
		Status:     common.STOPPING,
	}
	if err := db.Create(task).Error; err != nil {
		t.Fatalf("create task: %v", err)
	}

	ctx := t.Context()
	updated, err := NewIngestionTaskDAO().UpdateStatusIfCurrent(ctx, db, "task-1", []string{common.CREATED}, common.RUNNING)
	if err != nil {
		t.Fatalf("UpdateStatusIfCurrent failed: %v", err)
	}
	if updated {
		t.Fatal("expected update to be rejected")
	}

	reloaded, err := NewIngestionTaskDAO().GetByID(ctx, db, "task-1")
	if err != nil {
		t.Fatalf("reload task: %v", err)
	}
	if reloaded.Status != common.STOPPING {
		t.Fatalf("status = %q, want %q", reloaded.Status, common.STOPPING)
	}
}

func TestIngestionTaskDAOCountActiveByDatasetID(t *testing.T) {
	db := setupTaskTestDB(t)
	for _, task := range []*entity.IngestionTask{
		{ID: "task-active-1", UserID: "user-1", DocumentID: "doc-active-1", DatasetID: "kb-1", Status: common.SCHEDULED},
		{ID: "task-active-2", UserID: "user-1", DocumentID: "doc-active-2", DatasetID: "kb-1", Status: common.RUNNING},
		{ID: "task-active-3", UserID: "user-1", DocumentID: "doc-active-3", DatasetID: "kb-1", Status: common.CREATED},
		{ID: "task-active-4", UserID: "user-1", DocumentID: "doc-active-4", DatasetID: "kb-1", Status: common.STOPPING},
		{ID: "task-terminal", UserID: "user-1", DocumentID: "doc-terminal", DatasetID: "kb-1", Status: common.COMPLETED},
		{ID: "task-other-kb", UserID: "user-1", DocumentID: "doc-other", DatasetID: "kb-2", Status: common.SCHEDULED},
	} {
		if err := db.Create(task).Error; err != nil {
			t.Fatalf("create task %s: %v", task.ID, err)
		}
	}

	ctx := t.Context()
	count, err := NewIngestionTaskDAO().CountActiveByDatasetID(ctx, db, "kb-1")
	if err != nil {
		t.Fatalf("CountActiveByDatasetID: %v", err)
	}
	if count != 4 {
		t.Fatalf("active count for kb-1 = %d, want 4", count)
	}
	count, err = NewIngestionTaskDAO().CountActiveByDatasetID(ctx, db, "kb-2")
	if err != nil {
		t.Fatalf("CountActiveByDatasetID kb-2: %v", err)
	}
	if count != 1 {
		t.Fatalf("active count for kb-2 = %d, want 1", count)
	}
	count, err = NewIngestionTaskDAO().CountActiveByDatasetID(ctx, db, "kb-unknown")
	if err != nil {
		t.Fatalf("CountActiveByDatasetID unknown: %v", err)
	}
	if count != 0 {
		t.Fatalf("active count for unknown = %d, want 0", count)
	}
}

func TestIngestionTaskDAODeleteIfTerminalRequiresMatchingTerminalTask(t *testing.T) {
	db := setupTaskTestDB(t)
	for _, task := range []*entity.IngestionTask{
		{ID: "task-terminal", UserID: "user-1", DocumentID: "doc-terminal", DatasetID: "kb-1", Status: common.COMPLETED},
		{ID: "task-running", UserID: "user-1", DocumentID: "doc-running", DatasetID: "kb-1", Status: common.RUNNING},
		{ID: "task-queued", UserID: "user-1", DocumentID: "doc-queued", DatasetID: "kb-1", Status: common.SCHEDULED},
	} {
		if err := db.Create(task).Error; err != nil {
			t.Fatalf("create task %s: %v", task.ID, err)
		}
	}

	taskDAO := NewIngestionTaskDAO()
	deleted, err := taskDAO.DeleteIfTerminal(t.Context(), db, "task-terminal")
	if err != nil {
		t.Fatalf("delete terminal task: %v", err)
	}
	if deleted != 1 {
		t.Fatalf("deleted terminal rows = %d, want 1", deleted)
	}
	for _, taskID := range []string{"task-running", "task-queued"} {
		deleted, err := taskDAO.DeleteIfTerminal(t.Context(), db, taskID)
		if err != nil {
			t.Fatalf("delete %s: %v", taskID, err)
		}
		if deleted != 0 {
			t.Fatalf("deleted %s rows = %d, want 0", taskID, deleted)
		}
		if _, err := taskDAO.GetByID(t.Context(), db, taskID); err != nil {
			t.Fatalf("%s was removed despite non-terminal status: %v", taskID, err)
		}
	}
}

func TestIngestionTaskDAOCountActiveByDatasetIDUsesLatestTask(t *testing.T) {
	db := setupTaskTestDB(t)
	if err := db.Exec("DROP INDEX idx_ingestion_task_document_id").Error; err != nil {
		t.Fatalf("drop ingestion task unique index: %v", err)
	}

	oldTaskTime := int64(100)
	newTaskTime := int64(200)
	for _, task := range []*entity.IngestionTask{
		{
			ID:         "task-old-active",
			UserID:     "user-1",
			DocumentID: "doc-retried",
			DatasetID:  "kb-1",
			Status:     common.RUNNING,
			BaseModel:  entity.BaseModel{CreateTime: &oldTaskTime},
		},
		{
			ID:         "task-new-terminal",
			UserID:     "user-1",
			DocumentID: "doc-retried",
			DatasetID:  "kb-1",
			Status:     common.COMPLETED,
			BaseModel:  entity.BaseModel{CreateTime: &newTaskTime},
		},
		{
			ID:         "task-current-active",
			UserID:     "user-1",
			DocumentID: "doc-current",
			DatasetID:  "kb-1",
			Status:     common.SCHEDULED,
		},
	} {
		if err := db.Create(task).Error; err != nil {
			t.Fatalf("create task %s: %v", task.ID, err)
		}
	}

	count, err := NewIngestionTaskDAO().CountActiveByDatasetID(t.Context(), db, "kb-1")
	if err != nil {
		t.Fatalf("CountActiveByDatasetID: %v", err)
	}
	if count != 1 {
		t.Fatalf("active count for kb-1 = %d, want 1 (latest task per document)", count)
	}
}

func TestIngestionTaskDAOGetByDocumentIDUsesLatestTask(t *testing.T) {
	db := setupTaskTestDB(t)
	if err := db.Exec("DROP INDEX idx_ingestion_task_document_id").Error; err != nil {
		t.Fatalf("drop ingestion task unique index: %v", err)
	}

	oldTaskTime := int64(100)
	newTaskTime := int64(200)
	for _, task := range []*entity.IngestionTask{
		{
			ID:         "task-old",
			UserID:     "user-1",
			DocumentID: "doc-retried",
			DatasetID:  "kb-1",
			Status:     common.RUNNING,
			BaseModel:  entity.BaseModel{CreateTime: &oldTaskTime},
		},
		{
			ID:         "task-new",
			UserID:     "user-1",
			DocumentID: "doc-retried",
			DatasetID:  "kb-1",
			Status:     common.COMPLETED,
			BaseModel:  entity.BaseModel{CreateTime: &newTaskTime},
		},
	} {
		if err := db.Create(task).Error; err != nil {
			t.Fatalf("create task %s: %v", task.ID, err)
		}
	}

	task, err := NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-retried")
	if err != nil {
		t.Fatalf("GetByDocumentID: %v", err)
	}
	if task == nil || task.ID != "task-new" {
		t.Fatalf("latest task = %+v, want task-new", task)
	}
}

func TestIngestionTaskDAOGetByDocumentIDUsesIDAsTieBreaker(t *testing.T) {
	db := setupTaskTestDB(t)
	if err := db.Exec("DROP INDEX idx_ingestion_task_document_id").Error; err != nil {
		t.Fatalf("drop ingestion task unique index: %v", err)
	}

	createTime := int64(100)
	for _, task := range []*entity.IngestionTask{
		{
			ID:         "task-z",
			UserID:     "user-1",
			DocumentID: "doc-retried",
			DatasetID:  "kb-1",
			Status:     common.RUNNING,
			BaseModel:  entity.BaseModel{CreateTime: &createTime},
		},
		{
			ID:         "task-a",
			UserID:     "user-1",
			DocumentID: "doc-retried",
			DatasetID:  "kb-1",
			Status:     common.COMPLETED,
			BaseModel:  entity.BaseModel{CreateTime: &createTime},
		},
	} {
		if err := db.Create(task).Error; err != nil {
			t.Fatalf("create task %s: %v", task.ID, err)
		}
	}

	task, err := NewIngestionTaskDAO().GetByDocumentID(t.Context(), db, "doc-retried")
	if err != nil {
		t.Fatalf("GetByDocumentID: %v", err)
	}
	if task == nil || task.ID != "task-z" {
		t.Fatalf("latest task = %+v, want task-z", task)
	}
}

func TestIngestionTaskDAOHasDatasetStatusIndex(t *testing.T) {
	db := setupTaskTestDB(t)
	if !db.Migrator().HasIndex(&entity.IngestionTask{}, "idx_ingestion_task_dataset_status") {
		t.Fatal("expected composite dataset/status index on ingestion_task")
	}
}

// TestIngestionTaskDAOGetByDocumentID_ReturnsTask verifies the plain
// single-row lookup: with no task recorded for the document GetByDocumentID
// returns nil, and once one exists it returns that task. The test DB's
// AutoMigrate enforces the unique index on document_id, so this stays a
// one-row case — the ordering behaviour (newest first, id as tie-breaker) is
// covered by TestIngestionTaskDAOGetByDocumentIDUsesLatestTask and
// TestIngestionTaskDAOGetByDocumentIDUsesIDAsTieBreaker, both of which drop
// that index so they can create duplicate rows.
func TestIngestionTaskDAOGetByDocumentID_ReturnsTask(t *testing.T) {
	db := setupTaskTestDB(t)
	orig := DB
	DB = db
	t.Cleanup(func() { DB = orig })

	ctx := t.Context()
	dao := NewIngestionTaskDAO()

	// No task yet.
	task, err := dao.GetByDocumentID(ctx, db, "doc-1")
	if err != nil {
		t.Fatalf("GetByDocumentID empty: %v", err)
	}
	if task != nil {
		t.Fatalf("expected nil for doc-1, got %+v", task)
	}

	// Create one task.
	task = &entity.IngestionTask{
		ID:         "task-1",
		UserID:     "user-1",
		DocumentID: "doc-1",
		DatasetID:  "kb-1",
		Status:     common.CREATED,
	}
	if err := db.Create(task).Error; err != nil {
		t.Fatalf("create task: %v", err)
	}

	got, err := dao.GetByDocumentID(ctx, db, "doc-1")
	if err != nil {
		t.Fatalf("GetByDocumentID: %v", err)
	}
	if got == nil || got.ID != "task-1" {
		t.Fatalf("doc-1 task = %+v, want task-1", got)
	}
}