Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

93.1 KB
Raw Blame
package chunk

import (
	"bytes"
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"image"
	"image/color"
	_ "image/jpeg"
	"image/png"
	"ragflow/internal/common"
	"ragflow/internal/dao"
	"ragflow/internal/dao/transaction"
	"ragflow/internal/engine/types"
	"ragflow/internal/entity"
	"ragflow/internal/entity/models"
	"ragflow/internal/permission"
	"ragflow/internal/permission/testsupport"
	"ragflow/internal/service"
	"ragflow/internal/service/document"
	"ragflow/internal/storage"
	"reflect"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/glebarez/sqlite"
	"gorm.io/gorm"
)

func TestIsZeroVector(t *testing.T) {
	if !common.IsZeroVector([]float64{0, 0, 0}) {
		t.Error("all zeros should be true")
	}
	if common.IsZeroVector([]float64{0, 1, 0}) {
		t.Error("non-zero should be false")
	}
	if !common.IsZeroVector([]float64{}) {
		t.Error("empty should be true (treated as zero)")
	}
	if !common.IsZeroVector(nil) {
		t.Error("nil should be true")
	}
}

func TestHydrateChunkVectors_AllNonZero(t *testing.T) {
	chunks := []map[string]interface{}{
		{"id": "c1", "vector": []float64{1, 2, 3}},
		{"id": "c2", "vector": []float64{4, 5, 6}},
	}
	// No zero vectors → nothing to hydrate.
	hydrateChunkVectors(t.Context(), nil, chunks, nil, nil)
	if !reflect.DeepEqual(chunks[0]["vector"], []float64{1, 2, 3}) {
		t.Error("non-zero vector should not be changed")
	}
	if !reflect.DeepEqual(chunks[1]["vector"], []float64{4, 5, 6}) {
		t.Error("non-zero vector should not be changed")
	}
}

func TestHydrateChunkVectors_EmptyChunks(t *testing.T) {
	// Should not panic on empty or nil.
	hydrateChunkVectors(t.Context(), nil, nil, nil, nil)
	hydrateChunkVectors(t.Context(), nil, []map[string]interface{}{}, nil, nil)
}

func TestHydrateChunkVectors_MissingIDs(t *testing.T) {
	chunks := []map[string]interface{}{
		{"vector": []float64{1.0}}, // no id — skipped
	}
	hydrateChunkVectors(t.Context(), nil, chunks, nil, nil)
	// Should not change anything when engine is nil (FetchChunkVectors returns zero vectors).
	// The function doesn't panic — it just can't hydrate because dim is 0.
	// With nil engine, FetchChunkVectors returns zero vectors, so the zero stays zero.
}

func TestHydrateChunkVectors_NoDim(t *testing.T) {
	chunks := []map[string]interface{}{
		{"id": "c1", "vector": []float64{}},
	}
	hydrateChunkVectors(t.Context(), nil, chunks, []string{"kb1"}, []string{"t1"})
	// Empty vectors have dim=0 → early return. No crash.
}

func TestKnowledgebaseEmbeddingKey(t *testing.T) {
	tenantEmbdID := "42"
	staleTenantEmbdID := "stale-id"

	tests := []struct {
		name     string
		kb       *entity.Knowledgebase
		tenantID string
		want     string
	}{
		{
			name: "resolves tenant embedding id to base model name",
			kb: &entity.Knowledgebase{
				EmbdID:       "shared-model",
				TenantEmbdID: &tenantEmbdID,
			},
			want: "embd:BAAI/bge-m3",
		},
		{
			name: "stale tenant embedding id falls back to composite base name",
			kb: &entity.Knowledgebase{
				EmbdID:       "BAAI/bge-m3@1@SILICONFLOW",
				TenantEmbdID: &staleTenantEmbdID,
			},
			want: "embd:BAAI/bge-m3",
		},
		{
			name: "uses embd id without tenant embedding id",
			kb: &entity.Knowledgebase{
				EmbdID: "shared-model",
			},
			want: "embd:shared-model",
		},
		{
			name:     "uses tenant default when embedding id is empty",
			kb:       &entity.Knowledgebase{},
			tenantID: "tenant-1",
			want:     "default:tenant-1",
		},
		{
			name: "ignores non-positive tenant embedding id",
			kb: &entity.Knowledgebase{
				EmbdID:       "shared-model",
				TenantEmbdID: func() *string { s := ""; return &s }(),
			},
			want: "embd:shared-model",
		},
		{
			name: "legacy composite reduces to base model name",
			kb: &entity.Knowledgebase{
				EmbdID: "BAAI/bge-m3@2@SILICONFLOW",
			},
			want: "embd:BAAI/bge-m3",
		},
	}

	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
	if err != nil {
		t.Fatalf("failed to open sqlite: %v", err)
	}
	if err := db.AutoMigrate(&entity.TenantModel{}); err != nil {
		t.Fatalf("failed to migrate test schema: %v", err)
	}
	if err := db.Create(&entity.TenantModel{ID: "42", ModelName: "BAAI/bge-m3"}).Error; err != nil {
		t.Fatalf("failed to seed tenant_model: %v", err)
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			names := transaction.NewEmbeddingBaseNameResolver(dao.NewKnowledgebaseDAO().EmbeddingBaseName)
			if got := knowledgebaseEmbeddingKey(t.Context(), db, tt.kb, tt.tenantID, names); got != tt.want {
				t.Fatalf("knowledgebaseEmbeddingKey() = %q, want %q", got, tt.want)
			}
		})
	}
}

func TestParsePrevalidatesDocumentsBeforeMutating(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	userID := "user-1"
	datasetID := "kb-1"
	insertChunkTestKB(t, datasetID, userID)
	insertChunkTestKB(t, "kb-2", userID)
	insertChunkTestDoc(t, "doc-1", datasetID)
	insertChunkTestDoc(t, "doc-2", "kb-2")
	insertChunkTestTask(t, "task-1", "doc-1")

	svc := &ChunkService{
		docEngine:   nil,
		kbDAO:       dao.NewKnowledgebaseDAO(),
		documentDAO: dao.NewDocumentDAO(),
	}

	ctx := chunkTestContext(t, userID, userID)
	_, code, err := svc.Parse(ctx, datasetID, &service.ParseFileRequest{
		DocumentIDs: []string{"doc-1", "missing-doc", "doc-2"},
	})
	if err == nil {
		t.Fatal("expected parse to fail")
	}
	if code != common.CodeDataError {
		t.Fatalf("expected CodeDataError, got %v", code)
	}
	if !strings.Contains(err.Error(), "missing-doc") || !strings.Contains(err.Error(), "doc-2") {
		t.Fatalf("expected missing and foreign documents in error, got %q", err.Error())
	}

	var taskCount int64
	if err = db.Model(&entity.Task{}).Where("doc_id = ?", "doc-1").Count(&taskCount).Error; err != nil {
		t.Fatalf("count tasks: %v", err)
	}
	if taskCount != 1 {
		t.Fatalf("expected existing task to remain, got %d tasks", taskCount)
	}

	doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
	if err != nil {
		t.Fatalf("get doc: %v", err)
	}
	if doc.ChunkNum != 7 {
		t.Fatalf("expected chunk_num to remain 7, got %d", doc.ChunkNum)
	}
}

func TestParseRejectsInaccessibleDataset(t *testing.T) {
	svc := newParseTestService(t)
	svc.checkDatasetAccessFunc = func(string) error { return permission.ErrPermissionDenied }
	ctx := t.Context()
	_, code, err := svc.Parse(ctx, "kb-1", &service.ParseFileRequest{DocumentIDs: []string{"doc-1"}})
	if err == nil {
		t.Fatal("expected parse to fail")
	}
	if code != common.CodeForbidden {
		t.Fatalf("expected CodeForbidden, got %v", code)
	}
	if err.Error() != "Permission denied" {
		t.Fatalf("unexpected error: %v", err)
	}
}

func TestParseRequiresDocumentIDs(t *testing.T) {
	svc := newParseTestService(t)
	svc.checkDatasetAccessFunc = func(string) error { return nil }
	ctx := t.Context()
	for _, req := range []*service.ParseFileRequest{nil, {DocumentIDs: nil}, {DocumentIDs: []string{}}} {
		_, code, err := svc.Parse(ctx, "kb-1", req)
		if err == nil {
			t.Fatal("expected parse to fail")
		}
		if code != common.CodeDataError {
			t.Fatalf("expected CodeDataError, got %v", code)
		}
		if !strings.Contains(err.Error(), "document_ids") {
			t.Fatalf("unexpected error: %v", err)
		}
	}
}

func TestParseReturnsDataErrorWhenDatasetMissing(t *testing.T) {
	svc := newParseTestService(t)
	svc.checkDatasetAccessFunc = func(string) error { return nil }
	svc.getKnowledgebaseByIDFunc = func(string) (*entity.Knowledgebase, error) { return nil, nil }
	ctx := t.Context()
	_, code, err := svc.Parse(ctx, "kb-1", &service.ParseFileRequest{DocumentIDs: []string{"doc-1"}})
	if err == nil {
		t.Fatal("expected parse to fail")
	}
	if code != common.CodeDataError {
		t.Fatalf("expected CodeDataError, got %v", code)
	}
	if !strings.Contains(err.Error(), "dataset not found") {
		t.Fatalf("unexpected error: %v", err)
	}
}

func TestParseReturnsServerErrorWhenDocumentsQueryFails(t *testing.T) {
	svc := newParseTestService(t)
	queryErr := errors.New("documents query failed")
	svc.checkDatasetAccessFunc = func(string) error { return nil }
	svc.getKnowledgebaseByIDFunc = func(string) (*entity.Knowledgebase, error) {
		return &entity.Knowledgebase{ID: "kb-1", TenantID: "user-1"}, nil
	}
	svc.getDocumentsByIDsFunc = func([]string) ([]*entity.Document, error) {
		return nil, queryErr
	}
	ctx := t.Context()
	_, code, err := svc.Parse(ctx, "kb-1", &service.ParseFileRequest{DocumentIDs: []string{"doc-1"}})
	if !errors.Is(err, queryErr) {
		t.Fatalf("expected query error, got %v", err)
	}
	if code != common.CodeServerError {
		t.Fatalf("expected CodeServerError, got %v", code)
	}
}

func TestParseRejectsRunningDocument(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	userID := "user-1"
	datasetID := "kb-1"
	insertChunkTestKB(t, datasetID, userID)
	insertChunkTestDoc(t, "doc-1", datasetID)
	insertChunkTestIngestionTask(t, "task-1", userID, "doc-1", datasetID, common.RUNNING)

	svc := newParseTestService(t)
	ctx := chunkTestContext(t, userID, userID)
	_, code, err := svc.Parse(ctx, datasetID, &service.ParseFileRequest{DocumentIDs: []string{"doc-1"}})
	if err == nil {
		t.Fatal("expected parse to fail")
	}
	if code != common.CodeDataError {
		t.Fatalf("expected CodeDataError, got %v", code)
	}
	if !strings.Contains(err.Error(), "ingestion task task-1 is RUNNING") {
		t.Fatalf("unexpected error: %v", err)
	}
}

func TestListSortsChunksByDocumentPosition(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	userID := "user-1"
	tenantID := "tenant-1"
	datasetID := "kb-1"
	documentID := "doc-1"
	insertChunkTestUserTenant(t, userID, tenantID)
	insertChunkTestKB(t, datasetID, tenantID)
	insertChunkTestDoc(t, documentID, datasetID)

	engine := &listChunksSearchEngine{}
	svc := &ChunkService{
		docEngine:   engine,
		kbDAO:       dao.NewKnowledgebaseDAO(),
		documentDAO: dao.NewDocumentDAO(),
	}

	ctx := chunkTestContext(t, userID, tenantID)
	page := 1
	size := 30
	resp, err := svc.List(ctx, &service.ListChunksRequest{
		DatasetID: datasetID,
		DocID:     documentID,
		Page:      &page,
		Size:      &size,
	})
	if err != nil {
		t.Fatalf("List() error = %v", err)
	}
	if len(resp.Chunks) != 1 {
		t.Fatalf("chunks = %v, want one chunk", resp.Chunks)
	}
	if _, ok := resp.Chunks[0]["available_int"]; ok {
		t.Fatal("available_int should not be returned in chunk lists")
	}
	// Stored available_int (0 here) is surfaced as the `available` boolean.
	if got := resp.Chunks[0]["available"]; got != false {
		t.Fatalf("available = %#v, want false for a chunk stored with available_int=0", got)
	}

	if engine.searchReq == nil {
		t.Fatal("expected Search to be called")
	}
	if !engine.searchReq.IncludeUnavailable {
		t.Fatal("management chunk list must include disabled parent chunks")
	}
	// Parent rows must not be filtered out: they are listed after their children
	// on every engine (MaterializeParentChunks numbers them past the source
	// chunks), and excluding them by doc_type_kwd made the list engine-agnostic
	// but no longer showed the parent-child structure users see on ES.
	if _, exists := engine.searchReq.Filter["exists"]; exists {
		t.Fatalf("unexpected existence filter on the chunk list: %#v", engine.searchReq.Filter)
	}
	if engine.searchReq.OrderBy == nil {
		t.Fatal("expected OrderBy to be set")
	}
	want := []types.OrderByField{
		{Field: "chunk_order_int", Type: types.SortAsc},
		{Field: "page_num_int", Type: types.SortAsc},
		{Field: "top_int", Type: types.SortAsc},
		{Field: "create_timestamp_flt", Type: types.SortDesc},
	}
	if !reflect.DeepEqual(engine.searchReq.OrderBy.Fields, want) {
		t.Fatalf("OrderBy fields = %#v, want %#v", engine.searchReq.OrderBy.Fields, want)
	}
}

// TestListReportsChunkAvailability pins that stored available_int is surfaced as
// the `available` boolean (0 → false, missing → true).
func TestListReportsChunkAvailability(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	userID := "user-1"
	tenantID := "tenant-1"
	datasetID := "kb-1"
	documentID := "doc-1"
	insertChunkTestUserTenant(t, userID, tenantID)
	insertChunkTestKB(t, datasetID, tenantID)
	insertChunkTestDoc(t, documentID, datasetID)

	engine := &listChunksSearchEngine{
		chunks: []map[string]interface{}{
			{"id": "parent-1", "content_with_weight": "parent", "available_int": 0, "doc_id": documentID},
			{"id": "child-1", "content_with_weight": "child", "doc_id": documentID},
		},
	}
	svc := &ChunkService{
		docEngine:   engine,
		kbDAO:       dao.NewKnowledgebaseDAO(),
		documentDAO: dao.NewDocumentDAO(),
	}

	page := 1
	size := 30
	resp, err := svc.List(chunkTestContext(t, userID, tenantID), &service.ListChunksRequest{
		DatasetID: datasetID,
		DocID:     documentID,
		Page:      &page,
		Size:      &size,
	})
	if err != nil {
		t.Fatalf("List() error = %v", err)
	}
	if len(resp.Chunks) != 2 {
		t.Fatalf("chunks = %v, want two chunks", resp.Chunks)
	}
	if got := resp.Chunks[0]["available"]; got != false {
		t.Fatalf("parent available = %#v, want false (stored available_int=0)", got)
	}
	if got := resp.Chunks[1]["available"]; got != true {
		t.Fatalf("child available = %#v, want true (column absent)", got)
	}
	for i, chunk := range resp.Chunks {
		if _, ok := chunk["available_int"]; ok {
			t.Fatalf("chunk %d leaks the raw available_int column: %#v", i, chunk)
		}
	}
}

// TestListScopesSearchToDocumentDataset pins the dataset scope of the
// management chunk list: only the document's own KB may be searched. Passing
// every KB of the tenant made a document of dataset A list chunks that were
// written under dataset B's KB context (chunk rows carry img_id
// "<kb_id>-<chunk_id>"), which the UI surfaced as "occasionally seeing another
// dataset's chunks".
func TestListScopesSearchToDocumentDataset(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	userID := "user-1"
	tenantID := "tenant-1"
	datasetID := "kb-1"
	otherDatasetID := "kb-2"
	documentID := "doc-1"
	insertChunkTestUserTenant(t, userID, tenantID)
	insertChunkTestKB(t, datasetID, tenantID)
	insertChunkTestKB(t, otherDatasetID, tenantID)
	insertChunkTestDoc(t, documentID, datasetID)

	engine := &listChunksSearchEngine{}
	svc := &ChunkService{
		docEngine:   engine,
		kbDAO:       dao.NewKnowledgebaseDAO(),
		documentDAO: dao.NewDocumentDAO(),
	}

	ctx := chunkTestContext(t, userID, tenantID)
	page := 1
	size := 30
	if _, err := svc.List(ctx, &service.ListChunksRequest{
		DatasetID: datasetID,
		DocID:     documentID,
		Page:      &page,
		Size:      &size,
	}); err != nil {
		t.Fatalf("List() error = %v", err)
	}

	if engine.searchReq == nil {
		t.Fatal("expected Search to be called")
	}
	if !reflect.DeepEqual(engine.searchReq.KbIDs, []string{datasetID}) {
		t.Fatalf("KbIDs = %#v, want the document's dataset only (%q)", engine.searchReq.KbIDs, datasetID)
	}
}

// TestGetScopesChunkLookupToDocumentDataset pins the detail endpoint's scope:
// the chunk is read from the dataset named in the route only, never from every
// KB of the tenant.
func TestGetScopesChunkLookupToDocumentDataset(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	userID := "user-1"
	tenantID := "tenant-1"
	datasetID := "kb-1"
	otherDatasetID := "kb-2"
	documentID := "doc-1"
	insertChunkTestUserTenant(t, userID, tenantID)
	insertChunkTestKB(t, datasetID, tenantID)
	insertChunkTestKB(t, otherDatasetID, tenantID)
	insertChunkTestDoc(t, documentID, datasetID)

	engine := &getChunkTestEngine{chunk: map[string]interface{}{
		"id":                  "chunk-1",
		"doc_id":              documentID,
		"kb_id":               datasetID,
		"docnm_kwd":           "doc.txt",
		"content_with_weight": "invoice body",
		"content_ltks":        []string{"invoice", "body"},
		"available_int":       0,
	}}
	svc := &ChunkService{
		docEngine: engine,
		kbDAO:     dao.NewKnowledgebaseDAO(),
	}

	resp, err := svc.Get(chunkTestContext(t, userID, tenantID), &service.GetChunkRequest{
		DatasetID:  datasetID,
		DocumentID: documentID,
		ChunkID:    "chunk-1",
	})
	if err != nil {
		t.Fatalf("Get() error = %v", err)
	}
	if !reflect.DeepEqual(engine.datasetIDs, []string{datasetID}) {
		t.Fatalf("datasetIDs = %#v, want the route's dataset only (%q)", engine.datasetIDs, datasetID)
	}
	if engine.chunkID != "chunk-1" {
		t.Fatalf("chunkID = %q, want chunk-1", engine.chunkID)
	}
	if engine.indexName != "ragflow_"+tenantID {
		t.Fatalf("indexName = %q, want ragflow_%s", engine.indexName, tenantID)
	}
	if got := resp.Chunk["content_with_weight"]; got != "invoice body" {
		t.Fatalf("content_with_weight = %#v, want invoice body", got)
	}
	// Python returns the stored row (minus runtime fields), so the row identity
	// fields survive the round trip.
	if got := resp.Chunk["id"]; got != "chunk-1" {
		t.Fatalf("id = %#v, want chunk-1", got)
	}
	if got := resp.Chunk["docnm_kwd"]; got != "doc.txt" {
		t.Fatalf("docnm_kwd = %#v, want doc.txt", got)
	}
	if _, ok := resp.Chunk["content_ltks"]; ok {
		t.Fatal("content_ltks should be stripped like Python's _strip_chunk_runtime_fields")
	}
	if got := resp.Chunk["available_int"]; got != 0 {
		t.Fatalf("available_int = %v, want stored availability 0", got)
	}
	if engine.chunk["available_int"] != 0 {
		t.Fatal("stored availability should remain unchanged")
	}
}

// TestGetRejectsChunkOfAnotherDocument pins the doc_id guard: holding a chunk id
// is not authority to read it through a different document.
func TestGetRejectsChunkOfAnotherDocument(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	userID := "user-1"
	tenantID := "tenant-1"
	datasetID := "kb-1"
	documentID := "doc-1"
	insertChunkTestUserTenant(t, userID, tenantID)
	insertChunkTestKB(t, datasetID, tenantID)
	insertChunkTestDoc(t, documentID, datasetID)

	engine := &getChunkTestEngine{chunk: map[string]interface{}{
		"doc_id":              "doc-2",
		"content_with_weight": "other document body",
	}}
	svc := &ChunkService{
		docEngine: engine,
		kbDAO:     dao.NewKnowledgebaseDAO(),
	}

	_, err := svc.Get(chunkTestContext(t, userID, tenantID), &service.GetChunkRequest{
		DatasetID:  datasetID,
		DocumentID: documentID,
		ChunkID:    "chunk-1",
	})
	if err == nil || !strings.Contains(err.Error(), "chunk not found") {
		t.Fatalf("Get() error = %v, want chunk not found", err)
	}
}

// TestGetRequiresDocumentInRouteDataset pins the document guard: the chunk is
// only readable through a document that exists in the route's dataset,
// mirroring Python's DocumentService.query(id=document_id, kb_id=dataset_id).
func TestGetRequiresDocumentInRouteDataset(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	userID := "user-1"
	tenantID := "tenant-1"
	datasetID := "kb-1"
	otherDatasetID := "kb-2"
	insertChunkTestUserTenant(t, userID, tenantID)
	insertChunkTestKB(t, datasetID, tenantID)
	insertChunkTestKB(t, otherDatasetID, tenantID)
	insertChunkTestDoc(t, "doc-2", otherDatasetID)

	engine := &getChunkTestEngine{chunk: map[string]interface{}{
		"doc_id":              "doc-2",
		"content_with_weight": "other dataset document body",
	}}
	svc := &ChunkService{
		docEngine: engine,
		kbDAO:     dao.NewKnowledgebaseDAO(),
	}

	_, err := svc.Get(chunkTestContext(t, userID, tenantID), &service.GetChunkRequest{
		DatasetID:  datasetID,
		DocumentID: "doc-missing",
		ChunkID:    "chunk-1",
	})
	if err == nil || !strings.Contains(err.Error(), "document not found") {
		t.Fatalf("Get() error = %v, want document not found", err)
	}

	_, err = svc.Get(chunkTestContext(t, userID, tenantID), &service.GetChunkRequest{
		DatasetID:  datasetID,
		DocumentID: "doc-2",
		ChunkID:    "chunk-1",
	})
	if err == nil || !strings.Contains(err.Error(), "document does not belong to this dataset") {
		t.Fatalf("Get() error = %v, want document does not belong to this dataset", err)
	}
}

// TestIsChunkRuntimeField checks that tokenized/vector fields are dropped while
// other stored fields survive. Only _vec is end-anchored.
func TestIsChunkRuntimeField(t *testing.T) {
	cases := []struct {
		name string
		want bool
	}{
		{"content_ltks", true},
		{"content_sm_ltks", true},
		{"tokenized_content_ltks", true},
		{"title_tks", true},
		{"title_sm_tks", true},
		{"authors_sm_tks", true},
		{"important_tks", true},
		{"question_tks", true},
		{"name_tks", true},
		{"tags_tks", true},
		{"q_1024_vec", true},
		{"q_2_vec", true},
		// Only _vec is anchored at the end, so an index-shaped name is kept.
		{"q_2_vec_idx", false},
		{"content_with_weight", false},
		{"content", false},
		{"docnm_kwd", false},
		{"doc_id", false},
		{"kb_id", false},
		{"important_kwd", false},
		{"question_kwd", false},
		{"compile_kwd", false},
		{"tag_kwd", false},
		{"tag_feas", false},
		{"mom_id", false},
		{"position_int", false},
		{"img_id", false},
		{"available_int", false},
	}
	for _, tc := range cases {
		if got := isChunkRuntimeField(tc.name); got != tc.want {
			t.Errorf("isChunkRuntimeField(%q) = %v, want %v", tc.name, got, tc.want)
		}
	}
}

func TestListBuildsMatchTextExprForKeywords(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	userID := "user-1"
	tenantID := "tenant-1"
	datasetID := "kb-1"
	documentID := "doc-1"
	insertChunkTestUserTenant(t, userID, tenantID)
	insertChunkTestKB(t, datasetID, tenantID)
	insertChunkTestDoc(t, documentID, datasetID)

	engine := &listChunksSearchEngine{}
	svc := &ChunkService{
		docEngine:   engine,
		kbDAO:       dao.NewKnowledgebaseDAO(),
		documentDAO: dao.NewDocumentDAO(),
	}
	ctx := chunkTestContext(t, userID, tenantID)

	page := 2
	size := 5
	resp, err := svc.List(ctx, &service.ListChunksRequest{
		DatasetID: datasetID,
		DocID:     documentID,
		ChunkIDs:  []string{"chunk-1", "chunk-2"},
		Page:      &page,
		Size:      &size,
		Keywords:  "  invoice terms  ",
	})
	if err != nil {
		t.Fatalf("List() error = %v", err)
	}
	if resp.Total != 1 {
		t.Fatalf("total = %d, want 1", resp.Total)
	}
	if engine.searchReq == nil {
		t.Fatal("expected Search to be called")
	}
	if engine.searchReq.Offset != 5 || engine.searchReq.Limit != 5 {
		t.Fatalf("pagination = offset %d limit %d, want offset 5 limit 5", engine.searchReq.Offset, engine.searchReq.Limit)
	}
	if !reflect.DeepEqual(engine.searchReq.KbIDs, []string{datasetID}) {
		t.Fatalf("KbIDs = %#v, want %#v", engine.searchReq.KbIDs, []string{datasetID})
	}
	if got := engine.searchReq.Filter["doc_id"]; got != documentID {
		t.Fatalf("doc_id filter = %#v, want %q", got, documentID)
	}
	if got := engine.searchReq.Filter["id"]; !reflect.DeepEqual(got, []string{"chunk-1", "chunk-2"}) {
		t.Fatalf("id filter = %#v, want %#v", got, []string{"chunk-1", "chunk-2"})
	}
	mustNot, ok := engine.searchReq.Filter["must_not"].(map[string]interface{})
	if !ok || mustNot["exists"] != "compile_kwd" {
		t.Fatalf("must_not filter = %#v, want compile_kwd existence exclusion", engine.searchReq.Filter["must_not"])
	}
	if slices.Contains(engine.searchReq.SelectFields, "content") {
		t.Fatalf("SelectFields = %#v, should not request content with content_with_weight", engine.searchReq.SelectFields)
	}
	for _, field := range []string{"content_with_weight", "img_id", "position_int"} {
		if !slices.Contains(engine.searchReq.SelectFields, field) {
			t.Fatalf("SelectFields = %#v, missing %q", engine.searchReq.SelectFields, field)
		}
	}
	if got := resp.Chunks[0]["content_with_weight"]; got != "weighted invoice terms body" {
		t.Fatalf("formatted content_with_weight = %#v, want %q", got, "weighted invoice terms body")
	}
	if len(engine.searchReq.MatchExprs) != 1 {
		t.Fatalf("MatchExprs length = %d, want 1", len(engine.searchReq.MatchExprs))
	}
	matchText, ok := engine.searchReq.MatchExprs[0].(*types.MatchTextExpr)
	if !ok {
		t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr", engine.searchReq.MatchExprs[0])
	}
	if matchText.MatchingText == "invoice terms" || !strings.Contains(matchText.MatchingText, "invoice") || !strings.Contains(matchText.MatchingText, "terms") {
		t.Fatalf("MatchingText = %q, want a tokenized query containing invoice and terms", matchText.MatchingText)
	}
	if matchText.TopN != 100 {
		t.Fatalf("TopN = %d, want 100", matchText.TopN)
	}
	if got := matchText.ExtraOptions["original_query"]; got != "invoice terms" {
		t.Fatalf("original_query = %#v, want %q", got, "invoice terms")
	}
	if got := matchText.ExtraOptions["minimum_should_match"]; got != 0.3 {
		t.Fatalf("minimum_should_match = %#v, want 0.3", got)
	}
}

func TestUpdateChunkRejectsChunkFromAnotherDocument(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	insertChunkTestUserTenant(t, "user-1", "tenant-1")
	insertChunkTestKB(t, "kb-1", "tenant-1")
	insertChunkTestDoc(t, "doc-a", "kb-1")
	insertChunkTestDoc(t, "doc-b", "kb-1")

	engine := &updateChunkTestEngine{
		existingChunk: map[string]interface{}{"doc_id": "doc-b"},
	}
	svc := &ChunkService{
		docEngine: engine,
		kbDAO:     dao.NewKnowledgebaseDAO(),
	}

	err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
		DatasetID:  "kb-1",
		DocumentID: "doc-a",
		ChunkID:    "chunk-1",
	})
	if err == nil {
		t.Fatal("expected UpdateChunk to reject a chunk from another document")
	}
	if !strings.Contains(err.Error(), "chunk not found") {
		t.Fatalf("UpdateChunk error = %q, want chunk not found", err)
	}
	if len(engine.updateCalls) != 0 {
		t.Fatalf("UpdateChunks calls = %d, want 0", len(engine.updateCalls))
	}
}

func TestUpdateChunkUpdatesSameDocumentWithDocumentCondition(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)

	insertChunkTestUserTenant(t, "user-1", "tenant-1")
	insertChunkTestKB(t, "kb-1", "tenant-1")
	insertChunkTestDoc(t, "doc-a", "kb-1")

	engine := &updateChunkTestEngine{
		existingChunk: map[string]interface{}{
			"doc_id":              "doc-a",
			"content_with_weight": "existing content",
		},
	}
	svc := &ChunkService{
		docEngine: engine,
		kbDAO:     dao.NewKnowledgebaseDAO(),
	}

	err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
		DatasetID:  "kb-1",
		DocumentID: "doc-a",
		ChunkID:    "chunk-1",
	})
	if err != nil {
		t.Fatalf("UpdateChunk() error = %v", err)
	}
	if len(engine.updateCalls) != 1 {
		t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
	}
	call := engine.updateCalls[0]
	if !reflect.DeepEqual(call.condition, map[string]interface{}{
		"id":     "chunk-1",
		"doc_id": "doc-a",
	}) {
		t.Fatalf("UpdateChunks condition = %#v", call.condition)
	}
	if call.indexName != "ragflow_tenant-1" || call.datasetID != "kb-1" {
		t.Fatalf("UpdateChunks target index=%q dataset=%q", call.indexName, call.datasetID)
	}
}

func TestUpdateChunkStoresTagKwdOnlyWhenSent(t *testing.T) {
	tests := []struct {
		name    string
		tagKwd  []string
		want    []string
		present bool
	}{
		{name: "tags sent", tagKwd: []string{"alpha", "beta"}, want: []string{"alpha", "beta"}, present: true},
		{name: "empty list clears tags", tagKwd: []string{}, want: []string{}, present: true},
		{name: "absent leaves tags untouched", tagKwd: nil, present: false},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			db := setupChunkTestDB(t)
			pushChunkTestDB(t, db)

			insertChunkTestUserTenant(t, "user-1", "tenant-1")
			insertChunkTestKB(t, "kb-1", "tenant-1")
			insertChunkTestDoc(t, "doc-a", "kb-1")

			engine := &updateChunkTestEngine{
				existingChunk: map[string]interface{}{
					"doc_id":              "doc-a",
					"content_with_weight": "existing content",
				},
			}
			svc := &ChunkService{
				docEngine: engine,
				kbDAO:     dao.NewKnowledgebaseDAO(),
			}

			err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
				DatasetID:  "kb-1",
				DocumentID: "doc-a",
				ChunkID:    "chunk-1",
				TagKwd:     tt.tagKwd,
			})
			if err != nil {
				t.Fatalf("UpdateChunk() error = %v", err)
			}
			if len(engine.updateCalls) != 1 {
				t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
			}
			got, ok := engine.updateCalls[0].newValue["tag_kwd"]
			if ok != tt.present {
				t.Fatalf("tag_kwd present = %v, want %v (newValue=%#v)", ok, tt.present, engine.updateCalls[0].newValue)
			}
			if tt.present && !reflect.DeepEqual(got, tt.want) {
				t.Fatalf("tag_kwd = %#v, want %#v", got, tt.want)
			}
		})
	}
}

func TestUpdateChunkStoresImageAndFlagsImageChunk(t *testing.T) {
	cases := []struct {
		name     string
		mode     *string
		wantMode string
	}{
		{"mode defaults to append", nil, imageUpdateModeAppend},
		{"mode is normalized", strPtr(" REPLACE "), imageUpdateModeReplace},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			db := setupChunkTestDB(t)
			pushChunkTestDB(t, db)
			insertChunkTestUserTenant(t, "user-1", "tenant-1")
			insertChunkTestKB(t, "kb-1", "tenant-1")
			insertChunkTestDoc(t, "doc-a", "kb-1")

			engine := &updateChunkTestEngine{
				existingChunk: map[string]interface{}{
					"doc_id":              "doc-a",
					"content_with_weight": "existing content",
				},
			}
			var storedBucket, storedChunkID, storedMode string
			var storedBinary []byte
			svc := &ChunkService{
				docEngine: engine,
				kbDAO:     dao.NewKnowledgebaseDAO(),
				storeChunkImageFunc: func(bucket, chunkID string, imageBinary []byte, mode string) error {
					storedBucket, storedChunkID, storedMode, storedBinary = bucket, chunkID, mode, imageBinary
					return nil
				},
			}

			imageBase64 := base64.StdEncoding.EncodeToString([]byte("image-bytes"))
			err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
				DatasetID:       "kb-1",
				DocumentID:      "doc-a",
				ChunkID:         "chunk-1",
				ImageBase64:     &imageBase64,
				ImageUpdateMode: tc.mode,
			})
			if err != nil {
				t.Fatalf("UpdateChunk() error = %v", err)
			}

			if storedBucket != "kb-1" || storedChunkID != "chunk-1" || storedMode != tc.wantMode {
				t.Fatalf("store args = %q/%q/%q, want kb-1/chunk-1/%s", storedBucket, storedChunkID, storedMode, tc.wantMode)
			}
			if string(storedBinary) != "image-bytes" {
				t.Fatalf("stored binary = %q, want image-bytes", storedBinary)
			}

			if len(engine.updateCalls) != 1 {
				t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
			}
			updated := engine.updateCalls[0].newValue
			if updated["img_id"] != "kb-1-chunk-1" {
				t.Fatalf("img_id = %#v, want kb-1-chunk-1", updated["img_id"])
			}
			if updated["doc_type_kwd"] != "image" {
				t.Fatalf("doc_type_kwd = %#v, want image", updated["doc_type_kwd"])
			}
		})
	}
}

func TestUpdateChunkRemoveModeClearsImageAfterIndexUpdate(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestUserTenant(t, "user-1", "tenant-1")
	insertChunkTestKB(t, "kb-1", "tenant-1")
	insertChunkTestDoc(t, "doc-a", "kb-1")

	order := &[]string{}
	engine := &orderedUpdateChunkTestEngine{
		updateChunkTestEngine: updateChunkTestEngine{
			existingChunk: map[string]interface{}{
				"doc_id":              "doc-a",
				"content_with_weight": "existing content",
			},
		},
		order: order,
	}
	svc := &ChunkService{
		docEngine: engine,
		kbDAO:     dao.NewKnowledgebaseDAO(),
		storeChunkImageFunc: func(string, string, []byte, string) error {
			t.Fatal("remove mode must not store an image")
			return nil
		},
		removeChunkImageFunc: func(bucket, chunkID string) error {
			*order = append(*order, "storage")
			if bucket != "kb-1" || chunkID != "chunk-1" {
				t.Fatalf("remove args = %q/%q, want kb-1/chunk-1", bucket, chunkID)
			}
			return nil
		},
	}

	removeMode := imageUpdateModeRemove
	err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
		DatasetID:       "kb-1",
		DocumentID:      "doc-a",
		ChunkID:         "chunk-1",
		ImageUpdateMode: &removeMode,
	})
	if err != nil {
		t.Fatalf("UpdateChunk() error = %v", err)
	}

	if len(engine.updateCalls) != 1 {
		t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
	}
	updated := engine.updateCalls[0].newValue
	if imgID, ok := updated["img_id"]; !ok || imgID != "" {
		t.Fatalf("img_id = %#v, want empty string", updated["img_id"])
	}
	if updated["doc_type_kwd"] != "text" {
		t.Fatalf("doc_type_kwd = %#v, want text", updated["doc_type_kwd"])
	}
	if !reflect.DeepEqual(*order, []string{"index", "storage"}) {
		t.Fatalf("call order = %#v, want index before storage removal", *order)
	}
}

// A request that changes content and drops the image must still schedule the
// Wiki refresh when the stored object cannot be deleted: the index update has
// already committed by then.
func TestUpdateChunkMarksWikiDirtyWhenImageRemovalFails(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestUserTenant(t, "user-1", "tenant-1")
	insertChunkTestKB(t, "kb-1", "tenant-1")
	insertChunkTestDoc(t, "doc-a", "kb-1")

	engine := &updateChunkTestEngine{
		existingChunk: map[string]interface{}{
			"doc_id":              "doc-a",
			"content_with_weight": "existing content",
		},
	}
	wikiCalls := 0
	svc := &ChunkService{
		docEngine: engine,
		kbDAO:     dao.NewKnowledgebaseDAO(),
		removeChunkImageFunc: func(string, string) error {
			return errors.New("storage unavailable")
		},
		markWikiDirtyFunc: func(tenantID, datasetID, documentID string, chunkIDs []string) {
			wikiCalls++
		},
	}

	removeMode := imageUpdateModeRemove
	content := "updated content"
	err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
		DatasetID:       "kb-1",
		DocumentID:      "doc-a",
		ChunkID:         "chunk-1",
		Content:         &content,
		ImageUpdateMode: &removeMode,
	})
	if err == nil || !strings.Contains(err.Error(), "Failed to remove chunk image") {
		t.Fatalf("UpdateChunk() error = %v, want the removal failure", err)
	}
	if len(engine.updateCalls) != 1 {
		t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
	}
	if wikiCalls != 1 {
		t.Fatalf("markWikiDirty calls = %d, want 1 (the index update already committed)", wikiCalls)
	}
}

func TestUpdateChunkRejectsInvalidImageInput(t *testing.T) {
	cases := []struct {
		name      string
		imageB64  *string
		mode      *string
		wantError string
	}{
		{"undecodable base64", strPtr("not base64!!"), nil, "invalid `image_base64`"},
		{"empty base64", strPtr(""), nil, "`image_base64` must be a non-empty string"},
		{"unknown mode", strPtr("aGVsbG8="), strPtr("grow"), "`image_update_mode` must be one of: append, replace, remove"},
		{"empty mode", strPtr("aGVsbG8="), strPtr(""), "`image_update_mode` must be one of: append, replace, remove"},
		{"append without base64", nil, strPtr(imageUpdateModeAppend), "`image_base64` is required when `image_update_mode` is `append` or `replace`"},
		{"replace without base64", nil, strPtr(imageUpdateModeReplace), "`image_base64` is required when `image_update_mode` is `append` or `replace`"},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			db := setupChunkTestDB(t)
			pushChunkTestDB(t, db)
			insertChunkTestUserTenant(t, "user-1", "tenant-1")
			insertChunkTestKB(t, "kb-1", "tenant-1")
			insertChunkTestDoc(t, "doc-a", "kb-1")

			engine := &updateChunkTestEngine{
				existingChunk: map[string]interface{}{
					"doc_id":              "doc-a",
					"content_with_weight": "existing content",
				},
			}
			svc := &ChunkService{
				docEngine: engine,
				kbDAO:     dao.NewKnowledgebaseDAO(),
				storeChunkImageFunc: func(string, string, []byte, string) error {
					t.Fatal("invalid input must not reach storage")
					return nil
				},
			}

			err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
				DatasetID:       "kb-1",
				DocumentID:      "doc-a",
				ChunkID:         "chunk-1",
				ImageBase64:     tc.imageB64,
				ImageUpdateMode: tc.mode,
			})
			if err == nil || !strings.Contains(err.Error(), tc.wantError) {
				t.Fatalf("UpdateChunk() error = %v, want %q", err, tc.wantError)
			}
			var coded interface {
				Code() common.ErrorCode
			}
			if !errors.As(err, &coded) || coded.Code() != common.CodeDataError {
				t.Fatalf("error code = %v, want %d (Python: DATA_ERROR)", err, common.CodeDataError)
			}
			if len(engine.updateCalls) != 0 {
				t.Fatalf("UpdateChunks calls = %d, want 0", len(engine.updateCalls))
			}
		})
	}
}

type orderedUpdateChunkTestEngine struct {
	updateChunkTestEngine
	order *[]string
}

func (e *orderedUpdateChunkTestEngine) UpdateChunks(ctx context.Context, condition, newValue map[string]interface{}, indexName, datasetID string) error {
	*e.order = append(*e.order, "index")
	return e.updateChunkTestEngine.UpdateChunks(ctx, condition, newValue, indexName, datasetID)
}

func TestAddChunkSuccess(t *testing.T) {
	ctx := t.Context()
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	userID, datasetID, documentID := "user-1", "kb-1", "doc-1"
	insertChunkTestKB(t, datasetID, userID)
	insertChunkTestDoc(t, documentID, datasetID)

	// The document already holds source chunks whose highest order is 9; the
	// lookup page also carries rows the lookup must ignore (a parent-range value
	// and a row without any order), so the added chunk still continues from 9.
	engine := &addChunkTestEngine{
		searchResp: &types.SearchResult{
			Chunks: []map[string]interface{}{
				{"id": "parent-1", "chunk_order_int": float64(common.ParentChunkOrderBase + 3)},
				{"id": "raptor-1"},
				{"id": "chunk-9", "chunk_order_int": float64(9)},
			},
		},
	}
	var incrementTokenNum, incrementChunkNum int64
	language := "Slovak"
	var tokenizeLanguages []string
	svc := &ChunkService{
		docEngine:   engine,
		kbDAO:       dao.NewKnowledgebaseDAO(),
		documentDAO: dao.NewDocumentDAO(),
		checkDatasetAccessFunc: func(datasetIDArg string) error {
			if datasetIDArg == datasetID {
				return nil
			}
			return permission.ErrPermissionDenied
		},
		getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
			return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1", Language: &language, Pagerank: 7}, nil
		},
		getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
			driver := &stubEmbeddingDriver{
				embeddings: []models.EmbeddingData{
					{Embedding: []float64{1, 2}},
					{Embedding: []float64{3, 4}},
				},
			}
			modelName := "embed-1"
			return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil
		},
		incrementChunkStatsFunc: func(docID, kbID string, tokenNum, chunkNum int64, duration float64) error {
			if docID != documentID || kbID != datasetID || duration != 0 {
				t.Fatalf("unexpected increment args doc=%s kb=%s duration=%v", docID, kbID, duration)
			}
			incrementTokenNum = tokenNum
			incrementChunkNum = chunkNum
			return nil
		},
		tokenizeFunc: func(text, lang string) (string, error) {
			tokenizeLanguages = append(tokenizeLanguages, lang)
			return text, nil
		},
		fineGrainedTokenizeFunc: func(text, lang string) (string, error) {
			tokenizeLanguages = append(tokenizeLanguages, lang)
			return text + "_fg", nil
		},
		numTokensFunc: func(text string) int { return len(text) },
	}

	resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
		DatasetID:         datasetID,
		DocumentID:        documentID,
		Content:           "chunk body",
		ImportantKeywords: []string{"k1"},
		Questions:         []string{" q1 ", ""},
		TagFeas:           map[string]interface{}{"tag1": float64(0.5)},
	})
	if err != nil {
		t.Fatalf("AddChunk() error = %v", err)
	}
	if resp == nil || resp.Chunk == nil {
		t.Fatalf("expected chunk response, got %#v", resp)
	}
	if resp.Chunk["dataset_id"] != datasetID || resp.Chunk["document_id"] != documentID {
		t.Fatalf("unexpected response chunk: %#v", resp.Chunk)
	}
	if resp.Chunk["content"] != "chunk body" {
		t.Fatalf("content = %v, want chunk body", resp.Chunk["content"])
	}
	if len(engine.insertedChunks) != 1 {
		t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))
	}
	if got := engine.insertedChunks[0][common.PAGERANK_FLD]; got != int64(7) {
		t.Fatalf("pagerank_fea = %v, want the dataset's 7", got)
	}
	if resp.Chunk["document"] != "doc-1.txt" {
		t.Fatalf("document = %v, want doc-1.txt", resp.Chunk["document"])
	}
	if incrementChunkNum != 1 {
		t.Fatalf("increment chunk num = %d, want 1", incrementChunkNum)
	}
	if incrementTokenNum <= 0 {
		t.Fatalf("increment token num = %d, want > 0", incrementTokenNum)
	}
	if len(engine.insertedChunks) != 1 {
		t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))
	}
	inserted := engine.insertedChunks[0]
	if inserted["doc_id"] != documentID || inserted["kb_id"] != datasetID {
		t.Fatalf("unexpected inserted chunk: %#v", inserted)
	}
	if inserted["img_id"] != nil {
		t.Fatalf("did not expect image id in inserted chunk: %#v", inserted)
	}
	// A manual chunk is tokenized like the dataset's parsed chunks, and the
	// write that may create the chunk store carries the dataset language.
	if len(tokenizeLanguages) != 4 {
		t.Fatalf("tokenizer calls = %v, want 4", tokenizeLanguages)
	}
	for _, lang := range tokenizeLanguages {
		if lang != language {
			t.Fatalf("tokenized with language %q, want %q", lang, language)
		}
	}
	if engine.insertLanguage != language {
		t.Fatalf("InsertChunks language = %q, want %q", engine.insertLanguage, language)
	}

	// A manually added text chunk must carry doc_type_kwd (Python chunk_api.py:1225):
	// the management chunk list selects source chunks by that field, so a missing
	// value would hide the chunk from the list.
	if inserted["doc_type_kwd"] != "text" {
		t.Fatalf("doc_type_kwd = %#v, want text", inserted["doc_type_kwd"])
	}
	// The added chunk continues the document's source ordering (highest source
	// order + 1) and stays inside [0, common.ParentChunkOrderBase), so it lists
	// after the ingested chunks and still before the document's parent rows.
	if got := inserted["chunk_order_int"]; got != 10 {
		t.Fatalf("chunk_order_int = %#v, want 10 (highest source order + 1)", got)
	}
	if engine.searchReq == nil {
		t.Fatal("expected the document's highest source order to be looked up")
	}
	if got := engine.searchReq.Filter["exists"]; got != "doc_type_kwd" {
		t.Fatalf("order lookup exists filter = %#v, want doc_type_kwd", got)
	}
	if got := engine.searchReq.Filter["doc_id"]; got != documentID {
		t.Fatalf("order lookup doc_id = %#v, want %q", got, documentID)
	}
	if !engine.searchReq.IncludeUnavailable {
		t.Fatal("order lookup must count disabled source chunks")
	}
	if engine.searchReq.OrderBy == nil || len(engine.searchReq.OrderBy.Fields) != 1 ||
		engine.searchReq.OrderBy.Fields[0].Field != "chunk_order_int" ||
		engine.searchReq.OrderBy.Fields[0].Type != types.SortDesc {
		t.Fatalf("order lookup OrderBy = %#v, want descending chunk_order_int", engine.searchReq.OrderBy)
	}
	vec, ok := inserted["q_2_vec"].([]float64)
	if !ok {
		t.Fatalf("expected q_2_vec []float64, got %T", inserted["q_2_vec"])
	}
	if len(vec) != 2 || vec[0] < 2.7999 || vec[0] > 2.8001 || vec[1] < 3.7999 || vec[1] > 3.8001 {
		t.Fatalf("vector = %v, want approximately [2.8 3.8]", vec)
	}
}

// TestAddChunkPersistsTagKwdOnlyWhenProvided pins the Go/Python parity
// contract that mirrors api/apps/restful_apis/chunk_api.py: the `tag_kwd`
// field is only persisted on the inserted chunk when the caller actually
// supplied it, and is echoed back in the response when present. Issue
// #20138 documented that the Go path was silently dropping the field.
func TestAddChunkPersistsTagKwdOnlyWhenProvided(t *testing.T) {
	ctx := t.Context()
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	userID, datasetID, documentID := "user-1", "kb-1", "doc-1"
	insertChunkTestKB(t, datasetID, userID)
	insertChunkTestDoc(t, documentID, datasetID)

	makeService := func() (*ChunkService, *addChunkTestEngine) {
		engine := &addChunkTestEngine{}
		svc := &ChunkService{
			docEngine:   engine,
			kbDAO:       dao.NewKnowledgebaseDAO(),
			documentDAO: dao.NewDocumentDAO(),
			checkDatasetAccessFunc: func(datasetIDArg string) error {
				if datasetIDArg != datasetID {
					return permission.ErrPermissionDenied
				}
				return nil
			},
			getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
				return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil
			},
			getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
				driver := &stubEmbeddingDriver{
					embeddings: []models.EmbeddingData{
						{Embedding: []float64{1, 2}},
						{Embedding: []float64{3, 4}},
					},
				}
				modelName := "embed-1"
				return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil
			},
			incrementChunkStatsFunc: func(_, _ string, _, _ int64, _ float64) error { return nil },
			tokenizeFunc:            func(text, _ string) (string, error) { return text, nil },
			fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },
			numTokensFunc:           func(text string) int { return len(text) },
		}
		return svc, engine
	}

	t.Run("omitted by caller", func(t *testing.T) {
		svc, engine := makeService()
		resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
			DatasetID:  datasetID,
			DocumentID: documentID,
			Content:    "chunk body",
		})
		if err != nil {
			t.Fatalf("AddChunk() error = %v", err)
		}
		if len(engine.insertedChunks) != 1 {
			t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))
		}
		if _, ok := engine.insertedChunks[0]["tag_kwd"]; ok {
			t.Fatalf("expected tag_kwd to be absent when caller omits it; got %#v", engine.insertedChunks[0]["tag_kwd"])
		}
		if _, ok := resp.Chunk["tag_kwd"]; ok {
			t.Fatalf("response should not include tag_kwd when caller omits it; got %#v", resp.Chunk["tag_kwd"])
		}
	})

	t.Run("explicit empty list persists empty slice", func(t *testing.T) {
		svc, engine := makeService()
		resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
			DatasetID:  datasetID,
			DocumentID: documentID,
			Content:    "chunk body",
			TagKwd:     []string{},
		})
		if err != nil {
			t.Fatalf("AddChunk() error = %v", err)
		}
		got, ok := engine.insertedChunks[0]["tag_kwd"].([]string)
		if !ok {
			t.Fatalf("expected []string in tag_kwd, got %T", engine.insertedChunks[0]["tag_kwd"])
		}
		if len(got) != 0 {
			t.Fatalf("expected empty tag_kwd, got %v", got)
		}
		if echoed, _ := resp.Chunk["tag_kwd"].([]string); len(echoed) != 0 {
			t.Fatalf("response tag_kwd = %v, want empty", resp.Chunk["tag_kwd"])
		}
	})

	t.Run("values are persisted and echoed", func(t *testing.T) {
		svc, engine := makeService()
		want := []string{"alpha", "beta"}
		resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
			DatasetID:  datasetID,
			DocumentID: documentID,
			Content:    "chunk body",
			TagKwd:     want,
		})
		if err != nil {
			t.Fatalf("AddChunk() error = %v", err)
		}
		got, ok := engine.insertedChunks[0]["tag_kwd"].([]string)
		if !ok {
			t.Fatalf("expected []string in tag_kwd, got %T", engine.insertedChunks[0]["tag_kwd"])
		}
		if !reflect.DeepEqual(got, want) {
			t.Fatalf("inserted tag_kwd = %v, want %v", got, want)
		}
		echoed, ok := resp.Chunk["tag_kwd"].([]string)
		if !ok {
			t.Fatalf("response tag_kwd type = %T, want []string", resp.Chunk["tag_kwd"])
		}
		if !reflect.DeepEqual(echoed, want) {
			t.Fatalf("response tag_kwd = %v, want %v", echoed, want)
		}
	})
}

func TestAddChunkValidationErrors(t *testing.T) {
	ctx := t.Context()
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestKB(t, "kb-1", "user-1")
	insertChunkTestDoc(t, "doc-1", "kb-1")

	svc := &ChunkService{
		docEngine:   &addChunkTestEngine{},
		kbDAO:       dao.NewKnowledgebaseDAO(),
		documentDAO: dao.NewDocumentDAO(),
	}

	tests := []struct {
		name    string
		req     *service.AddChunkRequest
		setup   func(*ChunkService)
		wantMsg string
	}{
		{
			name:    "nil request",
			req:     nil,
			wantMsg: "invalid request payload",
		},
		{
			name: "empty content",
			req:  &service.AddChunkRequest{DatasetID: "kb-1", DocumentID: "doc-1", Content: " "},
			setup: func(svc *ChunkService) {
				svc.checkDatasetAccessFunc = func(string) error { return nil }
				svc.getKnowledgebaseByIDFunc = func(string) (*entity.Knowledgebase, error) {
					return &entity.Knowledgebase{ID: "kb-1", TenantID: "user-1", EmbdID: "embed-1"}, nil
				}
			},
			wantMsg: "`content` is required",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if tt.setup != nil {
				tt.setup(svc)
			}
			_, err := svc.AddChunk(ctx, tt.req)
			if err == nil || !strings.Contains(err.Error(), tt.wantMsg) {
				t.Fatalf("error = %v, want substring %q", err, tt.wantMsg)
			}
		})
	}
}

func TestAddChunkImageAndTagFeatureValidation(t *testing.T) {
	ctx := t.Context()
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	userID, datasetID, documentID := "user-1", "kb-1", "doc-1"
	insertChunkTestKB(t, datasetID, userID)
	insertChunkTestDoc(t, documentID, datasetID)

	storeCalls := 0
	svc := &ChunkService{
		docEngine:              &addChunkTestEngine{},
		kbDAO:                  dao.NewKnowledgebaseDAO(),
		documentDAO:            dao.NewDocumentDAO(),
		checkDatasetAccessFunc: func(string) error { return nil },
		getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
			return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil
		},
		tokenizeFunc:            func(text, _ string) (string, error) { return text, nil },
		fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },
		numTokensFunc:           func(text string) int { return len(text) },
		getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
			driver := &stubEmbeddingDriver{
				embeddings: []models.EmbeddingData{
					{Embedding: []float64{1, 1}},
					{Embedding: []float64{1, 1}},
				},
			}
			modelName := "embed-1"
			return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil
		},
		incrementChunkStatsFunc: func(string, string, int64, int64, float64) error { return nil },
		storeChunkImageFunc: func(bucket, chunkID string, imageBinary []byte, mode string) error {
			storeCalls++
			if bucket != datasetID || chunkID == "" || len(imageBinary) == 0 {
				t.Fatalf("unexpected store args bucket=%s chunkID=%s len=%d", bucket, chunkID, len(imageBinary))
			}
			if mode != imageUpdateModeAppend {
				t.Fatalf("store mode = %q, want %q", mode, imageUpdateModeAppend)
			}
			return nil
		},
	}

	_, err := svc.AddChunk(ctx, &service.AddChunkRequest{
		DatasetID:   datasetID,
		DocumentID:  documentID,
		Content:     "chunk body",
		ImageBase64: strPtr("not-base64"),
	})
	if err == nil || !strings.Contains(err.Error(), "invalid `image_base64`") {
		t.Fatalf("expected invalid image error, got %v", err)
	}

	validJPEG := "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO2pRZ0AAAAASUVORK5CYII="

	resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
		DatasetID:  datasetID,
		DocumentID: documentID,
		Content:    "chunk body",
		TagFeas:    map[string]interface{}{"tag1": "bad"},
	})
	if err == nil || !strings.Contains(err.Error(), "`tag_feas` values must be finite numbers") || resp != nil {
		t.Fatalf("expected tag_feas validation error, got resp=%#v err=%v", resp, err)
	}

	resp, err = svc.AddChunk(ctx, &service.AddChunkRequest{
		DatasetID:   datasetID,
		DocumentID:  documentID,
		Content:     "chunk body",
		TagFeas:     map[string]interface{}{"tag1": float64(1)},
		ImageBase64: strPtr(validJPEG),
	})
	if err != nil {
		t.Fatalf("AddChunk() with image error = %v", err)
	}
	if storeCalls != 1 {
		t.Fatalf("store image calls = %d, want 1", storeCalls)
	}
	if _, ok := resp.Chunk["image_id"]; !ok {
		t.Fatalf("expected image_id in response, got %#v", resp.Chunk)
	}
}

func TestAddChunkIncrementsStatsAfterInsert(t *testing.T) {
	ctx := t.Context()
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	userID, datasetID, documentID := "user-1", "kb-1", "doc-1"
	insertChunkTestKB(t, datasetID, userID)
	insertChunkTestDoc(t, documentID, datasetID)

	var incrementCalls int
	engine := &addChunkTestEngine{}
	svc := &ChunkService{
		docEngine:              engine,
		kbDAO:                  dao.NewKnowledgebaseDAO(),
		documentDAO:            dao.NewDocumentDAO(),
		checkDatasetAccessFunc: func(string) error { return nil },
		getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
			return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil
		},
		getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
			driver := &stubEmbeddingDriver{
				embeddings: []models.EmbeddingData{
					{Embedding: []float64{1, 2}},
					{Embedding: []float64{3, 4}},
				},
			}
			modelName := "embed-1"
			return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil
		},
		incrementChunkStatsFunc: func(string, string, int64, int64, float64) error {
			incrementCalls++
			return nil
		},
		tokenizeFunc:            func(text, _ string) (string, error) { return text, nil },
		fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },
		numTokensFunc:           func(text string) int { return len(text) },
	}

	_, err := svc.AddChunk(ctx, &service.AddChunkRequest{
		DatasetID:  datasetID,
		DocumentID: documentID,
		Content:    "chunk body",
	})
	if err != nil {
		t.Fatalf("AddChunk() error = %v", err)
	}
	if incrementCalls != 1 {
		t.Fatalf("increment calls = %d, want 1", incrementCalls)
	}
	if len(engine.insertedChunks) != 1 {
		t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))
	}
	importantKwd, ok := engine.insertedChunks[0]["important_kwd"].([]string)
	if !ok || len(importantKwd) != 0 {
		t.Fatalf("important_kwd = %#v, want empty []string", engine.insertedChunks[0]["important_kwd"])
	}
}

func TestStoreChunkImageMergesExistingImage(t *testing.T) {
	oldImage := mustEncodePNG(t, image.Rect(0, 0, 2, 2))
	newImage := mustEncodePNG(t, image.Rect(0, 0, 1, 1))
	mockStorage := &chunkImageStorage{
		exists:    true,
		oldBinary: oldImage,
	}
	ctx := t.Context()

	factory := storage.GetStorageFactory()
	originalStorage := factory.GetStorage()
	factory.SetStorage(mockStorage)
	t.Cleanup(func() {
		factory.SetStorage(originalStorage)
	})

	svc := &ChunkService{}
	if err := svc.storeChunkImage(ctx, "kb-1", "chunk-1", newImage, imageUpdateModeAppend); err != nil {
		t.Fatalf("storeChunkImage() error = %v", err)
	}
	if mockStorage.putCalls != 1 {
		t.Fatalf("put calls = %d, want 1", mockStorage.putCalls)
	}
	merged, format, err := image.Decode(bytes.NewReader(mockStorage.lastPut))
	if err != nil {
		t.Fatalf("decode stored image: %v", err)
	}
	if format != "jpeg" {
		t.Fatalf("stored format = %q, want jpeg", format)
	}
	// Append stacks the new image below the old one.
	if got, want := merged.Bounds().Dy(), 3; got != want {
		t.Fatalf("merged height = %d, want %d", got, want)
	}
}

func TestStoreChunkImageReplaceOverwritesExistingImage(t *testing.T) {
	oldImage := mustEncodePNG(t, image.Rect(0, 0, 2, 2))
	newImage := mustEncodePNG(t, image.Rect(0, 0, 1, 1))
	mockStorage := &chunkImageStorage{
		exists:    true,
		oldBinary: oldImage,
	}
	ctx := t.Context()

	factory := storage.GetStorageFactory()
	originalStorage := factory.GetStorage()
	factory.SetStorage(mockStorage)
	t.Cleanup(func() {
		factory.SetStorage(originalStorage)
	})

	svc := &ChunkService{}
	if err := svc.storeChunkImage(ctx, "kb-1", "chunk-1", newImage, imageUpdateModeReplace); err != nil {
		t.Fatalf("storeChunkImage() error = %v", err)
	}
	if mockStorage.putCalls != 1 {
		t.Fatalf("put calls = %d, want 1", mockStorage.putCalls)
	}
	if !bytes.Equal(mockStorage.lastPut, newImage) {
		t.Fatalf("replace stored %d bytes, want the new image verbatim (%d bytes)", len(mockStorage.lastPut), len(newImage))
	}
}

func TestRemoveChunkImageSkipsMissingObject(t *testing.T) {
	mockStorage := &chunkImageStorage{exists: false}
	factory := storage.GetStorageFactory()
	originalStorage := factory.GetStorage()
	factory.SetStorage(mockStorage)
	t.Cleanup(func() {
		factory.SetStorage(originalStorage)
	})

	svc := &ChunkService{}
	if err := svc.removeChunkImage(t.Context(), "kb-1", "chunk-1"); err != nil {
		t.Fatalf("removeChunkImage() error = %v", err)
	}
	if mockStorage.removeCalls != 0 {
		t.Fatalf("remove calls = %d, want 0 for a missing object", mockStorage.removeCalls)
	}

	mockStorage.exists = true
	if err := svc.removeChunkImage(t.Context(), "kb-1", "chunk-1"); err != nil {
		t.Fatalf("removeChunkImage() error = %v", err)
	}
	if mockStorage.removeCalls != 1 {
		t.Fatalf("remove calls = %d, want 1", mockStorage.removeCalls)
	}
}

// Every image mode has to serialize on the chunk's per-chunk lock: without it a
// replace (or a remove) can land between an append's read and write and be
// overwritten by the merged image.
func TestChunkImageOperationsShareThePerChunkLock(t *testing.T) {
	chunkID := "chunk-locked"
	mockStorage := &chunkImageStorage{
		exists:    true,
		oldBinary: mustEncodePNG(t, image.Rect(0, 0, 2, 2)),
	}
	factory := storage.GetStorageFactory()
	originalStorage := factory.GetStorage()
	factory.SetStorage(mockStorage)
	t.Cleanup(func() {
		factory.SetStorage(originalStorage)
	})

	lockKey := "kb-1/" + chunkID
	lock := acquireChunkImageLock(lockKey)
	lock.mu.Lock()
	defer releaseChunkImageLock(lockKey)

	type result struct {
		op  string
		err error
	}
	svc := &ChunkService{}
	ctx := t.Context()
	appendedImage := mustEncodePNG(t, image.Rect(0, 0, 1, 1))
	replacedImage := mustEncodePNG(t, image.Rect(0, 0, 3, 3))

	done := make(chan result, 3)
	run := func(op string, fn func() error) {
		go func() { done <- result{op: op, err: fn()} }()
	}
	run("append", func() error {
		return svc.storeChunkImage(ctx, "kb-1", chunkID, appendedImage, imageUpdateModeAppend)
	})
	run("replace", func() error {
		return svc.storeChunkImage(ctx, "kb-1", chunkID, replacedImage, imageUpdateModeReplace)
	})
	run("remove", func() error {
		return svc.removeChunkImage(ctx, "kb-1", chunkID)
	})

	select {
	case got := <-done:
		lock.mu.Unlock()
		t.Fatalf("%s finished while the chunk's image lock was held", got.op)
	case <-time.After(200 * time.Millisecond):
	}

	lock.mu.Unlock()
	for i := 0; i < 3; i++ {
		select {
		case got := <-done:
			if got.err != nil {
				t.Fatalf("%s: %v", got.op, got.err)
			}
		case <-time.After(5 * time.Second):
			t.Fatal("an image operation did not finish after the lock was released")
		}
	}
}

func TestRemoveChunksRejectsInvalidDatasetBeforeMutation(t *testing.T) {
	for _, datasetID := range []string{"", "kb-2"} {
		for _, deleteAll := range []bool{false, true} {
			t.Run(fmt.Sprintf("dataset=%s/delete_all=%t", datasetID, deleteAll), func(t *testing.T) {
				db := setupChunkTestDB(t)
				pushChunkTestDB(t, db)
				insertChunkTestKB(t, "kb-1", "owner")
				insertChunkTestDoc(t, "doc-1", "kb-1")
				engine := &parseTestDocEngine{deleteChunksCount: 1}
				svc := &ChunkService{
					docEngine: engine,
					decrementChunkStatsFunc: func(string, string, int64, int64, float64) error {
						t.Fatal("rejected deletion must not change statistics")
						return nil
					},
					markWikiDirtyFunc: func(string, string, string, []string) {
						t.Fatal("rejected deletion must not schedule wiki updates")
					},
				}
				req := &service.RemoveChunksRequest{DatasetID: datasetID, DocID: "doc-1", DeleteAll: deleteAll}
				if !deleteAll {
					req.ChunkIDs = []string{"chunk-1"}
				}
				count, err := svc.RemoveChunks(t.Context(), req)
				if err == nil || count != 0 {
					t.Fatalf("invalid dataset deletion = (%d, %v), want error and zero count", count, err)
				}
				if datasetID != "" && !errors.Is(err, permission.ErrResourceNotFound) {
					t.Fatalf("error = %v, want resource not found", err)
				}
				if engine.deleteChunksCalls != 0 {
					t.Fatalf("rejected deletion reached the engine %d times", engine.deleteChunksCalls)
				}
			})
		}
	}
}

func TestRemoveChunksDecrementsStatsAfterDelete(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestUserTenant(t, "user-1", "tenant-1")
	insertChunkTestKB(t, "kb-1", "tenant-1")
	insertChunkTestDoc(t, "doc-1", "kb-1")
	setChunkTestStats(t, "doc-1", "kb-1", 70, 7, 100, 10)

	engine := &parseTestDocEngine{deleteChunksCount: 3}
	svc := &ChunkService{
		docEngine: engine,
		kbDAO:     dao.NewKnowledgebaseDAO(),
	}
	ctx := chunkTestContext(t, "user-1", "tenant-1")
	deletedCount, err := svc.RemoveChunks(ctx, &service.RemoveChunksRequest{
		DatasetID: "kb-1",
		DocID:     "doc-1",
		ChunkIDs:  []string{"chunk-1", "chunk-2", "chunk-3"},
	})
	if err != nil {
		t.Fatalf("RemoveChunks() error = %v", err)
	}
	if deletedCount != 3 {
		t.Fatalf("deleted count = %d, want 3", deletedCount)
	}
	if engine.deleteChunksCalls != 1 {
		t.Fatalf("DeleteChunks calls = %d, want 1", engine.deleteChunksCalls)
	}
	if engine.deleteIndexName != "ragflow_tenant-1" || engine.deleteDatasetID != "kb-1" {
		t.Fatalf("unexpected delete target index=%q dataset=%q", engine.deleteIndexName, engine.deleteDatasetID)
	}
	if engine.deleteChunksCondition["doc_id"] != "doc-1" {
		t.Fatalf("delete doc_id condition = %#v, want doc-1", engine.deleteChunksCondition["doc_id"])
	}
	if !reflect.DeepEqual(engine.deleteChunksCondition["id"], []interface{}{"chunk-1", "chunk-2", "chunk-3"}) {
		t.Fatalf("delete id condition = %#v", engine.deleteChunksCondition["id"])
	}

	doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
	if err != nil {
		t.Fatalf("get doc: %v", err)
	}
	if doc.TokenNum != 70 {
		t.Fatalf("document token_num = %d, want 70", doc.TokenNum)
	}
	if doc.ChunkNum != 4 {
		t.Fatalf("document chunk_num = %d, want 4", doc.ChunkNum)
	}

	kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
	if err != nil {
		t.Fatalf("get kb: %v", err)
	}
	if kb.TokenNum != 100 {
		t.Fatalf("knowledgebase token_num = %d, want 100", kb.TokenNum)
	}
	if kb.ChunkNum != 7 {
		t.Fatalf("knowledgebase chunk_num = %d, want 7", kb.ChunkNum)
	}
}

func TestRemoveChunksSkipsStatsWhenNothingDeleted(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestUserTenant(t, "user-1", "tenant-1")
	insertChunkTestKB(t, "kb-1", "tenant-1")
	insertChunkTestDoc(t, "doc-1", "kb-1")
	setChunkTestStats(t, "doc-1", "kb-1", 70, 7, 100, 10)

	var decrementCalls int
	svc := &ChunkService{
		docEngine: &parseTestDocEngine{deleteChunksCount: 0},
		kbDAO:     dao.NewKnowledgebaseDAO(),
		decrementChunkStatsFunc: func(string, string, int64, int64, float64) error {
			decrementCalls++
			return nil
		},
	}
	ctx := chunkTestContext(t, "user-1", "tenant-1")
	deletedCount, err := svc.RemoveChunks(ctx, &service.RemoveChunksRequest{
		DatasetID: "kb-1",
		DocID:     "doc-1",
		DeleteAll: true,
	})
	if err != nil {
		t.Fatalf("RemoveChunks() error = %v", err)
	}
	if deletedCount != 0 {
		t.Fatalf("deleted count = %d, want 0", deletedCount)
	}
	if decrementCalls != 0 {
		t.Fatalf("decrement calls = %d, want 0", decrementCalls)
	}
}

func TestRemoveChunksReturnsStatsError(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestUserTenant(t, "user-1", "tenant-1")
	insertChunkTestKB(t, "kb-1", "tenant-1")
	insertChunkTestDoc(t, "doc-1", "kb-1")

	svc := &ChunkService{
		docEngine: &parseTestDocEngine{deleteChunksCount: 2},
		kbDAO:     dao.NewKnowledgebaseDAO(),
		decrementChunkStatsFunc: func(docID, kbID string, tokenNum, chunkNum int64, duration float64) error {
			if docID != "doc-1" || kbID != "kb-1" || tokenNum != 0 || chunkNum != 2 || duration != 0 {
				t.Fatalf("unexpected decrement args doc=%s kb=%s token=%d chunk=%d duration=%v", docID, kbID, tokenNum, chunkNum, duration)
			}
			return errors.New("stats update failed")
		},
	}
	ctx := chunkTestContext(t, "user-1", "tenant-1")
	deletedCount, err := svc.RemoveChunks(ctx, &service.RemoveChunksRequest{
		DatasetID: "kb-1",
		DocID:     "doc-1",
		ChunkIDs:  []string{"chunk-1", "chunk-2"},
	})
	if err == nil || !strings.Contains(err.Error(), "failed to update chunk stats") {
		t.Fatalf("expected stats update error, got count=%d err=%v", deletedCount, err)
	}
	if deletedCount != 2 {
		t.Fatalf("deleted count on error = %d, want 2", deletedCount)
	}
}

func TestDecrementChunkStatsClampsCounters(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestKB(t, "kb-1", "tenant-1")
	insertChunkTestDoc(t, "doc-1", "kb-1")
	setChunkTestStats(t, "doc-1", "kb-1", 2, 1, 3, 2)
	if err := dao.DB.Model(&entity.Document{}).
		Where("id = ?", "doc-1").
		Update("process_duration", 0.5).Error; err != nil {
		t.Fatalf("set process duration: %v", err)
	}

	svc := &ChunkService{}
	if err := svc.decrementChunkStats("doc-1", "kb-1", 5, 7, -1); err != nil {
		t.Fatalf("decrementChunkStats() error = %v", err)
	}
	ctx := t.Context()
	doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
	if err != nil {
		t.Fatalf("get doc: %v", err)
	}
	if doc.TokenNum != 0 || doc.ChunkNum != 0 || doc.ProcessDuration != 0 {
		t.Fatalf("document stats token=%d chunk=%d duration=%v, want zeros", doc.TokenNum, doc.ChunkNum, doc.ProcessDuration)
	}

	kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
	if err != nil {
		t.Fatalf("get kb: %v", err)
	}
	if kb.TokenNum != 0 || kb.ChunkNum != 0 {
		t.Fatalf("knowledgebase stats token=%d chunk=%d, want zeros", kb.TokenNum, kb.ChunkNum)
	}
}

func TestDecrementChunkStatsRollsBackWhenKnowledgebaseMissing(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestDoc(t, "doc-1", "missing-kb")

	svc := &ChunkService{}
	err := svc.decrementChunkStats("doc-1", "missing-kb", 0, 3, 0)
	if err == nil || !strings.Contains(err.Error(), "knowledgebase not found") {
		t.Fatalf("expected missing knowledgebase error, got %v", err)
	}
	ctx := t.Context()
	doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
	if err != nil {
		t.Fatalf("get doc: %v", err)
	}
	if doc.ChunkNum != 7 {
		t.Fatalf("document chunk_num after rollback = %d, want 7", doc.ChunkNum)
	}
}

func setupChunkTestDB(t *testing.T) *gorm.DB {
	t.Helper()

	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
		TranslateError: true,
	})
	if err != nil {
		t.Fatalf("failed to open sqlite: %v", err)
	}
	if err := db.AutoMigrate(
		&entity.UserTenant{},
		&entity.Knowledgebase{},
		&entity.Document{},
		&entity.Task{},
		&entity.IngestionTask{},
	); err != nil {
		t.Fatalf("failed to migrate: %v", err)
	}
	return db
}

func pushChunkTestDB(t *testing.T, testDB *gorm.DB) {
	t.Helper()

	orig := dao.DB
	dao.DB = testDB
	t.Cleanup(func() {
		dao.DB = orig
	})
}

func chunkTestContext(t *testing.T, userID, tenantID string, memberIDs ...string) context.Context {
	t.Helper()
	members := append([]string{userID}, memberIDs...)
	if err := testsupport.PrepareEnterprisePermissionFixture(dao.DB, tenantID, members...); err != nil {
		t.Fatalf("prepare permission fixture: %v", err)
	}
	return permission.WithIdentity(t.Context(), userID, tenantID)
}

func newParseTestService(t *testing.T) *ChunkService {
	t.Helper()

	return &ChunkService{
		docEngine:   nil,
		kbDAO:       dao.NewKnowledgebaseDAO(),
		documentDAO: dao.NewDocumentDAO(),
	}
}

func insertChunkTestUserTenant(t *testing.T, userID, tenantID string) {
	t.Helper()

	status := "1"
	role := "normal"
	if userID == tenantID {
		role = "owner"
	}
	userTenant := &entity.UserTenant{
		ID:        userID + "-" + tenantID,
		UserID:    userID,
		TenantID:  tenantID,
		Role:      role,
		InvitedBy: userID,
		Status:    &status,
	}
	if err := dao.DB.Create(userTenant).Error; err != nil {
		t.Fatalf("insert user tenant: %v", err)
	}
}

func insertChunkTestKB(t *testing.T, id, tenantID string) {
	t.Helper()

	var owner entity.UserTenant
	err := dao.DB.Where("tenant_id = ? AND role = ? AND status = ?", tenantID, "owner", "1").First(&owner).Error
	if errors.Is(err, gorm.ErrRecordNotFound) {
		insertChunkTestUserTenant(t, tenantID, tenantID)
	} else if err != nil {
		t.Fatalf("find tenant owner: %v", err)
	}

	status := string(entity.StatusValid)
	kb := &entity.Knowledgebase{
		ID:           id,
		TenantID:     tenantID,
		Name:         id,
		EmbdID:       "embedding-model",
		Permission:   string(entity.TenantPermissionTeam),
		CreatedBy:    tenantID,
		ParserConfig: entity.JSONMap{},
		Status:       &status,
	}
	if err := dao.DB.Create(kb).Error; err != nil {
		t.Fatalf("insert kb: %v", err)
	}
}

func insertChunkTestDoc(t *testing.T, id, kbID string) {
	t.Helper()

	location := id + ".txt"
	name := id + ".txt"
	status := string(entity.StatusValid)
	doc := &entity.Document{
		ID:           id,
		KbID:         kbID,
		ParserID:     string(entity.ParserTypeNaive),
		ParserConfig: entity.JSONMap{},
		SourceType:   string(entity.FileSourceLocal),
		Type:         "txt",
		CreatedBy:    "user-1",
		Name:         &name,
		Location:     &location,
		ChunkNum:     7,
		Suffix:       ".txt",
		Status:       &status,
	}
	if err := dao.DB.Create(doc).Error; err != nil {
		t.Fatalf("insert doc: %v", err)
	}
}

func insertChunkTestIngestionTask(t *testing.T, id, userID, docID, datasetID, status string) {
	t.Helper()
	task := &entity.IngestionTask{
		ID:         id,
		UserID:     userID,
		DocumentID: docID,
		DatasetID:  datasetID,
		Schema:     entity.JSONMap{},
		Status:     status,
	}
	if err := dao.DB.Create(task).Error; err != nil {
		t.Fatalf("insert ingestion task: %v", err)
	}
}

func setChunkTestStats(t *testing.T, docID, kbID string, docTokenNum, docChunkNum, kbTokenNum, kbChunkNum int64) {
	t.Helper()

	if err := dao.DB.Model(&entity.Document{}).
		Where("id = ?", docID).
		Updates(map[string]interface{}{"token_num": docTokenNum, "chunk_num": docChunkNum}).Error; err != nil {
		t.Fatalf("set doc stats: %v", err)
	}
	if err := dao.DB.Model(&entity.Knowledgebase{}).
		Where("id = ?", kbID).
		Updates(map[string]interface{}{"token_num": kbTokenNum, "chunk_num": kbChunkNum}).Error; err != nil {
		t.Fatalf("set kb stats: %v", err)
	}
}

func insertChunkTestTask(t *testing.T, id, docID string) {
	t.Helper()

	digest := "digest"
	task := &entity.Task{
		ID:     id,
		DocID:  docID,
		Digest: &digest,
	}
	if err := dao.DB.Create(task).Error; err != nil {
		t.Fatalf("insert task: %v", err)
	}
}

type parseTestDocEngine struct {
	deleteChunksErr       error
	deleteChunksCount     int64
	deleteChunksCalls     int
	deleteChunksCondition map[string]interface{}
	deleteIndexName       string
	deleteDatasetID       string
	chunkStoreExists      bool // if true, ChunkStoreExists returns true
}

func (e *parseTestDocEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {
	return nil
}

func (e *parseTestDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {
	return nil, nil
}

func (e *parseTestDocEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {
	return nil
}

func (e *parseTestDocEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {
	e.deleteChunksCalls++
	e.deleteChunksCondition = copyMap(condition)
	e.deleteIndexName = indexName
	e.deleteDatasetID = datasetID
	return e.deleteChunksCount, e.deleteChunksErr
}

func (e *parseTestDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {
	return nil, nil
}

func (e *parseTestDocEngine) GetChunk(context.Context, string, string, []string) (interface{}, error) {
	return nil, nil
}

func (e *parseTestDocEngine) DropChunkStore(context.Context, string, string) error {
	return nil
}

func (e *parseTestDocEngine) ChunkStoreExists(context.Context, string, string) (bool, error) {
	return e.chunkStoreExists, nil
}

func (e *parseTestDocEngine) CreateMetadataStore(context.Context, string) error {
	return nil
}

func (e *parseTestDocEngine) InsertMetadata(context.Context, []map[string]interface{}, string) ([]string, error) {
	return nil, nil
}

func (e *parseTestDocEngine) UpdateMetadata(context.Context, string, string, map[string]interface{}, string) error {
	return nil
}

func (e *parseTestDocEngine) DeleteMetadata(context.Context, map[string]interface{}, string) (int64, error) {
	return 0, nil
}

func (e *parseTestDocEngine) DeleteMetadataKeys(context.Context, string, string, []string, string) error {
	return nil
}

func (e *parseTestDocEngine) DropMetadataStore(context.Context, string) error {
	return nil
}

func (e *parseTestDocEngine) MetadataStoreExists(context.Context, string) (bool, error) {
	return false, nil
}

func (e *parseTestDocEngine) SearchMetadata(context.Context, *types.SearchMetadataRequest) (*types.SearchMetadataResult, error) {
	return nil, nil
}

func (e *parseTestDocEngine) IndexDocument(context.Context, string, string, interface{}) error {
	return nil
}

func (e *parseTestDocEngine) DeleteDocument(context.Context, string, string) error {
	return nil
}

func (e *parseTestDocEngine) BulkIndex(context.Context, string, []interface{}) (interface{}, error) {
	return nil, nil
}

func (e *parseTestDocEngine) GetFields([]map[string]interface{}, []string) map[string]map[string]interface{} {
	return nil
}

func (e *parseTestDocEngine) GetAggregation([]map[string]interface{}, string) []map[string]interface{} {
	return nil
}

func (e *parseTestDocEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
	return nil
}

func (e *parseTestDocEngine) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
	return nil, nil
}

func (e *parseTestDocEngine) GetChunkIDs([]map[string]interface{}) []string {
	return nil
}

type addChunkTestEngine struct {
	parseTestDocEngine
	insertedChunks []map[string]interface{}
	insertIndex    string
	insertDataset  string
	insertLanguage string
	insertErr      error
	searchReq      *types.SearchRequest
	searchResp     *types.SearchResult
}

func (e *addChunkTestEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {
	e.insertedChunks = chunks
	e.insertIndex = baseName
	e.insertDataset = datasetID
	e.insertLanguage = language
	return nil, e.insertErr
}

func (e *addChunkTestEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
	e.searchReq = req
	return e.searchResp, nil
}

type listChunksSearchEngine struct {
	parseTestDocEngine
	searchReq *types.SearchRequest
	// chunks optionally overrides the canned list rows.
	chunks []map[string]interface{}
}

type getChunkTestEngine struct {
	parseTestDocEngine
	chunk      map[string]interface{}
	err        error
	indexName  string
	chunkID    string
	datasetIDs []string
}

func (e *getChunkTestEngine) GetChunk(_ context.Context, baseName, chunkID string, datasetIDs []string) (interface{}, error) {
	e.indexName = baseName
	e.chunkID = chunkID
	e.datasetIDs = datasetIDs
	if e.err != nil {
		return nil, e.err
	}
	if e.chunk == nil {
		return nil, nil
	}
	return e.chunk, nil
}

func (e *listChunksSearchEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
	e.searchReq = req
	chunks := e.chunks
	if chunks == nil {
		chunks = []map[string]interface{}{
			{
				"id":                  "chunk-1",
				"content":             "plain invoice terms body",
				"content_with_weight": "weighted invoice terms body",
				"available_int":       0,
				"doc_id":              "doc-1",
			},
		}
	}
	return &types.SearchResult{
		Chunks: chunks,
		Total:  int64(len(chunks)),
	}, nil
}

type updateChunkTestEngine struct {
	parseTestDocEngine
	existingChunk interface{}
	updateCalls   []updateChunksCall
}

func (e *updateChunkTestEngine) GetChunk(context.Context, string, string, []string) (interface{}, error) {
	return e.existingChunk, nil
}

func (e *updateChunkTestEngine) UpdateChunks(_ context.Context, condition, newValue map[string]interface{}, indexName, datasetID string) error {
	e.updateCalls = append(e.updateCalls, updateChunksCall{
		condition: copyMap(condition),
		newValue:  copyMap(newValue),
		indexName: indexName,
		datasetID: datasetID,
	})
	return nil
}

type chunkImageStorage struct {
	exists      bool
	oldBinary   []byte
	putCalls    int
	removeCalls int
	lastPut     []byte
}

func (s *chunkImageStorage) Type() string                  { return "chunk_image_storage" }
func (s *chunkImageStorage) Health(_ context.Context) bool { return true }
func (s *chunkImageStorage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {
	s.putCalls++
	s.lastPut = binary
	return nil
}
func (s *chunkImageStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {
	return s.oldBinary, nil
}
func (s *chunkImageStorage) Remove(ctx context.Context, bucket, fnm string, tenantID ...string) error {
	s.removeCalls++
	return nil
}
func (s *chunkImageStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool {
	return s.exists
}

// ListObjects lists all objects in a bucket
func (s *chunkImageStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
	return []string{}, nil
}
func (s *chunkImageStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
	return "", nil
}
func (s *chunkImageStorage) BucketExists(ctx context.Context, bucket string) bool       { return true }
func (s *chunkImageStorage) RemoveBucket(ctx context.Context, bucket string) error      { return nil }
func (s *chunkImageStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error { return nil }
func (s *chunkImageStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
	return s.exists, nil
}
func (s *chunkImageStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
	return true, nil
}
func (s *chunkImageStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
	return false
}
func (s *chunkImageStorage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
	return false
}
func (s *chunkImageStorage) Close() error { return nil }

func mustEncodePNG(t *testing.T, rect image.Rectangle) []byte {
	t.Helper()

	img := image.NewRGBA(rect)
	for y := rect.Min.Y; y < rect.Max.Y; y++ {
		for x := rect.Min.X; x < rect.Max.X; x++ {
			img.Set(x, y, color.White)
		}
	}

	var buf bytes.Buffer
	if err := png.Encode(&buf, img); err != nil {
		t.Fatalf("encode png: %v", err)
	}
	return buf.Bytes()
}

type stubEmbeddingDriver struct {
	embeddings []models.EmbeddingData
	embedErr   error
}

func (d *stubEmbeddingDriver) NewInstance(map[string]string) models.ModelDriver { return d }
func (d *stubEmbeddingDriver) Name() string                                     { return "stub" }
func (d *stubEmbeddingDriver) ChatWithMessages(context.Context, string, []models.Message, *models.APIConfig, *models.ChatConfig, *common.ModelUsage) (*models.ChatResponse, error) {
	return nil, nil
}
func (d *stubEmbeddingDriver) ChatStreamlyWithSender(context.Context, string, []models.Message, *models.APIConfig, *models.ChatConfig, *common.ModelUsage, func(*string, *string) error) error {
	return nil
}
func (d *stubEmbeddingDriver) Embed(context.Context, *string, models.EmbedRequest, *models.APIConfig, *models.EmbeddingConfig, *common.ModelUsage) ([]models.EmbeddingData, error) {
	return d.embeddings, d.embedErr
}
func (d *stubEmbeddingDriver) Rerank(context.Context, *string, models.RerankRequest, *models.APIConfig, *models.RerankConfig, *common.ModelUsage) (*models.RerankResponse, error) {
	return nil, nil
}
func (d *stubEmbeddingDriver) TranscribeAudio(context.Context, *string, *string, *models.APIConfig, *models.ASRConfig, *common.ModelUsage) (*models.ASRResponse, error) {
	return nil, nil
}
func (d *stubEmbeddingDriver) TranscribeAudioWithSender(context.Context, *string, *string, *models.APIConfig, *models.ASRConfig, *common.ModelUsage, func(*string, *string) error) error {
	return nil
}
func (d *stubEmbeddingDriver) AudioSpeech(context.Context, *string, *string, *models.APIConfig, *models.TTSConfig, *common.ModelUsage) (*models.TTSResponse, error) {
	return nil, nil
}
func (d *stubEmbeddingDriver) AudioSpeechWithSender(context.Context, *string, *string, *models.APIConfig, *models.TTSConfig, *common.ModelUsage, func(*string, *string) error) error {
	return nil
}
func (d *stubEmbeddingDriver) OCRFile(context.Context, *string, []byte, *string, *models.APIConfig, *models.OCRConfig, *common.ModelUsage) (*models.OCRFileResponse, error) {
	return nil, nil
}
func (d *stubEmbeddingDriver) ParseFile(context.Context, *string, []byte, *string, *models.APIConfig, *models.ParseFileConfig, *common.ModelUsage) (*models.ParseFileResponse, error) {
	return nil, nil
}
func (d *stubEmbeddingDriver) ListModels(context.Context, *models.APIConfig) ([]models.ListModelResponse, error) {
	return nil, nil
}
func (d *stubEmbeddingDriver) Balance(context.Context, *models.APIConfig) (map[string]interface{}, error) {
	return nil, nil
}
func (d *stubEmbeddingDriver) CheckConnection(context.Context, *models.APIConfig) error { return nil }
func (d *stubEmbeddingDriver) ListTasks(context.Context, *models.APIConfig) ([]models.ListTaskStatus, error) {
	return nil, nil
}
func (d *stubEmbeddingDriver) ShowTask(context.Context, string, *models.APIConfig) (*models.TaskResponse, error) {
	return nil, nil
}
func (d *stubEmbeddingDriver) Decide(ctx context.Context, modelName *string, request *models.DecideRequest, apiConfig *models.APIConfig, modelUsage *common.ModelUsage) (*models.DecideResponse, error) {
	return nil, nil
}

func strPtr(v string) *string {
	return &v
}

func (e *parseTestDocEngine) KNNScores(context.Context, []map[string]interface{}, []float64, int) (map[string]interface{}, error) {
	return nil, nil
}

func (e *parseTestDocEngine) GetScores(map[string]interface{}) map[string]float64 {
	return nil
}

func (e *parseTestDocEngine) Ping(context.Context) error {
	return nil
}

func (e *parseTestDocEngine) Close() error {
	return nil
}

func (e *parseTestDocEngine) GetType() string {
	return "test"
}
func (e *parseTestDocEngine) SupportsPageRank() bool { return false }

func (e *parseTestDocEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []string, []map[string]interface{}, string) []string {
	return nil
}

func TestSwitchChunksUpdatesDocEngineWithAvailableInt(t *testing.T) {
	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
	if err != nil {
		t.Fatalf("failed to open sqlite: %v", err)
	}
	if err := db.AutoMigrate(&entity.UserTenant{}, &entity.Knowledgebase{}, &entity.Document{}); err != nil {
		t.Fatalf("failed to migrate sqlite: %v", err)
	}
	previousDB := dao.DB
	dao.DB = db
	t.Cleanup(func() { dao.DB = previousDB })

	valid := string(entity.StatusValid)
	if err := db.Create(&entity.UserTenant{
		ID:       "ut-1",
		UserID:   "user-1",
		TenantID: "tenant-1",
		Role:     "owner",
		Status:   &valid,
	}).Error; err != nil {
		t.Fatalf("failed to create user_tenant: %v", err)
	}
	if err := db.Create(&entity.Knowledgebase{
		ID:           "kb-1",
		TenantID:     "tenant-1",
		Name:         "dataset",
		EmbdID:       "embed",
		Permission:   string(entity.TenantPermissionTeam),
		CreatedBy:    "user-1",
		ParserID:     string(entity.ParserTypeNaive),
		ParserConfig: entity.JSONMap{},
		Status:       &valid,
	}).Error; err != nil {
		t.Fatalf("failed to create knowledgebase: %v", err)
	}
	if err := db.Create(&entity.Document{
		ID:           "doc-1",
		KbID:         "kb-1",
		ParserID:     string(entity.ParserTypeNaive),
		ParserConfig: entity.JSONMap{},
		SourceType:   "local",
		Type:         "doc",
		CreatedBy:    "user-1",
		Suffix:       "txt",
	}).Error; err != nil {
		t.Fatalf("failed to create document: %v", err)
	}

	engine := &switchChunksEngineMock{}
	svc := &ChunkService{
		docEngine: engine,
		kbDAO:     dao.NewKnowledgebaseDAO(),
	}

	ctx := chunkTestContext(t, "user-1", "tenant-1")
	if err = svc.SwitchChunks(ctx, "kb-1", "doc-1", 0, []string{"chunk-1", "chunk-2"}); err != nil {
		t.Fatalf("SwitchChunks() error = %v", err)
	}

	if len(engine.updateCalls) != 2 {
		t.Fatalf("UpdateChunks calls = %d, want 2", len(engine.updateCalls))
	}
	for i, call := range engine.updateCalls {
		if call.indexName != "ragflow_tenant-1" {
			t.Fatalf("call %d indexName = %q", i, call.indexName)
		}
		if call.datasetID != "kb-1" {
			t.Fatalf("call %d datasetID = %q", i, call.datasetID)
		}
		wantID := []string{"chunk-1", "chunk-2"}[i]
		if !reflect.DeepEqual(call.condition, map[string]interface{}{
			"id":     wantID,
			"doc_id": "doc-1",
		}) {
			t.Fatalf("call %d condition = %#v", i, call.condition)
		}
		if !reflect.DeepEqual(call.newValue, map[string]interface{}{"id": wantID, "available_int": 0}) {
			t.Fatalf("call %d newValue = %#v", i, call.newValue)
		}
	}
}

type updateChunksCall struct {
	condition map[string]interface{}
	newValue  map[string]interface{}
	indexName string
	datasetID string
}

type switchChunksEngineMock struct {
	updateCalls []updateChunksCall
}

func (m *switchChunksEngineMock) CreateChunkStore(context.Context, string, string, int, string, string) error {
	return nil
}
func (m *switchChunksEngineMock) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {
	return nil, nil
}
func (m *switchChunksEngineMock) UpdateChunks(_ context.Context, condition map[string]interface{}, newValue map[string]interface{}, indexName string, datasetID string) error {
	m.updateCalls = append(m.updateCalls, updateChunksCall{
		condition: copyMap(condition),
		newValue:  copyMap(newValue),
		indexName: indexName,
		datasetID: datasetID,
	})
	return nil
}
func (m *switchChunksEngineMock) DeleteChunks(context.Context, map[string]interface{}, string, string) (int64, error) {
	return 0, nil
}
func (m *switchChunksEngineMock) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {
	return nil, nil
}
func (m *switchChunksEngineMock) GetChunk(context.Context, string, string, []string) (interface{}, error) {
	return nil, nil
}
func (m *switchChunksEngineMock) DropChunkStore(context.Context, string, string) error { return nil }
func (m *switchChunksEngineMock) ChunkStoreExists(context.Context, string, string) (bool, error) {
	return false, nil
}
func (m *switchChunksEngineMock) CreateMetadataStore(context.Context, string) error { return nil }
func (m *switchChunksEngineMock) InsertMetadata(context.Context, []map[string]interface{}, string) ([]string, error) {
	return nil, nil
}
func (m *switchChunksEngineMock) UpdateMetadata(context.Context, string, string, map[string]interface{}, string) error {
	return nil
}
func (m *switchChunksEngineMock) DeleteMetadata(context.Context, map[string]interface{}, string) (int64, error) {
	return 0, nil
}
func (m *switchChunksEngineMock) DeleteMetadataKeys(context.Context, string, string, []string, string) error {
	return nil
}
func (m *switchChunksEngineMock) DropMetadataStore(context.Context, string) error { return nil }
func (m *switchChunksEngineMock) MetadataStoreExists(context.Context, string) (bool, error) {
	return false, nil
}
func (m *switchChunksEngineMock) SearchMetadata(context.Context, *types.SearchMetadataRequest) (*types.SearchMetadataResult, error) {
	return nil, nil
}
func (m *switchChunksEngineMock) IndexDocument(context.Context, string, string, interface{}) error {
	return nil
}
func (m *switchChunksEngineMock) DeleteDocument(context.Context, string, string) error { return nil }
func (m *switchChunksEngineMock) BulkIndex(context.Context, string, []interface{}) (interface{}, error) {
	return nil, nil
}
func (m *switchChunksEngineMock) GetFields([]map[string]interface{}, []string) map[string]map[string]interface{} {
	return nil
}
func (m *switchChunksEngineMock) GetAggregation([]map[string]interface{}, string) []map[string]interface{} {
	return nil
}
func (m *switchChunksEngineMock) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
	return nil
}
func (m *switchChunksEngineMock) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
	return nil, nil
}
func (m *switchChunksEngineMock) GetChunkIDs([]map[string]interface{}) []string { return nil }
func (m *switchChunksEngineMock) KNNScores(context.Context, []map[string]interface{}, []float64, int) (map[string]interface{}, error) {
	return nil, nil
}
func (m *switchChunksEngineMock) GetScores(map[string]interface{}) map[string]float64 { return nil }
func (m *switchChunksEngineMock) Ping(context.Context) error                          { return nil }
func (m *switchChunksEngineMock) Close() error                                        { return nil }
func (m *switchChunksEngineMock) GetType() string                                     { return "elasticsearch" }
func (m *switchChunksEngineMock) SupportsPageRank() bool                              { return true }
func (m *switchChunksEngineMock) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []string, []map[string]interface{}, string) []string {
	return nil
}

func copyMap(in map[string]interface{}) map[string]interface{} {
	out := make(map[string]interface{}, len(in))
	for k, v := range in {
		out[k] = v
	}
	return out
}

func TestChunkServiceParse_RejectsBatchWithRunningIngestionTask(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestUserTenant(t, "user-1", "tenant-1")
	insertChunkTestKB(t, "kb-1", "tenant-1")
	insertChunkTestDoc(t, "doc-1", "kb-1")
	insertChunkTestDoc(t, "doc-2", "kb-1")
	insertChunkTestIngestionTask(t, "task-2", "user-1", "doc-2", "kb-1", common.RUNNING)

	svc := newParseTestService(t)
	svc.checkDatasetAccessFunc = func(string) error { return nil }
	ctx := t.Context()
	_, _, err := svc.Parse(ctx, "kb-1", &service.ParseFileRequest{DocumentIDs: []string{"doc-1", "doc-2"}})
	if err == nil {
		t.Fatal("expected error for RUNNING ingestion task, got nil")
	}
}

func TestChunkServiceParse_CallsStartParseDocumentsWithRerunWithDelete(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestUserTenant(t, "user-1", "tenant-1")
	insertChunkTestKB(t, "kb-1", "tenant-1")
	insertChunkTestDoc(t, "doc-1", "kb-1")

	svc := newParseTestService(t)
	svc.checkDatasetAccessFunc = func(string) error { return nil }
	var calledOpts document.StartParseOptions
	var calledDocID string
	svc.startParseDocumentsFunc = func(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, opts document.StartParseOptions) error {
		calledOpts = opts
		calledDocID = doc.ID
		return nil
	}
	ctx := t.Context()
	_, code, err := svc.Parse(ctx, "kb-1", &service.ParseFileRequest{DocumentIDs: []string{"doc-1"}})
	if err != nil {
		t.Fatalf("Parse: %v", err)
	}
	if code != common.CodeSuccess {
		t.Fatalf("code = %d, want success", code)
	}
	if calledDocID != "doc-1" {
		t.Fatalf("startParseDocuments called with doc %s, want doc-1", calledDocID)
	}
	if !calledOpts.RerunWithDelete {
		t.Fatal("expected RerunWithDelete=true")
	}
}

func TestChunkServiceParse_ReturnsPartialSuccessForDuplicateDocumentIDs(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	userID, datasetID := "user-1", "kb-1"
	insertChunkTestKB(t, datasetID, userID)
	insertChunkTestDoc(t, "doc-1", datasetID)

	svc := newParseTestService(t)
	svc.startParseDocumentsFunc = func(context.Context, *entity.Document, *entity.Knowledgebase, document.StartParseOptions) error {
		return nil
	}

	ctx := chunkTestContext(t, userID, userID)
	result, code, err := svc.Parse(ctx, datasetID, &service.ParseFileRequest{
		DocumentIDs: []string{"doc-1", "doc-1"},
	})
	if err == nil {
		t.Fatal("expected duplicate warning error")
	}
	if code != common.CodeSuccess {
		t.Fatalf("expected CodeSuccess, got %v", code)
	}
	if result["success_count"] != 1 {
		t.Fatalf("expected one successful parse, got %v", result["success_count"])
	}
	errorsValue, ok := result["errors"].([]string)
	if !ok || len(errorsValue) != 1 || !strings.Contains(errorsValue[0], "Duplicate document ids: doc-1") {
		t.Fatalf("unexpected duplicate errors: %#v", result["errors"])
	}
}

func TestStopParsing_CallsCancelIngestionTask(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestKB(t, "kb-1", "user-1") // owner = user-1, Accessible passes
	insertChunkTestDoc(t, "doc-1", "kb-1")

	svc := newParseTestService(t)
	var calledDocID string
	svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error {
		calledDocID = doc.ID
		return nil
	}
	ctx := chunkTestContext(t, "user-1", "user-1")
	_, _, err := svc.StopParsing(ctx, "kb-1", service.StopParsingRequest{DocumentIDs: []string{"doc-1"}})
	if err != nil {
		t.Fatalf("StopParsing: %v", err)
	}
	if calledDocID != "doc-1" {
		t.Fatalf("cancelIngestionTaskFunc called with doc %s, want doc-1", calledDocID)
	}
}

func TestStopParsing_RejectsDocumentWithoutIngestionTask(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestKB(t, "kb-1", "user-1")
	insertChunkTestDoc(t, "doc-1", "kb-1")
	// No IngestionTask — should be rejected (nothing to stop).

	svc := newParseTestService(t)
	var called bool
	svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error {
		called = true
		return document.ErrParseNotRunning
	}
	ctx := chunkTestContext(t, "user-1", "user-1")
	_, code, err := svc.StopParsing(ctx, "kb-1", service.StopParsingRequest{DocumentIDs: []string{"doc-1"}})
	if err == nil {
		t.Fatal("expected error for document without ingestion task")
	}
	if code != common.CodeDataError {
		t.Fatalf("expected CodeDataError, got %v", code)
	}
	if !called {
		t.Fatal("StopParsing must delegate task-state validation to CancelDocParse")
	}
}

func TestStopParsing_DoesNotDeleteChunksOrResetCountersAfterCancel(t *testing.T) {
	db := setupChunkTestDB(t)
	pushChunkTestDB(t, db)
	insertChunkTestKB(t, "kb-1", "user-1")
	insertChunkTestDoc(t, "doc-1", "kb-1")
	// Pre-set counters on the doc so we can verify they are NOT reset.
	if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
		"token_num": int64(10),
		"chunk_num": int64(5),
	}).Error; err != nil {
		t.Fatalf("set counters: %v", err)
	}
	insertChunkTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)

	svc := newParseTestService(t)
	svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error {
		// Simulate CancelDocParse: stop task.
		return dao.DB.Model(&entity.IngestionTask{}).Where("document_id = ?", doc.ID).
			Update("status", common.STOPPED).Error
	}
	engine := &parseTestDocEngine{chunkStoreExists: true}
	svc.docEngine = engine
	ctx := chunkTestContext(t, "user-1", "user-1")
	_, _, err := svc.StopParsing(ctx, "kb-1", service.StopParsingRequest{DocumentIDs: []string{"doc-1"}})
	if err != nil {
		t.Fatalf("StopParsing: %v", err)
	}

	// Counters must NOT be reset.
	doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
	if err != nil {
		t.Fatalf("reload doc: %v", err)
	}
	if doc.TokenNum != 10 {
		t.Fatalf("token_num = %d, want 10 (not reset)", doc.TokenNum)
	}
	if doc.ChunkNum != 5 {
		t.Fatalf("chunk_num = %d, want 5 (not reset)", doc.ChunkNum)
	}

	// Chunks must NOT be deleted during cancel.
	if engine.deleteChunksCalls != 0 {
		t.Fatalf("DeleteChunks called %d times, want 0", engine.deleteChunksCalls)
	}
}