RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
package chunk
import (
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"image"
"image/color"
_ "image/jpeg"
"image/png"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/dao/transaction"
"ragflow/internal/engine/types"
"ragflow/internal/entity"
"ragflow/internal/entity/models"
"ragflow/internal/permission"
"ragflow/internal/permission/testsupport"
"ragflow/internal/service"
"ragflow/internal/service/document"
"ragflow/internal/storage"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
func TestIsZeroVector(t *testing.T) {
if !common.IsZeroVector([]float64{0, 0, 0}) {
t.Error("all zeros should be true")
}
if common.IsZeroVector([]float64{0, 1, 0}) {
t.Error("non-zero should be false")
}
if !common.IsZeroVector([]float64{}) {
t.Error("empty should be true (treated as zero)")
}
if !common.IsZeroVector(nil) {
t.Error("nil should be true")
}
}
func TestHydrateChunkVectors_AllNonZero(t *testing.T) {
chunks := []map[string]interface{}{
{"id": "c1", "vector": []float64{1, 2, 3}},
{"id": "c2", "vector": []float64{4, 5, 6}},
}
// No zero vectors → nothing to hydrate.
hydrateChunkVectors(t.Context(), nil, chunks, nil, nil)
if !reflect.DeepEqual(chunks[0]["vector"], []float64{1, 2, 3}) {
t.Error("non-zero vector should not be changed")
}
if !reflect.DeepEqual(chunks[1]["vector"], []float64{4, 5, 6}) {
t.Error("non-zero vector should not be changed")
}
}
func TestHydrateChunkVectors_EmptyChunks(t *testing.T) {
// Should not panic on empty or nil.
hydrateChunkVectors(t.Context(), nil, nil, nil, nil)
hydrateChunkVectors(t.Context(), nil, []map[string]interface{}{}, nil, nil)
}
func TestHydrateChunkVectors_MissingIDs(t *testing.T) {
chunks := []map[string]interface{}{
{"vector": []float64{1.0}}, // no id — skipped
}
hydrateChunkVectors(t.Context(), nil, chunks, nil, nil)
// Should not change anything when engine is nil (FetchChunkVectors returns zero vectors).
// The function doesn't panic — it just can't hydrate because dim is 0.
// With nil engine, FetchChunkVectors returns zero vectors, so the zero stays zero.
}
func TestHydrateChunkVectors_NoDim(t *testing.T) {
chunks := []map[string]interface{}{
{"id": "c1", "vector": []float64{}},
}
hydrateChunkVectors(t.Context(), nil, chunks, []string{"kb1"}, []string{"t1"})
// Empty vectors have dim=0 → early return. No crash.
}
func TestKnowledgebaseEmbeddingKey(t *testing.T) {
tenantEmbdID := "42"
staleTenantEmbdID := "stale-id"
tests := []struct {
name string
kb *entity.Knowledgebase
tenantID string
want string
}{
{
name: "resolves tenant embedding id to base model name",
kb: &entity.Knowledgebase{
EmbdID: "shared-model",
TenantEmbdID: &tenantEmbdID,
},
want: "embd:BAAI/bge-m3",
},
{
name: "stale tenant embedding id falls back to composite base name",
kb: &entity.Knowledgebase{
EmbdID: "BAAI/bge-m3@1@SILICONFLOW",
TenantEmbdID: &staleTenantEmbdID,
},
want: "embd:BAAI/bge-m3",
},
{
name: "uses embd id without tenant embedding id",
kb: &entity.Knowledgebase{
EmbdID: "shared-model",
},
want: "embd:shared-model",
},
{
name: "uses tenant default when embedding id is empty",
kb: &entity.Knowledgebase{},
tenantID: "tenant-1",
want: "default:tenant-1",
},
{
name: "ignores non-positive tenant embedding id",
kb: &entity.Knowledgebase{
EmbdID: "shared-model",
TenantEmbdID: func() *string { s := ""; return &s }(),
},
want: "embd:shared-model",
},
{
name: "legacy composite reduces to base model name",
kb: &entity.Knowledgebase{
EmbdID: "BAAI/bge-m3@2@SILICONFLOW",
},
want: "embd:BAAI/bge-m3",
},
}
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
if err != nil {
t.Fatalf("failed to open sqlite: %v", err)
}
if err := db.AutoMigrate(&entity.TenantModel{}); err != nil {
t.Fatalf("failed to migrate test schema: %v", err)
}
if err := db.Create(&entity.TenantModel{ID: "42", ModelName: "BAAI/bge-m3"}).Error; err != nil {
t.Fatalf("failed to seed tenant_model: %v", err)
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
names := transaction.NewEmbeddingBaseNameResolver(dao.NewKnowledgebaseDAO().EmbeddingBaseName)
if got := knowledgebaseEmbeddingKey(t.Context(), db, tt.kb, tt.tenantID, names); got != tt.want {
t.Fatalf("knowledgebaseEmbeddingKey() = %q, want %q", got, tt.want)
}
})
}
}
func TestParsePrevalidatesDocumentsBeforeMutating(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID := "user-1"
datasetID := "kb-1"
insertChunkTestKB(t, datasetID, userID)
insertChunkTestKB(t, "kb-2", userID)
insertChunkTestDoc(t, "doc-1", datasetID)
insertChunkTestDoc(t, "doc-2", "kb-2")
insertChunkTestTask(t, "task-1", "doc-1")
svc := &ChunkService{
docEngine: nil,
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
}
ctx := chunkTestContext(t, userID, userID)
_, code, err := svc.Parse(ctx, datasetID, &service.ParseFileRequest{
DocumentIDs: []string{"doc-1", "missing-doc", "doc-2"},
})
if err == nil {
t.Fatal("expected parse to fail")
}
if code != common.CodeDataError {
t.Fatalf("expected CodeDataError, got %v", code)
}
if !strings.Contains(err.Error(), "missing-doc") || !strings.Contains(err.Error(), "doc-2") {
t.Fatalf("expected missing and foreign documents in error, got %q", err.Error())
}
var taskCount int64
if err = db.Model(&entity.Task{}).Where("doc_id = ?", "doc-1").Count(&taskCount).Error; err != nil {
t.Fatalf("count tasks: %v", err)
}
if taskCount != 1 {
t.Fatalf("expected existing task to remain, got %d tasks", taskCount)
}
doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("get doc: %v", err)
}
if doc.ChunkNum != 7 {
t.Fatalf("expected chunk_num to remain 7, got %d", doc.ChunkNum)
}
}
func TestParseRejectsInaccessibleDataset(t *testing.T) {
svc := newParseTestService(t)
svc.checkDatasetAccessFunc = func(string) error { return permission.ErrPermissionDenied }
ctx := t.Context()
_, code, err := svc.Parse(ctx, "kb-1", &service.ParseFileRequest{DocumentIDs: []string{"doc-1"}})
if err == nil {
t.Fatal("expected parse to fail")
}
if code != common.CodeForbidden {
t.Fatalf("expected CodeForbidden, got %v", code)
}
if err.Error() != "Permission denied" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseRequiresDocumentIDs(t *testing.T) {
svc := newParseTestService(t)
svc.checkDatasetAccessFunc = func(string) error { return nil }
ctx := t.Context()
for _, req := range []*service.ParseFileRequest{nil, {DocumentIDs: nil}, {DocumentIDs: []string{}}} {
_, code, err := svc.Parse(ctx, "kb-1", req)
if err == nil {
t.Fatal("expected parse to fail")
}
if code != common.CodeDataError {
t.Fatalf("expected CodeDataError, got %v", code)
}
if !strings.Contains(err.Error(), "document_ids") {
t.Fatalf("unexpected error: %v", err)
}
}
}
func TestParseReturnsDataErrorWhenDatasetMissing(t *testing.T) {
svc := newParseTestService(t)
svc.checkDatasetAccessFunc = func(string) error { return nil }
svc.getKnowledgebaseByIDFunc = func(string) (*entity.Knowledgebase, error) { return nil, nil }
ctx := t.Context()
_, code, err := svc.Parse(ctx, "kb-1", &service.ParseFileRequest{DocumentIDs: []string{"doc-1"}})
if err == nil {
t.Fatal("expected parse to fail")
}
if code != common.CodeDataError {
t.Fatalf("expected CodeDataError, got %v", code)
}
if !strings.Contains(err.Error(), "dataset not found") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestParseReturnsServerErrorWhenDocumentsQueryFails(t *testing.T) {
svc := newParseTestService(t)
queryErr := errors.New("documents query failed")
svc.checkDatasetAccessFunc = func(string) error { return nil }
svc.getKnowledgebaseByIDFunc = func(string) (*entity.Knowledgebase, error) {
return &entity.Knowledgebase{ID: "kb-1", TenantID: "user-1"}, nil
}
svc.getDocumentsByIDsFunc = func([]string) ([]*entity.Document, error) {
return nil, queryErr
}
ctx := t.Context()
_, code, err := svc.Parse(ctx, "kb-1", &service.ParseFileRequest{DocumentIDs: []string{"doc-1"}})
if !errors.Is(err, queryErr) {
t.Fatalf("expected query error, got %v", err)
}
if code != common.CodeServerError {
t.Fatalf("expected CodeServerError, got %v", code)
}
}
func TestParseRejectsRunningDocument(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID := "user-1"
datasetID := "kb-1"
insertChunkTestKB(t, datasetID, userID)
insertChunkTestDoc(t, "doc-1", datasetID)
insertChunkTestIngestionTask(t, "task-1", userID, "doc-1", datasetID, common.RUNNING)
svc := newParseTestService(t)
ctx := chunkTestContext(t, userID, userID)
_, code, err := svc.Parse(ctx, datasetID, &service.ParseFileRequest{DocumentIDs: []string{"doc-1"}})
if err == nil {
t.Fatal("expected parse to fail")
}
if code != common.CodeDataError {
t.Fatalf("expected CodeDataError, got %v", code)
}
if !strings.Contains(err.Error(), "ingestion task task-1 is RUNNING") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestListSortsChunksByDocumentPosition(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID := "user-1"
tenantID := "tenant-1"
datasetID := "kb-1"
documentID := "doc-1"
insertChunkTestUserTenant(t, userID, tenantID)
insertChunkTestKB(t, datasetID, tenantID)
insertChunkTestDoc(t, documentID, datasetID)
engine := &listChunksSearchEngine{}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
}
ctx := chunkTestContext(t, userID, tenantID)
page := 1
size := 30
resp, err := svc.List(ctx, &service.ListChunksRequest{
DatasetID: datasetID,
DocID: documentID,
Page: &page,
Size: &size,
})
if err != nil {
t.Fatalf("List() error = %v", err)
}
if len(resp.Chunks) != 1 {
t.Fatalf("chunks = %v, want one chunk", resp.Chunks)
}
if _, ok := resp.Chunks[0]["available_int"]; ok {
t.Fatal("available_int should not be returned in chunk lists")
}
// Stored available_int (0 here) is surfaced as the `available` boolean.
if got := resp.Chunks[0]["available"]; got != false {
t.Fatalf("available = %#v, want false for a chunk stored with available_int=0", got)
}
if engine.searchReq == nil {
t.Fatal("expected Search to be called")
}
if !engine.searchReq.IncludeUnavailable {
t.Fatal("management chunk list must include disabled parent chunks")
}
// Parent rows must not be filtered out: they are listed after their children
// on every engine (MaterializeParentChunks numbers them past the source
// chunks), and excluding them by doc_type_kwd made the list engine-agnostic
// but no longer showed the parent-child structure users see on ES.
if _, exists := engine.searchReq.Filter["exists"]; exists {
t.Fatalf("unexpected existence filter on the chunk list: %#v", engine.searchReq.Filter)
}
if engine.searchReq.OrderBy == nil {
t.Fatal("expected OrderBy to be set")
}
want := []types.OrderByField{
{Field: "chunk_order_int", Type: types.SortAsc},
{Field: "page_num_int", Type: types.SortAsc},
{Field: "top_int", Type: types.SortAsc},
{Field: "create_timestamp_flt", Type: types.SortDesc},
}
if !reflect.DeepEqual(engine.searchReq.OrderBy.Fields, want) {
t.Fatalf("OrderBy fields = %#v, want %#v", engine.searchReq.OrderBy.Fields, want)
}
}
// TestListReportsChunkAvailability pins that stored available_int is surfaced as
// the `available` boolean (0 → false, missing → true).
func TestListReportsChunkAvailability(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID := "user-1"
tenantID := "tenant-1"
datasetID := "kb-1"
documentID := "doc-1"
insertChunkTestUserTenant(t, userID, tenantID)
insertChunkTestKB(t, datasetID, tenantID)
insertChunkTestDoc(t, documentID, datasetID)
engine := &listChunksSearchEngine{
chunks: []map[string]interface{}{
{"id": "parent-1", "content_with_weight": "parent", "available_int": 0, "doc_id": documentID},
{"id": "child-1", "content_with_weight": "child", "doc_id": documentID},
},
}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
}
page := 1
size := 30
resp, err := svc.List(chunkTestContext(t, userID, tenantID), &service.ListChunksRequest{
DatasetID: datasetID,
DocID: documentID,
Page: &page,
Size: &size,
})
if err != nil {
t.Fatalf("List() error = %v", err)
}
if len(resp.Chunks) != 2 {
t.Fatalf("chunks = %v, want two chunks", resp.Chunks)
}
if got := resp.Chunks[0]["available"]; got != false {
t.Fatalf("parent available = %#v, want false (stored available_int=0)", got)
}
if got := resp.Chunks[1]["available"]; got != true {
t.Fatalf("child available = %#v, want true (column absent)", got)
}
for i, chunk := range resp.Chunks {
if _, ok := chunk["available_int"]; ok {
t.Fatalf("chunk %d leaks the raw available_int column: %#v", i, chunk)
}
}
}
// TestListScopesSearchToDocumentDataset pins the dataset scope of the
// management chunk list: only the document's own KB may be searched. Passing
// every KB of the tenant made a document of dataset A list chunks that were
// written under dataset B's KB context (chunk rows carry img_id
// "<kb_id>-<chunk_id>"), which the UI surfaced as "occasionally seeing another
// dataset's chunks".
func TestListScopesSearchToDocumentDataset(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID := "user-1"
tenantID := "tenant-1"
datasetID := "kb-1"
otherDatasetID := "kb-2"
documentID := "doc-1"
insertChunkTestUserTenant(t, userID, tenantID)
insertChunkTestKB(t, datasetID, tenantID)
insertChunkTestKB(t, otherDatasetID, tenantID)
insertChunkTestDoc(t, documentID, datasetID)
engine := &listChunksSearchEngine{}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
}
ctx := chunkTestContext(t, userID, tenantID)
page := 1
size := 30
if _, err := svc.List(ctx, &service.ListChunksRequest{
DatasetID: datasetID,
DocID: documentID,
Page: &page,
Size: &size,
}); err != nil {
t.Fatalf("List() error = %v", err)
}
if engine.searchReq == nil {
t.Fatal("expected Search to be called")
}
if !reflect.DeepEqual(engine.searchReq.KbIDs, []string{datasetID}) {
t.Fatalf("KbIDs = %#v, want the document's dataset only (%q)", engine.searchReq.KbIDs, datasetID)
}
}
// TestGetScopesChunkLookupToDocumentDataset pins the detail endpoint's scope:
// the chunk is read from the dataset named in the route only, never from every
// KB of the tenant.
func TestGetScopesChunkLookupToDocumentDataset(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID := "user-1"
tenantID := "tenant-1"
datasetID := "kb-1"
otherDatasetID := "kb-2"
documentID := "doc-1"
insertChunkTestUserTenant(t, userID, tenantID)
insertChunkTestKB(t, datasetID, tenantID)
insertChunkTestKB(t, otherDatasetID, tenantID)
insertChunkTestDoc(t, documentID, datasetID)
engine := &getChunkTestEngine{chunk: map[string]interface{}{
"id": "chunk-1",
"doc_id": documentID,
"kb_id": datasetID,
"docnm_kwd": "doc.txt",
"content_with_weight": "invoice body",
"content_ltks": []string{"invoice", "body"},
"available_int": 0,
}}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
}
resp, err := svc.Get(chunkTestContext(t, userID, tenantID), &service.GetChunkRequest{
DatasetID: datasetID,
DocumentID: documentID,
ChunkID: "chunk-1",
})
if err != nil {
t.Fatalf("Get() error = %v", err)
}
if !reflect.DeepEqual(engine.datasetIDs, []string{datasetID}) {
t.Fatalf("datasetIDs = %#v, want the route's dataset only (%q)", engine.datasetIDs, datasetID)
}
if engine.chunkID != "chunk-1" {
t.Fatalf("chunkID = %q, want chunk-1", engine.chunkID)
}
if engine.indexName != "ragflow_"+tenantID {
t.Fatalf("indexName = %q, want ragflow_%s", engine.indexName, tenantID)
}
if got := resp.Chunk["content_with_weight"]; got != "invoice body" {
t.Fatalf("content_with_weight = %#v, want invoice body", got)
}
// Python returns the stored row (minus runtime fields), so the row identity
// fields survive the round trip.
if got := resp.Chunk["id"]; got != "chunk-1" {
t.Fatalf("id = %#v, want chunk-1", got)
}
if got := resp.Chunk["docnm_kwd"]; got != "doc.txt" {
t.Fatalf("docnm_kwd = %#v, want doc.txt", got)
}
if _, ok := resp.Chunk["content_ltks"]; ok {
t.Fatal("content_ltks should be stripped like Python's _strip_chunk_runtime_fields")
}
if got := resp.Chunk["available_int"]; got != 0 {
t.Fatalf("available_int = %v, want stored availability 0", got)
}
if engine.chunk["available_int"] != 0 {
t.Fatal("stored availability should remain unchanged")
}
}
// TestGetRejectsChunkOfAnotherDocument pins the doc_id guard: holding a chunk id
// is not authority to read it through a different document.
func TestGetRejectsChunkOfAnotherDocument(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID := "user-1"
tenantID := "tenant-1"
datasetID := "kb-1"
documentID := "doc-1"
insertChunkTestUserTenant(t, userID, tenantID)
insertChunkTestKB(t, datasetID, tenantID)
insertChunkTestDoc(t, documentID, datasetID)
engine := &getChunkTestEngine{chunk: map[string]interface{}{
"doc_id": "doc-2",
"content_with_weight": "other document body",
}}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
}
_, err := svc.Get(chunkTestContext(t, userID, tenantID), &service.GetChunkRequest{
DatasetID: datasetID,
DocumentID: documentID,
ChunkID: "chunk-1",
})
if err == nil || !strings.Contains(err.Error(), "chunk not found") {
t.Fatalf("Get() error = %v, want chunk not found", err)
}
}
// TestGetRequiresDocumentInRouteDataset pins the document guard: the chunk is
// only readable through a document that exists in the route's dataset,
// mirroring Python's DocumentService.query(id=document_id, kb_id=dataset_id).
func TestGetRequiresDocumentInRouteDataset(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID := "user-1"
tenantID := "tenant-1"
datasetID := "kb-1"
otherDatasetID := "kb-2"
insertChunkTestUserTenant(t, userID, tenantID)
insertChunkTestKB(t, datasetID, tenantID)
insertChunkTestKB(t, otherDatasetID, tenantID)
insertChunkTestDoc(t, "doc-2", otherDatasetID)
engine := &getChunkTestEngine{chunk: map[string]interface{}{
"doc_id": "doc-2",
"content_with_weight": "other dataset document body",
}}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
}
_, err := svc.Get(chunkTestContext(t, userID, tenantID), &service.GetChunkRequest{
DatasetID: datasetID,
DocumentID: "doc-missing",
ChunkID: "chunk-1",
})
if err == nil || !strings.Contains(err.Error(), "document not found") {
t.Fatalf("Get() error = %v, want document not found", err)
}
_, err = svc.Get(chunkTestContext(t, userID, tenantID), &service.GetChunkRequest{
DatasetID: datasetID,
DocumentID: "doc-2",
ChunkID: "chunk-1",
})
if err == nil || !strings.Contains(err.Error(), "document does not belong to this dataset") {
t.Fatalf("Get() error = %v, want document does not belong to this dataset", err)
}
}
// TestIsChunkRuntimeField checks that tokenized/vector fields are dropped while
// other stored fields survive. Only _vec is end-anchored.
func TestIsChunkRuntimeField(t *testing.T) {
cases := []struct {
name string
want bool
}{
{"content_ltks", true},
{"content_sm_ltks", true},
{"tokenized_content_ltks", true},
{"title_tks", true},
{"title_sm_tks", true},
{"authors_sm_tks", true},
{"important_tks", true},
{"question_tks", true},
{"name_tks", true},
{"tags_tks", true},
{"q_1024_vec", true},
{"q_2_vec", true},
// Only _vec is anchored at the end, so an index-shaped name is kept.
{"q_2_vec_idx", false},
{"content_with_weight", false},
{"content", false},
{"docnm_kwd", false},
{"doc_id", false},
{"kb_id", false},
{"important_kwd", false},
{"question_kwd", false},
{"compile_kwd", false},
{"tag_kwd", false},
{"tag_feas", false},
{"mom_id", false},
{"position_int", false},
{"img_id", false},
{"available_int", false},
}
for _, tc := range cases {
if got := isChunkRuntimeField(tc.name); got != tc.want {
t.Errorf("isChunkRuntimeField(%q) = %v, want %v", tc.name, got, tc.want)
}
}
}
func TestListBuildsMatchTextExprForKeywords(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID := "user-1"
tenantID := "tenant-1"
datasetID := "kb-1"
documentID := "doc-1"
insertChunkTestUserTenant(t, userID, tenantID)
insertChunkTestKB(t, datasetID, tenantID)
insertChunkTestDoc(t, documentID, datasetID)
engine := &listChunksSearchEngine{}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
}
ctx := chunkTestContext(t, userID, tenantID)
page := 2
size := 5
resp, err := svc.List(ctx, &service.ListChunksRequest{
DatasetID: datasetID,
DocID: documentID,
ChunkIDs: []string{"chunk-1", "chunk-2"},
Page: &page,
Size: &size,
Keywords: " invoice terms ",
})
if err != nil {
t.Fatalf("List() error = %v", err)
}
if resp.Total != 1 {
t.Fatalf("total = %d, want 1", resp.Total)
}
if engine.searchReq == nil {
t.Fatal("expected Search to be called")
}
if engine.searchReq.Offset != 5 || engine.searchReq.Limit != 5 {
t.Fatalf("pagination = offset %d limit %d, want offset 5 limit 5", engine.searchReq.Offset, engine.searchReq.Limit)
}
if !reflect.DeepEqual(engine.searchReq.KbIDs, []string{datasetID}) {
t.Fatalf("KbIDs = %#v, want %#v", engine.searchReq.KbIDs, []string{datasetID})
}
if got := engine.searchReq.Filter["doc_id"]; got != documentID {
t.Fatalf("doc_id filter = %#v, want %q", got, documentID)
}
if got := engine.searchReq.Filter["id"]; !reflect.DeepEqual(got, []string{"chunk-1", "chunk-2"}) {
t.Fatalf("id filter = %#v, want %#v", got, []string{"chunk-1", "chunk-2"})
}
mustNot, ok := engine.searchReq.Filter["must_not"].(map[string]interface{})
if !ok || mustNot["exists"] != "compile_kwd" {
t.Fatalf("must_not filter = %#v, want compile_kwd existence exclusion", engine.searchReq.Filter["must_not"])
}
if slices.Contains(engine.searchReq.SelectFields, "content") {
t.Fatalf("SelectFields = %#v, should not request content with content_with_weight", engine.searchReq.SelectFields)
}
for _, field := range []string{"content_with_weight", "img_id", "position_int"} {
if !slices.Contains(engine.searchReq.SelectFields, field) {
t.Fatalf("SelectFields = %#v, missing %q", engine.searchReq.SelectFields, field)
}
}
if got := resp.Chunks[0]["content_with_weight"]; got != "weighted invoice terms body" {
t.Fatalf("formatted content_with_weight = %#v, want %q", got, "weighted invoice terms body")
}
if len(engine.searchReq.MatchExprs) != 1 {
t.Fatalf("MatchExprs length = %d, want 1", len(engine.searchReq.MatchExprs))
}
matchText, ok := engine.searchReq.MatchExprs[0].(*types.MatchTextExpr)
if !ok {
t.Fatalf("MatchExprs[0] = %T, want *types.MatchTextExpr", engine.searchReq.MatchExprs[0])
}
if matchText.MatchingText == "invoice terms" || !strings.Contains(matchText.MatchingText, "invoice") || !strings.Contains(matchText.MatchingText, "terms") {
t.Fatalf("MatchingText = %q, want a tokenized query containing invoice and terms", matchText.MatchingText)
}
if matchText.TopN != 100 {
t.Fatalf("TopN = %d, want 100", matchText.TopN)
}
if got := matchText.ExtraOptions["original_query"]; got != "invoice terms" {
t.Fatalf("original_query = %#v, want %q", got, "invoice terms")
}
if got := matchText.ExtraOptions["minimum_should_match"]; got != 0.3 {
t.Fatalf("minimum_should_match = %#v, want 0.3", got)
}
}
func TestUpdateChunkRejectsChunkFromAnotherDocument(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-a", "kb-1")
insertChunkTestDoc(t, "doc-b", "kb-1")
engine := &updateChunkTestEngine{
existingChunk: map[string]interface{}{"doc_id": "doc-b"},
}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
}
err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
DatasetID: "kb-1",
DocumentID: "doc-a",
ChunkID: "chunk-1",
})
if err == nil {
t.Fatal("expected UpdateChunk to reject a chunk from another document")
}
if !strings.Contains(err.Error(), "chunk not found") {
t.Fatalf("UpdateChunk error = %q, want chunk not found", err)
}
if len(engine.updateCalls) != 0 {
t.Fatalf("UpdateChunks calls = %d, want 0", len(engine.updateCalls))
}
}
func TestUpdateChunkUpdatesSameDocumentWithDocumentCondition(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-a", "kb-1")
engine := &updateChunkTestEngine{
existingChunk: map[string]interface{}{
"doc_id": "doc-a",
"content_with_weight": "existing content",
},
}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
}
err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
DatasetID: "kb-1",
DocumentID: "doc-a",
ChunkID: "chunk-1",
})
if err != nil {
t.Fatalf("UpdateChunk() error = %v", err)
}
if len(engine.updateCalls) != 1 {
t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
}
call := engine.updateCalls[0]
if !reflect.DeepEqual(call.condition, map[string]interface{}{
"id": "chunk-1",
"doc_id": "doc-a",
}) {
t.Fatalf("UpdateChunks condition = %#v", call.condition)
}
if call.indexName != "ragflow_tenant-1" || call.datasetID != "kb-1" {
t.Fatalf("UpdateChunks target index=%q dataset=%q", call.indexName, call.datasetID)
}
}
func TestUpdateChunkStoresTagKwdOnlyWhenSent(t *testing.T) {
tests := []struct {
name string
tagKwd []string
want []string
present bool
}{
{name: "tags sent", tagKwd: []string{"alpha", "beta"}, want: []string{"alpha", "beta"}, present: true},
{name: "empty list clears tags", tagKwd: []string{}, want: []string{}, present: true},
{name: "absent leaves tags untouched", tagKwd: nil, present: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-a", "kb-1")
engine := &updateChunkTestEngine{
existingChunk: map[string]interface{}{
"doc_id": "doc-a",
"content_with_weight": "existing content",
},
}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
}
err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
DatasetID: "kb-1",
DocumentID: "doc-a",
ChunkID: "chunk-1",
TagKwd: tt.tagKwd,
})
if err != nil {
t.Fatalf("UpdateChunk() error = %v", err)
}
if len(engine.updateCalls) != 1 {
t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
}
got, ok := engine.updateCalls[0].newValue["tag_kwd"]
if ok != tt.present {
t.Fatalf("tag_kwd present = %v, want %v (newValue=%#v)", ok, tt.present, engine.updateCalls[0].newValue)
}
if tt.present && !reflect.DeepEqual(got, tt.want) {
t.Fatalf("tag_kwd = %#v, want %#v", got, tt.want)
}
})
}
}
func TestUpdateChunkStoresImageAndFlagsImageChunk(t *testing.T) {
cases := []struct {
name string
mode *string
wantMode string
}{
{"mode defaults to append", nil, imageUpdateModeAppend},
{"mode is normalized", strPtr(" REPLACE "), imageUpdateModeReplace},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-a", "kb-1")
engine := &updateChunkTestEngine{
existingChunk: map[string]interface{}{
"doc_id": "doc-a",
"content_with_weight": "existing content",
},
}
var storedBucket, storedChunkID, storedMode string
var storedBinary []byte
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
storeChunkImageFunc: func(bucket, chunkID string, imageBinary []byte, mode string) error {
storedBucket, storedChunkID, storedMode, storedBinary = bucket, chunkID, mode, imageBinary
return nil
},
}
imageBase64 := base64.StdEncoding.EncodeToString([]byte("image-bytes"))
err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
DatasetID: "kb-1",
DocumentID: "doc-a",
ChunkID: "chunk-1",
ImageBase64: &imageBase64,
ImageUpdateMode: tc.mode,
})
if err != nil {
t.Fatalf("UpdateChunk() error = %v", err)
}
if storedBucket != "kb-1" || storedChunkID != "chunk-1" || storedMode != tc.wantMode {
t.Fatalf("store args = %q/%q/%q, want kb-1/chunk-1/%s", storedBucket, storedChunkID, storedMode, tc.wantMode)
}
if string(storedBinary) != "image-bytes" {
t.Fatalf("stored binary = %q, want image-bytes", storedBinary)
}
if len(engine.updateCalls) != 1 {
t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
}
updated := engine.updateCalls[0].newValue
if updated["img_id"] != "kb-1-chunk-1" {
t.Fatalf("img_id = %#v, want kb-1-chunk-1", updated["img_id"])
}
if updated["doc_type_kwd"] != "image" {
t.Fatalf("doc_type_kwd = %#v, want image", updated["doc_type_kwd"])
}
})
}
}
func TestUpdateChunkRemoveModeClearsImageAfterIndexUpdate(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-a", "kb-1")
order := &[]string{}
engine := &orderedUpdateChunkTestEngine{
updateChunkTestEngine: updateChunkTestEngine{
existingChunk: map[string]interface{}{
"doc_id": "doc-a",
"content_with_weight": "existing content",
},
},
order: order,
}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
storeChunkImageFunc: func(string, string, []byte, string) error {
t.Fatal("remove mode must not store an image")
return nil
},
removeChunkImageFunc: func(bucket, chunkID string) error {
*order = append(*order, "storage")
if bucket != "kb-1" || chunkID != "chunk-1" {
t.Fatalf("remove args = %q/%q, want kb-1/chunk-1", bucket, chunkID)
}
return nil
},
}
removeMode := imageUpdateModeRemove
err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
DatasetID: "kb-1",
DocumentID: "doc-a",
ChunkID: "chunk-1",
ImageUpdateMode: &removeMode,
})
if err != nil {
t.Fatalf("UpdateChunk() error = %v", err)
}
if len(engine.updateCalls) != 1 {
t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
}
updated := engine.updateCalls[0].newValue
if imgID, ok := updated["img_id"]; !ok || imgID != "" {
t.Fatalf("img_id = %#v, want empty string", updated["img_id"])
}
if updated["doc_type_kwd"] != "text" {
t.Fatalf("doc_type_kwd = %#v, want text", updated["doc_type_kwd"])
}
if !reflect.DeepEqual(*order, []string{"index", "storage"}) {
t.Fatalf("call order = %#v, want index before storage removal", *order)
}
}
// A request that changes content and drops the image must still schedule the
// Wiki refresh when the stored object cannot be deleted: the index update has
// already committed by then.
func TestUpdateChunkMarksWikiDirtyWhenImageRemovalFails(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-a", "kb-1")
engine := &updateChunkTestEngine{
existingChunk: map[string]interface{}{
"doc_id": "doc-a",
"content_with_weight": "existing content",
},
}
wikiCalls := 0
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
removeChunkImageFunc: func(string, string) error {
return errors.New("storage unavailable")
},
markWikiDirtyFunc: func(tenantID, datasetID, documentID string, chunkIDs []string) {
wikiCalls++
},
}
removeMode := imageUpdateModeRemove
content := "updated content"
err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
DatasetID: "kb-1",
DocumentID: "doc-a",
ChunkID: "chunk-1",
Content: &content,
ImageUpdateMode: &removeMode,
})
if err == nil || !strings.Contains(err.Error(), "Failed to remove chunk image") {
t.Fatalf("UpdateChunk() error = %v, want the removal failure", err)
}
if len(engine.updateCalls) != 1 {
t.Fatalf("UpdateChunks calls = %d, want 1", len(engine.updateCalls))
}
if wikiCalls != 1 {
t.Fatalf("markWikiDirty calls = %d, want 1 (the index update already committed)", wikiCalls)
}
}
func TestUpdateChunkRejectsInvalidImageInput(t *testing.T) {
cases := []struct {
name string
imageB64 *string
mode *string
wantError string
}{
{"undecodable base64", strPtr("not base64!!"), nil, "invalid `image_base64`"},
{"empty base64", strPtr(""), nil, "`image_base64` must be a non-empty string"},
{"unknown mode", strPtr("aGVsbG8="), strPtr("grow"), "`image_update_mode` must be one of: append, replace, remove"},
{"empty mode", strPtr("aGVsbG8="), strPtr(""), "`image_update_mode` must be one of: append, replace, remove"},
{"append without base64", nil, strPtr(imageUpdateModeAppend), "`image_base64` is required when `image_update_mode` is `append` or `replace`"},
{"replace without base64", nil, strPtr(imageUpdateModeReplace), "`image_base64` is required when `image_update_mode` is `append` or `replace`"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-a", "kb-1")
engine := &updateChunkTestEngine{
existingChunk: map[string]interface{}{
"doc_id": "doc-a",
"content_with_weight": "existing content",
},
}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
storeChunkImageFunc: func(string, string, []byte, string) error {
t.Fatal("invalid input must not reach storage")
return nil
},
}
err := svc.UpdateChunk(chunkTestContext(t, "user-1", "tenant-1"), &service.UpdateChunkRequest{
DatasetID: "kb-1",
DocumentID: "doc-a",
ChunkID: "chunk-1",
ImageBase64: tc.imageB64,
ImageUpdateMode: tc.mode,
})
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
t.Fatalf("UpdateChunk() error = %v, want %q", err, tc.wantError)
}
var coded interface {
Code() common.ErrorCode
}
if !errors.As(err, &coded) || coded.Code() != common.CodeDataError {
t.Fatalf("error code = %v, want %d (Python: DATA_ERROR)", err, common.CodeDataError)
}
if len(engine.updateCalls) != 0 {
t.Fatalf("UpdateChunks calls = %d, want 0", len(engine.updateCalls))
}
})
}
}
type orderedUpdateChunkTestEngine struct {
updateChunkTestEngine
order *[]string
}
func (e *orderedUpdateChunkTestEngine) UpdateChunks(ctx context.Context, condition, newValue map[string]interface{}, indexName, datasetID string) error {
*e.order = append(*e.order, "index")
return e.updateChunkTestEngine.UpdateChunks(ctx, condition, newValue, indexName, datasetID)
}
func TestAddChunkSuccess(t *testing.T) {
ctx := t.Context()
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID, datasetID, documentID := "user-1", "kb-1", "doc-1"
insertChunkTestKB(t, datasetID, userID)
insertChunkTestDoc(t, documentID, datasetID)
// The document already holds source chunks whose highest order is 9; the
// lookup page also carries rows the lookup must ignore (a parent-range value
// and a row without any order), so the added chunk still continues from 9.
engine := &addChunkTestEngine{
searchResp: &types.SearchResult{
Chunks: []map[string]interface{}{
{"id": "parent-1", "chunk_order_int": float64(common.ParentChunkOrderBase + 3)},
{"id": "raptor-1"},
{"id": "chunk-9", "chunk_order_int": float64(9)},
},
},
}
var incrementTokenNum, incrementChunkNum int64
language := "Slovak"
var tokenizeLanguages []string
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
checkDatasetAccessFunc: func(datasetIDArg string) error {
if datasetIDArg == datasetID {
return nil
}
return permission.ErrPermissionDenied
},
getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1", Language: &language, Pagerank: 7}, nil
},
getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
driver := &stubEmbeddingDriver{
embeddings: []models.EmbeddingData{
{Embedding: []float64{1, 2}},
{Embedding: []float64{3, 4}},
},
}
modelName := "embed-1"
return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil
},
incrementChunkStatsFunc: func(docID, kbID string, tokenNum, chunkNum int64, duration float64) error {
if docID != documentID || kbID != datasetID || duration != 0 {
t.Fatalf("unexpected increment args doc=%s kb=%s duration=%v", docID, kbID, duration)
}
incrementTokenNum = tokenNum
incrementChunkNum = chunkNum
return nil
},
tokenizeFunc: func(text, lang string) (string, error) {
tokenizeLanguages = append(tokenizeLanguages, lang)
return text, nil
},
fineGrainedTokenizeFunc: func(text, lang string) (string, error) {
tokenizeLanguages = append(tokenizeLanguages, lang)
return text + "_fg", nil
},
numTokensFunc: func(text string) int { return len(text) },
}
resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
DatasetID: datasetID,
DocumentID: documentID,
Content: "chunk body",
ImportantKeywords: []string{"k1"},
Questions: []string{" q1 ", ""},
TagFeas: map[string]interface{}{"tag1": float64(0.5)},
})
if err != nil {
t.Fatalf("AddChunk() error = %v", err)
}
if resp == nil || resp.Chunk == nil {
t.Fatalf("expected chunk response, got %#v", resp)
}
if resp.Chunk["dataset_id"] != datasetID || resp.Chunk["document_id"] != documentID {
t.Fatalf("unexpected response chunk: %#v", resp.Chunk)
}
if resp.Chunk["content"] != "chunk body" {
t.Fatalf("content = %v, want chunk body", resp.Chunk["content"])
}
if len(engine.insertedChunks) != 1 {
t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))
}
if got := engine.insertedChunks[0][common.PAGERANK_FLD]; got != int64(7) {
t.Fatalf("pagerank_fea = %v, want the dataset's 7", got)
}
if resp.Chunk["document"] != "doc-1.txt" {
t.Fatalf("document = %v, want doc-1.txt", resp.Chunk["document"])
}
if incrementChunkNum != 1 {
t.Fatalf("increment chunk num = %d, want 1", incrementChunkNum)
}
if incrementTokenNum <= 0 {
t.Fatalf("increment token num = %d, want > 0", incrementTokenNum)
}
if len(engine.insertedChunks) != 1 {
t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))
}
inserted := engine.insertedChunks[0]
if inserted["doc_id"] != documentID || inserted["kb_id"] != datasetID {
t.Fatalf("unexpected inserted chunk: %#v", inserted)
}
if inserted["img_id"] != nil {
t.Fatalf("did not expect image id in inserted chunk: %#v", inserted)
}
// A manual chunk is tokenized like the dataset's parsed chunks, and the
// write that may create the chunk store carries the dataset language.
if len(tokenizeLanguages) != 4 {
t.Fatalf("tokenizer calls = %v, want 4", tokenizeLanguages)
}
for _, lang := range tokenizeLanguages {
if lang != language {
t.Fatalf("tokenized with language %q, want %q", lang, language)
}
}
if engine.insertLanguage != language {
t.Fatalf("InsertChunks language = %q, want %q", engine.insertLanguage, language)
}
// A manually added text chunk must carry doc_type_kwd (Python chunk_api.py:1225):
// the management chunk list selects source chunks by that field, so a missing
// value would hide the chunk from the list.
if inserted["doc_type_kwd"] != "text" {
t.Fatalf("doc_type_kwd = %#v, want text", inserted["doc_type_kwd"])
}
// The added chunk continues the document's source ordering (highest source
// order + 1) and stays inside [0, common.ParentChunkOrderBase), so it lists
// after the ingested chunks and still before the document's parent rows.
if got := inserted["chunk_order_int"]; got != 10 {
t.Fatalf("chunk_order_int = %#v, want 10 (highest source order + 1)", got)
}
if engine.searchReq == nil {
t.Fatal("expected the document's highest source order to be looked up")
}
if got := engine.searchReq.Filter["exists"]; got != "doc_type_kwd" {
t.Fatalf("order lookup exists filter = %#v, want doc_type_kwd", got)
}
if got := engine.searchReq.Filter["doc_id"]; got != documentID {
t.Fatalf("order lookup doc_id = %#v, want %q", got, documentID)
}
if !engine.searchReq.IncludeUnavailable {
t.Fatal("order lookup must count disabled source chunks")
}
if engine.searchReq.OrderBy == nil || len(engine.searchReq.OrderBy.Fields) != 1 ||
engine.searchReq.OrderBy.Fields[0].Field != "chunk_order_int" ||
engine.searchReq.OrderBy.Fields[0].Type != types.SortDesc {
t.Fatalf("order lookup OrderBy = %#v, want descending chunk_order_int", engine.searchReq.OrderBy)
}
vec, ok := inserted["q_2_vec"].([]float64)
if !ok {
t.Fatalf("expected q_2_vec []float64, got %T", inserted["q_2_vec"])
}
if len(vec) != 2 || vec[0] < 2.7999 || vec[0] > 2.8001 || vec[1] < 3.7999 || vec[1] > 3.8001 {
t.Fatalf("vector = %v, want approximately [2.8 3.8]", vec)
}
}
// TestAddChunkPersistsTagKwdOnlyWhenProvided pins the Go/Python parity
// contract that mirrors api/apps/restful_apis/chunk_api.py: the `tag_kwd`
// field is only persisted on the inserted chunk when the caller actually
// supplied it, and is echoed back in the response when present. Issue
// #20138 documented that the Go path was silently dropping the field.
func TestAddChunkPersistsTagKwdOnlyWhenProvided(t *testing.T) {
ctx := t.Context()
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID, datasetID, documentID := "user-1", "kb-1", "doc-1"
insertChunkTestKB(t, datasetID, userID)
insertChunkTestDoc(t, documentID, datasetID)
makeService := func() (*ChunkService, *addChunkTestEngine) {
engine := &addChunkTestEngine{}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
checkDatasetAccessFunc: func(datasetIDArg string) error {
if datasetIDArg != datasetID {
return permission.ErrPermissionDenied
}
return nil
},
getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil
},
getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
driver := &stubEmbeddingDriver{
embeddings: []models.EmbeddingData{
{Embedding: []float64{1, 2}},
{Embedding: []float64{3, 4}},
},
}
modelName := "embed-1"
return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil
},
incrementChunkStatsFunc: func(_, _ string, _, _ int64, _ float64) error { return nil },
tokenizeFunc: func(text, _ string) (string, error) { return text, nil },
fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },
numTokensFunc: func(text string) int { return len(text) },
}
return svc, engine
}
t.Run("omitted by caller", func(t *testing.T) {
svc, engine := makeService()
resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
DatasetID: datasetID,
DocumentID: documentID,
Content: "chunk body",
})
if err != nil {
t.Fatalf("AddChunk() error = %v", err)
}
if len(engine.insertedChunks) != 1 {
t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))
}
if _, ok := engine.insertedChunks[0]["tag_kwd"]; ok {
t.Fatalf("expected tag_kwd to be absent when caller omits it; got %#v", engine.insertedChunks[0]["tag_kwd"])
}
if _, ok := resp.Chunk["tag_kwd"]; ok {
t.Fatalf("response should not include tag_kwd when caller omits it; got %#v", resp.Chunk["tag_kwd"])
}
})
t.Run("explicit empty list persists empty slice", func(t *testing.T) {
svc, engine := makeService()
resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
DatasetID: datasetID,
DocumentID: documentID,
Content: "chunk body",
TagKwd: []string{},
})
if err != nil {
t.Fatalf("AddChunk() error = %v", err)
}
got, ok := engine.insertedChunks[0]["tag_kwd"].([]string)
if !ok {
t.Fatalf("expected []string in tag_kwd, got %T", engine.insertedChunks[0]["tag_kwd"])
}
if len(got) != 0 {
t.Fatalf("expected empty tag_kwd, got %v", got)
}
if echoed, _ := resp.Chunk["tag_kwd"].([]string); len(echoed) != 0 {
t.Fatalf("response tag_kwd = %v, want empty", resp.Chunk["tag_kwd"])
}
})
t.Run("values are persisted and echoed", func(t *testing.T) {
svc, engine := makeService()
want := []string{"alpha", "beta"}
resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
DatasetID: datasetID,
DocumentID: documentID,
Content: "chunk body",
TagKwd: want,
})
if err != nil {
t.Fatalf("AddChunk() error = %v", err)
}
got, ok := engine.insertedChunks[0]["tag_kwd"].([]string)
if !ok {
t.Fatalf("expected []string in tag_kwd, got %T", engine.insertedChunks[0]["tag_kwd"])
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("inserted tag_kwd = %v, want %v", got, want)
}
echoed, ok := resp.Chunk["tag_kwd"].([]string)
if !ok {
t.Fatalf("response tag_kwd type = %T, want []string", resp.Chunk["tag_kwd"])
}
if !reflect.DeepEqual(echoed, want) {
t.Fatalf("response tag_kwd = %v, want %v", echoed, want)
}
})
}
func TestAddChunkValidationErrors(t *testing.T) {
ctx := t.Context()
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestKB(t, "kb-1", "user-1")
insertChunkTestDoc(t, "doc-1", "kb-1")
svc := &ChunkService{
docEngine: &addChunkTestEngine{},
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
}
tests := []struct {
name string
req *service.AddChunkRequest
setup func(*ChunkService)
wantMsg string
}{
{
name: "nil request",
req: nil,
wantMsg: "invalid request payload",
},
{
name: "empty content",
req: &service.AddChunkRequest{DatasetID: "kb-1", DocumentID: "doc-1", Content: " "},
setup: func(svc *ChunkService) {
svc.checkDatasetAccessFunc = func(string) error { return nil }
svc.getKnowledgebaseByIDFunc = func(string) (*entity.Knowledgebase, error) {
return &entity.Knowledgebase{ID: "kb-1", TenantID: "user-1", EmbdID: "embed-1"}, nil
}
},
wantMsg: "`content` is required",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.setup != nil {
tt.setup(svc)
}
_, err := svc.AddChunk(ctx, tt.req)
if err == nil || !strings.Contains(err.Error(), tt.wantMsg) {
t.Fatalf("error = %v, want substring %q", err, tt.wantMsg)
}
})
}
}
func TestAddChunkImageAndTagFeatureValidation(t *testing.T) {
ctx := t.Context()
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID, datasetID, documentID := "user-1", "kb-1", "doc-1"
insertChunkTestKB(t, datasetID, userID)
insertChunkTestDoc(t, documentID, datasetID)
storeCalls := 0
svc := &ChunkService{
docEngine: &addChunkTestEngine{},
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
checkDatasetAccessFunc: func(string) error { return nil },
getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil
},
tokenizeFunc: func(text, _ string) (string, error) { return text, nil },
fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },
numTokensFunc: func(text string) int { return len(text) },
getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
driver := &stubEmbeddingDriver{
embeddings: []models.EmbeddingData{
{Embedding: []float64{1, 1}},
{Embedding: []float64{1, 1}},
},
}
modelName := "embed-1"
return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil
},
incrementChunkStatsFunc: func(string, string, int64, int64, float64) error { return nil },
storeChunkImageFunc: func(bucket, chunkID string, imageBinary []byte, mode string) error {
storeCalls++
if bucket != datasetID || chunkID == "" || len(imageBinary) == 0 {
t.Fatalf("unexpected store args bucket=%s chunkID=%s len=%d", bucket, chunkID, len(imageBinary))
}
if mode != imageUpdateModeAppend {
t.Fatalf("store mode = %q, want %q", mode, imageUpdateModeAppend)
}
return nil
},
}
_, err := svc.AddChunk(ctx, &service.AddChunkRequest{
DatasetID: datasetID,
DocumentID: documentID,
Content: "chunk body",
ImageBase64: strPtr("not-base64"),
})
if err == nil || !strings.Contains(err.Error(), "invalid `image_base64`") {
t.Fatalf("expected invalid image error, got %v", err)
}
validJPEG := "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO2pRZ0AAAAASUVORK5CYII="
resp, err := svc.AddChunk(ctx, &service.AddChunkRequest{
DatasetID: datasetID,
DocumentID: documentID,
Content: "chunk body",
TagFeas: map[string]interface{}{"tag1": "bad"},
})
if err == nil || !strings.Contains(err.Error(), "`tag_feas` values must be finite numbers") || resp != nil {
t.Fatalf("expected tag_feas validation error, got resp=%#v err=%v", resp, err)
}
resp, err = svc.AddChunk(ctx, &service.AddChunkRequest{
DatasetID: datasetID,
DocumentID: documentID,
Content: "chunk body",
TagFeas: map[string]interface{}{"tag1": float64(1)},
ImageBase64: strPtr(validJPEG),
})
if err != nil {
t.Fatalf("AddChunk() with image error = %v", err)
}
if storeCalls != 1 {
t.Fatalf("store image calls = %d, want 1", storeCalls)
}
if _, ok := resp.Chunk["image_id"]; !ok {
t.Fatalf("expected image_id in response, got %#v", resp.Chunk)
}
}
func TestAddChunkIncrementsStatsAfterInsert(t *testing.T) {
ctx := t.Context()
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID, datasetID, documentID := "user-1", "kb-1", "doc-1"
insertChunkTestKB(t, datasetID, userID)
insertChunkTestDoc(t, documentID, datasetID)
var incrementCalls int
engine := &addChunkTestEngine{}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
checkDatasetAccessFunc: func(string) error { return nil },
getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1"}, nil
},
getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
driver := &stubEmbeddingDriver{
embeddings: []models.EmbeddingData{
{Embedding: []float64{1, 2}},
{Embedding: []float64{3, 4}},
},
}
modelName := "embed-1"
return models.NewEmbeddingModel(driver, &modelName, &models.APIConfig{}, 0), nil
},
incrementChunkStatsFunc: func(string, string, int64, int64, float64) error {
incrementCalls++
return nil
},
tokenizeFunc: func(text, _ string) (string, error) { return text, nil },
fineGrainedTokenizeFunc: func(text, _ string) (string, error) { return text + "_fg", nil },
numTokensFunc: func(text string) int { return len(text) },
}
_, err := svc.AddChunk(ctx, &service.AddChunkRequest{
DatasetID: datasetID,
DocumentID: documentID,
Content: "chunk body",
})
if err != nil {
t.Fatalf("AddChunk() error = %v", err)
}
if incrementCalls != 1 {
t.Fatalf("increment calls = %d, want 1", incrementCalls)
}
if len(engine.insertedChunks) != 1 {
t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))
}
importantKwd, ok := engine.insertedChunks[0]["important_kwd"].([]string)
if !ok || len(importantKwd) != 0 {
t.Fatalf("important_kwd = %#v, want empty []string", engine.insertedChunks[0]["important_kwd"])
}
}
func TestStoreChunkImageMergesExistingImage(t *testing.T) {
oldImage := mustEncodePNG(t, image.Rect(0, 0, 2, 2))
newImage := mustEncodePNG(t, image.Rect(0, 0, 1, 1))
mockStorage := &chunkImageStorage{
exists: true,
oldBinary: oldImage,
}
ctx := t.Context()
factory := storage.GetStorageFactory()
originalStorage := factory.GetStorage()
factory.SetStorage(mockStorage)
t.Cleanup(func() {
factory.SetStorage(originalStorage)
})
svc := &ChunkService{}
if err := svc.storeChunkImage(ctx, "kb-1", "chunk-1", newImage, imageUpdateModeAppend); err != nil {
t.Fatalf("storeChunkImage() error = %v", err)
}
if mockStorage.putCalls != 1 {
t.Fatalf("put calls = %d, want 1", mockStorage.putCalls)
}
merged, format, err := image.Decode(bytes.NewReader(mockStorage.lastPut))
if err != nil {
t.Fatalf("decode stored image: %v", err)
}
if format != "jpeg" {
t.Fatalf("stored format = %q, want jpeg", format)
}
// Append stacks the new image below the old one.
if got, want := merged.Bounds().Dy(), 3; got != want {
t.Fatalf("merged height = %d, want %d", got, want)
}
}
func TestStoreChunkImageReplaceOverwritesExistingImage(t *testing.T) {
oldImage := mustEncodePNG(t, image.Rect(0, 0, 2, 2))
newImage := mustEncodePNG(t, image.Rect(0, 0, 1, 1))
mockStorage := &chunkImageStorage{
exists: true,
oldBinary: oldImage,
}
ctx := t.Context()
factory := storage.GetStorageFactory()
originalStorage := factory.GetStorage()
factory.SetStorage(mockStorage)
t.Cleanup(func() {
factory.SetStorage(originalStorage)
})
svc := &ChunkService{}
if err := svc.storeChunkImage(ctx, "kb-1", "chunk-1", newImage, imageUpdateModeReplace); err != nil {
t.Fatalf("storeChunkImage() error = %v", err)
}
if mockStorage.putCalls != 1 {
t.Fatalf("put calls = %d, want 1", mockStorage.putCalls)
}
if !bytes.Equal(mockStorage.lastPut, newImage) {
t.Fatalf("replace stored %d bytes, want the new image verbatim (%d bytes)", len(mockStorage.lastPut), len(newImage))
}
}
func TestRemoveChunkImageSkipsMissingObject(t *testing.T) {
mockStorage := &chunkImageStorage{exists: false}
factory := storage.GetStorageFactory()
originalStorage := factory.GetStorage()
factory.SetStorage(mockStorage)
t.Cleanup(func() {
factory.SetStorage(originalStorage)
})
svc := &ChunkService{}
if err := svc.removeChunkImage(t.Context(), "kb-1", "chunk-1"); err != nil {
t.Fatalf("removeChunkImage() error = %v", err)
}
if mockStorage.removeCalls != 0 {
t.Fatalf("remove calls = %d, want 0 for a missing object", mockStorage.removeCalls)
}
mockStorage.exists = true
if err := svc.removeChunkImage(t.Context(), "kb-1", "chunk-1"); err != nil {
t.Fatalf("removeChunkImage() error = %v", err)
}
if mockStorage.removeCalls != 1 {
t.Fatalf("remove calls = %d, want 1", mockStorage.removeCalls)
}
}
// Every image mode has to serialize on the chunk's per-chunk lock: without it a
// replace (or a remove) can land between an append's read and write and be
// overwritten by the merged image.
func TestChunkImageOperationsShareThePerChunkLock(t *testing.T) {
chunkID := "chunk-locked"
mockStorage := &chunkImageStorage{
exists: true,
oldBinary: mustEncodePNG(t, image.Rect(0, 0, 2, 2)),
}
factory := storage.GetStorageFactory()
originalStorage := factory.GetStorage()
factory.SetStorage(mockStorage)
t.Cleanup(func() {
factory.SetStorage(originalStorage)
})
lockKey := "kb-1/" + chunkID
lock := acquireChunkImageLock(lockKey)
lock.mu.Lock()
defer releaseChunkImageLock(lockKey)
type result struct {
op string
err error
}
svc := &ChunkService{}
ctx := t.Context()
appendedImage := mustEncodePNG(t, image.Rect(0, 0, 1, 1))
replacedImage := mustEncodePNG(t, image.Rect(0, 0, 3, 3))
done := make(chan result, 3)
run := func(op string, fn func() error) {
go func() { done <- result{op: op, err: fn()} }()
}
run("append", func() error {
return svc.storeChunkImage(ctx, "kb-1", chunkID, appendedImage, imageUpdateModeAppend)
})
run("replace", func() error {
return svc.storeChunkImage(ctx, "kb-1", chunkID, replacedImage, imageUpdateModeReplace)
})
run("remove", func() error {
return svc.removeChunkImage(ctx, "kb-1", chunkID)
})
select {
case got := <-done:
lock.mu.Unlock()
t.Fatalf("%s finished while the chunk's image lock was held", got.op)
case <-time.After(200 * time.Millisecond):
}
lock.mu.Unlock()
for i := 0; i < 3; i++ {
select {
case got := <-done:
if got.err != nil {
t.Fatalf("%s: %v", got.op, got.err)
}
case <-time.After(5 * time.Second):
t.Fatal("an image operation did not finish after the lock was released")
}
}
}
func TestRemoveChunksRejectsInvalidDatasetBeforeMutation(t *testing.T) {
for _, datasetID := range []string{"", "kb-2"} {
for _, deleteAll := range []bool{false, true} {
t.Run(fmt.Sprintf("dataset=%s/delete_all=%t", datasetID, deleteAll), func(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestKB(t, "kb-1", "owner")
insertChunkTestDoc(t, "doc-1", "kb-1")
engine := &parseTestDocEngine{deleteChunksCount: 1}
svc := &ChunkService{
docEngine: engine,
decrementChunkStatsFunc: func(string, string, int64, int64, float64) error {
t.Fatal("rejected deletion must not change statistics")
return nil
},
markWikiDirtyFunc: func(string, string, string, []string) {
t.Fatal("rejected deletion must not schedule wiki updates")
},
}
req := &service.RemoveChunksRequest{DatasetID: datasetID, DocID: "doc-1", DeleteAll: deleteAll}
if !deleteAll {
req.ChunkIDs = []string{"chunk-1"}
}
count, err := svc.RemoveChunks(t.Context(), req)
if err == nil || count != 0 {
t.Fatalf("invalid dataset deletion = (%d, %v), want error and zero count", count, err)
}
if datasetID != "" && !errors.Is(err, permission.ErrResourceNotFound) {
t.Fatalf("error = %v, want resource not found", err)
}
if engine.deleteChunksCalls != 0 {
t.Fatalf("rejected deletion reached the engine %d times", engine.deleteChunksCalls)
}
})
}
}
}
func TestRemoveChunksDecrementsStatsAfterDelete(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-1", "kb-1")
setChunkTestStats(t, "doc-1", "kb-1", 70, 7, 100, 10)
engine := &parseTestDocEngine{deleteChunksCount: 3}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
}
ctx := chunkTestContext(t, "user-1", "tenant-1")
deletedCount, err := svc.RemoveChunks(ctx, &service.RemoveChunksRequest{
DatasetID: "kb-1",
DocID: "doc-1",
ChunkIDs: []string{"chunk-1", "chunk-2", "chunk-3"},
})
if err != nil {
t.Fatalf("RemoveChunks() error = %v", err)
}
if deletedCount != 3 {
t.Fatalf("deleted count = %d, want 3", deletedCount)
}
if engine.deleteChunksCalls != 1 {
t.Fatalf("DeleteChunks calls = %d, want 1", engine.deleteChunksCalls)
}
if engine.deleteIndexName != "ragflow_tenant-1" || engine.deleteDatasetID != "kb-1" {
t.Fatalf("unexpected delete target index=%q dataset=%q", engine.deleteIndexName, engine.deleteDatasetID)
}
if engine.deleteChunksCondition["doc_id"] != "doc-1" {
t.Fatalf("delete doc_id condition = %#v, want doc-1", engine.deleteChunksCondition["doc_id"])
}
if !reflect.DeepEqual(engine.deleteChunksCondition["id"], []interface{}{"chunk-1", "chunk-2", "chunk-3"}) {
t.Fatalf("delete id condition = %#v", engine.deleteChunksCondition["id"])
}
doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("get doc: %v", err)
}
if doc.TokenNum != 70 {
t.Fatalf("document token_num = %d, want 70", doc.TokenNum)
}
if doc.ChunkNum != 4 {
t.Fatalf("document chunk_num = %d, want 4", doc.ChunkNum)
}
kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
if err != nil {
t.Fatalf("get kb: %v", err)
}
if kb.TokenNum != 100 {
t.Fatalf("knowledgebase token_num = %d, want 100", kb.TokenNum)
}
if kb.ChunkNum != 7 {
t.Fatalf("knowledgebase chunk_num = %d, want 7", kb.ChunkNum)
}
}
func TestRemoveChunksSkipsStatsWhenNothingDeleted(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-1", "kb-1")
setChunkTestStats(t, "doc-1", "kb-1", 70, 7, 100, 10)
var decrementCalls int
svc := &ChunkService{
docEngine: &parseTestDocEngine{deleteChunksCount: 0},
kbDAO: dao.NewKnowledgebaseDAO(),
decrementChunkStatsFunc: func(string, string, int64, int64, float64) error {
decrementCalls++
return nil
},
}
ctx := chunkTestContext(t, "user-1", "tenant-1")
deletedCount, err := svc.RemoveChunks(ctx, &service.RemoveChunksRequest{
DatasetID: "kb-1",
DocID: "doc-1",
DeleteAll: true,
})
if err != nil {
t.Fatalf("RemoveChunks() error = %v", err)
}
if deletedCount != 0 {
t.Fatalf("deleted count = %d, want 0", deletedCount)
}
if decrementCalls != 0 {
t.Fatalf("decrement calls = %d, want 0", decrementCalls)
}
}
func TestRemoveChunksReturnsStatsError(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-1", "kb-1")
svc := &ChunkService{
docEngine: &parseTestDocEngine{deleteChunksCount: 2},
kbDAO: dao.NewKnowledgebaseDAO(),
decrementChunkStatsFunc: func(docID, kbID string, tokenNum, chunkNum int64, duration float64) error {
if docID != "doc-1" || kbID != "kb-1" || tokenNum != 0 || chunkNum != 2 || duration != 0 {
t.Fatalf("unexpected decrement args doc=%s kb=%s token=%d chunk=%d duration=%v", docID, kbID, tokenNum, chunkNum, duration)
}
return errors.New("stats update failed")
},
}
ctx := chunkTestContext(t, "user-1", "tenant-1")
deletedCount, err := svc.RemoveChunks(ctx, &service.RemoveChunksRequest{
DatasetID: "kb-1",
DocID: "doc-1",
ChunkIDs: []string{"chunk-1", "chunk-2"},
})
if err == nil || !strings.Contains(err.Error(), "failed to update chunk stats") {
t.Fatalf("expected stats update error, got count=%d err=%v", deletedCount, err)
}
if deletedCount != 2 {
t.Fatalf("deleted count on error = %d, want 2", deletedCount)
}
}
func TestDecrementChunkStatsClampsCounters(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-1", "kb-1")
setChunkTestStats(t, "doc-1", "kb-1", 2, 1, 3, 2)
if err := dao.DB.Model(&entity.Document{}).
Where("id = ?", "doc-1").
Update("process_duration", 0.5).Error; err != nil {
t.Fatalf("set process duration: %v", err)
}
svc := &ChunkService{}
if err := svc.decrementChunkStats("doc-1", "kb-1", 5, 7, -1); err != nil {
t.Fatalf("decrementChunkStats() error = %v", err)
}
ctx := t.Context()
doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("get doc: %v", err)
}
if doc.TokenNum != 0 || doc.ChunkNum != 0 || doc.ProcessDuration != 0 {
t.Fatalf("document stats token=%d chunk=%d duration=%v, want zeros", doc.TokenNum, doc.ChunkNum, doc.ProcessDuration)
}
kb, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")
if err != nil {
t.Fatalf("get kb: %v", err)
}
if kb.TokenNum != 0 || kb.ChunkNum != 0 {
t.Fatalf("knowledgebase stats token=%d chunk=%d, want zeros", kb.TokenNum, kb.ChunkNum)
}
}
func TestDecrementChunkStatsRollsBackWhenKnowledgebaseMissing(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestDoc(t, "doc-1", "missing-kb")
svc := &ChunkService{}
err := svc.decrementChunkStats("doc-1", "missing-kb", 0, 3, 0)
if err == nil || !strings.Contains(err.Error(), "knowledgebase not found") {
t.Fatalf("expected missing knowledgebase error, got %v", err)
}
ctx := t.Context()
doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("get doc: %v", err)
}
if doc.ChunkNum != 7 {
t.Fatalf("document chunk_num after rollback = %d, want 7", doc.ChunkNum)
}
}
func setupChunkTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
TranslateError: true,
})
if err != nil {
t.Fatalf("failed to open sqlite: %v", err)
}
if err := db.AutoMigrate(
&entity.UserTenant{},
&entity.Knowledgebase{},
&entity.Document{},
&entity.Task{},
&entity.IngestionTask{},
); err != nil {
t.Fatalf("failed to migrate: %v", err)
}
return db
}
func pushChunkTestDB(t *testing.T, testDB *gorm.DB) {
t.Helper()
orig := dao.DB
dao.DB = testDB
t.Cleanup(func() {
dao.DB = orig
})
}
func chunkTestContext(t *testing.T, userID, tenantID string, memberIDs ...string) context.Context {
t.Helper()
members := append([]string{userID}, memberIDs...)
if err := testsupport.PrepareEnterprisePermissionFixture(dao.DB, tenantID, members...); err != nil {
t.Fatalf("prepare permission fixture: %v", err)
}
return permission.WithIdentity(t.Context(), userID, tenantID)
}
func newParseTestService(t *testing.T) *ChunkService {
t.Helper()
return &ChunkService{
docEngine: nil,
kbDAO: dao.NewKnowledgebaseDAO(),
documentDAO: dao.NewDocumentDAO(),
}
}
func insertChunkTestUserTenant(t *testing.T, userID, tenantID string) {
t.Helper()
status := "1"
role := "normal"
if userID == tenantID {
role = "owner"
}
userTenant := &entity.UserTenant{
ID: userID + "-" + tenantID,
UserID: userID,
TenantID: tenantID,
Role: role,
InvitedBy: userID,
Status: &status,
}
if err := dao.DB.Create(userTenant).Error; err != nil {
t.Fatalf("insert user tenant: %v", err)
}
}
func insertChunkTestKB(t *testing.T, id, tenantID string) {
t.Helper()
var owner entity.UserTenant
err := dao.DB.Where("tenant_id = ? AND role = ? AND status = ?", tenantID, "owner", "1").First(&owner).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
insertChunkTestUserTenant(t, tenantID, tenantID)
} else if err != nil {
t.Fatalf("find tenant owner: %v", err)
}
status := string(entity.StatusValid)
kb := &entity.Knowledgebase{
ID: id,
TenantID: tenantID,
Name: id,
EmbdID: "embedding-model",
Permission: string(entity.TenantPermissionTeam),
CreatedBy: tenantID,
ParserConfig: entity.JSONMap{},
Status: &status,
}
if err := dao.DB.Create(kb).Error; err != nil {
t.Fatalf("insert kb: %v", err)
}
}
func insertChunkTestDoc(t *testing.T, id, kbID string) {
t.Helper()
location := id + ".txt"
name := id + ".txt"
status := string(entity.StatusValid)
doc := &entity.Document{
ID: id,
KbID: kbID,
ParserID: string(entity.ParserTypeNaive),
ParserConfig: entity.JSONMap{},
SourceType: string(entity.FileSourceLocal),
Type: "txt",
CreatedBy: "user-1",
Name: &name,
Location: &location,
ChunkNum: 7,
Suffix: ".txt",
Status: &status,
}
if err := dao.DB.Create(doc).Error; err != nil {
t.Fatalf("insert doc: %v", err)
}
}
func insertChunkTestIngestionTask(t *testing.T, id, userID, docID, datasetID, status string) {
t.Helper()
task := &entity.IngestionTask{
ID: id,
UserID: userID,
DocumentID: docID,
DatasetID: datasetID,
Schema: entity.JSONMap{},
Status: status,
}
if err := dao.DB.Create(task).Error; err != nil {
t.Fatalf("insert ingestion task: %v", err)
}
}
func setChunkTestStats(t *testing.T, docID, kbID string, docTokenNum, docChunkNum, kbTokenNum, kbChunkNum int64) {
t.Helper()
if err := dao.DB.Model(&entity.Document{}).
Where("id = ?", docID).
Updates(map[string]interface{}{"token_num": docTokenNum, "chunk_num": docChunkNum}).Error; err != nil {
t.Fatalf("set doc stats: %v", err)
}
if err := dao.DB.Model(&entity.Knowledgebase{}).
Where("id = ?", kbID).
Updates(map[string]interface{}{"token_num": kbTokenNum, "chunk_num": kbChunkNum}).Error; err != nil {
t.Fatalf("set kb stats: %v", err)
}
}
func insertChunkTestTask(t *testing.T, id, docID string) {
t.Helper()
digest := "digest"
task := &entity.Task{
ID: id,
DocID: docID,
Digest: &digest,
}
if err := dao.DB.Create(task).Error; err != nil {
t.Fatalf("insert task: %v", err)
}
}
type parseTestDocEngine struct {
deleteChunksErr error
deleteChunksCount int64
deleteChunksCalls int
deleteChunksCondition map[string]interface{}
deleteIndexName string
deleteDatasetID string
chunkStoreExists bool // if true, ChunkStoreExists returns true
}
func (e *parseTestDocEngine) CreateChunkStore(context.Context, string, string, int, string, string) error {
return nil
}
func (e *parseTestDocEngine) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {
return nil, nil
}
func (e *parseTestDocEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {
return nil
}
func (e *parseTestDocEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {
e.deleteChunksCalls++
e.deleteChunksCondition = copyMap(condition)
e.deleteIndexName = indexName
e.deleteDatasetID = datasetID
return e.deleteChunksCount, e.deleteChunksErr
}
func (e *parseTestDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {
return nil, nil
}
func (e *parseTestDocEngine) GetChunk(context.Context, string, string, []string) (interface{}, error) {
return nil, nil
}
func (e *parseTestDocEngine) DropChunkStore(context.Context, string, string) error {
return nil
}
func (e *parseTestDocEngine) ChunkStoreExists(context.Context, string, string) (bool, error) {
return e.chunkStoreExists, nil
}
func (e *parseTestDocEngine) CreateMetadataStore(context.Context, string) error {
return nil
}
func (e *parseTestDocEngine) InsertMetadata(context.Context, []map[string]interface{}, string) ([]string, error) {
return nil, nil
}
func (e *parseTestDocEngine) UpdateMetadata(context.Context, string, string, map[string]interface{}, string) error {
return nil
}
func (e *parseTestDocEngine) DeleteMetadata(context.Context, map[string]interface{}, string) (int64, error) {
return 0, nil
}
func (e *parseTestDocEngine) DeleteMetadataKeys(context.Context, string, string, []string, string) error {
return nil
}
func (e *parseTestDocEngine) DropMetadataStore(context.Context, string) error {
return nil
}
func (e *parseTestDocEngine) MetadataStoreExists(context.Context, string) (bool, error) {
return false, nil
}
func (e *parseTestDocEngine) SearchMetadata(context.Context, *types.SearchMetadataRequest) (*types.SearchMetadataResult, error) {
return nil, nil
}
func (e *parseTestDocEngine) IndexDocument(context.Context, string, string, interface{}) error {
return nil
}
func (e *parseTestDocEngine) DeleteDocument(context.Context, string, string) error {
return nil
}
func (e *parseTestDocEngine) BulkIndex(context.Context, string, []interface{}) (interface{}, error) {
return nil, nil
}
func (e *parseTestDocEngine) GetFields([]map[string]interface{}, []string) map[string]map[string]interface{} {
return nil
}
func (e *parseTestDocEngine) GetAggregation([]map[string]interface{}, string) []map[string]interface{} {
return nil
}
func (e *parseTestDocEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
return nil
}
func (e *parseTestDocEngine) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
return nil, nil
}
func (e *parseTestDocEngine) GetChunkIDs([]map[string]interface{}) []string {
return nil
}
type addChunkTestEngine struct {
parseTestDocEngine
insertedChunks []map[string]interface{}
insertIndex string
insertDataset string
insertLanguage string
insertErr error
searchReq *types.SearchRequest
searchResp *types.SearchResult
}
func (e *addChunkTestEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {
e.insertedChunks = chunks
e.insertIndex = baseName
e.insertDataset = datasetID
e.insertLanguage = language
return nil, e.insertErr
}
func (e *addChunkTestEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
e.searchReq = req
return e.searchResp, nil
}
type listChunksSearchEngine struct {
parseTestDocEngine
searchReq *types.SearchRequest
// chunks optionally overrides the canned list rows.
chunks []map[string]interface{}
}
type getChunkTestEngine struct {
parseTestDocEngine
chunk map[string]interface{}
err error
indexName string
chunkID string
datasetIDs []string
}
func (e *getChunkTestEngine) GetChunk(_ context.Context, baseName, chunkID string, datasetIDs []string) (interface{}, error) {
e.indexName = baseName
e.chunkID = chunkID
e.datasetIDs = datasetIDs
if e.err != nil {
return nil, e.err
}
if e.chunk == nil {
return nil, nil
}
return e.chunk, nil
}
func (e *listChunksSearchEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
e.searchReq = req
chunks := e.chunks
if chunks == nil {
chunks = []map[string]interface{}{
{
"id": "chunk-1",
"content": "plain invoice terms body",
"content_with_weight": "weighted invoice terms body",
"available_int": 0,
"doc_id": "doc-1",
},
}
}
return &types.SearchResult{
Chunks: chunks,
Total: int64(len(chunks)),
}, nil
}
type updateChunkTestEngine struct {
parseTestDocEngine
existingChunk interface{}
updateCalls []updateChunksCall
}
func (e *updateChunkTestEngine) GetChunk(context.Context, string, string, []string) (interface{}, error) {
return e.existingChunk, nil
}
func (e *updateChunkTestEngine) UpdateChunks(_ context.Context, condition, newValue map[string]interface{}, indexName, datasetID string) error {
e.updateCalls = append(e.updateCalls, updateChunksCall{
condition: copyMap(condition),
newValue: copyMap(newValue),
indexName: indexName,
datasetID: datasetID,
})
return nil
}
type chunkImageStorage struct {
exists bool
oldBinary []byte
putCalls int
removeCalls int
lastPut []byte
}
func (s *chunkImageStorage) Type() string { return "chunk_image_storage" }
func (s *chunkImageStorage) Health(_ context.Context) bool { return true }
func (s *chunkImageStorage) Put(ctx context.Context, bucket, fnm string, binary []byte, tenantID ...string) error {
s.putCalls++
s.lastPut = binary
return nil
}
func (s *chunkImageStorage) Get(ctx context.Context, bucket, fnm string, tenantID ...string) ([]byte, error) {
return s.oldBinary, nil
}
func (s *chunkImageStorage) Remove(ctx context.Context, bucket, fnm string, tenantID ...string) error {
s.removeCalls++
return nil
}
func (s *chunkImageStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool {
return s.exists
}
// ListObjects lists all objects in a bucket
func (s *chunkImageStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
return []string{}, nil
}
func (s *chunkImageStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
return "", nil
}
func (s *chunkImageStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
func (s *chunkImageStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
func (s *chunkImageStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error { return nil }
func (s *chunkImageStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
return s.exists, nil
}
func (s *chunkImageStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
return true, nil
}
func (s *chunkImageStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
return false
}
func (s *chunkImageStorage) Move(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
return false
}
func (s *chunkImageStorage) Close() error { return nil }
func mustEncodePNG(t *testing.T, rect image.Rectangle) []byte {
t.Helper()
img := image.NewRGBA(rect)
for y := rect.Min.Y; y < rect.Max.Y; y++ {
for x := rect.Min.X; x < rect.Max.X; x++ {
img.Set(x, y, color.White)
}
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatalf("encode png: %v", err)
}
return buf.Bytes()
}
type stubEmbeddingDriver struct {
embeddings []models.EmbeddingData
embedErr error
}
func (d *stubEmbeddingDriver) NewInstance(map[string]string) models.ModelDriver { return d }
func (d *stubEmbeddingDriver) Name() string { return "stub" }
func (d *stubEmbeddingDriver) ChatWithMessages(context.Context, string, []models.Message, *models.APIConfig, *models.ChatConfig, *common.ModelUsage) (*models.ChatResponse, error) {
return nil, nil
}
func (d *stubEmbeddingDriver) ChatStreamlyWithSender(context.Context, string, []models.Message, *models.APIConfig, *models.ChatConfig, *common.ModelUsage, func(*string, *string) error) error {
return nil
}
func (d *stubEmbeddingDriver) Embed(context.Context, *string, models.EmbedRequest, *models.APIConfig, *models.EmbeddingConfig, *common.ModelUsage) ([]models.EmbeddingData, error) {
return d.embeddings, d.embedErr
}
func (d *stubEmbeddingDriver) Rerank(context.Context, *string, models.RerankRequest, *models.APIConfig, *models.RerankConfig, *common.ModelUsage) (*models.RerankResponse, error) {
return nil, nil
}
func (d *stubEmbeddingDriver) TranscribeAudio(context.Context, *string, *string, *models.APIConfig, *models.ASRConfig, *common.ModelUsage) (*models.ASRResponse, error) {
return nil, nil
}
func (d *stubEmbeddingDriver) TranscribeAudioWithSender(context.Context, *string, *string, *models.APIConfig, *models.ASRConfig, *common.ModelUsage, func(*string, *string) error) error {
return nil
}
func (d *stubEmbeddingDriver) AudioSpeech(context.Context, *string, *string, *models.APIConfig, *models.TTSConfig, *common.ModelUsage) (*models.TTSResponse, error) {
return nil, nil
}
func (d *stubEmbeddingDriver) AudioSpeechWithSender(context.Context, *string, *string, *models.APIConfig, *models.TTSConfig, *common.ModelUsage, func(*string, *string) error) error {
return nil
}
func (d *stubEmbeddingDriver) OCRFile(context.Context, *string, []byte, *string, *models.APIConfig, *models.OCRConfig, *common.ModelUsage) (*models.OCRFileResponse, error) {
return nil, nil
}
func (d *stubEmbeddingDriver) ParseFile(context.Context, *string, []byte, *string, *models.APIConfig, *models.ParseFileConfig, *common.ModelUsage) (*models.ParseFileResponse, error) {
return nil, nil
}
func (d *stubEmbeddingDriver) ListModels(context.Context, *models.APIConfig) ([]models.ListModelResponse, error) {
return nil, nil
}
func (d *stubEmbeddingDriver) Balance(context.Context, *models.APIConfig) (map[string]interface{}, error) {
return nil, nil
}
func (d *stubEmbeddingDriver) CheckConnection(context.Context, *models.APIConfig) error { return nil }
func (d *stubEmbeddingDriver) ListTasks(context.Context, *models.APIConfig) ([]models.ListTaskStatus, error) {
return nil, nil
}
func (d *stubEmbeddingDriver) ShowTask(context.Context, string, *models.APIConfig) (*models.TaskResponse, error) {
return nil, nil
}
func (d *stubEmbeddingDriver) Decide(ctx context.Context, modelName *string, request *models.DecideRequest, apiConfig *models.APIConfig, modelUsage *common.ModelUsage) (*models.DecideResponse, error) {
return nil, nil
}
func strPtr(v string) *string {
return &v
}
func (e *parseTestDocEngine) KNNScores(context.Context, []map[string]interface{}, []float64, int) (map[string]interface{}, error) {
return nil, nil
}
func (e *parseTestDocEngine) GetScores(map[string]interface{}) map[string]float64 {
return nil
}
func (e *parseTestDocEngine) Ping(context.Context) error {
return nil
}
func (e *parseTestDocEngine) Close() error {
return nil
}
func (e *parseTestDocEngine) GetType() string {
return "test"
}
func (e *parseTestDocEngine) SupportsPageRank() bool { return false }
func (e *parseTestDocEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []string, []map[string]interface{}, string) []string {
return nil
}
func TestSwitchChunksUpdatesDocEngineWithAvailableInt(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatalf("failed to open sqlite: %v", err)
}
if err := db.AutoMigrate(&entity.UserTenant{}, &entity.Knowledgebase{}, &entity.Document{}); err != nil {
t.Fatalf("failed to migrate sqlite: %v", err)
}
previousDB := dao.DB
dao.DB = db
t.Cleanup(func() { dao.DB = previousDB })
valid := string(entity.StatusValid)
if err := db.Create(&entity.UserTenant{
ID: "ut-1",
UserID: "user-1",
TenantID: "tenant-1",
Role: "owner",
Status: &valid,
}).Error; err != nil {
t.Fatalf("failed to create user_tenant: %v", err)
}
if err := db.Create(&entity.Knowledgebase{
ID: "kb-1",
TenantID: "tenant-1",
Name: "dataset",
EmbdID: "embed",
Permission: string(entity.TenantPermissionTeam),
CreatedBy: "user-1",
ParserID: string(entity.ParserTypeNaive),
ParserConfig: entity.JSONMap{},
Status: &valid,
}).Error; err != nil {
t.Fatalf("failed to create knowledgebase: %v", err)
}
if err := db.Create(&entity.Document{
ID: "doc-1",
KbID: "kb-1",
ParserID: string(entity.ParserTypeNaive),
ParserConfig: entity.JSONMap{},
SourceType: "local",
Type: "doc",
CreatedBy: "user-1",
Suffix: "txt",
}).Error; err != nil {
t.Fatalf("failed to create document: %v", err)
}
engine := &switchChunksEngineMock{}
svc := &ChunkService{
docEngine: engine,
kbDAO: dao.NewKnowledgebaseDAO(),
}
ctx := chunkTestContext(t, "user-1", "tenant-1")
if err = svc.SwitchChunks(ctx, "kb-1", "doc-1", 0, []string{"chunk-1", "chunk-2"}); err != nil {
t.Fatalf("SwitchChunks() error = %v", err)
}
if len(engine.updateCalls) != 2 {
t.Fatalf("UpdateChunks calls = %d, want 2", len(engine.updateCalls))
}
for i, call := range engine.updateCalls {
if call.indexName != "ragflow_tenant-1" {
t.Fatalf("call %d indexName = %q", i, call.indexName)
}
if call.datasetID != "kb-1" {
t.Fatalf("call %d datasetID = %q", i, call.datasetID)
}
wantID := []string{"chunk-1", "chunk-2"}[i]
if !reflect.DeepEqual(call.condition, map[string]interface{}{
"id": wantID,
"doc_id": "doc-1",
}) {
t.Fatalf("call %d condition = %#v", i, call.condition)
}
if !reflect.DeepEqual(call.newValue, map[string]interface{}{"id": wantID, "available_int": 0}) {
t.Fatalf("call %d newValue = %#v", i, call.newValue)
}
}
}
type updateChunksCall struct {
condition map[string]interface{}
newValue map[string]interface{}
indexName string
datasetID string
}
type switchChunksEngineMock struct {
updateCalls []updateChunksCall
}
func (m *switchChunksEngineMock) CreateChunkStore(context.Context, string, string, int, string, string) error {
return nil
}
func (m *switchChunksEngineMock) InsertChunks(context.Context, []map[string]interface{}, string, string, string) ([]string, error) {
return nil, nil
}
func (m *switchChunksEngineMock) UpdateChunks(_ context.Context, condition map[string]interface{}, newValue map[string]interface{}, indexName string, datasetID string) error {
m.updateCalls = append(m.updateCalls, updateChunksCall{
condition: copyMap(condition),
newValue: copyMap(newValue),
indexName: indexName,
datasetID: datasetID,
})
return nil
}
func (m *switchChunksEngineMock) DeleteChunks(context.Context, map[string]interface{}, string, string) (int64, error) {
return 0, nil
}
func (m *switchChunksEngineMock) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {
return nil, nil
}
func (m *switchChunksEngineMock) GetChunk(context.Context, string, string, []string) (interface{}, error) {
return nil, nil
}
func (m *switchChunksEngineMock) DropChunkStore(context.Context, string, string) error { return nil }
func (m *switchChunksEngineMock) ChunkStoreExists(context.Context, string, string) (bool, error) {
return false, nil
}
func (m *switchChunksEngineMock) CreateMetadataStore(context.Context, string) error { return nil }
func (m *switchChunksEngineMock) InsertMetadata(context.Context, []map[string]interface{}, string) ([]string, error) {
return nil, nil
}
func (m *switchChunksEngineMock) UpdateMetadata(context.Context, string, string, map[string]interface{}, string) error {
return nil
}
func (m *switchChunksEngineMock) DeleteMetadata(context.Context, map[string]interface{}, string) (int64, error) {
return 0, nil
}
func (m *switchChunksEngineMock) DeleteMetadataKeys(context.Context, string, string, []string, string) error {
return nil
}
func (m *switchChunksEngineMock) DropMetadataStore(context.Context, string) error { return nil }
func (m *switchChunksEngineMock) MetadataStoreExists(context.Context, string) (bool, error) {
return false, nil
}
func (m *switchChunksEngineMock) SearchMetadata(context.Context, *types.SearchMetadataRequest) (*types.SearchMetadataResult, error) {
return nil, nil
}
func (m *switchChunksEngineMock) IndexDocument(context.Context, string, string, interface{}) error {
return nil
}
func (m *switchChunksEngineMock) DeleteDocument(context.Context, string, string) error { return nil }
func (m *switchChunksEngineMock) BulkIndex(context.Context, string, []interface{}) (interface{}, error) {
return nil, nil
}
func (m *switchChunksEngineMock) GetFields([]map[string]interface{}, []string) map[string]map[string]interface{} {
return nil
}
func (m *switchChunksEngineMock) GetAggregation([]map[string]interface{}, string) []map[string]interface{} {
return nil
}
func (m *switchChunksEngineMock) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
return nil
}
func (m *switchChunksEngineMock) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
return nil, nil
}
func (m *switchChunksEngineMock) GetChunkIDs([]map[string]interface{}) []string { return nil }
func (m *switchChunksEngineMock) KNNScores(context.Context, []map[string]interface{}, []float64, int) (map[string]interface{}, error) {
return nil, nil
}
func (m *switchChunksEngineMock) GetScores(map[string]interface{}) map[string]float64 { return nil }
func (m *switchChunksEngineMock) Ping(context.Context) error { return nil }
func (m *switchChunksEngineMock) Close() error { return nil }
func (m *switchChunksEngineMock) GetType() string { return "elasticsearch" }
func (m *switchChunksEngineMock) SupportsPageRank() bool { return true }
func (m *switchChunksEngineMock) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []string, []map[string]interface{}, string) []string {
return nil
}
func copyMap(in map[string]interface{}) map[string]interface{} {
out := make(map[string]interface{}, len(in))
for k, v := range in {
out[k] = v
}
return out
}
func TestChunkServiceParse_RejectsBatchWithRunningIngestionTask(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-1", "kb-1")
insertChunkTestDoc(t, "doc-2", "kb-1")
insertChunkTestIngestionTask(t, "task-2", "user-1", "doc-2", "kb-1", common.RUNNING)
svc := newParseTestService(t)
svc.checkDatasetAccessFunc = func(string) error { return nil }
ctx := t.Context()
_, _, err := svc.Parse(ctx, "kb-1", &service.ParseFileRequest{DocumentIDs: []string{"doc-1", "doc-2"}})
if err == nil {
t.Fatal("expected error for RUNNING ingestion task, got nil")
}
}
func TestChunkServiceParse_CallsStartParseDocumentsWithRerunWithDelete(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestUserTenant(t, "user-1", "tenant-1")
insertChunkTestKB(t, "kb-1", "tenant-1")
insertChunkTestDoc(t, "doc-1", "kb-1")
svc := newParseTestService(t)
svc.checkDatasetAccessFunc = func(string) error { return nil }
var calledOpts document.StartParseOptions
var calledDocID string
svc.startParseDocumentsFunc = func(ctx context.Context, doc *entity.Document, kb *entity.Knowledgebase, opts document.StartParseOptions) error {
calledOpts = opts
calledDocID = doc.ID
return nil
}
ctx := t.Context()
_, code, err := svc.Parse(ctx, "kb-1", &service.ParseFileRequest{DocumentIDs: []string{"doc-1"}})
if err != nil {
t.Fatalf("Parse: %v", err)
}
if code != common.CodeSuccess {
t.Fatalf("code = %d, want success", code)
}
if calledDocID != "doc-1" {
t.Fatalf("startParseDocuments called with doc %s, want doc-1", calledDocID)
}
if !calledOpts.RerunWithDelete {
t.Fatal("expected RerunWithDelete=true")
}
}
func TestChunkServiceParse_ReturnsPartialSuccessForDuplicateDocumentIDs(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
userID, datasetID := "user-1", "kb-1"
insertChunkTestKB(t, datasetID, userID)
insertChunkTestDoc(t, "doc-1", datasetID)
svc := newParseTestService(t)
svc.startParseDocumentsFunc = func(context.Context, *entity.Document, *entity.Knowledgebase, document.StartParseOptions) error {
return nil
}
ctx := chunkTestContext(t, userID, userID)
result, code, err := svc.Parse(ctx, datasetID, &service.ParseFileRequest{
DocumentIDs: []string{"doc-1", "doc-1"},
})
if err == nil {
t.Fatal("expected duplicate warning error")
}
if code != common.CodeSuccess {
t.Fatalf("expected CodeSuccess, got %v", code)
}
if result["success_count"] != 1 {
t.Fatalf("expected one successful parse, got %v", result["success_count"])
}
errorsValue, ok := result["errors"].([]string)
if !ok || len(errorsValue) != 1 || !strings.Contains(errorsValue[0], "Duplicate document ids: doc-1") {
t.Fatalf("unexpected duplicate errors: %#v", result["errors"])
}
}
func TestStopParsing_CallsCancelIngestionTask(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestKB(t, "kb-1", "user-1") // owner = user-1, Accessible passes
insertChunkTestDoc(t, "doc-1", "kb-1")
svc := newParseTestService(t)
var calledDocID string
svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error {
calledDocID = doc.ID
return nil
}
ctx := chunkTestContext(t, "user-1", "user-1")
_, _, err := svc.StopParsing(ctx, "kb-1", service.StopParsingRequest{DocumentIDs: []string{"doc-1"}})
if err != nil {
t.Fatalf("StopParsing: %v", err)
}
if calledDocID != "doc-1" {
t.Fatalf("cancelIngestionTaskFunc called with doc %s, want doc-1", calledDocID)
}
}
func TestStopParsing_RejectsDocumentWithoutIngestionTask(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestKB(t, "kb-1", "user-1")
insertChunkTestDoc(t, "doc-1", "kb-1")
// No IngestionTask — should be rejected (nothing to stop).
svc := newParseTestService(t)
var called bool
svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error {
called = true
return document.ErrParseNotRunning
}
ctx := chunkTestContext(t, "user-1", "user-1")
_, code, err := svc.StopParsing(ctx, "kb-1", service.StopParsingRequest{DocumentIDs: []string{"doc-1"}})
if err == nil {
t.Fatal("expected error for document without ingestion task")
}
if code != common.CodeDataError {
t.Fatalf("expected CodeDataError, got %v", code)
}
if !called {
t.Fatal("StopParsing must delegate task-state validation to CancelDocParse")
}
}
func TestStopParsing_DoesNotDeleteChunksOrResetCountersAfterCancel(t *testing.T) {
db := setupChunkTestDB(t)
pushChunkTestDB(t, db)
insertChunkTestKB(t, "kb-1", "user-1")
insertChunkTestDoc(t, "doc-1", "kb-1")
// Pre-set counters on the doc so we can verify they are NOT reset.
if err := dao.DB.Model(&entity.Document{}).Where("id = ?", "doc-1").Updates(map[string]interface{}{
"token_num": int64(10),
"chunk_num": int64(5),
}).Error; err != nil {
t.Fatalf("set counters: %v", err)
}
insertChunkTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1", common.RUNNING)
svc := newParseTestService(t)
svc.cancelIngestionTaskFunc = func(ctx context.Context, doc *entity.Document) error {
// Simulate CancelDocParse: stop task.
return dao.DB.Model(&entity.IngestionTask{}).Where("document_id = ?", doc.ID).
Update("status", common.STOPPED).Error
}
engine := &parseTestDocEngine{chunkStoreExists: true}
svc.docEngine = engine
ctx := chunkTestContext(t, "user-1", "user-1")
_, _, err := svc.StopParsing(ctx, "kb-1", service.StopParsingRequest{DocumentIDs: []string{"doc-1"}})
if err != nil {
t.Fatalf("StopParsing: %v", err)
}
// Counters must NOT be reset.
doc, err := dao.NewDocumentDAO().GetByID(ctx, db, "doc-1")
if err != nil {
t.Fatalf("reload doc: %v", err)
}
if doc.TokenNum != 10 {
t.Fatalf("token_num = %d, want 10 (not reset)", doc.TokenNum)
}
if doc.ChunkNum != 5 {
t.Fatalf("chunk_num = %d, want 5 (not reset)", doc.ChunkNum)
}
// Chunks must NOT be deleted during cancel.
if engine.deleteChunksCalls != 0 {
t.Fatalf("DeleteChunks called %d times, want 0", engine.deleteChunksCalls)
}
}