RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(dataset): read document-declared tag sources in tag aggregation (#20119)
jay77721b227b950dbde
diff --git a/internal/dao/document.go b/internal/dao/document.goindex d706ecf37d..cbb8a339e2 100644--- a/internal/dao/document.go+++ b/internal/dao/document.go@@ -434,6 +434,45 @@ func (dao *DocumentDAO) GetAllDocIDsByKBIDs(ctx context.Context, db *gorm.DB, kbreturn result, nil}+// ListParserConfigsByKBIDs returns each dataset's distinct document+// parser_config that declares a tag source file, keyed by dataset ID.+//+// Because parser_config is a LONGTEXT column and also carries per-document+// state such as page ranges, an unconstrained DISTINCT across all documents+// forces disk temporary tables and filesort over large JSON blobs. Filtering+// by `parser_config LIKE '%tag_file_id%'` drops the vast majority of+// documents that carry no tag configuration before distinct deduplication,+// avoiding OOM and slow queries on large datasets.+func (dao *DocumentDAO) ListParserConfigsByKBIDs(ctx context.Context, db *gorm.DB, kbIDs []string) (map[string][]entity.JSONMap, error) {+ if len(kbIDs) == 0 {+ return nil, nil+ }+ var rows []struct {+ KbID string `gorm:"column:kb_id"`+ ParserConfig entity.JSONMap `gorm:"column:parser_config;type:longtext"`+ }+ query := db.WithContext(ctx).Table("document").+ Distinct("kb_id", "parser_config").+ Where("kb_id IN ?", kbIDs)+ if db.Dialector.Name() == "sqlite" {+ query = query.Where("CAST(parser_config AS TEXT) LIKE '%tag_file_id%'")+ } else {+ query = query.Where("parser_config LIKE '%tag_file_id%'")+ }+ if err := query.Find(&rows).Error; err != nil {+ return nil, err+ }++ result := make(map[string][]entity.JSONMap, len(kbIDs))+ for _, row := range rows {+ if len(row.ParserConfig) == 0 {+ continue+ }+ result[row.KbID] = append(result[row.KbID], row.ParserConfig)+ }+ return result, nil+}+// GetByIDs retrieves documents by multiple IDsfunc (dao *DocumentDAO) GetByIDs(ctx context.Context, db *gorm.DB, ids []string) ([]*entity.Document, error) {if len(ids) == 0 {diff --git a/internal/dao/document_test.go b/internal/dao/document_test.goindex 141ab6c01f..2bd47c4c6b 100644--- a/internal/dao/document_test.go+++ b/internal/dao/document_test.go@@ -452,3 +452,74 @@ func TestDocumentDAOGetParsingStatusByKBID(t *testing.T) {}func sp(s string) *string { return &s }++func TestDocumentListParserConfigsByKBIDs(t *testing.T) {+ db := setupDocumentTestDB(t)+ dao := NewDocumentDAO()+ ctx := t.Context()++ // Empty kbIDs returns nil, nil+ res, err := dao.ListParserConfigsByKBIDs(ctx, db, nil)+ if err != nil || res != nil {+ t.Fatalf("expected nil, nil for empty kbIDs, got res=%v, err=%v", res, err)+ }++ // Doc 1: no tag_file_id (should be filtered out by SQL query)+ db.Create(&entity.Document{+ ID: "doc-no-tag",+ KbID: "kb-1",+ ParserConfig: entity.JSONMap{"pages": []any{1, 5}},+ })+ // Doc 2: empty parser_config+ db.Create(&entity.Document{+ ID: "doc-empty-config",+ KbID: "kb-1",+ ParserConfig: entity.JSONMap{},+ })+ // Doc 3: has tag_file_id+ db.Create(&entity.Document{+ ID: "doc-with-tag-1",+ KbID: "kb-1",+ ParserConfig: entity.JSONMap{+ "tags": map[string]any{"tag_file_id": "file-1"},+ },+ })+ // Doc 4: identical parser_config in kb-1 (should collapse via DISTINCT)+ db.Create(&entity.Document{+ ID: "doc-with-tag-1-dup",+ KbID: "kb-1",+ ParserConfig: entity.JSONMap{+ "tags": map[string]any{"tag_file_id": "file-1"},+ },+ })+ // Doc 5: distinct tag in kb-1+ db.Create(&entity.Document{+ ID: "doc-with-tag-2",+ KbID: "kb-1",+ ParserConfig: entity.JSONMap{+ "tags": map[string]any{"tag_file_id": "file-2"},+ },+ })+ // Doc 6: tag in kb-2+ db.Create(&entity.Document{+ ID: "doc-with-tag-kb2",+ KbID: "kb-2",+ ParserConfig: entity.JSONMap{+ "Extractor:Auto": map[string]any{+ "tags": map[string]any{"tag_file_id": "file-3"},+ },+ },+ })++ res, err = dao.ListParserConfigsByKBIDs(ctx, db, []string{"kb-1", "kb-2"})+ if err != nil {+ t.Fatalf("ListParserConfigsByKBIDs failed: %v", err)+ }++ if len(res["kb-1"]) != 2 {+ t.Fatalf("expected 2 distinct configs for kb-1, got %d: %v", len(res["kb-1"]), res["kb-1"])+ }+ if len(res["kb-2"]) != 1 {+ t.Fatalf("expected 1 config for kb-2, got %d: %v", len(res["kb-2"]), res["kb-2"])+ }+}diff --git a/internal/ingestion/component/tag_source.go b/internal/ingestion/component/tag_source.goindex bb16af45e6..0d0a78a4d4 100644--- a/internal/ingestion/component/tag_source.go+++ b/internal/ingestion/component/tag_source.go@@ -736,6 +736,23 @@ func (c *tagSourceFileRowCache) store(key string, f *entity.File, now time.Time)var tagSourceFileRows = &tagSourceFileRowCache{items: make(map[string]tagSourceFileRow, 64)}+// errTagSourceNotFound marks a failure to resolve one tag source file id+// against its owning tenant. It deliberately covers only the resolution step —+// no row, a row without a storage location — and NOT the storage step: a+// missing backend or a failed object read says nothing about this particular+// id and must stay a hard failure.+var errTagSourceNotFound = errors.New("tag source file not found")++// IsTagSourceNotFound reports whether err means this specific tag source file+// id cannot be resolved, as opposed to a transient database failure or the+// storage layer being unavailable. Callers that treat a source as optional —+// one inherited from a document, where a stale reference is expected once the+// file has been deleted — may skip it; a source the dataset configured+// explicitly, and any database or storage failure, must still fail loudly.+func IsTagSourceNotFound(err error) bool {+ return errors.Is(err, errTagSourceNotFound)+}+// resolveTagSourceFile looks up a tag source file inside ownerTenantID and// validates that it is a readable stored object. Resolved rows are reused for// tagSourceFileRowTTL so repeated lookups of the same file do not each cost a@@ -748,15 +765,19 @@ func resolveTagSourceFile(ctx context.Context, tagFileID, ownerTenantID string)}f, err := dao.NewFileDAO().GetByIDAndTenant(ctx, dao.DB, tagFileID, ownerTenantID)if err != nil {- return nil, fmt.Errorf("tag source file %q not found: %w", tagFileID, err)+ // The DAO returns database errors unchanged, and only a genuine miss+ // means this id is dead: wrapping a transient failure as well would+ // let a caller skip it and report an incomplete vocabulary as success.+ if dao.IsNotFoundErr(err) {+ return nil, fmt.Errorf("tag source file %q not found: %w: %w", tagFileID, errTagSourceNotFound, err)+ }+ return nil, fmt.Errorf("resolve tag source file %q: %w", tagFileID, err)}- // The DAO can return (nil, nil) for a missing row, so the "%w" form is only- // used when there is an error to wrap.if f == nil {- return nil, fmt.Errorf("tag source file %q not found in tenant %q", tagFileID, ownerTenantID)+ return nil, fmt.Errorf("tag source file %q not found in tenant %q: %w", tagFileID, ownerTenantID, errTagSourceNotFound)}if f.Location == nil || *f.Location == "" {- return nil, fmt.Errorf("tag source file %q has no storage location", tagFileID)+ return nil, fmt.Errorf("tag source file %q has no storage location: %w", tagFileID, errTagSourceNotFound)}common.Info(fmt.Sprintf("tag source: file_id=%q name=%q parent_id=%q location=%q",tagFileID, f.Name, f.ParentID, *f.Location))diff --git a/internal/ingestion/component/tag_source_test.go b/internal/ingestion/component/tag_source_test.goindex 439d0bc0ca..7795bb9e9e 100644--- a/internal/ingestion/component/tag_source_test.go+++ b/internal/ingestion/component/tag_source_test.go@@ -16,7 +16,16 @@package component-import "testing"+import (+ "context"+ "testing"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/dao"+ "ragflow/internal/entity"+)func TestBoundedTagCacheEvictsOldest(t *testing.T) {c := newBoundedTagCache(2)@@ -92,3 +101,38 @@ func TestBoundedTagCacheEvictionSurvivesRecentDrift(t *testing.T) {t.Fatalf("cache exceeded its cap: %d > %d", len(c.items), c.cap)}}++// GetByIDAndTenant returns database errors unchanged, so only a genuine+// not-found may carry the sentinel. Wrapping a transient failure too would let+// AggregateTags treat "the database blipped" as "the file was deleted", skip it+// for a document-only source and report an incomplete vocabulary as success.+func TestIsTagSourceNotFoundExcludesDatabaseFailures(t *testing.T) {+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ if err = db.AutoMigrate(&entity.File{}); err != nil {+ t.Fatalf("migrate sqlite: %v", err)+ }+ prev := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = prev })++ sqlDB, err := db.DB()+ if err != nil {+ t.Fatalf("sql db: %v", err)+ }+ // A closed handle fails every query with a database error and never a+ // not-found — the transient case this test exists for.+ if err = sqlDB.Close(); err != nil {+ t.Fatalf("close: %v", err)+ }++ _, err = resolveTagSourceFile(context.Background(), "file-transient-db", "tenant-1")+ if err == nil {+ t.Fatal("expected the query against a closed database to fail")+ }+ if IsTagSourceNotFound(err) {+ t.Fatalf("a transient database error was classified as not-found: %v", err)+ }+}diff --git a/internal/service/dataset/tags.go b/internal/service/dataset/tags.goindex 179d3bb9b7..b1b8cb6a68 100644--- a/internal/service/dataset/tags.go+++ b/internal/service/dataset/tags.go@@ -4,6 +4,7 @@ import ("context""errors""fmt"+ "sort""strings""ragflow/internal/common"@@ -15,10 +16,19 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,if len(datasetIDs) == 0 {return nil, common.CodeDataError, errors.New("Lack of dataset_ids in query parameters")}- // merged holds the selectable-tag vocabulary sourced exclusively from each- // dataset's configured tag source file (parser_config.tags.tag_file_id). The- // count for a tag is the number of times it appears in that source file.- // The Go backend has no Python-style tag-library datasets, so there is no+ // merged holds the selectable-tag vocabulary sourced from the tag source+ // files (parser_config.tags.tag_file_id) declared by the dataset's+ // documents. A document copies the dataset config at upload and the+ // document parser dialog may then override it, and that document-level copy+ // is the one the extractor reads at parse time (see+ // ingestion/task/pipeline_executor.go) — so it, not the dataset row, is what+ // describes the tags actually in effect. A dataset-level value no document+ // carries has not been applied to anything yet and is deliberately not+ // reported: this endpoint answers "in effect", not "configured", so a+ // dataset configured after its documents were uploaded reads empty until+ // those documents are re-uploaded or configured individually. The count for+ // a tag is the number of times it appears in that source file. The Go+ // backend has no Python-style tag-library datasets, so there is no// chunk-level aggregation.loader := d.tagVocabularyLoaderif loader == nil {@@ -26,11 +36,12 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,}merged := make(map[string]int)// The handler accepts repeated IDs, and distinct raw IDs can normalize to- // the same dataset (hyphenated vs. compact form). Track the normalized IDs- // so each dataset is loaded and merged exactly once — otherwise a repeated- // dataset (e.g. dataset_ids=A,A) would load the same vocabulary twice and- // double every count.+ // the same dataset (hyphenated vs. compact form). Normalize up front so+ // each dataset is authorized, loaded and merged exactly once — otherwise a+ // repeated dataset (e.g. dataset_ids=A,A) would load the same vocabulary+ // twice and double every count.seen := make(map[string]struct{}, len(datasetIDs))+ orderedIDs := make([]string, 0, len(datasetIDs))for _, rawID := range datasetIDs {rawID = strings.TrimSpace(rawID)if rawID == "" {@@ -44,6 +55,17 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,continue}seen[datasetID] = struct{}{}+ orderedIDs = append(orderedIDs, datasetID)+ }++ // Authorize and load every dataset before reading any document config, so+ // nothing belonging to an inaccessible dataset is fetched.+ type authorizedDataset struct {+ id string+ tenant string+ }+ authorized := make([]authorizedDataset, 0, len(orderedIDs))+ for _, datasetID := range orderedIDs {if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {return nil, common.CodeDataError, fmt.Errorf("No authorization for dataset '%s'", datasetID)}@@ -54,16 +76,61 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,}return nil, common.CodeServerError, errors.New("Database operation failed")}+ authorized = append(authorized, authorizedDataset{+ id: datasetID,+ tenant: kb.TenantID,+ })+ }++ docConfigs, err := d.documentDAO.ListParserConfigsByKBIDs(ctx, dao.DB, orderedIDs)+ if err != nil {+ return nil, common.CodeServerError, errors.New("Database operation failed")+ }++ // Every source below comes from a document — that is the copy the extractor+ // reads, so only it can describe tags in effect. A source file is scoped to+ // its owning tenant and is normally referenced by every document seeded from+ // the dataset, so load each (tenant, file) once or its counts add up again.+ loaded := make(map[string]struct{})+ for _, ds := range authorized {+ sources := make(map[string]struct{}, len(docConfigs[ds.id]))+ for _, docConfig := range docConfigs[ds.id] {+ if id := component.TagFileIDFromParserConfig(map[string]any(docConfig)); id != "" {+ sources[id] = struct{}{}+ }+ }+ // Sorted so the vocabulary and any load error are deterministic when a+ // dataset carries more than one source file.+ ids := make([]string, 0, len(sources))+ for id := range sources {+ ids = append(ids, id)+ }+ sort.Strings(ids)- // Include the dataset's tag-source vocabulary. The count for each tag is- // the number of times it occurs in the tag source file. The file is- // resolved against the dataset's own tenant: tag_file_id is user-writable,- // so a foreign file ID must not resolve (IDOR, CWE-639).- if tagFileID := component.TagFileIDFromParserConfig(map[string]any(kb.ParserConfig)); tagFileID != "" {- counts, vErr := loader(ctx, tagFileID, kb.TenantID)+ for _, tagFileID := range ids {+ key := ds.tenant + "\x00" + tagFileID+ if _, dup := loaded[key]; dup {+ continue+ }+ loaded[key] = struct{}{}+ // The file is resolved against the dataset's own tenant:+ // tag_file_id is user-writable, so a foreign file ID must not+ // resolve (IDOR, CWE-639).+ counts, vErr := loader(ctx, tagFileID, ds.tenant)if vErr != nil {+ // Nothing clears a stale reference when a tag file is deleted, and+ // a document's parser_config is never validated against the file+ // table, so an unresolvable id is skipped instead of letting it+ // discard every other dataset in the request. A database or+ // storage failure still fails loudly: it says nothing about this+ // id and may be environment-wide.+ if component.IsTagSourceNotFound(vErr) {+ common.Warn(fmt.Sprintf("tag_vocab: skipping unresolvable tag source %q for dataset %q: %v",+ tagFileID, ds.id, vErr))+ continue+ }return nil, common.CodeServerError,- fmt.Errorf("load tag vocabulary for dataset %q: %w", datasetID, vErr)+ fmt.Errorf("load tag vocabulary for dataset %q: %w", ds.id, vErr)}for tag, c := range counts {merged[tag] += cdiff --git a/internal/service/dataset/tags_aggregate_test.go b/internal/service/dataset/tags_aggregate_test.goindex 6ca0c81548..2a640ce348 100644--- a/internal/service/dataset/tags_aggregate_test.go+++ b/internal/service/dataset/tags_aggregate_test.go@@ -2,22 +2,48 @@ package datasetimport ("context"+ "errors""strings""testing""ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity"+ "ragflow/internal/ingestion/component")func testDatasetServiceForAggregateTags(t *testing.T, loader func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error)) *DatasetService {t.Helper()return &DatasetService{kbDAO: dao.NewKnowledgebaseDAO(),+ documentDAO: dao.NewDocumentDAO(),tagVocabularyLoader: loader,}}+// insertAggregateTagsDoc seeds a document whose parser_config declares a tag+// source file at the document level, mirroring what the document parser dialog+// writes. An empty tagFileID seeds the inherited dataset-level shape instead.+func insertAggregateTagsDoc(t *testing.T, docID, datasetID, tagFileID string) {+ t.Helper()+ parserConfig := entity.JSONMap{}+ if tagFileID != "" {+ parserConfig["Extractor:AutoExtractDefault"] = map[string]any{+ "tags": map[string]any{"tag_file_id": tagFileID, "top_n": 3},+ }+ }+ doc := &entity.Document{+ ID: docID,+ KbID: datasetID,+ Name: sptr("doc-" + docID[:6]),+ CreatedBy: datasetID,+ ParserConfig: parserConfig,+ }+ if err := dao.DB.Create(doc).Error; err != nil {+ t.Fatalf("insert test document: %v", err)+ }+}+func insertAggregateTagsKB(t *testing.T, datasetID, tenantID, permission, tagFileID string, docNum int64) {t.Helper()parserConfig := entity.JSONMap{}@@ -68,7 +94,9 @@ func aggregateTagsResultMap(rows []map[string]interface{}) map[string]int {// TestDatasetServiceAggregateTagsSeedsFromTagFile verifies that the tag// vocabulary (and its counts) come solely from the configured tag source file,-// with no chunk-level aggregation.+// with no chunk-level aggregation. The file is configured on the dataset and+// reaches the endpoint through the copy upload puts on its document — the+// endpoint reads documents, not the dataset row.func TestDatasetServiceAggregateTagsSeedsFromTagFile(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)@@ -76,6 +104,7 @@ func TestDatasetServiceAggregateTagsSeedsFromTagFile(t *testing.T) {kbInput := "123e4567-e89b-12d3-a456-426614174000"kbID := strings.ReplaceAll(kbInput, "-", "")insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)+ insertAggregateTagsDoc(t, "doc-seed", kbID, "file-1")loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {if tagFileID == "file-1" {@@ -117,6 +146,8 @@ func TestDatasetServiceAggregateTagsMergesAcrossTagFiles(t *testing.T) {kb2ID := strings.ReplaceAll(kb2Input, "-", "")insertAggregateTagsKB(t, kb1ID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)insertAggregateTagsKB(t, kb2ID, "user-1", string(entity.TenantPermissionMe), "file-2", 0)+ insertAggregateTagsDoc(t, "doc-merge-1", kb1ID, "file-1")+ insertAggregateTagsDoc(t, "doc-merge-2", kb2ID, "file-2")loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {switch tagFileID {@@ -210,6 +241,7 @@ func TestDatasetServiceAggregateTagsScopesLoaderToDatasetTenant(t *testing.T) {kbID := strings.ReplaceAll(kbInput, "-", "")const kbTenant = "tenant-owner"insertAggregateTagsKB(t, kbID, kbTenant, string(entity.TenantPermissionTeam), "file-foreign", 0)+ insertAggregateTagsDoc(t, "doc-scope", kbID, "file-foreign")insertAggregateTagsMembership(t, kbTenant, "user-1")ctx := t.Context()@@ -229,3 +261,280 @@ func TestDatasetServiceAggregateTagsScopesLoaderToDatasetTenant(t *testing.T) {t.Fatalf("loader ownerTenantID = %q, want %q (must be the dataset tenant, not the user)", gotOwner, kbTenant)}}++// assertTagCounts fails unless the aggregated result holds exactly want.+func assertTagCounts(t *testing.T, result []map[string]interface{}, want map[string]int) {+ t.Helper()+ got := aggregateTagsResultMap(result)+ if len(got) != len(want) {+ t.Fatalf("result len=%d want=%d result=%v", len(got), len(want), got)+ }+ for tag, wantCount := range want {+ if got[tag] != wantCount {+ t.Fatalf("tag %q count=%d want=%d all=%v", tag, got[tag], wantCount, got)+ }+ }+}++// TestDatasetServiceAggregateTagsReadsDocumentTagFile is the reported+// scenario: the dataset keeps the template default (empty tag_file_id) while+// the tag source is declared on a document by the document parser dialog. The+// extractor reads that document-level copy at parse time, so aggregation must+// return its vocabulary as well.+func TestDatasetServiceAggregateTagsReadsDocumentTagFile(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ kbInput := "623e4567-e89b-12d3-a456-426614174005"+ kbID := strings.ReplaceAll(kbInput, "-", "")+ insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "", 0)+ insertAggregateTagsDoc(t, "doc-tag-a", kbID, "file-1")++ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ if ownerTenantID != "user-1" {+ t.Fatalf("ownerTenantID = %q, want user-1", ownerTenantID)+ }+ if tagFileID == "file-1" {+ return map[string]int{"finance": 2, "urgent": 1}, nil+ }+ return nil, nil+ }++ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1")+ if err != nil {+ t.Fatalf("AggregateTags failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)+ }+ assertTagCounts(t, result, map[string]int{"finance": 2, "urgent": 1})+}++// TestDatasetServiceAggregateTagsCountsSnapshotOnce: every document inherits+// the dataset's tag source at upload, so the same file is reached both from+// the dataset and from each document — it must be loaded and counted once.+func TestDatasetServiceAggregateTagsCountsSnapshotOnce(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ kbInput := "723e4567-e89b-12d3-a456-426614174006"+ kbID := strings.ReplaceAll(kbInput, "-", "")+ insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)+ insertAggregateTagsDoc(t, "doc-tag-b", kbID, "file-1")+ insertAggregateTagsDoc(t, "doc-tag-c", kbID, "file-1")++ loads := 0+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ if tagFileID == "file-1" {+ loads+++ return map[string]int{"finance": 2}, nil+ }+ return nil, nil+ }++ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1")+ if err != nil {+ t.Fatalf("AggregateTags failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)+ }+ if loads != 1 {+ t.Fatalf("loader called %d times, want 1", loads)+ }+ assertTagCounts(t, result, map[string]int{"finance": 2})+}++// TestDatasetServiceAggregateTagsUnionsDocumentTagFiles: documents may+// override the dataset's tag source individually, so a dataset can be served+// by more than one file and all of their vocabularies must be merged.+func TestDatasetServiceAggregateTagsUnionsDocumentTagFiles(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ kbInput := "823e4567-e89b-12d3-a456-426614174007"+ kbID := strings.ReplaceAll(kbInput, "-", "")+ insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)+ insertAggregateTagsDoc(t, "doc-tag-d", kbID, "file-1")+ insertAggregateTagsDoc(t, "doc-tag-e", kbID, "file-2")++ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ switch tagFileID {+ case "file-1":+ return map[string]int{"finance": 2}, nil+ case "file-2":+ return map[string]int{"urgent": 3, "internal": 1}, nil+ }+ return nil, nil+ }++ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1")+ if err != nil {+ t.Fatalf("AggregateTags failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)+ }+ assertTagCounts(t, result, map[string]int{"finance": 2, "urgent": 3, "internal": 1})+}++// TestDatasetServiceAggregateTagsCountsSharedFileOnce: two datasets of the+// same tenant can declare the same source file. The count for a tag is the+// number of times it occurs in that file, so the file is loaded once for the+// whole request rather than once per dataset.+func TestDatasetServiceAggregateTagsCountsSharedFileOnce(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ kb1Input := "923e4567-e89b-12d3-a456-426614174008"+ kb2Input := "a23e4567-e89b-12d3-a456-426614174009"+ kb1ID := strings.ReplaceAll(kb1Input, "-", "")+ kb2ID := strings.ReplaceAll(kb2Input, "-", "")+ insertAggregateTagsKB(t, kb1ID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)+ insertAggregateTagsKB(t, kb2ID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)+ insertAggregateTagsDoc(t, "doc-shared-1", kb1ID, "file-1")+ insertAggregateTagsDoc(t, "doc-shared-2", kb2ID, "file-1")++ loads := 0+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ if tagFileID == "file-1" {+ loads+++ return map[string]int{"finance": 2}, nil+ }+ return nil, nil+ }++ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kb1Input, kb2Input}, "user-1")+ if err != nil {+ t.Fatalf("AggregateTags failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)+ }+ if loads != 1 {+ t.Fatalf("loader called %d times, want 1", loads)+ }+ assertTagCounts(t, result, map[string]int{"finance": 2})+}++// missingTagSourceLoader resolves ids that no longer exist through the real+// loader, so the error carries the production sentinel rather than a stub's+// approximation of it.+func missingTagSourceLoader(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ return component.TagVocabularyFromTagFileID(ctx, tagFileID, ownerTenantID)+}++// TestDatasetServiceAggregateTagsSkipsUnresolvableDocumentTagSource is the+// availability regression the review flagged: a document keeps referencing a+// tag file that has been deleted, and nothing clears that reference. One dead+// document-level source must not discard the vocabulary of every other dataset+// in the same request.+func TestDatasetServiceAggregateTagsSkipsUnresolvableDocumentTagSource(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ deadKB := "b23e4567-e89b-12d3-a456-426614174010"+ liveKB := "c23e4567-e89b-12d3-a456-426614174011"+ insertAggregateTagsKB(t, strings.ReplaceAll(deadKB, "-", ""), "user-1", string(entity.TenantPermissionMe), "", 0)+ insertAggregateTagsDoc(t, "doc-dead", strings.ReplaceAll(deadKB, "-", ""), "file-dead")+ insertAggregateTagsKB(t, strings.ReplaceAll(liveKB, "-", ""), "user-1", string(entity.TenantPermissionMe), "", 0)+ insertAggregateTagsDoc(t, "doc-live", strings.ReplaceAll(liveKB, "-", ""), "file-live")++ loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ if tagFileID == "file-dead" {+ return missingTagSourceLoader(ctx, tagFileID, ownerTenantID)+ }+ return map[string]int{"finance": 2}, nil+ }++ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{deadKB, liveKB}, "user-1")+ if err != nil {+ t.Fatalf("AggregateTags failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)+ }+ assertTagCounts(t, result, map[string]int{"finance": 2})+}++// TestDatasetServiceAggregateTagsIgnoresUnappliedDatasetTagSource pins the+// chosen semantics: this endpoint reports what is in effect, not what is+// configured. A source only the dataset row carries has not been applied to+// any chunk, so it is not reported. That is the deliberate cost of answering+// at document level — a dataset configured after its documents were uploaded+// reads empty until those documents are re-uploaded or configured individually.+func TestDatasetServiceAggregateTagsIgnoresUnappliedDatasetTagSource(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ kbInput := "d23e4567-e89b-12d3-a456-426614174012"+ insertAggregateTagsKB(t, strings.ReplaceAll(kbInput, "-", ""), "user-1", string(entity.TenantPermissionMe), "file-1", 0)++ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ t.Fatalf("loader must not be called for a source no document carries, got %q", tagFileID)+ return nil, nil+ }++ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1")+ if err != nil {+ t.Fatalf("AggregateTags failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)+ }+ if len(result) != 0 {+ t.Fatalf("result=%v want empty: a dataset-level source with no document carrying it is not in effect", result)+ }+}++// TestDatasetServiceAggregateTagsFailsOnStorageErrorForDocumentTagSource pins+// the boundary of the skip: only a source that cannot be resolved by id may be+// dropped. A storage-layer failure says nothing about that id and must stay a+// hard failure even for a document-level source.+func TestDatasetServiceAggregateTagsFailsOnStorageErrorForDocumentTagSource(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ kbInput := "e23e4567-e89b-12d3-a456-426614174013"+ insertAggregateTagsKB(t, strings.ReplaceAll(kbInput, "-", ""), "user-1", string(entity.TenantPermissionMe), "", 0)+ insertAggregateTagsDoc(t, "doc-storage", strings.ReplaceAll(kbInput, "-", ""), "file-1")++ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ return nil, errors.New("tag source file: no storage backend registered")+ }++ _, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1")+ if err == nil {+ t.Fatal("expected an error for a storage failure on a document-level tag source")+ }+ if code != common.CodeServerError {+ t.Fatalf("code=%d want=%d", code, common.CodeServerError)+ }+}++// TestDatasetServiceAggregateTagsScopesDocumentTagSourceToDatasetTenant: the+// document-level id is reached through the same union, so it must be resolved+// against the dataset's tenant rather than the caller's (IDOR, CWE-639).+func TestDatasetServiceAggregateTagsScopesDocumentTagSourceToDatasetTenant(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ kbInput := "f23e4567-e89b-12d3-a456-426614174014"+ const kbTenant = "tenant-owner"+ insertAggregateTagsKB(t, strings.ReplaceAll(kbInput, "-", ""), kbTenant, string(entity.TenantPermissionTeam), "", 0)+ insertAggregateTagsMembership(t, kbTenant, "user-1")+ insertAggregateTagsDoc(t, "doc-owned", strings.ReplaceAll(kbInput, "-", ""), "file-foreign")++ var gotOwner string+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ gotOwner = ownerTenantID+ return map[string]int{"finance": 1}, nil+ }++ if _, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1"); err != nil {+ t.Fatalf("AggregateTags failed: code=%d err=%v", code, err)+ }+ if gotOwner != kbTenant {+ t.Fatalf("loader ownerTenantID = %q, want %q (document sources must resolve against the dataset tenant)", gotOwner, kbTenant)+ }+}