Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(dataset): read document-declared tag sources in tag aggregation (#20119)

jay77721b227b950dbde
diff --git a/internal/dao/document.go b/internal/dao/document.go
index d706ecf37d..cbb8a339e2 100644
--- a/internal/dao/document.go
+++ b/internal/dao/document.go
@@ -434,6 +434,45 @@ func (dao *DocumentDAO) GetAllDocIDsByKBIDs(ctx context.Context, db *gorm.DB, kb
return result, nil
}
+// ListParserConfigsByKBIDs returns each dataset's distinct document
+// parser_config that declares a tag source file, keyed by dataset ID.
+//
+// Because parser_config is a LONGTEXT column and also carries per-document
+// state such as page ranges, an unconstrained DISTINCT across all documents
+// forces disk temporary tables and filesort over large JSON blobs. Filtering
+// by `parser_config LIKE '%tag_file_id%'` drops the vast majority of
+// documents that carry no tag configuration before distinct deduplication,
+// avoiding OOM and slow queries on large datasets.
+func (dao *DocumentDAO) ListParserConfigsByKBIDs(ctx context.Context, db *gorm.DB, kbIDs []string) (map[string][]entity.JSONMap, error) {
+ if len(kbIDs) == 0 {
+ return nil, nil
+ }
+ var rows []struct {
+ KbID string `gorm:"column:kb_id"`
+ ParserConfig entity.JSONMap `gorm:"column:parser_config;type:longtext"`
+ }
+ query := db.WithContext(ctx).Table("document").
+ Distinct("kb_id", "parser_config").
+ Where("kb_id IN ?", kbIDs)
+ if db.Dialector.Name() == "sqlite" {
+ query = query.Where("CAST(parser_config AS TEXT) LIKE '%tag_file_id%'")
+ } else {
+ query = query.Where("parser_config LIKE '%tag_file_id%'")
+ }
+ if err := query.Find(&rows).Error; err != nil {
+ return nil, err
+ }
+
+ result := make(map[string][]entity.JSONMap, len(kbIDs))
+ for _, row := range rows {
+ if len(row.ParserConfig) == 0 {
+ continue
+ }
+ result[row.KbID] = append(result[row.KbID], row.ParserConfig)
+ }
+ return result, nil
+}
+
// GetByIDs retrieves documents by multiple IDs
func (dao *DocumentDAO) GetByIDs(ctx context.Context, db *gorm.DB, ids []string) ([]*entity.Document, error) {
if len(ids) == 0 {
diff --git a/internal/dao/document_test.go b/internal/dao/document_test.go
index 141ab6c01f..2bd47c4c6b 100644
--- a/internal/dao/document_test.go
+++ b/internal/dao/document_test.go
@@ -452,3 +452,74 @@ func TestDocumentDAOGetParsingStatusByKBID(t *testing.T) {
}
func sp(s string) *string { return &s }
+
+func TestDocumentListParserConfigsByKBIDs(t *testing.T) {
+ db := setupDocumentTestDB(t)
+ dao := NewDocumentDAO()
+ ctx := t.Context()
+
+ // Empty kbIDs returns nil, nil
+ res, err := dao.ListParserConfigsByKBIDs(ctx, db, nil)
+ if err != nil || res != nil {
+ t.Fatalf("expected nil, nil for empty kbIDs, got res=%v, err=%v", res, err)
+ }
+
+ // Doc 1: no tag_file_id (should be filtered out by SQL query)
+ db.Create(&entity.Document{
+ ID: "doc-no-tag",
+ KbID: "kb-1",
+ ParserConfig: entity.JSONMap{"pages": []any{1, 5}},
+ })
+ // Doc 2: empty parser_config
+ db.Create(&entity.Document{
+ ID: "doc-empty-config",
+ KbID: "kb-1",
+ ParserConfig: entity.JSONMap{},
+ })
+ // Doc 3: has tag_file_id
+ db.Create(&entity.Document{
+ ID: "doc-with-tag-1",
+ KbID: "kb-1",
+ ParserConfig: entity.JSONMap{
+ "tags": map[string]any{"tag_file_id": "file-1"},
+ },
+ })
+ // Doc 4: identical parser_config in kb-1 (should collapse via DISTINCT)
+ db.Create(&entity.Document{
+ ID: "doc-with-tag-1-dup",
+ KbID: "kb-1",
+ ParserConfig: entity.JSONMap{
+ "tags": map[string]any{"tag_file_id": "file-1"},
+ },
+ })
+ // Doc 5: distinct tag in kb-1
+ db.Create(&entity.Document{
+ ID: "doc-with-tag-2",
+ KbID: "kb-1",
+ ParserConfig: entity.JSONMap{
+ "tags": map[string]any{"tag_file_id": "file-2"},
+ },
+ })
+ // Doc 6: tag in kb-2
+ db.Create(&entity.Document{
+ ID: "doc-with-tag-kb2",
+ KbID: "kb-2",
+ ParserConfig: entity.JSONMap{
+ "Extractor:Auto": map[string]any{
+ "tags": map[string]any{"tag_file_id": "file-3"},
+ },
+ },
+ })
+
+ res, err = dao.ListParserConfigsByKBIDs(ctx, db, []string{"kb-1", "kb-2"})
+ if err != nil {
+ t.Fatalf("ListParserConfigsByKBIDs failed: %v", err)
+ }
+
+ if len(res["kb-1"]) != 2 {
+ t.Fatalf("expected 2 distinct configs for kb-1, got %d: %v", len(res["kb-1"]), res["kb-1"])
+ }
+ if len(res["kb-2"]) != 1 {
+ t.Fatalf("expected 1 config for kb-2, got %d: %v", len(res["kb-2"]), res["kb-2"])
+ }
+}
diff --git a/internal/ingestion/component/tag_source.go b/internal/ingestion/component/tag_source.go
index bb16af45e6..0d0a78a4d4 100644
--- a/internal/ingestion/component/tag_source.go
+++ b/internal/ingestion/component/tag_source.go
@@ -736,6 +736,23 @@ func (c *tagSourceFileRowCache) store(key string, f *entity.File, now time.Time)
var tagSourceFileRows = &tagSourceFileRowCache{items: make(map[string]tagSourceFileRow, 64)}
+// errTagSourceNotFound marks a failure to resolve one tag source file id
+// against its owning tenant. It deliberately covers only the resolution step —
+// no row, a row without a storage location — and NOT the storage step: a
+// missing backend or a failed object read says nothing about this particular
+// id and must stay a hard failure.
+var errTagSourceNotFound = errors.New("tag source file not found")
+
+// IsTagSourceNotFound reports whether err means this specific tag source file
+// id cannot be resolved, as opposed to a transient database failure or the
+// storage layer being unavailable. Callers that treat a source as optional —
+// one inherited from a document, where a stale reference is expected once the
+// file has been deleted — may skip it; a source the dataset configured
+// explicitly, and any database or storage failure, must still fail loudly.
+func IsTagSourceNotFound(err error) bool {
+ return errors.Is(err, errTagSourceNotFound)
+}
+
// resolveTagSourceFile looks up a tag source file inside ownerTenantID and
// validates that it is a readable stored object. Resolved rows are reused for
// tagSourceFileRowTTL so repeated lookups of the same file do not each cost a
@@ -748,15 +765,19 @@ func resolveTagSourceFile(ctx context.Context, tagFileID, ownerTenantID string)
}
f, err := dao.NewFileDAO().GetByIDAndTenant(ctx, dao.DB, tagFileID, ownerTenantID)
if err != nil {
- return nil, fmt.Errorf("tag source file %q not found: %w", tagFileID, err)
+ // The DAO returns database errors unchanged, and only a genuine miss
+ // means this id is dead: wrapping a transient failure as well would
+ // let a caller skip it and report an incomplete vocabulary as success.
+ if dao.IsNotFoundErr(err) {
+ return nil, fmt.Errorf("tag source file %q not found: %w: %w", tagFileID, errTagSourceNotFound, err)
+ }
+ return nil, fmt.Errorf("resolve tag source file %q: %w", tagFileID, err)
}
- // The DAO can return (nil, nil) for a missing row, so the "%w" form is only
- // used when there is an error to wrap.
if f == nil {
- return nil, fmt.Errorf("tag source file %q not found in tenant %q", tagFileID, ownerTenantID)
+ return nil, fmt.Errorf("tag source file %q not found in tenant %q: %w", tagFileID, ownerTenantID, errTagSourceNotFound)
}
if f.Location == nil || *f.Location == "" {
- return nil, fmt.Errorf("tag source file %q has no storage location", tagFileID)
+ return nil, fmt.Errorf("tag source file %q has no storage location: %w", tagFileID, errTagSourceNotFound)
}
common.Info(fmt.Sprintf("tag source: file_id=%q name=%q parent_id=%q location=%q",
tagFileID, f.Name, f.ParentID, *f.Location))
diff --git a/internal/ingestion/component/tag_source_test.go b/internal/ingestion/component/tag_source_test.go
index 439d0bc0ca..7795bb9e9e 100644
--- a/internal/ingestion/component/tag_source_test.go
+++ b/internal/ingestion/component/tag_source_test.go
@@ -16,7 +16,16 @@
package component
-import "testing"
+import (
+ "context"
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+)
func TestBoundedTagCacheEvictsOldest(t *testing.T) {
c := newBoundedTagCache(2)
@@ -92,3 +101,38 @@ func TestBoundedTagCacheEvictionSurvivesRecentDrift(t *testing.T) {
t.Fatalf("cache exceeded its cap: %d > %d", len(c.items), c.cap)
}
}
+
+// GetByIDAndTenant returns database errors unchanged, so only a genuine
+// not-found may carry the sentinel. Wrapping a transient failure too would let
+// AggregateTags treat "the database blipped" as "the file was deleted", skip it
+// for a document-only source and report an incomplete vocabulary as success.
+func TestIsTagSourceNotFoundExcludesDatabaseFailures(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err = db.AutoMigrate(&entity.File{}); err != nil {
+ t.Fatalf("migrate sqlite: %v", err)
+ }
+ prev := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = prev })
+
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatalf("sql db: %v", err)
+ }
+ // A closed handle fails every query with a database error and never a
+ // not-found — the transient case this test exists for.
+ if err = sqlDB.Close(); err != nil {
+ t.Fatalf("close: %v", err)
+ }
+
+ _, err = resolveTagSourceFile(context.Background(), "file-transient-db", "tenant-1")
+ if err == nil {
+ t.Fatal("expected the query against a closed database to fail")
+ }
+ if IsTagSourceNotFound(err) {
+ t.Fatalf("a transient database error was classified as not-found: %v", err)
+ }
+}
diff --git a/internal/service/dataset/tags.go b/internal/service/dataset/tags.go
index 179d3bb9b7..b1b8cb6a68 100644
--- a/internal/service/dataset/tags.go
+++ b/internal/service/dataset/tags.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "sort"
"strings"
"ragflow/internal/common"
@@ -15,10 +16,19 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,
if len(datasetIDs) == 0 {
return nil, common.CodeDataError, errors.New("Lack of dataset_ids in query parameters")
}
- // merged holds the selectable-tag vocabulary sourced exclusively from each
- // dataset's configured tag source file (parser_config.tags.tag_file_id). The
- // count for a tag is the number of times it appears in that source file.
- // The Go backend has no Python-style tag-library datasets, so there is no
+ // merged holds the selectable-tag vocabulary sourced from the tag source
+ // files (parser_config.tags.tag_file_id) declared by the dataset's
+ // documents. A document copies the dataset config at upload and the
+ // document parser dialog may then override it, and that document-level copy
+ // is the one the extractor reads at parse time (see
+ // ingestion/task/pipeline_executor.go) — so it, not the dataset row, is what
+ // describes the tags actually in effect. A dataset-level value no document
+ // carries has not been applied to anything yet and is deliberately not
+ // reported: this endpoint answers "in effect", not "configured", so a
+ // dataset configured after its documents were uploaded reads empty until
+ // those documents are re-uploaded or configured individually. The count for
+ // a tag is the number of times it appears in that source file. The Go
+ // backend has no Python-style tag-library datasets, so there is no
// chunk-level aggregation.
loader := d.tagVocabularyLoader
if loader == nil {
@@ -26,11 +36,12 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,
}
merged := make(map[string]int)
// The handler accepts repeated IDs, and distinct raw IDs can normalize to
- // the same dataset (hyphenated vs. compact form). Track the normalized IDs
- // so each dataset is loaded and merged exactly once — otherwise a repeated
- // dataset (e.g. dataset_ids=A,A) would load the same vocabulary twice and
- // double every count.
+ // the same dataset (hyphenated vs. compact form). Normalize up front so
+ // each dataset is authorized, loaded and merged exactly once — otherwise a
+ // repeated dataset (e.g. dataset_ids=A,A) would load the same vocabulary
+ // twice and double every count.
seen := make(map[string]struct{}, len(datasetIDs))
+ orderedIDs := make([]string, 0, len(datasetIDs))
for _, rawID := range datasetIDs {
rawID = strings.TrimSpace(rawID)
if rawID == "" {
@@ -44,6 +55,17 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,
continue
}
seen[datasetID] = struct{}{}
+ orderedIDs = append(orderedIDs, datasetID)
+ }
+
+ // Authorize and load every dataset before reading any document config, so
+ // nothing belonging to an inaccessible dataset is fetched.
+ type authorizedDataset struct {
+ id string
+ tenant string
+ }
+ authorized := make([]authorizedDataset, 0, len(orderedIDs))
+ for _, datasetID := range orderedIDs {
if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {
return nil, common.CodeDataError, fmt.Errorf("No authorization for dataset '%s'", datasetID)
}
@@ -54,16 +76,61 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,
}
return nil, common.CodeServerError, errors.New("Database operation failed")
}
+ authorized = append(authorized, authorizedDataset{
+ id: datasetID,
+ tenant: kb.TenantID,
+ })
+ }
+
+ docConfigs, err := d.documentDAO.ListParserConfigsByKBIDs(ctx, dao.DB, orderedIDs)
+ if err != nil {
+ return nil, common.CodeServerError, errors.New("Database operation failed")
+ }
+
+ // Every source below comes from a document — that is the copy the extractor
+ // reads, so only it can describe tags in effect. A source file is scoped to
+ // its owning tenant and is normally referenced by every document seeded from
+ // the dataset, so load each (tenant, file) once or its counts add up again.
+ loaded := make(map[string]struct{})
+ for _, ds := range authorized {
+ sources := make(map[string]struct{}, len(docConfigs[ds.id]))
+ for _, docConfig := range docConfigs[ds.id] {
+ if id := component.TagFileIDFromParserConfig(map[string]any(docConfig)); id != "" {
+ sources[id] = struct{}{}
+ }
+ }
+ // Sorted so the vocabulary and any load error are deterministic when a
+ // dataset carries more than one source file.
+ ids := make([]string, 0, len(sources))
+ for id := range sources {
+ ids = append(ids, id)
+ }
+ sort.Strings(ids)
- // Include the dataset's tag-source vocabulary. The count for each tag is
- // the number of times it occurs in the tag source file. The file is
- // resolved against the dataset's own tenant: tag_file_id is user-writable,
- // so a foreign file ID must not resolve (IDOR, CWE-639).
- if tagFileID := component.TagFileIDFromParserConfig(map[string]any(kb.ParserConfig)); tagFileID != "" {
- counts, vErr := loader(ctx, tagFileID, kb.TenantID)
+ for _, tagFileID := range ids {
+ key := ds.tenant + "\x00" + tagFileID
+ if _, dup := loaded[key]; dup {
+ continue
+ }
+ loaded[key] = struct{}{}
+ // The file is resolved against the dataset's own tenant:
+ // tag_file_id is user-writable, so a foreign file ID must not
+ // resolve (IDOR, CWE-639).
+ counts, vErr := loader(ctx, tagFileID, ds.tenant)
if vErr != nil {
+ // Nothing clears a stale reference when a tag file is deleted, and
+ // a document's parser_config is never validated against the file
+ // table, so an unresolvable id is skipped instead of letting it
+ // discard every other dataset in the request. A database or
+ // storage failure still fails loudly: it says nothing about this
+ // id and may be environment-wide.
+ if component.IsTagSourceNotFound(vErr) {
+ common.Warn(fmt.Sprintf("tag_vocab: skipping unresolvable tag source %q for dataset %q: %v",
+ tagFileID, ds.id, vErr))
+ continue
+ }
return nil, common.CodeServerError,
- fmt.Errorf("load tag vocabulary for dataset %q: %w", datasetID, vErr)
+ fmt.Errorf("load tag vocabulary for dataset %q: %w", ds.id, vErr)
}
for tag, c := range counts {
merged[tag] += c
diff --git a/internal/service/dataset/tags_aggregate_test.go b/internal/service/dataset/tags_aggregate_test.go
index 6ca0c81548..2a640ce348 100644
--- a/internal/service/dataset/tags_aggregate_test.go
+++ b/internal/service/dataset/tags_aggregate_test.go
@@ -2,22 +2,48 @@ package dataset
import (
"context"
+ "errors"
"strings"
"testing"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion/component"
)
func testDatasetServiceForAggregateTags(t *testing.T, loader func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error)) *DatasetService {
t.Helper()
return &DatasetService{
kbDAO: dao.NewKnowledgebaseDAO(),
+ documentDAO: dao.NewDocumentDAO(),
tagVocabularyLoader: loader,
}
}
+// insertAggregateTagsDoc seeds a document whose parser_config declares a tag
+// source file at the document level, mirroring what the document parser dialog
+// writes. An empty tagFileID seeds the inherited dataset-level shape instead.
+func insertAggregateTagsDoc(t *testing.T, docID, datasetID, tagFileID string) {
+ t.Helper()
+ parserConfig := entity.JSONMap{}
+ if tagFileID != "" {
+ parserConfig["Extractor:AutoExtractDefault"] = map[string]any{
+ "tags": map[string]any{"tag_file_id": tagFileID, "top_n": 3},
+ }
+ }
+ doc := &entity.Document{
+ ID: docID,
+ KbID: datasetID,
+ Name: sptr("doc-" + docID[:6]),
+ CreatedBy: datasetID,
+ ParserConfig: parserConfig,
+ }
+ if err := dao.DB.Create(doc).Error; err != nil {
+ t.Fatalf("insert test document: %v", err)
+ }
+}
+
func insertAggregateTagsKB(t *testing.T, datasetID, tenantID, permission, tagFileID string, docNum int64) {
t.Helper()
parserConfig := entity.JSONMap{}
@@ -68,7 +94,9 @@ func aggregateTagsResultMap(rows []map[string]interface{}) map[string]int {
// TestDatasetServiceAggregateTagsSeedsFromTagFile verifies that the tag
// vocabulary (and its counts) come solely from the configured tag source file,
-// with no chunk-level aggregation.
+// with no chunk-level aggregation. The file is configured on the dataset and
+// reaches the endpoint through the copy upload puts on its document — the
+// endpoint reads documents, not the dataset row.
func TestDatasetServiceAggregateTagsSeedsFromTagFile(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
@@ -76,6 +104,7 @@ func TestDatasetServiceAggregateTagsSeedsFromTagFile(t *testing.T) {
kbInput := "123e4567-e89b-12d3-a456-426614174000"
kbID := strings.ReplaceAll(kbInput, "-", "")
insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)
+ insertAggregateTagsDoc(t, "doc-seed", kbID, "file-1")
loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
if tagFileID == "file-1" {
@@ -117,6 +146,8 @@ func TestDatasetServiceAggregateTagsMergesAcrossTagFiles(t *testing.T) {
kb2ID := strings.ReplaceAll(kb2Input, "-", "")
insertAggregateTagsKB(t, kb1ID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)
insertAggregateTagsKB(t, kb2ID, "user-1", string(entity.TenantPermissionMe), "file-2", 0)
+ insertAggregateTagsDoc(t, "doc-merge-1", kb1ID, "file-1")
+ insertAggregateTagsDoc(t, "doc-merge-2", kb2ID, "file-2")
loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
switch tagFileID {
@@ -210,6 +241,7 @@ func TestDatasetServiceAggregateTagsScopesLoaderToDatasetTenant(t *testing.T) {
kbID := strings.ReplaceAll(kbInput, "-", "")
const kbTenant = "tenant-owner"
insertAggregateTagsKB(t, kbID, kbTenant, string(entity.TenantPermissionTeam), "file-foreign", 0)
+ insertAggregateTagsDoc(t, "doc-scope", kbID, "file-foreign")
insertAggregateTagsMembership(t, kbTenant, "user-1")
ctx := t.Context()
@@ -229,3 +261,280 @@ func TestDatasetServiceAggregateTagsScopesLoaderToDatasetTenant(t *testing.T) {
t.Fatalf("loader ownerTenantID = %q, want %q (must be the dataset tenant, not the user)", gotOwner, kbTenant)
}
}
+
+// assertTagCounts fails unless the aggregated result holds exactly want.
+func assertTagCounts(t *testing.T, result []map[string]interface{}, want map[string]int) {
+ t.Helper()
+ got := aggregateTagsResultMap(result)
+ if len(got) != len(want) {
+ t.Fatalf("result len=%d want=%d result=%v", len(got), len(want), got)
+ }
+ for tag, wantCount := range want {
+ if got[tag] != wantCount {
+ t.Fatalf("tag %q count=%d want=%d all=%v", tag, got[tag], wantCount, got)
+ }
+ }
+}
+
+// TestDatasetServiceAggregateTagsReadsDocumentTagFile is the reported
+// scenario: the dataset keeps the template default (empty tag_file_id) while
+// the tag source is declared on a document by the document parser dialog. The
+// extractor reads that document-level copy at parse time, so aggregation must
+// return its vocabulary as well.
+func TestDatasetServiceAggregateTagsReadsDocumentTagFile(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ kbInput := "623e4567-e89b-12d3-a456-426614174005"
+ kbID := strings.ReplaceAll(kbInput, "-", "")
+ insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "", 0)
+ insertAggregateTagsDoc(t, "doc-tag-a", kbID, "file-1")
+
+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ if ownerTenantID != "user-1" {
+ t.Fatalf("ownerTenantID = %q, want user-1", ownerTenantID)
+ }
+ if tagFileID == "file-1" {
+ return map[string]int{"finance": 2, "urgent": 1}, nil
+ }
+ return nil, nil
+ }
+
+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1")
+ if err != nil {
+ t.Fatalf("AggregateTags failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)
+ }
+ assertTagCounts(t, result, map[string]int{"finance": 2, "urgent": 1})
+}
+
+// TestDatasetServiceAggregateTagsCountsSnapshotOnce: every document inherits
+// the dataset's tag source at upload, so the same file is reached both from
+// the dataset and from each document — it must be loaded and counted once.
+func TestDatasetServiceAggregateTagsCountsSnapshotOnce(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ kbInput := "723e4567-e89b-12d3-a456-426614174006"
+ kbID := strings.ReplaceAll(kbInput, "-", "")
+ insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)
+ insertAggregateTagsDoc(t, "doc-tag-b", kbID, "file-1")
+ insertAggregateTagsDoc(t, "doc-tag-c", kbID, "file-1")
+
+ loads := 0
+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ if tagFileID == "file-1" {
+ loads++
+ return map[string]int{"finance": 2}, nil
+ }
+ return nil, nil
+ }
+
+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1")
+ if err != nil {
+ t.Fatalf("AggregateTags failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)
+ }
+ if loads != 1 {
+ t.Fatalf("loader called %d times, want 1", loads)
+ }
+ assertTagCounts(t, result, map[string]int{"finance": 2})
+}
+
+// TestDatasetServiceAggregateTagsUnionsDocumentTagFiles: documents may
+// override the dataset's tag source individually, so a dataset can be served
+// by more than one file and all of their vocabularies must be merged.
+func TestDatasetServiceAggregateTagsUnionsDocumentTagFiles(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ kbInput := "823e4567-e89b-12d3-a456-426614174007"
+ kbID := strings.ReplaceAll(kbInput, "-", "")
+ insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)
+ insertAggregateTagsDoc(t, "doc-tag-d", kbID, "file-1")
+ insertAggregateTagsDoc(t, "doc-tag-e", kbID, "file-2")
+
+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ switch tagFileID {
+ case "file-1":
+ return map[string]int{"finance": 2}, nil
+ case "file-2":
+ return map[string]int{"urgent": 3, "internal": 1}, nil
+ }
+ return nil, nil
+ }
+
+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1")
+ if err != nil {
+ t.Fatalf("AggregateTags failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)
+ }
+ assertTagCounts(t, result, map[string]int{"finance": 2, "urgent": 3, "internal": 1})
+}
+
+// TestDatasetServiceAggregateTagsCountsSharedFileOnce: two datasets of the
+// same tenant can declare the same source file. The count for a tag is the
+// number of times it occurs in that file, so the file is loaded once for the
+// whole request rather than once per dataset.
+func TestDatasetServiceAggregateTagsCountsSharedFileOnce(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ kb1Input := "923e4567-e89b-12d3-a456-426614174008"
+ kb2Input := "a23e4567-e89b-12d3-a456-426614174009"
+ kb1ID := strings.ReplaceAll(kb1Input, "-", "")
+ kb2ID := strings.ReplaceAll(kb2Input, "-", "")
+ insertAggregateTagsKB(t, kb1ID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)
+ insertAggregateTagsKB(t, kb2ID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)
+ insertAggregateTagsDoc(t, "doc-shared-1", kb1ID, "file-1")
+ insertAggregateTagsDoc(t, "doc-shared-2", kb2ID, "file-1")
+
+ loads := 0
+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ if tagFileID == "file-1" {
+ loads++
+ return map[string]int{"finance": 2}, nil
+ }
+ return nil, nil
+ }
+
+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kb1Input, kb2Input}, "user-1")
+ if err != nil {
+ t.Fatalf("AggregateTags failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)
+ }
+ if loads != 1 {
+ t.Fatalf("loader called %d times, want 1", loads)
+ }
+ assertTagCounts(t, result, map[string]int{"finance": 2})
+}
+
+// missingTagSourceLoader resolves ids that no longer exist through the real
+// loader, so the error carries the production sentinel rather than a stub's
+// approximation of it.
+func missingTagSourceLoader(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ return component.TagVocabularyFromTagFileID(ctx, tagFileID, ownerTenantID)
+}
+
+// TestDatasetServiceAggregateTagsSkipsUnresolvableDocumentTagSource is the
+// availability regression the review flagged: a document keeps referencing a
+// tag file that has been deleted, and nothing clears that reference. One dead
+// document-level source must not discard the vocabulary of every other dataset
+// in the same request.
+func TestDatasetServiceAggregateTagsSkipsUnresolvableDocumentTagSource(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ deadKB := "b23e4567-e89b-12d3-a456-426614174010"
+ liveKB := "c23e4567-e89b-12d3-a456-426614174011"
+ insertAggregateTagsKB(t, strings.ReplaceAll(deadKB, "-", ""), "user-1", string(entity.TenantPermissionMe), "", 0)
+ insertAggregateTagsDoc(t, "doc-dead", strings.ReplaceAll(deadKB, "-", ""), "file-dead")
+ insertAggregateTagsKB(t, strings.ReplaceAll(liveKB, "-", ""), "user-1", string(entity.TenantPermissionMe), "", 0)
+ insertAggregateTagsDoc(t, "doc-live", strings.ReplaceAll(liveKB, "-", ""), "file-live")
+
+ loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ if tagFileID == "file-dead" {
+ return missingTagSourceLoader(ctx, tagFileID, ownerTenantID)
+ }
+ return map[string]int{"finance": 2}, nil
+ }
+
+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{deadKB, liveKB}, "user-1")
+ if err != nil {
+ t.Fatalf("AggregateTags failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)
+ }
+ assertTagCounts(t, result, map[string]int{"finance": 2})
+}
+
+// TestDatasetServiceAggregateTagsIgnoresUnappliedDatasetTagSource pins the
+// chosen semantics: this endpoint reports what is in effect, not what is
+// configured. A source only the dataset row carries has not been applied to
+// any chunk, so it is not reported. That is the deliberate cost of answering
+// at document level — a dataset configured after its documents were uploaded
+// reads empty until those documents are re-uploaded or configured individually.
+func TestDatasetServiceAggregateTagsIgnoresUnappliedDatasetTagSource(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ kbInput := "d23e4567-e89b-12d3-a456-426614174012"
+ insertAggregateTagsKB(t, strings.ReplaceAll(kbInput, "-", ""), "user-1", string(entity.TenantPermissionMe), "file-1", 0)
+
+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ t.Fatalf("loader must not be called for a source no document carries, got %q", tagFileID)
+ return nil, nil
+ }
+
+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1")
+ if err != nil {
+ t.Fatalf("AggregateTags failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)
+ }
+ if len(result) != 0 {
+ t.Fatalf("result=%v want empty: a dataset-level source with no document carrying it is not in effect", result)
+ }
+}
+
+// TestDatasetServiceAggregateTagsFailsOnStorageErrorForDocumentTagSource pins
+// the boundary of the skip: only a source that cannot be resolved by id may be
+// dropped. A storage-layer failure says nothing about that id and must stay a
+// hard failure even for a document-level source.
+func TestDatasetServiceAggregateTagsFailsOnStorageErrorForDocumentTagSource(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ kbInput := "e23e4567-e89b-12d3-a456-426614174013"
+ insertAggregateTagsKB(t, strings.ReplaceAll(kbInput, "-", ""), "user-1", string(entity.TenantPermissionMe), "", 0)
+ insertAggregateTagsDoc(t, "doc-storage", strings.ReplaceAll(kbInput, "-", ""), "file-1")
+
+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ return nil, errors.New("tag source file: no storage backend registered")
+ }
+
+ _, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1")
+ if err == nil {
+ t.Fatal("expected an error for a storage failure on a document-level tag source")
+ }
+ if code != common.CodeServerError {
+ t.Fatalf("code=%d want=%d", code, common.CodeServerError)
+ }
+}
+
+// TestDatasetServiceAggregateTagsScopesDocumentTagSourceToDatasetTenant: the
+// document-level id is reached through the same union, so it must be resolved
+// against the dataset's tenant rather than the caller's (IDOR, CWE-639).
+func TestDatasetServiceAggregateTagsScopesDocumentTagSourceToDatasetTenant(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ kbInput := "f23e4567-e89b-12d3-a456-426614174014"
+ const kbTenant = "tenant-owner"
+ insertAggregateTagsKB(t, strings.ReplaceAll(kbInput, "-", ""), kbTenant, string(entity.TenantPermissionTeam), "", 0)
+ insertAggregateTagsMembership(t, kbTenant, "user-1")
+ insertAggregateTagsDoc(t, "doc-owned", strings.ReplaceAll(kbInput, "-", ""), "file-foreign")
+
+ var gotOwner string
+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ gotOwner = ownerTenantID
+ return map[string]int{"finance": 1}, nil
+ }
+
+ if _, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(t.Context(), []string{kbInput}, "user-1"); err != nil {
+ t.Fatalf("AggregateTags failed: code=%d err=%v", code, err)
+ }
+ if gotOwner != kbTenant {
+ t.Fatalf("loader ownerTenantID = %q, want %q (document sources must resolve against the dataset tenant)", gotOwner, kbTenant)
+ }
+}