Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Implement endpoint GET /datasets/tags/aggregation in GO (#19626)

qinling0210ebe33eaa8b25
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index 0e085d38d6..4a1ff44762 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -926,8 +926,11 @@ func startServer(ctx context.Context) {
// Load the KB objects (mirroring Python RAGTools' self.kbs via
// KnowledgebaseService.get_by_ids(kb_ids)) so the agentic tool can
- // derive rank features from parser_config.tag_kb_ids. Best-effort: a
- // load failure leaves KBs empty and the adapter resolves them itself.
+ // derive rank features. The Go tag extractor (extractor_tag.go) writes
+ // both tag_kwd (the list of tag names) and tag_feas (per-tag weights)
+ // onto each chunk at parse time; the labeler aggregates tag_kwd to build
+ // the tag vocabulary and the retriever ranks with tag_feas. Best-effort:
+ // a load failure leaves KBs empty and the adapter resolves them itself.
var kbs []*entity.Knowledgebase
if len(req.DatasetIDs) > 0 {
if loaded, lErr := dao.NewKnowledgebaseDAO().GetByIDs(ctx, dao.DB, req.DatasetIDs); lErr == nil {
diff --git a/internal/dao/file.go b/internal/dao/file.go
index 281d6c032c..131644d887 100644
--- a/internal/dao/file.go
+++ b/internal/dao/file.go
@@ -45,6 +45,24 @@ func (dao *FileDAO) GetByID(ctx context.Context, db *gorm.DB, id string) (*entit
return &file, nil
}
+// GetByIDAndTenant gets a file by ID scoped to the given tenant. Callers that
+// resolve a user-supplied file ID (e.g. parser_config.tags.tag_file_id, which
+// the dataset update API accepts from the client) MUST use this instead of
+// GetByID: an unscoped ID lookup crosses tenant boundaries and is an IDOR
+// (CWE-639). An empty id or tenantID fails closed with gorm.ErrRecordNotFound
+// so callers see the same "not found" as a missing row.
+func (dao *FileDAO) GetByIDAndTenant(ctx context.Context, db *gorm.DB, id, tenantID string) (*entity.File, error) {
+ if id == "" || tenantID == "" {
+ return nil, gorm.ErrRecordNotFound
+ }
+ var file entity.File
+ err := db.WithContext(ctx).Where("id = ? AND tenant_id = ?", id, tenantID).First(&file).Error
+ if err != nil {
+ return nil, err
+ }
+ return &file, nil
+}
+
// GetByPfID gets files by parent folder ID with pagination and filtering.
// When keywords is empty, only direct children of pfID are listed; when
// keywords is non-empty, the search covers the whole subtree under pfID so
diff --git a/internal/dao/file_test.go b/internal/dao/file_test.go
index 82da00431a..1cd68dbacc 100644
--- a/internal/dao/file_test.go
+++ b/internal/dao/file_test.go
@@ -147,3 +147,32 @@ func TestFileDAO_GetByPfID_NoKeywordsListsDirectChildren(t *testing.T) {
}
}
}
+
+func TestFileDAO_GetByIDAndTenant(t *testing.T) {
+ db := setupFileTestDB(t)
+ seedFileTree(t, db) // f-t2 belongs to tenant t2
+ d := NewFileDAO()
+ ctx := t.Context()
+
+ got, err := d.GetByIDAndTenant(ctx, db, "f-t2", "t2")
+ if err != nil {
+ t.Fatalf("owner tenant lookup failed: %v", err)
+ }
+ if got.ID != "f-t2" {
+ t.Fatalf("id = %q, want f-t2", got.ID)
+ }
+
+ // Another tenant, an empty tenant (fail closed) and a case-different tenant
+ // must all miss.
+ for _, tenantID := range []string{"t1", "", "T2"} {
+ if _, err := d.GetByIDAndTenant(ctx, db, "f-t2", tenantID); !IsNotFoundErr(err) {
+ t.Fatalf("tenant %q: err = %v, want gorm.ErrRecordNotFound", tenantID, err)
+ }
+ }
+
+ // The unscoped GetByID still returns the row across tenants: that is exactly
+ // the lookup the tag-source loader must not use with a user-supplied ID.
+ if _, err := d.GetByID(ctx, db, "f-t2"); err != nil {
+ t.Fatalf("unscoped GetByID should still see the row: %v", err)
+ }
+}
diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.go
index d83cae3dbb..80edf8e3d2 100644
--- a/internal/ingestion/component/extractor_tag.go
+++ b/internal/ingestion/component/extractor_tag.go
@@ -645,7 +645,11 @@ func (c *ExtractorComponent) resolveTagSource(ctx context.Context, lang string)
}
func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context, lang string) (*MemoryTagIndex, bool) {
- f, err := dao.NewFileDAO().GetByID(ctx, dao.DB, c.Param.Tags.TagFileID)
+ // Same IDOR guard as TagVocabularyFromTagFileID: tag_file_id is
+ // user-controlled through parser_config, so it must resolve inside the
+ // caller's tenant before the bytes are read.
+ tenantID := globals.GlobalOrInput(ctx, nil, "tenant_id", "")
+ f, err := dao.NewFileDAO().GetByIDAndTenant(ctx, dao.DB, c.Param.Tags.TagFileID, tenantID)
if err != nil || f == nil || f.Location == nil || *f.Location == "" {
common.Warn(fmt.Sprintf("extractor tags: resolve tag_file_id %q: %v", c.Param.Tags.TagFileID, err))
return nil, false
@@ -664,7 +668,6 @@ func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context, lang string
common.Warn("extractor tags: no storage backend registered")
return nil, false
}
- tenantID := globals.GlobalOrInput(ctx, nil, "tenant_id", "")
data, err := stg.Get(ctx, f.ParentID, *f.Location, tenantID)
if err != nil {
common.Warn(fmt.Sprintf("extractor tags: load tag source %q/%q: %v", f.ParentID, *f.Location, err))
@@ -733,6 +736,75 @@ func parseTagSourceByFilename(data []byte, filename string) ([]schema.TagLabel,
}
}
+// TagVocabularyFromBytes is the storage/DAO-free core of
+// TagVocabularyFromTagFileID. It parses a tag source file's raw bytes (mirroring
+// rag/app/tag.py's chunk format) and returns the vocabulary as tag -> number of
+// source examples that mention the tag; a tag repeated within one source example
+// counts once, matching buildMemoryTagIndex's per-sample deduplication. Note: the
+// Go tag extractor
+// (matchAndTagChunk) DOES write tag_kwd onto chunks at parse time, but this
+// vocabulary is the authoritative selectable-tag list for the Go backend (the
+// tag-options/aggregation API sources the list from the tag source file, not
+// from chunk usage).
+func TagVocabularyFromBytes(data []byte, filename string) (map[string]int, error) {
+ labels, err := parseTagSourceByFilename(data, filename)
+ if err != nil {
+ return nil, err
+ }
+ counts := make(map[string]int)
+ for _, lbl := range labels {
+ seen := make(map[string]struct{}, len(lbl.Tags))
+ for _, t := range lbl.Tags {
+ t = strings.TrimSpace(strings.ReplaceAll(t, ".", "_"))
+ if t != "" {
+ if _, exists := seen[t]; exists {
+ continue
+ }
+ seen[t] = struct{}{}
+ counts[t]++
+ }
+ }
+ }
+ return counts, nil
+}
+
+// TagVocabularyFromTagFileID loads a tag source file (parser_config.tags.tag_file_id)
+// and returns the tag vocabulary it defines. It is the Go-native source of the
+// selectable-tag list surfaced by the tag-options API. Although the Go tag
+// extractor (matchAndTagChunk) writes tag_kwd onto chunks during parsing, the
+// selectable-tag list is taken from this vocabulary (the tag source file), not
+// from chunk usage.
+//
+// It returns (nil, nil) when tagFileID is empty.
+//
+// ownerTenantID is the tenant of the dataset that configured the file and is
+// required: tag_file_id is user-writable through parser_config, so the file must
+// be proven to belong to that tenant before its bytes are read (IDOR, CWE-639).
+// An empty ownerTenantID fails closed. Note that storage.Get's tenant argument
+// does not establish file ownership by itself.
+func TagVocabularyFromTagFileID(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ if tagFileID == "" {
+ return nil, nil
+ }
+ common.Info(fmt.Sprintf("tag_vocab: loading tag source file_id=%q", tagFileID))
+ f, err := dao.NewFileDAO().GetByIDAndTenant(ctx, dao.DB, tagFileID, ownerTenantID)
+ if err != nil || f == nil || f.Location == nil || *f.Location == "" {
+ return nil, fmt.Errorf("tag source file %q not found: %w", tagFileID, err)
+ }
+ common.Info(fmt.Sprintf("tag_vocab: file_id=%q name=%q parent_id=%q location=%q",
+ tagFileID, f.Name, f.ParentID, *f.Location))
+ stg := resolveStorage()
+ if stg == nil {
+ return nil, fmt.Errorf("tag source file %q: no storage backend registered", tagFileID)
+ }
+ data, err := stg.Get(ctx, f.ParentID, *f.Location, ownerTenantID)
+ if err != nil {
+ return nil, fmt.Errorf("load tag source file %q/%q: %w", f.ParentID, *f.Location, err)
+ }
+ common.Info(fmt.Sprintf("tag_vocab: file_id=%q loaded %d bytes", tagFileID, len(data)))
+ return TagVocabularyFromBytes(data, f.Name)
+}
+
func parseCSVTagSource(text string) []schema.TagLabel {
return parseCSVTagSourceBytes([]byte(text))
}
diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.go
index 7bd908c0cd..173541fea0 100644
--- a/internal/ingestion/component/extractor_tag_test.go
+++ b/internal/ingestion/component/extractor_tag_test.go
@@ -12,6 +12,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
+ "ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
)
@@ -117,6 +118,130 @@ func TestExtractorTags_WithKeywords(t *testing.T) {
}
}
+func TestTagVocabularyFromBytes(t *testing.T) {
+ // Each line is [content, tags]; the tags column may hold a comma-separated
+ // list, which requires quoting (or a tab delimiter) to survive CSV parsing.
+ csv := []byte("\"some content\",\"finance,urgent\"\n\"other content\",\"finance\"\n\"just a line\",\"legal\"")
+ vocab, err := TagVocabularyFromBytes(csv, "tags.csv")
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := map[string]int{"finance": 2, "urgent": 1, "legal": 1}
+ if len(vocab) != len(want) {
+ t.Fatalf("vocab=%v want=%v", vocab, want)
+ }
+ for tag, c := range want {
+ if vocab[tag] != c {
+ t.Fatalf("tag %q count=%d want=%d", tag, vocab[tag], c)
+ }
+ }
+
+ // Dots in tags are normalized to underscores, matching the extractor.
+ csv2 := []byte("\"x\",\"Alpha.Beta\"\n\"y\",\"Alpha.Beta\"")
+ vocab2, err := TagVocabularyFromBytes(csv2, "tags.csv")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if vocab2["Alpha_Beta"] != 2 {
+ t.Fatalf("expected Alpha_Beta=2, got %v", vocab2)
+ }
+
+ // A tag repeated inside one source example counts once: the count is the
+ // number of source examples mentioning the tag, consistent with
+ // buildMemoryTagIndex's per-sample deduplication.
+ csv3 := []byte("\"dup content\",\"finance,finance,urgent\"\n\"other content\",\"finance\"")
+ vocab3, err := TagVocabularyFromBytes(csv3, "tags.csv")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if vocab3["finance"] != 2 { // two examples, not three occurrences
+ t.Fatalf("expected finance=2 (per source example), got %v", vocab3)
+ }
+ if vocab3["urgent"] != 1 {
+ t.Fatalf("expected urgent=1, got %v", vocab3)
+ }
+ if len(vocab3) != 2 {
+ t.Fatalf("expected 2 distinct tags, got %v", vocab3)
+ }
+
+ // Unsupported extension is rejected.
+ if _, err := TagVocabularyFromBytes([]byte("x"), "tags.pdf"); err == nil {
+ t.Fatal("expected error for unsupported extension")
+ }
+}
+
+// TestTagVocabularyFromTagFileID_RejectsForeignTenantFile pins the IDOR guard.
+// parser_config.tags.tag_file_id is user-controlled (the dataset update API
+// accepts parser_config), so the file it names must belong to the caller's
+// tenant before its bytes are read. The tag source bytes are deliberately
+// seeded in storage: without the ownership check the loader would happily
+// return them to the other tenant.
+func TestTagVocabularyFromTagFileID_RejectsForeignTenantFile(t *testing.T) {
+ db := withFileComponentTestDB(t)
+ ms := withMemoryStorage(t)
+
+ const (
+ ownerTenant = "tenant-owner"
+ attackerTenant = "tenant-attacker"
+ fileID = "tag-file-foreign"
+ bucket = "kb-bucket-1"
+ location = "tags/tags.csv"
+ )
+
+ csv := []byte("\"some content\",\"finance,urgent\"\n\"other content\",\"finance\"")
+ loc := location
+ if err := db.Create(&entity.File{
+ ID: fileID,
+ ParentID: bucket,
+ TenantID: ownerTenant,
+ CreatedBy: ownerTenant,
+ Name: "tags.csv",
+ Type: "doc",
+ Location: &loc,
+ }).Error; err != nil {
+ t.Fatalf("seed file row: %v", err)
+ }
+ if err := ms.Put(t.Context(), bucket, location, csv); err != nil {
+ t.Fatalf("seed storage: %v", err)
+ }
+
+ // A dataset whose tenant does not own the file must be denied, and must not
+ // get the bytes.
+ vocab, err := TagVocabularyFromTagFileID(t.Context(), fileID, attackerTenant)
+ if err == nil {
+ t.Fatalf("IDOR: foreign tenant read the tag source file, vocab=%v", vocab)
+ }
+ if vocab != nil {
+ t.Fatalf("IDOR: expected no vocabulary for foreign tenant, got %v", vocab)
+ }
+
+ // An empty owner tenant fails closed as well.
+ if _, err := TagVocabularyFromTagFileID(t.Context(), fileID, ""); err == nil {
+ t.Fatal("empty owner tenant must fail closed")
+ }
+
+ // The owning tenant still gets the vocabulary.
+ vocab, err = TagVocabularyFromTagFileID(t.Context(), fileID, ownerTenant)
+ if err != nil {
+ t.Fatalf("owner tenant load failed: %v", err)
+ }
+ if vocab["finance"] != 2 || vocab["urgent"] != 1 {
+ t.Fatalf("owner vocab = %v, want finance=2 urgent=1", vocab)
+ }
+
+ // Sharing is intentional: the guard is tenant-scoped, not dataset-scoped (the
+ // file's ParentID is never compared against the referencing dataset), so any
+ // number of datasets in ownerTenant can point at this same tag source file —
+ // each passes its own dataset tenant (kb.TenantID) here.
+ shared, err := TagVocabularyFromTagFileID(t.Context(), fileID, ownerTenant)
+ if err != nil {
+ t.Fatalf("a second dataset in the same tenant must resolve the shared file: %v", err)
+ }
+ if shared["finance"] != 2 {
+ t.Fatalf("shared vocab = %v, want finance=2", shared)
+ }
+}
+
func TestExtractorTags_ComponentRegistration(t *testing.T) {
factory, cat, md, ok := runtime.DefaultRegistry.Lookup(componentNameExtractor)
if !ok {
diff --git a/internal/rag/advanced_rag/harness/tool_search.go b/internal/rag/advanced_rag/harness/tool_search.go
index df9fa80913..33406f6130 100644
--- a/internal/rag/advanced_rag/harness/tool_search.go
+++ b/internal/rag/advanced_rag/harness/tool_search.go
@@ -444,10 +444,14 @@ type SearchDeps struct {
}
// QuestionLabeler mirrors Python rag.app.tag.label_question(question, kbs)
-// (agentic_rag.py:retrieve → tag.py). Given the query and the KB objects (which
-// carry parser_config.tag_kb_ids), it returns a map of question-type tag →
-// weight the retriever uses to rank results. The production implementation is
-// internal/service.MetadataService.LabelQuestion; tests supply a stub.
+// (agentic_rag.py:retrieve → tag.py). Given the query and the KB objects, it
+// returns a map of question-type tag → weight the retriever uses to rank
+// results. The Go extractor (extractor_tag.go) writes tag_kwd (the tag-name
+// list) and tag_feas (per-tag weights) onto each chunk; the labeler aggregates
+// tag_kwd to build the vocabulary and the retriever ranks with tag_feas. It
+// does not rely on a separate Python-only tag-library dataset. The production
+// implementation is internal/service.MetadataService.LabelQuestion; tests
+// supply a stub.
type QuestionLabeler interface {
LabelQuestion(ctx context.Context, question string, kbs []*entity.Knowledgebase) map[string]float64
}
diff --git a/internal/service/dataset/service.go b/internal/service/dataset/service.go
index 28081e9133..e0116aa201 100644
--- a/internal/service/dataset/service.go
+++ b/internal/service/dataset/service.go
@@ -1,6 +1,8 @@
package dataset
import (
+ "context"
+
"ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/service"
@@ -20,6 +22,13 @@ type DatasetService struct {
searchService *service.SearchService
docEngine engine.DocEngine
embeddingCache *utility.EmbeddingLRU
+ // tagVocabularyLoader resolves the selectable-tag vocabulary for a tag
+ // source file (parser_config.tags.tag_file_id). It is a field so tests can
+ // inject a fake; in production it defaults to
+ // component.TagVocabularyFromTagFileID (see AggregateTags). The ownerTenantID
+ // it receives is the dataset's tenant: the loader must prove the file belongs
+ // to it, since tag_file_id is user-writable (IDOR, CWE-639).
+ tagVocabularyLoader func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error)
}
// NewDatasetService creates a new datasets service.
diff --git a/internal/service/dataset/tags.go b/internal/service/dataset/tags.go
index 54b6ee51fd..e6a81c175d 100644
--- a/internal/service/dataset/tags.go
+++ b/internal/service/dataset/tags.go
@@ -11,17 +11,110 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
enginetypes "ragflow/internal/engine/types"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/component"
)
+// tagFileIDFromTagsMap pulls tag_file_id out of a raw "tags" config value
+// (i.e. parser_config["tags"] in the legacy flat form, or
+// parser_config["Extractor:<name>"]["tags"] in the Go pipeline scoped form),
+// accepting either a map[string]any or an entity.JSONMap. Returns "" when the
+// value is not a tags map or has no tag_file_id. In the Go backend the
+// authoritative tag vocabulary lives in this tag source file (parsed by the
+// extractor in extractor_tag.go). Note: the Go tag extractor (matchAndTagChunk)
+// DOES write tag_kwd onto chunks at parse time, but the selectable-tag list is
+// sourced from this vocabulary, not from chunk usage. Returns "" when unset.
+func tagFileIDFromTagsMap(raw any) string {
+ if raw == nil {
+ return ""
+ }
+ var tags map[string]any
+ switch v := raw.(type) {
+ case map[string]any:
+ tags = v
+ case entity.JSONMap:
+ tags = map[string]any(v)
+ default:
+ return ""
+ }
+ if v, ok := tags["tag_file_id"].(string); ok {
+ return strings.TrimSpace(v)
+ }
+ return ""
+}
+
+// extractTagFileID returns the tag source file ID declared in a dataset's
+// parser_config. The tags config lives in two possible shapes depending on how
+// the dataset was persisted:
+//
+// - Legacy / flat form: parser_config["tags"]["tag_file_id"].
+// - Go pipeline scoped form: parser_config["Extractor:<name>"]["tags"]
+// ["tag_file_id"] (e.g. "Extractor:AutoExtractDefault"). In this form the
+// component config is keyed by "<Component>:<InstanceName>" and the
+// sub-feature maps (tags/keywords/questions/summary/metadata) sit at the
+// top level of that value — mirroring how NewExtractorComponent reads its
+// params.
+//
+// The Go backend sources the authoritative tag vocabulary from this tag source
+// file (parsed by the extractor in extractor_tag.go). Note: the Go tag extractor
+// (matchAndTagChunk) DOES write tag_kwd onto chunks at parse time, but the
+// selectable-tag list is taken from this vocabulary, not from chunk usage.
+// Returns "" when unset.
+func extractTagFileID(parserConfig entity.JSONMap) string {
+ if parserConfig == nil {
+ return ""
+ }
+ // 1. Legacy / flat form.
+ if v := tagFileIDFromTagsMap(parserConfig["tags"]); v != "" {
+ return v
+ }
+ // 2. Go pipeline scoped form: any key shaped "<Component>:<Instance>" whose
+ // component is Extractor.
+ for key, raw := range parserConfig {
+ if key == "Extractor" || strings.HasPrefix(key, "Extractor:") {
+ extractorMap, ok := raw.(map[string]any)
+ if !ok {
+ // entity.JSONMap is an alias of map[string]any, so the assertion
+ // above already covers it; this guard handles any other wrapper.
+ if jm, ok2 := raw.(entity.JSONMap); ok2 {
+ extractorMap = map[string]any(jm)
+ } else {
+ continue
+ }
+ }
+ // The tags config sits at the top level of the Extractor node value,
+ // i.e. parser_config["Extractor:<name>"]["tags"], NOT at the node
+ // value itself. Pass that nested "tags" map to tagFileIDFromTagsMap.
+ if v := tagFileIDFromTagsMap(extractorMap["tags"]); v != "" {
+ return v
+ }
+ }
+ }
+ return ""
+}
+
func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string, userID string) ([]map[string]interface{}, common.ErrorCode, error) {
if len(datasetIDs) == 0 {
return nil, common.CodeDataError, errors.New("Lack of dataset_ids in query parameters")
}
- if d.docEngine == nil {
- return nil, common.CodeServerError, errors.New("Document engine is not initialized")
+ // merged holds the selectable-tag vocabulary sourced exclusively from each
+ // dataset's configured tag source file (parser_config.tags.tag_file_id). The
+ // count for a tag is the number of times it appears in that source file. The
+ // Go tag extractor (matchAndTagChunk) does write tag_kwd onto chunks at parse
+ // time, but by requirement the count reflects only the tag-file occurrence
+ // count, so there is intentionally no chunk-level aggregation here. The Go
+ // backend has no Python-style tag-library datasets.
+ loader := d.tagVocabularyLoader
+ if loader == nil {
+ loader = component.TagVocabularyFromTagFileID
}
-
- datasetIDsByTenant := make(map[string][]string)
+ merged := make(map[string]int)
+ // The handler accepts repeated IDs, and distinct raw IDs can normalize to
+ // the same dataset (hyphenated vs. compact form). Track the normalized IDs
+ // so each dataset is loaded and merged exactly once — otherwise a repeated
+ // dataset (e.g. dataset_ids=A,A) would load the same vocabulary twice and
+ // double every count.
+ seen := make(map[string]struct{}, len(datasetIDs))
for _, rawID := range datasetIDs {
rawID = strings.TrimSpace(rawID)
if rawID == "" {
@@ -31,6 +124,10 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,
if err != nil {
return nil, common.CodeDataError, err
}
+ if _, dup := seen[datasetID]; dup {
+ continue
+ }
+ seen[datasetID] = struct{}{}
if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {
return nil, common.CodeDataError, fmt.Errorf("No authorization for dataset '%s'", datasetID)
}
@@ -41,48 +138,19 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,
}
return nil, common.CodeServerError, errors.New("Database operation failed")
}
- if kb.DocNum <= 0 {
- continue
- }
- datasetIDsByTenant[kb.TenantID] = append(datasetIDsByTenant[kb.TenantID], datasetID)
- }
- const pageSize = 10000
- merged := make(map[string]int)
- for tenantID, kbIDs := range datasetIDsByTenant {
- for offset := 0; ; offset += pageSize {
- searchResp, err := d.docEngine.Search(ctx, &enginetypes.SearchRequest{
- IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
- KbIDs: kbIDs,
- Offset: offset,
- Limit: pageSize,
- SelectFields: []string{"tag_kwd"},
- })
- if err != nil {
- return nil, common.CodeServerError, fmt.Errorf("failed to aggregate tags: %w", err)
- }
- for _, agg := range d.docEngine.GetAggregation(searchResp.Chunks, "tag_kwd") {
- tag, _ := agg["key"].(string)
- if tag == "" {
- continue
- }
- switch count := agg["count"].(type) {
- case int:
- merged[tag] += count
- case int32:
- merged[tag] += int(count)
- case int64:
- merged[tag] += int(count)
- case float64:
- merged[tag] += int(count)
- }
- }
- chunkCount := len(searchResp.Chunks)
- if chunkCount == 0 || chunkCount < pageSize {
- break
+ // Include the dataset's tag-source vocabulary. The count for each tag is
+ // the number of times it occurs in the tag source file. The file is
+ // resolved against the dataset's own tenant: tag_file_id is user-writable,
+ // so a foreign file ID must not resolve (IDOR, CWE-639).
+ if tagFileID := extractTagFileID(kb.ParserConfig); tagFileID != "" {
+ counts, vErr := loader(ctx, tagFileID, kb.TenantID)
+ if vErr != nil {
+ return nil, common.CodeServerError,
+ fmt.Errorf("load tag vocabulary for dataset %q: %w", datasetID, vErr)
}
- if searchResp.Total > 0 && int64(offset+chunkCount) >= searchResp.Total {
- break
+ for tag, c := range counts {
+ merged[tag] += c
}
}
}
diff --git a/internal/service/dataset/tags_aggregate_test.go b/internal/service/dataset/tags_aggregate_test.go
index e560a06e87..6ca0c81548 100644
--- a/internal/service/dataset/tags_aggregate_test.go
+++ b/internal/service/dataset/tags_aggregate_test.go
@@ -2,103 +2,28 @@ package dataset
import (
"context"
- "errors"
"strings"
"testing"
"ragflow/internal/common"
"ragflow/internal/dao"
- "ragflow/internal/engine"
- "ragflow/internal/engine/types"
"ragflow/internal/entity"
)
-type aggregateTagsMockEngine struct {
- engine.DocEngine
- searchResults map[string]*types.SearchResult
- pagedSearchResults map[string]map[int]*types.SearchResult
- searchErr error
- requests []*types.SearchRequest
-}
-
-func (m *aggregateTagsMockEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
- if m.searchErr != nil {
- return nil, m.searchErr
- }
- cloned := &types.SearchRequest{
- IndexNames: append([]string(nil), req.IndexNames...),
- KbIDs: append([]string(nil), req.KbIDs...),
- Offset: req.Offset,
- Limit: req.Limit,
- SelectFields: append([]string(nil), req.SelectFields...),
- }
- m.requests = append(m.requests, cloned)
- if len(req.IndexNames) == 0 {
- return &types.SearchResult{}, nil
- }
- if byOffset, ok := m.pagedSearchResults[req.IndexNames[0]]; ok {
- if res, ok := byOffset[req.Offset]; ok {
- return res, nil
- }
- return &types.SearchResult{}, nil
- }
- if res, ok := m.searchResults[req.IndexNames[0]]; ok {
- return res, nil
- }
- return &types.SearchResult{}, nil
-}
-
-func (m *aggregateTagsMockEngine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} {
- counts := make(map[string]int)
- for _, chunk := range chunks {
- raw, ok := chunk[fieldName]
- if !ok || raw == nil {
- continue
- }
- switch value := raw.(type) {
- case string:
- for _, tag := range strings.Split(value, "###") {
- tag = strings.TrimSpace(tag)
- if tag != "" {
- counts[tag]++
- }
- }
- case []string:
- for _, tag := range value {
- tag = strings.TrimSpace(tag)
- if tag != "" {
- counts[tag]++
- }
- }
- }
- }
-
- result := make([]map[string]interface{}, 0, len(counts))
- for tag, count := range counts {
- result = append(result, map[string]interface{}{
- "key": tag,
- "count": count,
- })
- }
- return result
-}
-
-func (m *aggregateTagsMockEngine) GetType() string { return "mock" }
-
-func (m *aggregateTagsMockEngine) Ping(ctx context.Context) error { return nil }
-
-func (m *aggregateTagsMockEngine) Close() error { return nil }
-
-func testDatasetServiceForAggregateTags(t *testing.T, docEngine engine.DocEngine) *DatasetService {
+func testDatasetServiceForAggregateTags(t *testing.T, loader func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error)) *DatasetService {
t.Helper()
return &DatasetService{
- kbDAO: dao.NewKnowledgebaseDAO(),
- docEngine: docEngine,
+ kbDAO: dao.NewKnowledgebaseDAO(),
+ tagVocabularyLoader: loader,
}
}
-func insertAggregateTagsKB(t *testing.T, datasetID, tenantID, permission string, docNum int64) {
+func insertAggregateTagsKB(t *testing.T, datasetID, tenantID, permission, tagFileID string, docNum int64) {
t.Helper()
+ parserConfig := entity.JSONMap{}
+ if tagFileID != "" {
+ parserConfig["tags"] = map[string]any{"tag_file_id": tagFileID}
+ }
kb := &entity.Knowledgebase{
ID: datasetID,
TenantID: tenantID,
@@ -107,7 +32,7 @@ func insertAggregateTagsKB(t *testing.T, datasetID, tenantID, permission string,
CreatedBy: tenantID,
Permission: permission,
ParserID: "naive",
- ParserConfig: entity.JSONMap{},
+ ParserConfig: parserConfig,
DocNum: docNum,
Status: sptr(string(entity.StatusValid)),
}
@@ -141,37 +66,26 @@ func aggregateTagsResultMap(rows []map[string]interface{}) map[string]int {
return result
}
-func TestDatasetServiceAggregateTagsMergesAcrossTenants(t *testing.T) {
+// TestDatasetServiceAggregateTagsSeedsFromTagFile verifies that the tag
+// vocabulary (and its counts) come solely from the configured tag source file,
+// with no chunk-level aggregation.
+func TestDatasetServiceAggregateTagsSeedsFromTagFile(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
- kb1Input := "123e4567-e89b-12d3-a456-426614174000"
- kb2Input := "223e4567-e89b-12d3-a456-426614174001"
- kb1ID := strings.ReplaceAll(kb1Input, "-", "")
- kb2ID := strings.ReplaceAll(kb2Input, "-", "")
-
- insertAggregateTagsKB(t, kb1ID, "user-1", string(entity.TenantPermissionMe), 2)
- insertAggregateTagsKB(t, kb2ID, "tenant-2", string(entity.TenantPermissionTeam), 1)
- insertAggregateTagsMembership(t, "tenant-2", "user-1")
+ kbInput := "123e4567-e89b-12d3-a456-426614174000"
+ kbID := strings.ReplaceAll(kbInput, "-", "")
+ insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)
- docEngine := &aggregateTagsMockEngine{
- searchResults: map[string]*types.SearchResult{
- "ragflow_user-1": {
- Chunks: []map[string]interface{}{
- {"tag_kwd": "finance###urgent"},
- {"tag_kwd": "finance"},
- },
- },
- "ragflow_tenant-2": {
- Chunks: []map[string]interface{}{
- {"tag_kwd": "urgent###internal"},
- },
- },
- },
+ loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ if tagFileID == "file-1" {
+ return map[string]int{"finance": 2, "urgent": 1}, nil
+ }
+ return nil, nil
}
ctx := t.Context()
- result, code, err := testDatasetServiceForAggregateTags(t, docEngine).AggregateTags(ctx, []string{kb1Input, kb2Input}, "user-1")
+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(ctx, []string{kbInput}, "user-1")
if err != nil {
t.Fatalf("AggregateTags failed: %v", err)
}
@@ -180,11 +94,7 @@ func TestDatasetServiceAggregateTagsMergesAcrossTenants(t *testing.T) {
}
got := aggregateTagsResultMap(result)
- want := map[string]int{
- "finance": 2,
- "urgent": 2,
- "internal": 1,
- }
+ want := map[string]int{"finance": 2, "urgent": 1}
if len(got) != len(want) {
t.Fatalf("result len=%d want=%d result=%v", len(got), len(want), got)
}
@@ -193,92 +103,78 @@ func TestDatasetServiceAggregateTagsMergesAcrossTenants(t *testing.T) {
t.Fatalf("tag %q count=%d want=%d all=%v", tag, got[tag], wantCount, got)
}
}
+}
- if len(docEngine.requests) != 2 {
- t.Fatalf("search requests=%d want=2", len(docEngine.requests))
- }
+// TestDatasetServiceAggregateTagsMergesAcrossTagFiles verifies that counts from
+// multiple datasets' tag source files are merged by tag name.
+func TestDatasetServiceAggregateTagsMergesAcrossTagFiles(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
- requestByIndex := make(map[string]*types.SearchRequest, len(docEngine.requests))
- for _, req := range docEngine.requests {
- if len(req.IndexNames) != 1 {
- t.Fatalf("IndexNames=%v want single entry", req.IndexNames)
- }
- requestByIndex[req.IndexNames[0]] = req
- if req.Offset != 0 {
- t.Fatalf("Offset=%d want=0", req.Offset)
- }
- if req.Limit != 10000 {
- t.Fatalf("Limit=%d want=10000", req.Limit)
- }
- if len(req.SelectFields) != 1 || req.SelectFields[0] != "tag_kwd" {
- t.Fatalf("SelectFields=%v want [tag_kwd]", req.SelectFields)
+ kb1Input := "123e4567-e89b-12d3-a456-426614174000"
+ kb2Input := "223e4567-e89b-12d3-a456-426614174001"
+ kb1ID := strings.ReplaceAll(kb1Input, "-", "")
+ kb2ID := strings.ReplaceAll(kb2Input, "-", "")
+ insertAggregateTagsKB(t, kb1ID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)
+ insertAggregateTagsKB(t, kb2ID, "user-1", string(entity.TenantPermissionMe), "file-2", 0)
+
+ loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ switch tagFileID {
+ case "file-1":
+ return map[string]int{"finance": 2, "urgent": 1}, nil
+ case "file-2":
+ return map[string]int{"finance": 1, "urgent": 3, "internal": 1}, nil
}
+ return nil, nil
+ }
+ ctx := t.Context()
+
+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(ctx, []string{kb1Input, kb2Input}, "user-1")
+ if err != nil {
+ t.Fatalf("AggregateTags failed: %v", err)
+ }
+ if code != common.CodeSuccess {
+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)
}
- if req := requestByIndex["ragflow_user-1"]; req == nil || len(req.KbIDs) != 1 || req.KbIDs[0] != kb1ID {
- t.Fatalf("request for ragflow_user-1 = %#v, want kbIDs=[%s]", req, kb1ID)
+ got := aggregateTagsResultMap(result)
+ want := map[string]int{"finance": 3, "urgent": 4, "internal": 1}
+ if len(got) != len(want) {
+ t.Fatalf("result len=%d want=%d result=%v", len(got), len(want), got)
}
- if req := requestByIndex["ragflow_tenant-2"]; req == nil || len(req.KbIDs) != 1 || req.KbIDs[0] != kb2ID {
- t.Fatalf("request for ragflow_tenant-2 = %#v, want kbIDs=[%s]", req, kb2ID)
+ for tag, wantCount := range want {
+ if got[tag] != wantCount {
+ t.Fatalf("tag %q count=%d want=%d all=%v", tag, got[tag], wantCount, got)
+ }
}
}
-func TestDatasetServiceAggregateTagsPagesThroughAllChunks(t *testing.T) {
+// TestDatasetServiceAggregateTagsEmptyWhenNoTagFile verifies that a dataset
+// without a configured tag source file yields an empty vocabulary and does not
+// invoke the loader.
+func TestDatasetServiceAggregateTagsEmptyWhenNoTagFile(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
- kbInput := "723e4567-e89b-12d3-a456-426614174006"
+ kbInput := "323e4567-e89b-12d3-a456-426614174002"
kbID := strings.ReplaceAll(kbInput, "-", "")
- insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), 10002)
+ insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "", 0)
- firstPage := make([]map[string]interface{}, 10000)
- for i := range firstPage {
- firstPage[i] = map[string]interface{}{"tag_kwd": "finance"}
- }
-
- docEngine := &aggregateTagsMockEngine{
- pagedSearchResults: map[string]map[int]*types.SearchResult{
- "ragflow_user-1": {
- 0: {
- Chunks: firstPage,
- Total: 10002,
- },
- 10000: {
- Chunks: []map[string]interface{}{
- {"tag_kwd": "finance"},
- {"tag_kwd": "urgent"},
- },
- Total: 10002,
- },
- },
- },
+ loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ t.Fatalf("loader should not be called when no tag_file_id is set")
+ return nil, nil
}
ctx := t.Context()
- result, code, err := testDatasetServiceForAggregateTags(t, docEngine).AggregateTags(ctx, []string{kbInput}, "user-1")
+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(ctx, []string{kbInput}, "user-1")
if err != nil {
t.Fatalf("AggregateTags failed: %v", err)
}
if code != common.CodeSuccess {
t.Fatalf("code=%d want=%d", code, common.CodeSuccess)
}
-
- got := aggregateTagsResultMap(result)
- if got["finance"] != 10001 {
- t.Fatalf("finance count=%d want=10001 result=%v", got["finance"], got)
- }
- if got["urgent"] != 1 {
- t.Fatalf("urgent count=%d want=1 result=%v", got["urgent"], got)
- }
-
- if len(docEngine.requests) != 2 {
- t.Fatalf("search requests=%d want=2", len(docEngine.requests))
- }
- if docEngine.requests[0].Offset != 0 {
- t.Fatalf("first request offset=%d want=0", docEngine.requests[0].Offset)
- }
- if docEngine.requests[1].Offset != 10000 {
- t.Fatalf("second request offset=%d want=10000", docEngine.requests[1].Offset)
+ if len(result) != 0 {
+ t.Fatalf("result=%v want empty", result)
}
}
@@ -286,12 +182,12 @@ func TestDatasetServiceAggregateTagsRejectsUnauthorizedDataset(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
- kbInput := "323e4567-e89b-12d3-a456-426614174002"
+ kbInput := "423e4567-e89b-12d3-a456-426614174003"
kbID := strings.ReplaceAll(kbInput, "-", "")
- insertAggregateTagsKB(t, kbID, "tenant-9", string(entity.TenantPermissionMe), 1)
+ insertAggregateTagsKB(t, kbID, "tenant-9", string(entity.TenantPermissionMe), "", 0)
ctx := t.Context()
- _, code, err := testDatasetServiceForAggregateTags(t, &aggregateTagsMockEngine{}).AggregateTags(ctx, []string{kbInput}, "user-1")
+ _, code, err := testDatasetServiceForAggregateTags(t, nil).AggregateTags(ctx, []string{kbInput}, "user-1")
if err == nil {
t.Fatal("expected authorization error")
}
@@ -303,81 +199,33 @@ func TestDatasetServiceAggregateTagsRejectsUnauthorizedDataset(t *testing.T) {
}
}
-func TestDatasetServiceAggregateTagsRequiresDocumentEngine(t *testing.T) {
- ctx := t.Context()
- _, code, err := testDatasetServiceForAggregateTags(t, nil).AggregateTags(ctx, []string{"123e4567-e89b-12d3-a456-426614174000"}, "user-1")
- if err == nil {
- t.Fatal("expected missing doc engine error")
- }
- if code != common.CodeServerError {
- t.Fatalf("code=%d want=%d", code, common.CodeServerError)
- }
- if err.Error() != "Document engine is not initialized" {
- t.Fatalf("unexpected error: %v", err)
- }
-}
-
-func TestDatasetServiceAggregateTagsSkipsDatasetsWithoutDocuments(t *testing.T) {
+// TestDatasetServiceAggregateTagsScopesLoaderToDatasetTenant verifies the loader
+// receives the dataset's tenant — the authorization scope used to resolve the
+// user-writable tag_file_id — rather than the caller's user ID.
+func TestDatasetServiceAggregateTagsScopesLoaderToDatasetTenant(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
- emptyInput := "423e4567-e89b-12d3-a456-426614174003"
- liveInput := "523e4567-e89b-12d3-a456-426614174004"
- emptyID := strings.ReplaceAll(emptyInput, "-", "")
- liveID := strings.ReplaceAll(liveInput, "-", "")
-
- insertAggregateTagsKB(t, emptyID, "user-1", string(entity.TenantPermissionMe), 0)
- insertAggregateTagsKB(t, liveID, "user-1", string(entity.TenantPermissionMe), 1)
-
- docEngine := &aggregateTagsMockEngine{
- searchResults: map[string]*types.SearchResult{
- "ragflow_user-1": {
- Chunks: []map[string]interface{}{
- {"tag_kwd": "alpha###beta"},
- },
- },
- },
- }
- ctx := t.Context()
-
- result, code, err := testDatasetServiceForAggregateTags(t, docEngine).AggregateTags(ctx, []string{emptyInput, liveInput}, "user-1")
- if err != nil {
- t.Fatalf("AggregateTags failed: %v", err)
- }
- if code != common.CodeSuccess {
- t.Fatalf("code=%d want=%d", code, common.CodeSuccess)
- }
- if len(docEngine.requests) != 1 {
- t.Fatalf("search requests=%d want=1", len(docEngine.requests))
- }
- if len(docEngine.requests[0].KbIDs) != 1 || docEngine.requests[0].KbIDs[0] != liveID {
- t.Fatalf("KbIDs=%v want [%s]", docEngine.requests[0].KbIDs, liveID)
- }
-
- got := aggregateTagsResultMap(result)
- if got["alpha"] != 1 || got["beta"] != 1 {
- t.Fatalf("unexpected result=%v", got)
- }
-}
-
-func TestDatasetServiceAggregateTagsReturnsSearchError(t *testing.T) {
- db := setupServiceTestDB(t)
- pushServiceDB(t, db)
-
- kbInput := "623e4567-e89b-12d3-a456-426614174005"
+ kbInput := "523e4567-e89b-12d3-a456-426614174004"
kbID := strings.ReplaceAll(kbInput, "-", "")
- insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), 1)
+ const kbTenant = "tenant-owner"
+ insertAggregateTagsKB(t, kbID, kbTenant, string(entity.TenantPermissionTeam), "file-foreign", 0)
+ insertAggregateTagsMembership(t, kbTenant, "user-1")
ctx := t.Context()
- docEngine := &aggregateTagsMockEngine{searchErr: errors.New("boom")}
- _, code, err := testDatasetServiceForAggregateTags(t, docEngine).AggregateTags(ctx, []string{kbInput}, "user-1")
- if err == nil {
- t.Fatal("expected search error")
+ var gotOwner string
+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {
+ if tagFileID != "file-foreign" {
+ t.Fatalf("tagFileID = %q, want file-foreign", tagFileID)
+ }
+ gotOwner = ownerTenantID
+ return map[string]int{"finance": 1}, nil
}
- if code != common.CodeServerError {
- t.Fatalf("code=%d want=%d", code, common.CodeServerError)
+
+ if _, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(ctx, []string{kbInput}, "user-1"); err != nil {
+ t.Fatalf("AggregateTags failed: code=%d err=%v", code, err)
}
- if !strings.Contains(err.Error(), "failed to aggregate tags: boom") {
- t.Fatalf("unexpected error: %v", err)
+ if gotOwner != kbTenant {
+ t.Fatalf("loader ownerTenantID = %q, want %q (must be the dataset tenant, not the user)", gotOwner, kbTenant)
}
}
diff --git a/internal/service/tag.go b/internal/service/tag.go
index 444f1c0eb6..2e2525ae70 100644
--- a/internal/service/tag.go
+++ b/internal/service/tag.go
@@ -28,7 +28,6 @@ import (
"go.uber.org/zap"
- "ragflow/internal/dao"
"ragflow/internal/engine/types"
"ragflow/internal/entity"
"ragflow/internal/service/nlp"
@@ -115,16 +114,24 @@ func SetTagsToCache(ctx context.Context, kbIDs []string, tags map[string]float64
type Knowledgebase = entity.Knowledgebase
// GetAllTagsInPortion returns all tag_kwd values and their occurrence counts
-// for documents belonging to the given kbIDs.
-func (s *MetadataService) GetAllTagsInPortion(ctx context.Context, tenantID string, kbIDs []string) (map[string]float64, error) {
- if len(kbIDs) == 0 {
+// for documents belonging to the given kbIDs, aggregated across every tenant
+// index in tenantIDs — the same tenant scope TagQuery searches. The dataset
+// and chunk-search callers can supply authorized knowledgebases from multiple
+// tenants, so restricting the aggregation to a single tenant index would omit
+// tags that exist only in the other tenants' indices (leaving them absent from
+// allTags and scored with the 0.0001 fallback, which inflates their scores).
+func (s *MetadataService) GetAllTagsInPortion(ctx context.Context, tenantIDs []string, kbIDs []string) (map[string]float64, error) {
+ if len(kbIDs) == 0 || len(tenantIDs) == 0 {
return make(map[string]float64), nil
}
- indexName := fmt.Sprintf("ragflow_%s", tenantID)
+ indexNames := make([]string, len(tenantIDs))
+ for i, tenantID := range tenantIDs {
+ indexNames[i] = fmt.Sprintf("ragflow_%s", tenantID)
+ }
searchReq := &types.SearchRequest{
- IndexNames: []string{indexName},
+ IndexNames: indexNames,
KbIDs: kbIDs,
Offset: 0,
// Python passes limit=0 ("unlimited") which Go SearchRequest treats
@@ -253,78 +260,73 @@ func (s *MetadataService) TagQuery(ctx context.Context, question string, tenantI
return resultTags, nil
}
-// LabelQuestion returns rank features for a question based on KB's tag configuration.
+// LabelQuestion returns rank features for a question based on the dataset's own
+// tags.
//
// Flow:
-// 1. Collect tag_kb_ids from KBs' parser_config
+// 1. Use the queried KBs themselves as the tag source and search scope
// 2. Try to get all_tags from cache (via GetTagsFromCache)
// 3. If cache miss, call GetAllTagsInPortion and cache the result (via SetTagsToCache)
-// 4. Get tag KBs by IDs
-// 5. Call TagQuery to get weighted tag features for the question
+// 4. Call TagQuery to get weighted tag features for the question
+//
+// The Go backend has no separate tag-library dataset as the Python backend
+// does. Instead the Go extractor (extractor_tag.go) writes tag_kwd onto each
+// chunk during ingestion, so the authoritative tag vocabulary lives on the
+// dataset's own chunks and is aggregated from the KBs being queried.
func (s *MetadataService) LabelQuestion(ctx context.Context, question string, kbs []*Knowledgebase) map[string]float64 {
if len(kbs) == 0 || question == "" {
return nil
}
- // Collect tag_kb_ids from KBs' parser_config and track last KB
- var tagKBIDs []string
+ // Use the queried KBs themselves as both the tag source and the search
+ // scope. Track the last KB (for tenant_id / topn_tags) and the unique
+ // tenant IDs spanned by the KBs.
+ var kbIDs []string
var lastKB *Knowledgebase
+ tenantIDSet := make(map[string]bool)
for _, kb := range kbs {
- if kb.ParserConfig == nil {
+ if kb == nil {
continue
}
lastKB = kb
- if rawTagKBIDs, ok := kb.ParserConfig["tag_kb_ids"].([]interface{}); ok {
- for _, id := range rawTagKBIDs {
- if idStr, ok := id.(string); ok {
- tagKBIDs = append(tagKBIDs, idStr)
- }
- }
+ kbIDs = append(kbIDs, kb.ID)
+ if kb.TenantID != "" {
+ tenantIDSet[kb.TenantID] = true
}
}
-
- if len(tagKBIDs) == 0 {
+ if len(kbIDs) == 0 {
+ return nil
+ }
+ uniqueTenantIDs := make([]string, 0, len(tenantIDSet))
+ for tid := range tenantIDSet {
+ uniqueTenantIDs = append(uniqueTenantIDs, tid)
+ }
+ if len(uniqueTenantIDs) == 0 {
return nil
}
- common.Debug("tag_kb_ids found in parser_config", zap.Strings("tag_kb_ids", tagKBIDs))
-
- // Get all tags from cache or compute and cache
- allTags, err := GetTagsFromCache(ctx, tagKBIDs)
+ // Aggregate tag_kwd across the dataset's own chunks (cached by KB set).
+ allTags, err := GetTagsFromCache(ctx, kbIDs)
if err != nil {
common.Warn("Failed to get tags from cache", zap.Error(err))
}
if allTags == nil {
// Cache miss - compute all_tags_in_portion
- allTags, err = s.GetAllTagsInPortion(ctx, lastKB.TenantID, tagKBIDs)
+ allTags, err = s.GetAllTagsInPortion(ctx, uniqueTenantIDs, kbIDs)
if err != nil {
common.Warn("Failed to get all tags in portion", zap.Error(err))
return nil
}
// Store in cache for future lookups
- if err = SetTagsToCache(ctx, tagKBIDs, allTags); err != nil {
+ if err = SetTagsToCache(ctx, kbIDs, allTags); err != nil {
common.Warn("Failed to set tags cache", zap.Error(err))
}
}
- // Get tag_kbs by IDs
- kbDAO := dao.NewKnowledgebaseDAO()
- tagKBs, err := kbDAO.GetByIDs(ctx, dao.DB, tagKBIDs)
- if err != nil || len(tagKBs) == 0 {
- // Return nil if no tag_kbs found
- return nil
- }
-
- // Get unique tenant IDs from tag_kbs
- tenantIDSet := make(map[string]bool)
- for _, kb := range tagKBs {
- tenantIDSet[kb.TenantID] = true
- }
- var uniqueTenantIDs []string
- for tid := range tenantIDSet {
- uniqueTenantIDs = append(uniqueTenantIDs, tid)
- }
- if len(uniqueTenantIDs) == 0 {
+ // No tag_kwd on the dataset's chunks means tagging is not in effect for
+ // this query: return nil so the caller applies no tag boost (Python's
+ // label_question returning None).
+ if len(allTags) == 0 {
return nil
}
@@ -346,14 +348,14 @@ func (s *MetadataService) LabelQuestion(ctx context.Context, question string, kb
}
}
- // Query tags for the question using unique tenant IDs
- tagFeatures, err := s.TagQuery(ctx, question, uniqueTenantIDs, tagKBIDs, allTags, topnTags)
+ // Query tags for the question across the dataset's own chunks.
+ tagFeatures, err := s.TagQuery(ctx, question, uniqueTenantIDs, kbIDs, allTags, topnTags)
if err != nil {
return nil
}
if len(tagFeatures) == 0 {
- // Tag kb exists but returned no matching tags - return empty map (not nil)
- // so caller knows tag kb was configured vs not configured at all
+ // Tags configured but the question matched none - return empty map
+ // (not nil) so the caller knows tagging was active for this dataset.
return make(map[string]float64)
}
diff --git a/test/api/dataset_api_test.go b/test/api/dataset_api_test.go
index c191879073..36c8b39ca6 100644
--- a/test/api/dataset_api_test.go
+++ b/test/api/dataset_api_test.go
@@ -397,10 +397,10 @@ func TestDatasetUpdateParserConfigValidMatrixContract(t *testing.T) {
{name: "html4excel_false", parserConfig: map[string]interface{}{"html4excel": false}},
{name: "layout_recognize_DeepDOC", parserConfig: map[string]interface{}{"layout_recognize": "DeepDOC"}},
{name: "layout_recognize_navie", parserConfig: map[string]interface{}{"layout_recognize": "Plain Text"}},
- {name: "tag_kb_ids", parserConfig: map[string]interface{}{"tag_kb_ids": []interface{}{"1", "2"}}},
{name: "topn_tags_min", parserConfig: map[string]interface{}{"topn_tags": 1}},
{name: "topn_tags_mid", parserConfig: map[string]interface{}{"topn_tags": 5}},
{name: "topn_tags_max", parserConfig: map[string]interface{}{"topn_tags": 10}},
+ {name: "tags_tag_file_id", parserConfig: map[string]interface{}{"tags": map[string]interface{}{"tag_file_id": "file-123"}}},
{name: "filename_embd_weight_min", parserConfig: map[string]interface{}{"filename_embd_weight": 0.1}},
{name: "filename_embd_weight_mid", parserConfig: map[string]interface{}{"filename_embd_weight": 0.5}},
{name: "filename_embd_weight_max", parserConfig: map[string]interface{}{"filename_embd_weight": 1.0}},