RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Implement endpoint GET /datasets/tags/aggregation in GO (#19626)
qinling0210ebe33eaa8b25
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex 0e085d38d6..4a1ff44762 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -926,8 +926,11 @@ func startServer(ctx context.Context) {// Load the KB objects (mirroring Python RAGTools' self.kbs via// KnowledgebaseService.get_by_ids(kb_ids)) so the agentic tool can- // derive rank features from parser_config.tag_kb_ids. Best-effort: a- // load failure leaves KBs empty and the adapter resolves them itself.+ // derive rank features. The Go tag extractor (extractor_tag.go) writes+ // both tag_kwd (the list of tag names) and tag_feas (per-tag weights)+ // onto each chunk at parse time; the labeler aggregates tag_kwd to build+ // the tag vocabulary and the retriever ranks with tag_feas. Best-effort:+ // a load failure leaves KBs empty and the adapter resolves them itself.var kbs []*entity.Knowledgebaseif len(req.DatasetIDs) > 0 {if loaded, lErr := dao.NewKnowledgebaseDAO().GetByIDs(ctx, dao.DB, req.DatasetIDs); lErr == nil {diff --git a/internal/dao/file.go b/internal/dao/file.goindex 281d6c032c..131644d887 100644--- a/internal/dao/file.go+++ b/internal/dao/file.go@@ -45,6 +45,24 @@ func (dao *FileDAO) GetByID(ctx context.Context, db *gorm.DB, id string) (*entitreturn &file, nil}+// GetByIDAndTenant gets a file by ID scoped to the given tenant. Callers that+// resolve a user-supplied file ID (e.g. parser_config.tags.tag_file_id, which+// the dataset update API accepts from the client) MUST use this instead of+// GetByID: an unscoped ID lookup crosses tenant boundaries and is an IDOR+// (CWE-639). An empty id or tenantID fails closed with gorm.ErrRecordNotFound+// so callers see the same "not found" as a missing row.+func (dao *FileDAO) GetByIDAndTenant(ctx context.Context, db *gorm.DB, id, tenantID string) (*entity.File, error) {+ if id == "" || tenantID == "" {+ return nil, gorm.ErrRecordNotFound+ }+ var file entity.File+ err := db.WithContext(ctx).Where("id = ? AND tenant_id = ?", id, tenantID).First(&file).Error+ if err != nil {+ return nil, err+ }+ return &file, nil+}+// GetByPfID gets files by parent folder ID with pagination and filtering.// When keywords is empty, only direct children of pfID are listed; when// keywords is non-empty, the search covers the whole subtree under pfID sodiff --git a/internal/dao/file_test.go b/internal/dao/file_test.goindex 82da00431a..1cd68dbacc 100644--- a/internal/dao/file_test.go+++ b/internal/dao/file_test.go@@ -147,3 +147,32 @@ func TestFileDAO_GetByPfID_NoKeywordsListsDirectChildren(t *testing.T) {}}}++func TestFileDAO_GetByIDAndTenant(t *testing.T) {+ db := setupFileTestDB(t)+ seedFileTree(t, db) // f-t2 belongs to tenant t2+ d := NewFileDAO()+ ctx := t.Context()++ got, err := d.GetByIDAndTenant(ctx, db, "f-t2", "t2")+ if err != nil {+ t.Fatalf("owner tenant lookup failed: %v", err)+ }+ if got.ID != "f-t2" {+ t.Fatalf("id = %q, want f-t2", got.ID)+ }++ // Another tenant, an empty tenant (fail closed) and a case-different tenant+ // must all miss.+ for _, tenantID := range []string{"t1", "", "T2"} {+ if _, err := d.GetByIDAndTenant(ctx, db, "f-t2", tenantID); !IsNotFoundErr(err) {+ t.Fatalf("tenant %q: err = %v, want gorm.ErrRecordNotFound", tenantID, err)+ }+ }++ // The unscoped GetByID still returns the row across tenants: that is exactly+ // the lookup the tag-source loader must not use with a user-supplied ID.+ if _, err := d.GetByID(ctx, db, "f-t2"); err != nil {+ t.Fatalf("unscoped GetByID should still see the row: %v", err)+ }+}diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.goindex d83cae3dbb..80edf8e3d2 100644--- a/internal/ingestion/component/extractor_tag.go+++ b/internal/ingestion/component/extractor_tag.go@@ -645,7 +645,11 @@ func (c *ExtractorComponent) resolveTagSource(ctx context.Context, lang string)}func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context, lang string) (*MemoryTagIndex, bool) {- f, err := dao.NewFileDAO().GetByID(ctx, dao.DB, c.Param.Tags.TagFileID)+ // Same IDOR guard as TagVocabularyFromTagFileID: tag_file_id is+ // user-controlled through parser_config, so it must resolve inside the+ // caller's tenant before the bytes are read.+ tenantID := globals.GlobalOrInput(ctx, nil, "tenant_id", "")+ f, err := dao.NewFileDAO().GetByIDAndTenant(ctx, dao.DB, c.Param.Tags.TagFileID, tenantID)if err != nil || f == nil || f.Location == nil || *f.Location == "" {common.Warn(fmt.Sprintf("extractor tags: resolve tag_file_id %q: %v", c.Param.Tags.TagFileID, err))return nil, false@@ -664,7 +668,6 @@ func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context, lang stringcommon.Warn("extractor tags: no storage backend registered")return nil, false}- tenantID := globals.GlobalOrInput(ctx, nil, "tenant_id", "")data, err := stg.Get(ctx, f.ParentID, *f.Location, tenantID)if err != nil {common.Warn(fmt.Sprintf("extractor tags: load tag source %q/%q: %v", f.ParentID, *f.Location, err))@@ -733,6 +736,75 @@ func parseTagSourceByFilename(data []byte, filename string) ([]schema.TagLabel,}}+// TagVocabularyFromBytes is the storage/DAO-free core of+// TagVocabularyFromTagFileID. It parses a tag source file's raw bytes (mirroring+// rag/app/tag.py's chunk format) and returns the vocabulary as tag -> number of+// source examples that mention the tag; a tag repeated within one source example+// counts once, matching buildMemoryTagIndex's per-sample deduplication. Note: the+// Go tag extractor+// (matchAndTagChunk) DOES write tag_kwd onto chunks at parse time, but this+// vocabulary is the authoritative selectable-tag list for the Go backend (the+// tag-options/aggregation API sources the list from the tag source file, not+// from chunk usage).+func TagVocabularyFromBytes(data []byte, filename string) (map[string]int, error) {+ labels, err := parseTagSourceByFilename(data, filename)+ if err != nil {+ return nil, err+ }+ counts := make(map[string]int)+ for _, lbl := range labels {+ seen := make(map[string]struct{}, len(lbl.Tags))+ for _, t := range lbl.Tags {+ t = strings.TrimSpace(strings.ReplaceAll(t, ".", "_"))+ if t != "" {+ if _, exists := seen[t]; exists {+ continue+ }+ seen[t] = struct{}{}+ counts[t]+++ }+ }+ }+ return counts, nil+}++// TagVocabularyFromTagFileID loads a tag source file (parser_config.tags.tag_file_id)+// and returns the tag vocabulary it defines. It is the Go-native source of the+// selectable-tag list surfaced by the tag-options API. Although the Go tag+// extractor (matchAndTagChunk) writes tag_kwd onto chunks during parsing, the+// selectable-tag list is taken from this vocabulary (the tag source file), not+// from chunk usage.+//+// It returns (nil, nil) when tagFileID is empty.+//+// ownerTenantID is the tenant of the dataset that configured the file and is+// required: tag_file_id is user-writable through parser_config, so the file must+// be proven to belong to that tenant before its bytes are read (IDOR, CWE-639).+// An empty ownerTenantID fails closed. Note that storage.Get's tenant argument+// does not establish file ownership by itself.+func TagVocabularyFromTagFileID(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ if tagFileID == "" {+ return nil, nil+ }+ common.Info(fmt.Sprintf("tag_vocab: loading tag source file_id=%q", tagFileID))+ f, err := dao.NewFileDAO().GetByIDAndTenant(ctx, dao.DB, tagFileID, ownerTenantID)+ if err != nil || f == nil || f.Location == nil || *f.Location == "" {+ return nil, fmt.Errorf("tag source file %q not found: %w", tagFileID, err)+ }+ common.Info(fmt.Sprintf("tag_vocab: file_id=%q name=%q parent_id=%q location=%q",+ tagFileID, f.Name, f.ParentID, *f.Location))+ stg := resolveStorage()+ if stg == nil {+ return nil, fmt.Errorf("tag source file %q: no storage backend registered", tagFileID)+ }+ data, err := stg.Get(ctx, f.ParentID, *f.Location, ownerTenantID)+ if err != nil {+ return nil, fmt.Errorf("load tag source file %q/%q: %w", f.ParentID, *f.Location, err)+ }+ common.Info(fmt.Sprintf("tag_vocab: file_id=%q loaded %d bytes", tagFileID, len(data)))+ return TagVocabularyFromBytes(data, f.Name)+}+func parseCSVTagSource(text string) []schema.TagLabel {return parseCSVTagSourceBytes([]byte(text))}diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.goindex 7bd908c0cd..173541fea0 100644--- a/internal/ingestion/component/extractor_tag_test.go+++ b/internal/ingestion/component/extractor_tag_test.go@@ -12,6 +12,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common"+ "ragflow/internal/entity""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer")@@ -117,6 +118,130 @@ func TestExtractorTags_WithKeywords(t *testing.T) {}}+func TestTagVocabularyFromBytes(t *testing.T) {+ // Each line is [content, tags]; the tags column may hold a comma-separated+ // list, which requires quoting (or a tab delimiter) to survive CSV parsing.+ csv := []byte("\"some content\",\"finance,urgent\"\n\"other content\",\"finance\"\n\"just a line\",\"legal\"")+ vocab, err := TagVocabularyFromBytes(csv, "tags.csv")+ if err != nil {+ t.Fatal(err)+ }+ want := map[string]int{"finance": 2, "urgent": 1, "legal": 1}+ if len(vocab) != len(want) {+ t.Fatalf("vocab=%v want=%v", vocab, want)+ }+ for tag, c := range want {+ if vocab[tag] != c {+ t.Fatalf("tag %q count=%d want=%d", tag, vocab[tag], c)+ }+ }++ // Dots in tags are normalized to underscores, matching the extractor.+ csv2 := []byte("\"x\",\"Alpha.Beta\"\n\"y\",\"Alpha.Beta\"")+ vocab2, err := TagVocabularyFromBytes(csv2, "tags.csv")+ if err != nil {+ t.Fatal(err)+ }+ if vocab2["Alpha_Beta"] != 2 {+ t.Fatalf("expected Alpha_Beta=2, got %v", vocab2)+ }++ // A tag repeated inside one source example counts once: the count is the+ // number of source examples mentioning the tag, consistent with+ // buildMemoryTagIndex's per-sample deduplication.+ csv3 := []byte("\"dup content\",\"finance,finance,urgent\"\n\"other content\",\"finance\"")+ vocab3, err := TagVocabularyFromBytes(csv3, "tags.csv")+ if err != nil {+ t.Fatal(err)+ }+ if vocab3["finance"] != 2 { // two examples, not three occurrences+ t.Fatalf("expected finance=2 (per source example), got %v", vocab3)+ }+ if vocab3["urgent"] != 1 {+ t.Fatalf("expected urgent=1, got %v", vocab3)+ }+ if len(vocab3) != 2 {+ t.Fatalf("expected 2 distinct tags, got %v", vocab3)+ }++ // Unsupported extension is rejected.+ if _, err := TagVocabularyFromBytes([]byte("x"), "tags.pdf"); err == nil {+ t.Fatal("expected error for unsupported extension")+ }+}++// TestTagVocabularyFromTagFileID_RejectsForeignTenantFile pins the IDOR guard.+// parser_config.tags.tag_file_id is user-controlled (the dataset update API+// accepts parser_config), so the file it names must belong to the caller's+// tenant before its bytes are read. The tag source bytes are deliberately+// seeded in storage: without the ownership check the loader would happily+// return them to the other tenant.+func TestTagVocabularyFromTagFileID_RejectsForeignTenantFile(t *testing.T) {+ db := withFileComponentTestDB(t)+ ms := withMemoryStorage(t)++ const (+ ownerTenant = "tenant-owner"+ attackerTenant = "tenant-attacker"+ fileID = "tag-file-foreign"+ bucket = "kb-bucket-1"+ location = "tags/tags.csv"+ )++ csv := []byte("\"some content\",\"finance,urgent\"\n\"other content\",\"finance\"")+ loc := location+ if err := db.Create(&entity.File{+ ID: fileID,+ ParentID: bucket,+ TenantID: ownerTenant,+ CreatedBy: ownerTenant,+ Name: "tags.csv",+ Type: "doc",+ Location: &loc,+ }).Error; err != nil {+ t.Fatalf("seed file row: %v", err)+ }+ if err := ms.Put(t.Context(), bucket, location, csv); err != nil {+ t.Fatalf("seed storage: %v", err)+ }++ // A dataset whose tenant does not own the file must be denied, and must not+ // get the bytes.+ vocab, err := TagVocabularyFromTagFileID(t.Context(), fileID, attackerTenant)+ if err == nil {+ t.Fatalf("IDOR: foreign tenant read the tag source file, vocab=%v", vocab)+ }+ if vocab != nil {+ t.Fatalf("IDOR: expected no vocabulary for foreign tenant, got %v", vocab)+ }++ // An empty owner tenant fails closed as well.+ if _, err := TagVocabularyFromTagFileID(t.Context(), fileID, ""); err == nil {+ t.Fatal("empty owner tenant must fail closed")+ }++ // The owning tenant still gets the vocabulary.+ vocab, err = TagVocabularyFromTagFileID(t.Context(), fileID, ownerTenant)+ if err != nil {+ t.Fatalf("owner tenant load failed: %v", err)+ }+ if vocab["finance"] != 2 || vocab["urgent"] != 1 {+ t.Fatalf("owner vocab = %v, want finance=2 urgent=1", vocab)+ }++ // Sharing is intentional: the guard is tenant-scoped, not dataset-scoped (the+ // file's ParentID is never compared against the referencing dataset), so any+ // number of datasets in ownerTenant can point at this same tag source file —+ // each passes its own dataset tenant (kb.TenantID) here.+ shared, err := TagVocabularyFromTagFileID(t.Context(), fileID, ownerTenant)+ if err != nil {+ t.Fatalf("a second dataset in the same tenant must resolve the shared file: %v", err)+ }+ if shared["finance"] != 2 {+ t.Fatalf("shared vocab = %v, want finance=2", shared)+ }+}+func TestExtractorTags_ComponentRegistration(t *testing.T) {factory, cat, md, ok := runtime.DefaultRegistry.Lookup(componentNameExtractor)if !ok {diff --git a/internal/rag/advanced_rag/harness/tool_search.go b/internal/rag/advanced_rag/harness/tool_search.goindex df9fa80913..33406f6130 100644--- a/internal/rag/advanced_rag/harness/tool_search.go+++ b/internal/rag/advanced_rag/harness/tool_search.go@@ -444,10 +444,14 @@ type SearchDeps struct {}// QuestionLabeler mirrors Python rag.app.tag.label_question(question, kbs)-// (agentic_rag.py:retrieve → tag.py). Given the query and the KB objects (which-// carry parser_config.tag_kb_ids), it returns a map of question-type tag →-// weight the retriever uses to rank results. The production implementation is-// internal/service.MetadataService.LabelQuestion; tests supply a stub.+// (agentic_rag.py:retrieve → tag.py). Given the query and the KB objects, it+// returns a map of question-type tag → weight the retriever uses to rank+// results. The Go extractor (extractor_tag.go) writes tag_kwd (the tag-name+// list) and tag_feas (per-tag weights) onto each chunk; the labeler aggregates+// tag_kwd to build the vocabulary and the retriever ranks with tag_feas. It+// does not rely on a separate Python-only tag-library dataset. The production+// implementation is internal/service.MetadataService.LabelQuestion; tests+// supply a stub.type QuestionLabeler interface {LabelQuestion(ctx context.Context, question string, kbs []*entity.Knowledgebase) map[string]float64}diff --git a/internal/service/dataset/service.go b/internal/service/dataset/service.goindex 28081e9133..e0116aa201 100644--- a/internal/service/dataset/service.go+++ b/internal/service/dataset/service.go@@ -1,6 +1,8 @@package datasetimport (+ "context"+"ragflow/internal/dao""ragflow/internal/engine""ragflow/internal/service"@@ -20,6 +22,13 @@ type DatasetService struct {searchService *service.SearchServicedocEngine engine.DocEngineembeddingCache *utility.EmbeddingLRU+ // tagVocabularyLoader resolves the selectable-tag vocabulary for a tag+ // source file (parser_config.tags.tag_file_id). It is a field so tests can+ // inject a fake; in production it defaults to+ // component.TagVocabularyFromTagFileID (see AggregateTags). The ownerTenantID+ // it receives is the dataset's tenant: the loader must prove the file belongs+ // to it, since tag_file_id is user-writable (IDOR, CWE-639).+ tagVocabularyLoader func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error)}// NewDatasetService creates a new datasets service.diff --git a/internal/service/dataset/tags.go b/internal/service/dataset/tags.goindex 54b6ee51fd..e6a81c175d 100644--- a/internal/service/dataset/tags.go+++ b/internal/service/dataset/tags.go@@ -11,17 +11,110 @@ import ("ragflow/internal/common""ragflow/internal/dao"enginetypes "ragflow/internal/engine/types"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/component")+// tagFileIDFromTagsMap pulls tag_file_id out of a raw "tags" config value+// (i.e. parser_config["tags"] in the legacy flat form, or+// parser_config["Extractor:<name>"]["tags"] in the Go pipeline scoped form),+// accepting either a map[string]any or an entity.JSONMap. Returns "" when the+// value is not a tags map or has no tag_file_id. In the Go backend the+// authoritative tag vocabulary lives in this tag source file (parsed by the+// extractor in extractor_tag.go). Note: the Go tag extractor (matchAndTagChunk)+// DOES write tag_kwd onto chunks at parse time, but the selectable-tag list is+// sourced from this vocabulary, not from chunk usage. Returns "" when unset.+func tagFileIDFromTagsMap(raw any) string {+ if raw == nil {+ return ""+ }+ var tags map[string]any+ switch v := raw.(type) {+ case map[string]any:+ tags = v+ case entity.JSONMap:+ tags = map[string]any(v)+ default:+ return ""+ }+ if v, ok := tags["tag_file_id"].(string); ok {+ return strings.TrimSpace(v)+ }+ return ""+}++// extractTagFileID returns the tag source file ID declared in a dataset's+// parser_config. The tags config lives in two possible shapes depending on how+// the dataset was persisted:+//+// - Legacy / flat form: parser_config["tags"]["tag_file_id"].+// - Go pipeline scoped form: parser_config["Extractor:<name>"]["tags"]+// ["tag_file_id"] (e.g. "Extractor:AutoExtractDefault"). In this form the+// component config is keyed by "<Component>:<InstanceName>" and the+// sub-feature maps (tags/keywords/questions/summary/metadata) sit at the+// top level of that value — mirroring how NewExtractorComponent reads its+// params.+//+// The Go backend sources the authoritative tag vocabulary from this tag source+// file (parsed by the extractor in extractor_tag.go). Note: the Go tag extractor+// (matchAndTagChunk) DOES write tag_kwd onto chunks at parse time, but the+// selectable-tag list is taken from this vocabulary, not from chunk usage.+// Returns "" when unset.+func extractTagFileID(parserConfig entity.JSONMap) string {+ if parserConfig == nil {+ return ""+ }+ // 1. Legacy / flat form.+ if v := tagFileIDFromTagsMap(parserConfig["tags"]); v != "" {+ return v+ }+ // 2. Go pipeline scoped form: any key shaped "<Component>:<Instance>" whose+ // component is Extractor.+ for key, raw := range parserConfig {+ if key == "Extractor" || strings.HasPrefix(key, "Extractor:") {+ extractorMap, ok := raw.(map[string]any)+ if !ok {+ // entity.JSONMap is an alias of map[string]any, so the assertion+ // above already covers it; this guard handles any other wrapper.+ if jm, ok2 := raw.(entity.JSONMap); ok2 {+ extractorMap = map[string]any(jm)+ } else {+ continue+ }+ }+ // The tags config sits at the top level of the Extractor node value,+ // i.e. parser_config["Extractor:<name>"]["tags"], NOT at the node+ // value itself. Pass that nested "tags" map to tagFileIDFromTagsMap.+ if v := tagFileIDFromTagsMap(extractorMap["tags"]); v != "" {+ return v+ }+ }+ }+ return ""+}+func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string, userID string) ([]map[string]interface{}, common.ErrorCode, error) {if len(datasetIDs) == 0 {return nil, common.CodeDataError, errors.New("Lack of dataset_ids in query parameters")}- if d.docEngine == nil {- return nil, common.CodeServerError, errors.New("Document engine is not initialized")+ // merged holds the selectable-tag vocabulary sourced exclusively from each+ // dataset's configured tag source file (parser_config.tags.tag_file_id). The+ // count for a tag is the number of times it appears in that source file. The+ // Go tag extractor (matchAndTagChunk) does write tag_kwd onto chunks at parse+ // time, but by requirement the count reflects only the tag-file occurrence+ // count, so there is intentionally no chunk-level aggregation here. The Go+ // backend has no Python-style tag-library datasets.+ loader := d.tagVocabularyLoader+ if loader == nil {+ loader = component.TagVocabularyFromTagFileID}-- datasetIDsByTenant := make(map[string][]string)+ merged := make(map[string]int)+ // The handler accepts repeated IDs, and distinct raw IDs can normalize to+ // the same dataset (hyphenated vs. compact form). Track the normalized IDs+ // so each dataset is loaded and merged exactly once — otherwise a repeated+ // dataset (e.g. dataset_ids=A,A) would load the same vocabulary twice and+ // double every count.+ seen := make(map[string]struct{}, len(datasetIDs))for _, rawID := range datasetIDs {rawID = strings.TrimSpace(rawID)if rawID == "" {@@ -31,6 +124,10 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,if err != nil {return nil, common.CodeDataError, err}+ if _, dup := seen[datasetID]; dup {+ continue+ }+ seen[datasetID] = struct{}{}if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {return nil, common.CodeDataError, fmt.Errorf("No authorization for dataset '%s'", datasetID)}@@ -41,48 +138,19 @@ func (d *DatasetService) AggregateTags(ctx context.Context, datasetIDs []string,}return nil, common.CodeServerError, errors.New("Database operation failed")}- if kb.DocNum <= 0 {- continue- }- datasetIDsByTenant[kb.TenantID] = append(datasetIDsByTenant[kb.TenantID], datasetID)- }- const pageSize = 10000- merged := make(map[string]int)- for tenantID, kbIDs := range datasetIDsByTenant {- for offset := 0; ; offset += pageSize {- searchResp, err := d.docEngine.Search(ctx, &enginetypes.SearchRequest{- IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},- KbIDs: kbIDs,- Offset: offset,- Limit: pageSize,- SelectFields: []string{"tag_kwd"},- })- if err != nil {- return nil, common.CodeServerError, fmt.Errorf("failed to aggregate tags: %w", err)- }- for _, agg := range d.docEngine.GetAggregation(searchResp.Chunks, "tag_kwd") {- tag, _ := agg["key"].(string)- if tag == "" {- continue- }- switch count := agg["count"].(type) {- case int:- merged[tag] += count- case int32:- merged[tag] += int(count)- case int64:- merged[tag] += int(count)- case float64:- merged[tag] += int(count)- }- }- chunkCount := len(searchResp.Chunks)- if chunkCount == 0 || chunkCount < pageSize {- break+ // Include the dataset's tag-source vocabulary. The count for each tag is+ // the number of times it occurs in the tag source file. The file is+ // resolved against the dataset's own tenant: tag_file_id is user-writable,+ // so a foreign file ID must not resolve (IDOR, CWE-639).+ if tagFileID := extractTagFileID(kb.ParserConfig); tagFileID != "" {+ counts, vErr := loader(ctx, tagFileID, kb.TenantID)+ if vErr != nil {+ return nil, common.CodeServerError,+ fmt.Errorf("load tag vocabulary for dataset %q: %w", datasetID, vErr)}- if searchResp.Total > 0 && int64(offset+chunkCount) >= searchResp.Total {- break+ for tag, c := range counts {+ merged[tag] += c}}}diff --git a/internal/service/dataset/tags_aggregate_test.go b/internal/service/dataset/tags_aggregate_test.goindex e560a06e87..6ca0c81548 100644--- a/internal/service/dataset/tags_aggregate_test.go+++ b/internal/service/dataset/tags_aggregate_test.go@@ -2,103 +2,28 @@ package datasetimport ("context"- "errors""strings""testing""ragflow/internal/common""ragflow/internal/dao"- "ragflow/internal/engine"- "ragflow/internal/engine/types""ragflow/internal/entity")-type aggregateTagsMockEngine struct {- engine.DocEngine- searchResults map[string]*types.SearchResult- pagedSearchResults map[string]map[int]*types.SearchResult- searchErr error- requests []*types.SearchRequest-}--func (m *aggregateTagsMockEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {- if m.searchErr != nil {- return nil, m.searchErr- }- cloned := &types.SearchRequest{- IndexNames: append([]string(nil), req.IndexNames...),- KbIDs: append([]string(nil), req.KbIDs...),- Offset: req.Offset,- Limit: req.Limit,- SelectFields: append([]string(nil), req.SelectFields...),- }- m.requests = append(m.requests, cloned)- if len(req.IndexNames) == 0 {- return &types.SearchResult{}, nil- }- if byOffset, ok := m.pagedSearchResults[req.IndexNames[0]]; ok {- if res, ok := byOffset[req.Offset]; ok {- return res, nil- }- return &types.SearchResult{}, nil- }- if res, ok := m.searchResults[req.IndexNames[0]]; ok {- return res, nil- }- return &types.SearchResult{}, nil-}--func (m *aggregateTagsMockEngine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} {- counts := make(map[string]int)- for _, chunk := range chunks {- raw, ok := chunk[fieldName]- if !ok || raw == nil {- continue- }- switch value := raw.(type) {- case string:- for _, tag := range strings.Split(value, "###") {- tag = strings.TrimSpace(tag)- if tag != "" {- counts[tag]++- }- }- case []string:- for _, tag := range value {- tag = strings.TrimSpace(tag)- if tag != "" {- counts[tag]++- }- }- }- }-- result := make([]map[string]interface{}, 0, len(counts))- for tag, count := range counts {- result = append(result, map[string]interface{}{- "key": tag,- "count": count,- })- }- return result-}--func (m *aggregateTagsMockEngine) GetType() string { return "mock" }--func (m *aggregateTagsMockEngine) Ping(ctx context.Context) error { return nil }--func (m *aggregateTagsMockEngine) Close() error { return nil }--func testDatasetServiceForAggregateTags(t *testing.T, docEngine engine.DocEngine) *DatasetService {+func testDatasetServiceForAggregateTags(t *testing.T, loader func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error)) *DatasetService {t.Helper()return &DatasetService{- kbDAO: dao.NewKnowledgebaseDAO(),- docEngine: docEngine,+ kbDAO: dao.NewKnowledgebaseDAO(),+ tagVocabularyLoader: loader,}}-func insertAggregateTagsKB(t *testing.T, datasetID, tenantID, permission string, docNum int64) {+func insertAggregateTagsKB(t *testing.T, datasetID, tenantID, permission, tagFileID string, docNum int64) {t.Helper()+ parserConfig := entity.JSONMap{}+ if tagFileID != "" {+ parserConfig["tags"] = map[string]any{"tag_file_id": tagFileID}+ }kb := &entity.Knowledgebase{ID: datasetID,TenantID: tenantID,@@ -107,7 +32,7 @@ func insertAggregateTagsKB(t *testing.T, datasetID, tenantID, permission string,CreatedBy: tenantID,Permission: permission,ParserID: "naive",- ParserConfig: entity.JSONMap{},+ ParserConfig: parserConfig,DocNum: docNum,Status: sptr(string(entity.StatusValid)),}@@ -141,37 +66,26 @@ func aggregateTagsResultMap(rows []map[string]interface{}) map[string]int {return result}-func TestDatasetServiceAggregateTagsMergesAcrossTenants(t *testing.T) {+// TestDatasetServiceAggregateTagsSeedsFromTagFile verifies that the tag+// vocabulary (and its counts) come solely from the configured tag source file,+// with no chunk-level aggregation.+func TestDatasetServiceAggregateTagsSeedsFromTagFile(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)- kb1Input := "123e4567-e89b-12d3-a456-426614174000"- kb2Input := "223e4567-e89b-12d3-a456-426614174001"- kb1ID := strings.ReplaceAll(kb1Input, "-", "")- kb2ID := strings.ReplaceAll(kb2Input, "-", "")-- insertAggregateTagsKB(t, kb1ID, "user-1", string(entity.TenantPermissionMe), 2)- insertAggregateTagsKB(t, kb2ID, "tenant-2", string(entity.TenantPermissionTeam), 1)- insertAggregateTagsMembership(t, "tenant-2", "user-1")+ kbInput := "123e4567-e89b-12d3-a456-426614174000"+ kbID := strings.ReplaceAll(kbInput, "-", "")+ insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)- docEngine := &aggregateTagsMockEngine{- searchResults: map[string]*types.SearchResult{- "ragflow_user-1": {- Chunks: []map[string]interface{}{- {"tag_kwd": "finance###urgent"},- {"tag_kwd": "finance"},- },- },- "ragflow_tenant-2": {- Chunks: []map[string]interface{}{- {"tag_kwd": "urgent###internal"},- },- },- },+ loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ if tagFileID == "file-1" {+ return map[string]int{"finance": 2, "urgent": 1}, nil+ }+ return nil, nil}ctx := t.Context()- result, code, err := testDatasetServiceForAggregateTags(t, docEngine).AggregateTags(ctx, []string{kb1Input, kb2Input}, "user-1")+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(ctx, []string{kbInput}, "user-1")if err != nil {t.Fatalf("AggregateTags failed: %v", err)}@@ -180,11 +94,7 @@ func TestDatasetServiceAggregateTagsMergesAcrossTenants(t *testing.T) {}got := aggregateTagsResultMap(result)- want := map[string]int{- "finance": 2,- "urgent": 2,- "internal": 1,- }+ want := map[string]int{"finance": 2, "urgent": 1}if len(got) != len(want) {t.Fatalf("result len=%d want=%d result=%v", len(got), len(want), got)}@@ -193,92 +103,78 @@ func TestDatasetServiceAggregateTagsMergesAcrossTenants(t *testing.T) {t.Fatalf("tag %q count=%d want=%d all=%v", tag, got[tag], wantCount, got)}}+}- if len(docEngine.requests) != 2 {- t.Fatalf("search requests=%d want=2", len(docEngine.requests))- }+// TestDatasetServiceAggregateTagsMergesAcrossTagFiles verifies that counts from+// multiple datasets' tag source files are merged by tag name.+func TestDatasetServiceAggregateTagsMergesAcrossTagFiles(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)- requestByIndex := make(map[string]*types.SearchRequest, len(docEngine.requests))- for _, req := range docEngine.requests {- if len(req.IndexNames) != 1 {- t.Fatalf("IndexNames=%v want single entry", req.IndexNames)- }- requestByIndex[req.IndexNames[0]] = req- if req.Offset != 0 {- t.Fatalf("Offset=%d want=0", req.Offset)- }- if req.Limit != 10000 {- t.Fatalf("Limit=%d want=10000", req.Limit)- }- if len(req.SelectFields) != 1 || req.SelectFields[0] != "tag_kwd" {- t.Fatalf("SelectFields=%v want [tag_kwd]", req.SelectFields)+ kb1Input := "123e4567-e89b-12d3-a456-426614174000"+ kb2Input := "223e4567-e89b-12d3-a456-426614174001"+ kb1ID := strings.ReplaceAll(kb1Input, "-", "")+ kb2ID := strings.ReplaceAll(kb2Input, "-", "")+ insertAggregateTagsKB(t, kb1ID, "user-1", string(entity.TenantPermissionMe), "file-1", 0)+ insertAggregateTagsKB(t, kb2ID, "user-1", string(entity.TenantPermissionMe), "file-2", 0)++ loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ switch tagFileID {+ case "file-1":+ return map[string]int{"finance": 2, "urgent": 1}, nil+ case "file-2":+ return map[string]int{"finance": 1, "urgent": 3, "internal": 1}, nil}+ return nil, nil+ }+ ctx := t.Context()++ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(ctx, []string{kb1Input, kb2Input}, "user-1")+ if err != nil {+ t.Fatalf("AggregateTags failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("code=%d want=%d", code, common.CodeSuccess)}- if req := requestByIndex["ragflow_user-1"]; req == nil || len(req.KbIDs) != 1 || req.KbIDs[0] != kb1ID {- t.Fatalf("request for ragflow_user-1 = %#v, want kbIDs=[%s]", req, kb1ID)+ got := aggregateTagsResultMap(result)+ want := map[string]int{"finance": 3, "urgent": 4, "internal": 1}+ if len(got) != len(want) {+ t.Fatalf("result len=%d want=%d result=%v", len(got), len(want), got)}- if req := requestByIndex["ragflow_tenant-2"]; req == nil || len(req.KbIDs) != 1 || req.KbIDs[0] != kb2ID {- t.Fatalf("request for ragflow_tenant-2 = %#v, want kbIDs=[%s]", req, kb2ID)+ for tag, wantCount := range want {+ if got[tag] != wantCount {+ t.Fatalf("tag %q count=%d want=%d all=%v", tag, got[tag], wantCount, got)+ }}}-func TestDatasetServiceAggregateTagsPagesThroughAllChunks(t *testing.T) {+// TestDatasetServiceAggregateTagsEmptyWhenNoTagFile verifies that a dataset+// without a configured tag source file yields an empty vocabulary and does not+// invoke the loader.+func TestDatasetServiceAggregateTagsEmptyWhenNoTagFile(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)- kbInput := "723e4567-e89b-12d3-a456-426614174006"+ kbInput := "323e4567-e89b-12d3-a456-426614174002"kbID := strings.ReplaceAll(kbInput, "-", "")- insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), 10002)+ insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), "", 0)- firstPage := make([]map[string]interface{}, 10000)- for i := range firstPage {- firstPage[i] = map[string]interface{}{"tag_kwd": "finance"}- }-- docEngine := &aggregateTagsMockEngine{- pagedSearchResults: map[string]map[int]*types.SearchResult{- "ragflow_user-1": {- 0: {- Chunks: firstPage,- Total: 10002,- },- 10000: {- Chunks: []map[string]interface{}{- {"tag_kwd": "finance"},- {"tag_kwd": "urgent"},- },- Total: 10002,- },- },- },+ loader := func(ctx context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ t.Fatalf("loader should not be called when no tag_file_id is set")+ return nil, nil}ctx := t.Context()- result, code, err := testDatasetServiceForAggregateTags(t, docEngine).AggregateTags(ctx, []string{kbInput}, "user-1")+ result, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(ctx, []string{kbInput}, "user-1")if err != nil {t.Fatalf("AggregateTags failed: %v", err)}if code != common.CodeSuccess {t.Fatalf("code=%d want=%d", code, common.CodeSuccess)}-- got := aggregateTagsResultMap(result)- if got["finance"] != 10001 {- t.Fatalf("finance count=%d want=10001 result=%v", got["finance"], got)- }- if got["urgent"] != 1 {- t.Fatalf("urgent count=%d want=1 result=%v", got["urgent"], got)- }-- if len(docEngine.requests) != 2 {- t.Fatalf("search requests=%d want=2", len(docEngine.requests))- }- if docEngine.requests[0].Offset != 0 {- t.Fatalf("first request offset=%d want=0", docEngine.requests[0].Offset)- }- if docEngine.requests[1].Offset != 10000 {- t.Fatalf("second request offset=%d want=10000", docEngine.requests[1].Offset)+ if len(result) != 0 {+ t.Fatalf("result=%v want empty", result)}}@@ -286,12 +182,12 @@ func TestDatasetServiceAggregateTagsRejectsUnauthorizedDataset(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)- kbInput := "323e4567-e89b-12d3-a456-426614174002"+ kbInput := "423e4567-e89b-12d3-a456-426614174003"kbID := strings.ReplaceAll(kbInput, "-", "")- insertAggregateTagsKB(t, kbID, "tenant-9", string(entity.TenantPermissionMe), 1)+ insertAggregateTagsKB(t, kbID, "tenant-9", string(entity.TenantPermissionMe), "", 0)ctx := t.Context()- _, code, err := testDatasetServiceForAggregateTags(t, &aggregateTagsMockEngine{}).AggregateTags(ctx, []string{kbInput}, "user-1")+ _, code, err := testDatasetServiceForAggregateTags(t, nil).AggregateTags(ctx, []string{kbInput}, "user-1")if err == nil {t.Fatal("expected authorization error")}@@ -303,81 +199,33 @@ func TestDatasetServiceAggregateTagsRejectsUnauthorizedDataset(t *testing.T) {}}-func TestDatasetServiceAggregateTagsRequiresDocumentEngine(t *testing.T) {- ctx := t.Context()- _, code, err := testDatasetServiceForAggregateTags(t, nil).AggregateTags(ctx, []string{"123e4567-e89b-12d3-a456-426614174000"}, "user-1")- if err == nil {- t.Fatal("expected missing doc engine error")- }- if code != common.CodeServerError {- t.Fatalf("code=%d want=%d", code, common.CodeServerError)- }- if err.Error() != "Document engine is not initialized" {- t.Fatalf("unexpected error: %v", err)- }-}--func TestDatasetServiceAggregateTagsSkipsDatasetsWithoutDocuments(t *testing.T) {+// TestDatasetServiceAggregateTagsScopesLoaderToDatasetTenant verifies the loader+// receives the dataset's tenant — the authorization scope used to resolve the+// user-writable tag_file_id — rather than the caller's user ID.+func TestDatasetServiceAggregateTagsScopesLoaderToDatasetTenant(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)- emptyInput := "423e4567-e89b-12d3-a456-426614174003"- liveInput := "523e4567-e89b-12d3-a456-426614174004"- emptyID := strings.ReplaceAll(emptyInput, "-", "")- liveID := strings.ReplaceAll(liveInput, "-", "")-- insertAggregateTagsKB(t, emptyID, "user-1", string(entity.TenantPermissionMe), 0)- insertAggregateTagsKB(t, liveID, "user-1", string(entity.TenantPermissionMe), 1)-- docEngine := &aggregateTagsMockEngine{- searchResults: map[string]*types.SearchResult{- "ragflow_user-1": {- Chunks: []map[string]interface{}{- {"tag_kwd": "alpha###beta"},- },- },- },- }- ctx := t.Context()-- result, code, err := testDatasetServiceForAggregateTags(t, docEngine).AggregateTags(ctx, []string{emptyInput, liveInput}, "user-1")- if err != nil {- t.Fatalf("AggregateTags failed: %v", err)- }- if code != common.CodeSuccess {- t.Fatalf("code=%d want=%d", code, common.CodeSuccess)- }- if len(docEngine.requests) != 1 {- t.Fatalf("search requests=%d want=1", len(docEngine.requests))- }- if len(docEngine.requests[0].KbIDs) != 1 || docEngine.requests[0].KbIDs[0] != liveID {- t.Fatalf("KbIDs=%v want [%s]", docEngine.requests[0].KbIDs, liveID)- }-- got := aggregateTagsResultMap(result)- if got["alpha"] != 1 || got["beta"] != 1 {- t.Fatalf("unexpected result=%v", got)- }-}--func TestDatasetServiceAggregateTagsReturnsSearchError(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)-- kbInput := "623e4567-e89b-12d3-a456-426614174005"+ kbInput := "523e4567-e89b-12d3-a456-426614174004"kbID := strings.ReplaceAll(kbInput, "-", "")- insertAggregateTagsKB(t, kbID, "user-1", string(entity.TenantPermissionMe), 1)+ const kbTenant = "tenant-owner"+ insertAggregateTagsKB(t, kbID, kbTenant, string(entity.TenantPermissionTeam), "file-foreign", 0)+ insertAggregateTagsMembership(t, kbTenant, "user-1")ctx := t.Context()- docEngine := &aggregateTagsMockEngine{searchErr: errors.New("boom")}- _, code, err := testDatasetServiceForAggregateTags(t, docEngine).AggregateTags(ctx, []string{kbInput}, "user-1")- if err == nil {- t.Fatal("expected search error")+ var gotOwner string+ loader := func(_ context.Context, tagFileID, ownerTenantID string) (map[string]int, error) {+ if tagFileID != "file-foreign" {+ t.Fatalf("tagFileID = %q, want file-foreign", tagFileID)+ }+ gotOwner = ownerTenantID+ return map[string]int{"finance": 1}, nil}- if code != common.CodeServerError {- t.Fatalf("code=%d want=%d", code, common.CodeServerError)++ if _, code, err := testDatasetServiceForAggregateTags(t, loader).AggregateTags(ctx, []string{kbInput}, "user-1"); err != nil {+ t.Fatalf("AggregateTags failed: code=%d err=%v", code, err)}- if !strings.Contains(err.Error(), "failed to aggregate tags: boom") {- t.Fatalf("unexpected error: %v", err)+ if gotOwner != kbTenant {+ t.Fatalf("loader ownerTenantID = %q, want %q (must be the dataset tenant, not the user)", gotOwner, kbTenant)}}diff --git a/internal/service/tag.go b/internal/service/tag.goindex 444f1c0eb6..2e2525ae70 100644--- a/internal/service/tag.go+++ b/internal/service/tag.go@@ -28,7 +28,6 @@ import ("go.uber.org/zap"- "ragflow/internal/dao""ragflow/internal/engine/types""ragflow/internal/entity""ragflow/internal/service/nlp"@@ -115,16 +114,24 @@ func SetTagsToCache(ctx context.Context, kbIDs []string, tags map[string]float64type Knowledgebase = entity.Knowledgebase// GetAllTagsInPortion returns all tag_kwd values and their occurrence counts-// for documents belonging to the given kbIDs.-func (s *MetadataService) GetAllTagsInPortion(ctx context.Context, tenantID string, kbIDs []string) (map[string]float64, error) {- if len(kbIDs) == 0 {+// for documents belonging to the given kbIDs, aggregated across every tenant+// index in tenantIDs — the same tenant scope TagQuery searches. The dataset+// and chunk-search callers can supply authorized knowledgebases from multiple+// tenants, so restricting the aggregation to a single tenant index would omit+// tags that exist only in the other tenants' indices (leaving them absent from+// allTags and scored with the 0.0001 fallback, which inflates their scores).+func (s *MetadataService) GetAllTagsInPortion(ctx context.Context, tenantIDs []string, kbIDs []string) (map[string]float64, error) {+ if len(kbIDs) == 0 || len(tenantIDs) == 0 {return make(map[string]float64), nil}- indexName := fmt.Sprintf("ragflow_%s", tenantID)+ indexNames := make([]string, len(tenantIDs))+ for i, tenantID := range tenantIDs {+ indexNames[i] = fmt.Sprintf("ragflow_%s", tenantID)+ }searchReq := &types.SearchRequest{- IndexNames: []string{indexName},+ IndexNames: indexNames,KbIDs: kbIDs,Offset: 0,// Python passes limit=0 ("unlimited") which Go SearchRequest treats@@ -253,78 +260,73 @@ func (s *MetadataService) TagQuery(ctx context.Context, question string, tenantIreturn resultTags, nil}-// LabelQuestion returns rank features for a question based on KB's tag configuration.+// LabelQuestion returns rank features for a question based on the dataset's own+// tags.//// Flow:-// 1. Collect tag_kb_ids from KBs' parser_config+// 1. Use the queried KBs themselves as the tag source and search scope// 2. Try to get all_tags from cache (via GetTagsFromCache)// 3. If cache miss, call GetAllTagsInPortion and cache the result (via SetTagsToCache)-// 4. Get tag KBs by IDs-// 5. Call TagQuery to get weighted tag features for the question+// 4. Call TagQuery to get weighted tag features for the question+//+// The Go backend has no separate tag-library dataset as the Python backend+// does. Instead the Go extractor (extractor_tag.go) writes tag_kwd onto each+// chunk during ingestion, so the authoritative tag vocabulary lives on the+// dataset's own chunks and is aggregated from the KBs being queried.func (s *MetadataService) LabelQuestion(ctx context.Context, question string, kbs []*Knowledgebase) map[string]float64 {if len(kbs) == 0 || question == "" {return nil}- // Collect tag_kb_ids from KBs' parser_config and track last KB- var tagKBIDs []string+ // Use the queried KBs themselves as both the tag source and the search+ // scope. Track the last KB (for tenant_id / topn_tags) and the unique+ // tenant IDs spanned by the KBs.+ var kbIDs []stringvar lastKB *Knowledgebase+ tenantIDSet := make(map[string]bool)for _, kb := range kbs {- if kb.ParserConfig == nil {+ if kb == nil {continue}lastKB = kb- if rawTagKBIDs, ok := kb.ParserConfig["tag_kb_ids"].([]interface{}); ok {- for _, id := range rawTagKBIDs {- if idStr, ok := id.(string); ok {- tagKBIDs = append(tagKBIDs, idStr)- }- }+ kbIDs = append(kbIDs, kb.ID)+ if kb.TenantID != "" {+ tenantIDSet[kb.TenantID] = true}}-- if len(tagKBIDs) == 0 {+ if len(kbIDs) == 0 {+ return nil+ }+ uniqueTenantIDs := make([]string, 0, len(tenantIDSet))+ for tid := range tenantIDSet {+ uniqueTenantIDs = append(uniqueTenantIDs, tid)+ }+ if len(uniqueTenantIDs) == 0 {return nil}- common.Debug("tag_kb_ids found in parser_config", zap.Strings("tag_kb_ids", tagKBIDs))-- // Get all tags from cache or compute and cache- allTags, err := GetTagsFromCache(ctx, tagKBIDs)+ // Aggregate tag_kwd across the dataset's own chunks (cached by KB set).+ allTags, err := GetTagsFromCache(ctx, kbIDs)if err != nil {common.Warn("Failed to get tags from cache", zap.Error(err))}if allTags == nil {// Cache miss - compute all_tags_in_portion- allTags, err = s.GetAllTagsInPortion(ctx, lastKB.TenantID, tagKBIDs)+ allTags, err = s.GetAllTagsInPortion(ctx, uniqueTenantIDs, kbIDs)if err != nil {common.Warn("Failed to get all tags in portion", zap.Error(err))return nil}// Store in cache for future lookups- if err = SetTagsToCache(ctx, tagKBIDs, allTags); err != nil {+ if err = SetTagsToCache(ctx, kbIDs, allTags); err != nil {common.Warn("Failed to set tags cache", zap.Error(err))}}- // Get tag_kbs by IDs- kbDAO := dao.NewKnowledgebaseDAO()- tagKBs, err := kbDAO.GetByIDs(ctx, dao.DB, tagKBIDs)- if err != nil || len(tagKBs) == 0 {- // Return nil if no tag_kbs found- return nil- }-- // Get unique tenant IDs from tag_kbs- tenantIDSet := make(map[string]bool)- for _, kb := range tagKBs {- tenantIDSet[kb.TenantID] = true- }- var uniqueTenantIDs []string- for tid := range tenantIDSet {- uniqueTenantIDs = append(uniqueTenantIDs, tid)- }- if len(uniqueTenantIDs) == 0 {+ // No tag_kwd on the dataset's chunks means tagging is not in effect for+ // this query: return nil so the caller applies no tag boost (Python's+ // label_question returning None).+ if len(allTags) == 0 {return nil}@@ -346,14 +348,14 @@ func (s *MetadataService) LabelQuestion(ctx context.Context, question string, kb}}- // Query tags for the question using unique tenant IDs- tagFeatures, err := s.TagQuery(ctx, question, uniqueTenantIDs, tagKBIDs, allTags, topnTags)+ // Query tags for the question across the dataset's own chunks.+ tagFeatures, err := s.TagQuery(ctx, question, uniqueTenantIDs, kbIDs, allTags, topnTags)if err != nil {return nil}if len(tagFeatures) == 0 {- // Tag kb exists but returned no matching tags - return empty map (not nil)- // so caller knows tag kb was configured vs not configured at all+ // Tags configured but the question matched none - return empty map+ // (not nil) so the caller knows tagging was active for this dataset.return make(map[string]float64)}diff --git a/test/api/dataset_api_test.go b/test/api/dataset_api_test.goindex c191879073..36c8b39ca6 100644--- a/test/api/dataset_api_test.go+++ b/test/api/dataset_api_test.go@@ -397,10 +397,10 @@ func TestDatasetUpdateParserConfigValidMatrixContract(t *testing.T) {{name: "html4excel_false", parserConfig: map[string]interface{}{"html4excel": false}},{name: "layout_recognize_DeepDOC", parserConfig: map[string]interface{}{"layout_recognize": "DeepDOC"}},{name: "layout_recognize_navie", parserConfig: map[string]interface{}{"layout_recognize": "Plain Text"}},- {name: "tag_kb_ids", parserConfig: map[string]interface{}{"tag_kb_ids": []interface{}{"1", "2"}}},{name: "topn_tags_min", parserConfig: map[string]interface{}{"topn_tags": 1}},{name: "topn_tags_mid", parserConfig: map[string]interface{}{"topn_tags": 5}},{name: "topn_tags_max", parserConfig: map[string]interface{}{"topn_tags": 10}},+ {name: "tags_tag_file_id", parserConfig: map[string]interface{}{"tags": map[string]interface{}{"tag_file_id": "file-123"}}},{name: "filename_embd_weight_min", parserConfig: map[string]interface{}{"filename_embd_weight": 0.1}},{name: "filename_embd_weight_mid", parserConfig: map[string]interface{}{"filename_embd_weight": 0.5}},{name: "filename_embd_weight_max", parserConfig: map[string]interface{}{"filename_embd_weight": 1.0}},