Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): refactor tag extractor with memory index, asymmetric coverage, multi-source enrichment and worker pool sequencing (#18607)

jay7772183ae771399a5
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index 735d121d7f..15b80500e3 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -209,6 +209,9 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
// Document line: work with a copy to avoid mutating the original
docCopy := copyFields(doc)
docCopy["kb_id"] = datasetID
+ if raw, ok := formatOrderedTagFeas(docCopy["tag_feas"]); ok {
+ docCopy["tag_feas"] = raw
+ }
if err := jsonIterator.NewEncoder(&buf).Encode(docCopy); err != nil {
return nil, fmt.Errorf("failed to encode document: %w", err)
}
@@ -608,6 +611,9 @@ func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID strin
// Update document fields if any remain
if len(doc) > 0 {
+ if raw, ok := formatOrderedTagFeas(doc["tag_feas"]); ok {
+ doc["tag_feas"] = raw
+ }
updateBody := map[string]interface{}{"doc": doc}
body, _ := json.Marshal(updateBody)
req := esapi.UpdateRequest{
@@ -782,6 +788,90 @@ func copyFields(m map[string]interface{}) map[string]interface{} {
return result
}
+// formatOrderedTagFeas formats tag_feas as json.RawMessage with keys ordered
+// strictly descending by score, preventing jsoniter map-key randomized output.
+func formatOrderedTagFeas(v any) (json.RawMessage, bool) {
+ if v == nil {
+ return nil, false
+ }
+ if raw, ok := v.(json.RawMessage); ok {
+ return raw, true
+ }
+ if marshaler, ok := v.(json.Marshaler); ok {
+ b, err := marshaler.MarshalJSON()
+ if err == nil {
+ return json.RawMessage(b), true
+ }
+ }
+ type kv struct {
+ k string
+ v int
+ }
+ var kvs []kv
+ roundScore := func(f float64) int {
+ if f < 0 {
+ return int(f - 0.5)
+ }
+ return int(f + 0.5)
+ }
+
+ switch m := v.(type) {
+ case map[string]int:
+ kvs = make([]kv, 0, len(m))
+ for k, val := range m {
+ kvs = append(kvs, kv{k, val})
+ }
+ case map[string]float64:
+ kvs = make([]kv, 0, len(m))
+ for k, val := range m {
+ kvs = append(kvs, kv{k, roundScore(val)})
+ }
+ case map[string]any:
+ kvs = make([]kv, 0, len(m))
+ for k, val := range m {
+ switch score := val.(type) {
+ case int:
+ kvs = append(kvs, kv{k, score})
+ case float64:
+ kvs = append(kvs, kv{k, roundScore(score)})
+ case int64:
+ kvs = append(kvs, kv{k, int(score)})
+ case json.Number:
+ if f, err := score.Float64(); err == nil {
+ kvs = append(kvs, kv{k, roundScore(f)})
+ }
+ }
+ }
+ default:
+ return nil, false
+ }
+
+ if len(kvs) == 0 {
+ return json.RawMessage("{}"), true
+ }
+
+ sort.Slice(kvs, func(i, j int) bool {
+ if kvs[i].v != kvs[j].v {
+ return kvs[i].v > kvs[j].v
+ }
+ return kvs[i].k < kvs[j].k
+ })
+
+ var buf bytes.Buffer
+ buf.WriteByte('{')
+ for i, item := range kvs {
+ if i > 0 {
+ buf.WriteByte(',')
+ }
+ kb, _ := json.Marshal(item.k)
+ buf.Write(kb)
+ buf.WriteByte(':')
+ buf.WriteString(strconv.Itoa(item.v))
+ }
+ buf.WriteByte('}')
+ return json.RawMessage(buf.Bytes()), true
+}
+
// DeleteChunks deletes chunks from a dataset index by condition
func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {
// For ES, index name is just indexName (e.g., "ragflow_{tenantID}"), not indexName_datasetID
diff --git a/internal/engine/elasticsearch/chunk_test.go b/internal/engine/elasticsearch/chunk_test.go
index 2eeebd3841..7d159806e0 100644
--- a/internal/engine/elasticsearch/chunk_test.go
+++ b/internal/engine/elasticsearch/chunk_test.go
@@ -17,7 +17,10 @@
package elasticsearch
import (
+ "bytes"
"context"
+ "encoding/json"
+ "strings"
"testing"
"ragflow/internal/common"
@@ -528,3 +531,53 @@ func TestCalculatePaginationReportedRegression(t *testing.T) {
}
}
}
+
+func TestFormatOrderedTagFeas(t *testing.T) {
+ tagFeas := map[string]any{
+ "价格咨询": 4,
+ "活动咨询": 9,
+ "正面评价": 3,
+ "服务投诉": 6,
+ "质量投诉": 3,
+ }
+
+ raw, ok := formatOrderedTagFeas(tagFeas)
+ if !ok {
+ t.Fatal("expected formatOrderedTagFeas to succeed")
+ }
+
+ expectedJSON := `{"活动咨询":9,"服务投诉":6,"价格咨询":4,"正面评价":3,"质量投诉":3}`
+ if string(raw) != expectedJSON {
+ t.Fatalf("formatOrderedTagFeas output mismatch:\ngot: %s\nwant: %s", string(raw), expectedJSON)
+ }
+
+ // Test json.Number and float rounding support
+ tagFeasWithJSONNumber := map[string]any{
+ "LowTag": json.Number("3.2"),
+ "HighTag": json.Number("8.7"),
+ "MidTag": float64(5.6),
+ }
+ rawNum, okNum := formatOrderedTagFeas(tagFeasWithJSONNumber)
+ if !okNum {
+ t.Fatal("expected formatOrderedTagFeas with json.Number to succeed")
+ }
+ expectedNumJSON := `{"HighTag":9,"MidTag":6,"LowTag":3}`
+ if string(rawNum) != expectedNumJSON {
+ t.Fatalf("formatOrderedTagFeas with json.Number mismatch:\ngot: %s\nwant: %s", string(rawNum), expectedNumJSON)
+ }
+
+ // Verify that jsonIterator preserves the raw byte order inside docCopy
+ docCopy := map[string]any{
+ "doc_id": "doc-1",
+ "tag_feas": raw,
+ }
+ var buf bytes.Buffer
+ if err := jsonIterator.NewEncoder(&buf).Encode(docCopy); err != nil {
+ t.Fatalf("jsonIterator Encode failed: %v", err)
+ }
+
+ encodedStr := buf.String()
+ if !strings.Contains(encodedStr, `"tag_feas":{"活动咨询":9,"服务投诉":6,"价格咨询":4,"正面评价":3,"质量投诉":3}`) {
+ t.Fatalf("encoded JSON does not preserve score-descending order:\n%s", encodedStr)
+ }
+}
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index b5646e1fe9..d2989909b6 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -607,7 +607,14 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
}
if err := runtime.WithTimeout(ctx, extractorTimeout, func(timeoutCtx context.Context) error {
- // Tag phase: run when tags.top_n > 0 and we have chunks.
+ // Phase 1: Keywords extraction (if enabled), running across chunks via extractorPool.
+ if c.Param.Keywords.TopN > 0 {
+ if err := c.runAutoKeywordsPool(timeoutCtx, db, in); err != nil {
+ return err
+ }
+ }
+
+ // Phase 2: Tag phase (if enabled), benefiting from title and freshly extracted keywords.
if c.Param.Tags.TopN > 0 {
tagged, tagErr := c.runAutoTags(timeoutCtx, db, in)
if tagErr != nil {
@@ -616,7 +623,8 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
in.chunks = tagged
}
- return c.runAutoExtractions(timeoutCtx, db, in)
+ // Phase 3: Remaining extractions (Questions, Summary, Metadata) via extractorPool.
+ return c.runRemainingExtractions(timeoutCtx, db, in)
}); err != nil {
return nil, fmt.Errorf("extractor: %w", err)
}
@@ -751,15 +759,10 @@ func (c *ExtractorComponent) runAutoSummary(ctx context.Context, db *gorm.DB, in
return nil
}
-// runAutoExtractions dispatches the auto keyword / question / summary / metadata
-// extraction for every chunk to the process-wide extractorPool so the
-// work runs with bounded cross-chunk concurrency (mirrors Python's
-// chat_limiter). The pool only bounds concurrency; per-invocation
-// completion and first-error collection happen here with a local
-// sync.WaitGroup, so concurrent Invoke calls do not disturb each other.
-// Each chunk job owns its own chunk map, so no per-chunk mutex is needed.
-func (c *ExtractorComponent) runAutoExtractions(ctx context.Context, db *gorm.DB, in extractorInputs) error {
- if c.Param.Keywords.TopN <= 0 && c.Param.Questions.TopN <= 0 && !c.Param.Summary.Enabled && !c.Param.Metadata.Enabled {
+// runAutoKeywordsPool dispatches keyword extraction across all chunks concurrently
+// using extractorPool before the tagging stage.
+func (c *ExtractorComponent) runAutoKeywordsPool(ctx context.Context, db *gorm.DB, in extractorInputs) error {
+ if c.Param.Keywords.TopN <= 0 || len(in.chunks) == 0 {
return nil
}
futs := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(in.chunks))
@@ -769,13 +772,66 @@ func (c *ExtractorComponent) runAutoExtractions(ctx context.Context, db *gorm.DB
if strings.TrimSpace(text) == "" {
text, _ = ck["text"].(string)
}
- fn := c.autoExtractionJob(ctx, db, in, i, ck, text)
+ fn := func() error {
+ if err := c.runAutoKeywords(ctx, db, in, ck, text); err != nil {
+ return fmt.Errorf("chunk %d keywords: %w", i, err)
+ }
+ return nil
+ }
f, err := extractorPool.Submit(ctx, fn)
if err != nil {
return err
}
futs = append(futs, f)
}
+ return awaitFutures(ctx, futs)
+}
+
+// runRemainingExtractions dispatches auto questions / summary / metadata
+// extractions across all chunks concurrently using extractorPool.
+func (c *ExtractorComponent) runRemainingExtractions(ctx context.Context, db *gorm.DB, in extractorInputs) error {
+ if c.Param.Questions.TopN <= 0 && !c.Param.Summary.Enabled && !c.Param.Metadata.Enabled {
+ return nil
+ }
+ futs := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(in.chunks))
+ for i, ck := range in.chunks {
+ i, ck := i, ck
+ text, _ := ck["content_with_weight"].(string)
+ if strings.TrimSpace(text) == "" {
+ text, _ = ck["text"].(string)
+ }
+ fn := c.remainingExtractionJob(ctx, db, in, i, ck, text)
+ f, err := extractorPool.Submit(ctx, fn)
+ if err != nil {
+ return err
+ }
+ futs = append(futs, f)
+ }
+ return awaitFutures(ctx, futs)
+}
+
+func (c *ExtractorComponent) remainingExtractionJob(ctx context.Context, db *gorm.DB, in extractorInputs, idx int, ck map[string]any, chunkText string) extractorJob {
+ return func() error {
+ if c.Param.Questions.TopN > 0 {
+ if err := c.runAutoQuestions(ctx, db, in, ck, chunkText); err != nil {
+ return fmt.Errorf("chunk %d questions: %w", idx, err)
+ }
+ }
+ if c.Param.Summary.Enabled {
+ if err := c.runAutoSummary(ctx, db, in, ck, chunkText); err != nil {
+ return fmt.Errorf("chunk %d summary: %w", idx, err)
+ }
+ }
+ if c.Param.Metadata.Enabled {
+ if err := c.runEnableMetadata(ctx, db, in, ck, chunkText); err != nil {
+ return fmt.Errorf("chunk %d metadata: %w", idx, err)
+ }
+ }
+ return nil
+ }
+}
+
+func awaitFutures(ctx context.Context, futs []utility.WorkerPoolFuture[extractorJob, struct{}]) error {
var firstErr error
var emu sync.Mutex
var wg sync.WaitGroup
@@ -805,37 +861,6 @@ func (c *ExtractorComponent) runAutoExtractions(ctx context.Context, db *gorm.DB
return firstErr
}
-// autoExtractionJob returns the unit of work for one chunk: it runs the
-// enabled auto-extractions (keywords, questions, summary, metadata) sequentially
-// on the chunk's own map. Running them sequentially inside the job keeps
-// the code free of per-chunk mutexes — cross-chunk parallelism is provided
-// by the pool, not by intra-chunk goroutines.
-func (c *ExtractorComponent) autoExtractionJob(ctx context.Context, db *gorm.DB, in extractorInputs, idx int, ck map[string]any, chunkText string) extractorJob {
- return func() error {
- if c.Param.Keywords.TopN > 0 {
- if err := c.runAutoKeywords(ctx, db, in, ck, chunkText); err != nil {
- return fmt.Errorf("chunk %d keywords: %w", idx, err)
- }
- }
- if c.Param.Questions.TopN > 0 {
- if err := c.runAutoQuestions(ctx, db, in, ck, chunkText); err != nil {
- return fmt.Errorf("chunk %d questions: %w", idx, err)
- }
- }
- if c.Param.Summary.Enabled {
- if err := c.runAutoSummary(ctx, db, in, ck, chunkText); err != nil {
- return fmt.Errorf("chunk %d summary: %w", idx, err)
- }
- }
- if c.Param.Metadata.Enabled {
- if err := c.runEnableMetadata(ctx, db, in, ck, chunkText); err != nil {
- return fmt.Errorf("chunk %d metadata: %w", idx, err)
- }
- }
- return nil
- }
-}
-
// runEnableMetadata extracts structured metadata for the current chunk and
// merges the parsed JSON object into ck["metadata"]. It mirrors the
// runAutoKeywords/runAutoQuestions shape but parses a JSON object and
diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.go
index ec82e16a69..bc3192cd8c 100644
--- a/internal/ingestion/component/extractor_tag.go
+++ b/internal/ingestion/component/extractor_tag.go
@@ -8,19 +8,20 @@ import (
"encoding/json"
"fmt"
"io"
+ "math"
"math/rand/v2"
"path/filepath"
"sort"
+ "strconv"
"strings"
"sync"
"time"
- "github.com/xuri/excelize/v2"
- "gorm.io/gorm"
-
"github.com/cespare/xxhash/v2"
eschema "github.com/cloudwego/eino/schema"
+ "github.com/xuri/excelize/v2"
"go.uber.org/zap"
+ "gorm.io/gorm"
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
@@ -32,7 +33,37 @@ import (
"ragflow/internal/tokenizer"
)
-const matchOverlapThreshold = 0.5
+const (
+ // defaultMatchCoverageThreshold is the asymmetric coverage threshold for Phase 1 (55%).
+ defaultMatchCoverageThreshold = 0.55
+
+ // singleTokenCoverageThreshold is the full-coverage threshold required for 1-token rules (99.9%).
+ singleTokenCoverageThreshold = 0.999
+
+ // singleTokenMinIDF is the minimum total IDF required for 1-token rules to match.
+ singleTokenMinIDF = 1.5
+
+ // defaultTopK is the maximum number of matched example candidates to aggregate in Phase 1.
+ defaultTopK = 5
+
+ // bgSmoothing is the Dirichlet smoothing constant for background prior tag probabilities.
+ bgSmoothing = 10.0
+
+ // rankDecayTierDropRelativeThreshold is the relative drop threshold to descend an effective rank tier.
+ rankDecayTierDropRelativeThreshold = 0.05
+
+ // rankDecayBaseMax is the initial rank base score for the top effective rank.
+ rankDecayBaseMax = 8.5
+
+ // rankDecayBaseMin is the floor for rank base score.
+ rankDecayBaseMin = 3.0
+
+ // rankDecayStep is the score decay per effective rank tier.
+ rankDecayStep = 2.0
+
+ // rankDecayWeightPower is the power exponent applied to relative tag weight.
+ rankDecayWeightPower = 0.4
+)
const taggerLLMConcurrency = 8
@@ -74,29 +105,407 @@ Output:
`
-type indexedTagSource struct {
- examples []schema.TagLabel
- allTags map[string]float64
- tagTokens [][]string
+// MemoryTagIndex is an in-memory inverted index of tag examples (immutable after construction, safe for concurrent use).
+type MemoryTagIndex struct {
+ examples []schema.TagLabel
+ postings map[string][]int // word -> doc_ids (compact slice without per-doc map overhead)
+ idfs map[string]float64 // word -> idf
+ exTotalIDF []float64 // doc_id -> sum(IDF(w))
+ exTokenCount []int // doc_id -> number of unique tokens
+ allTags map[string]float64 // tag -> background prob (S=10)
+}
+
+func buildMemoryTagIndex(rawExamples []schema.TagLabel, tok tokenizer.Tokenizer) *MemoryTagIndex {
+ if len(rawExamples) == 0 {
+ return nil
+ }
+
+ cleanExamples := make([]schema.TagLabel, 0, len(rawExamples))
+ exWordSets := make([]map[string]struct{}, 0, len(rawExamples))
+ allTagCounts := make(map[string]int)
+ totalTagCount := 0
+
+ // 1. Filter empty contents, deduplicate tags per sample, and normalize dots with deep copies.
+ for _, ex := range rawExamples {
+ content := strings.TrimSpace(ex.Content)
+ if content == "" {
+ continue // Skip empty content.
+ }
+ tks, err := tok.Tokenize(content)
+ if err != nil {
+ common.Warn(fmt.Sprintf("extractor tags: tokenize example failed: %v", err))
+ continue
+ }
+ fields := strings.Fields(tks)
+ if len(fields) == 0 {
+ continue // Skip samples with no tokens.
+ }
+ wordSet := make(map[string]struct{}, len(fields))
+ for _, w := range fields {
+ if w = strings.TrimSpace(w); w != "" {
+ wordSet[w] = struct{}{}
+ }
+ }
+ if len(wordSet) == 0 {
+ continue
+ }
+
+ tagSet := make(map[string]struct{}, len(ex.Tags))
+ cleanTags := make([]string, 0, len(ex.Tags))
+ for _, t := range ex.Tags {
+ t = strings.TrimSpace(strings.ReplaceAll(t, ".", "_"))
+ if t != "" {
+ if _, exists := tagSet[t]; !exists {
+ tagSet[t] = struct{}{}
+ cleanTags = append(cleanTags, t)
+ }
+ }
+ }
+ if len(cleanTags) == 0 {
+ continue // Skip examples with no usable tags.
+ }
+ for _, t := range cleanTags {
+ allTagCounts[t]++
+ totalTagCount++
+ }
+
+ cleanExamples = append(cleanExamples, schema.TagLabel{
+ Content: content,
+ Tags: cleanTags,
+ })
+ exWordSets = append(exWordSets, wordSet)
+ }
+
+ N := float64(len(cleanExamples))
+ if N == 0 || totalTagCount == 0 {
+ return nil // Guard against all samples having empty tags.
+ }
+
+ docFreq := make(map[string]int)
+ postings := make(map[string][]int)
+
+ for i, wordSet := range exWordSets {
+ // Invariant: each docID is added to postings[w] at most once.
+ for w := range wordSet {
+ postings[w] = append(postings[w], i)
+ docFreq[w]++ // Each document contributes at most 1 to document frequency.
+ }
+ }
+
+ // 2. Standard smoothed IDF (terms covering all documents smoothly approach 0).
+ idfs := make(map[string]float64, len(docFreq))
+ for w, df := range docFreq {
+ idfs[w] = math.Log(1.0 + (N-float64(df)+0.5)/(float64(df)+0.5))
+ }
+
+ // 3. Precompute total IDF and token count per valid example (short-text TF=1, avoiding per-example TF maps).
+ exTotalIDF := make([]float64, len(cleanExamples))
+ exTokenCount := make([]int, len(cleanExamples))
+ for i, wordSet := range exWordSets {
+ var sum float64
+ for w := range wordSet {
+ sum += idfs[w]
+ }
+ exTotalIDF[i] = sum
+ exTokenCount[i] = len(wordSet)
+ }
+
+ // 4. Background prior probability distribution (smoothed Dirichlet prior).
+ alpha := bgSmoothing / float64(len(allTagCounts))
+ bgProportions := make(map[string]float64, len(allTagCounts))
+ for t, count := range allTagCounts {
+ bgProportions[t] = (float64(count) + alpha) / (float64(totalTagCount) + bgSmoothing)
+ }
+
+ return &MemoryTagIndex{
+ examples: cleanExamples,
+ postings: postings,
+ idfs: idfs,
+ exTotalIDF: exTotalIDF,
+ exTokenCount: exTokenCount,
+ allTags: bgProportions,
+ }
+}
+
+// OrderedTagWeights represents a tag-to-weight mapping that preserves
+// score-descending order when marshaled to JSON for Elasticsearch storage.
+type OrderedTagWeights map[string]int
+
+func (m OrderedTagWeights) MarshalJSON() ([]byte, error) {
+ if m == nil {
+ return []byte("null"), nil
+ }
+ type kv struct {
+ k string
+ v int
+ }
+ kvs := make([]kv, 0, len(m))
+ for k, v := range m {
+ kvs = append(kvs, kv{k, v})
+ }
+ sort.Slice(kvs, func(i, j int) bool {
+ if kvs[i].v != kvs[j].v {
+ return kvs[i].v > kvs[j].v
+ }
+ return kvs[i].k < kvs[j].k
+ })
+
+ var buf bytes.Buffer
+ buf.WriteByte('{')
+ for i, item := range kvs {
+ if i > 0 {
+ buf.WriteByte(',')
+ }
+ keyBytes, _ := json.Marshal(item.k)
+ buf.Write(keyBytes)
+ buf.WriteByte(':')
+ buf.WriteString(strconv.Itoa(item.v))
+ }
+ buf.WriteByte('}')
+ return buf.Bytes(), nil
+}
+
+// ----------------------------------------------------------------------
+// Phase 1 local matching
+// ----------------------------------------------------------------------
+
+func matchAndTagChunk(
+ chunk map[string]any,
+ idx *MemoryTagIndex,
+ tok tokenizer.Tokenizer,
+ topN int,
+) *schema.TaggedChunk {
+ text := getChunkText(chunk)
+ if text == "" || idx == nil || len(idx.allTags) == 0 {
+ return nil
+ }
+
+ tokens, err := tok.Tokenize(text)
+ if err != nil || tokens == "" {
+ return nil
+ }
+
+ chunkWordFreq := make(map[string]int)
+ for _, w := range strings.Fields(tokens) {
+ if w = strings.TrimSpace(w); w != "" {
+ chunkWordFreq[w]++
+ }
+ }
+
+ // 1. Inverted index lookup (skips irrelevant samples).
+ candidateInterIDF := make(map[int]float64)
+ candidateTFScore := make(map[int]float64)
+ candidateMatchedTokens := make(map[int]int)
+ for w, freq := range chunkWordFreq {
+ idf, exists := idx.idfs[w]
+ if !exists || idf <= 1e-6 {
+ continue
+ }
+ tfBoost := 1.0 + 0.3*math.Log(1.0+float64(freq))
+ for _, docID := range idx.postings[w] {
+ candidateInterIDF[docID] += idf // Pure IDF sum, ensures coverage <= 1.0
+ candidateTFScore[docID] += idf * tfBoost // Saliency weighted by TF
+ candidateMatchedTokens[docID]++
+ }
+ }
+ if len(candidateInterIDF) == 0 {
+ return nil
+ }
+
+ // 2. Compute asymmetric coverage (Coverage(E) = InterIDF / ExTotalIDF in [0.0, 1.0]).
+ type candidateScore struct {
+ docID int
+ coverage float64
+ tfWeight float64
+ }
+ passed := make([]candidateScore, 0, len(candidateInterIDF))
+ for docID, interIDF := range candidateInterIDF {
+ exTotal := idx.exTotalIDF[docID]
+ if exTotal <= 0 {
+ continue
+ }
+ matchedTokens := candidateMatchedTokens[docID]
+ exTokens := 0
+ if docID < len(idx.exTokenCount) {
+ exTokens = idx.exTokenCount[docID]
+ }
+
+ if exTokens >= 2 {
+ if matchedTokens < 2 {
+ continue
+ }
+ } else {
+ // For 1-token examples, require high IDF (exTotal >= singleTokenMinIDF)
+ if exTotal < singleTokenMinIDF {
+ continue
+ }
+ }
+
+ cov := min(1.0, interIDF/exTotal)
+ if exTokens < 2 {
+ if cov < singleTokenCoverageThreshold {
+ continue
+ }
+ } else {
+ if cov < defaultMatchCoverageThreshold {
+ continue
+ }
+ }
+ tfFactor := candidateTFScore[docID] / max(1e-6, interIDF)
+ passed = append(passed, candidateScore{
+ docID: docID,
+ coverage: cov,
+ tfWeight: cov * tfFactor,
+ })
+ }
+ if len(passed) == 0 {
+ return nil
+ }
+
+ // 3. Aggregate Top-K samples weighted by Coverage to eliminate label pollution.
+ sort.Slice(passed, func(i, j int) bool {
+ if passed[i].coverage != passed[j].coverage {
+ return passed[i].coverage > passed[j].coverage
+ }
+ return passed[i].docID < passed[j].docID
+ })
+ topK := min(defaultTopK, len(passed))
+
+ tagWeightedCounts := make(map[string]float64)
+ var totalWeightSum float64
+
+ for i := 0; i < topK; i++ {
+ weight := passed[i].tfWeight
+ cov := passed[i].coverage
+ ex := idx.examples[passed[i].docID]
+ totalWeightSum += cov
+ tagCount := float64(max(1, len(ex.Tags)))
+ for _, t := range ex.Tags {
+ tagWeightedCounts[t] += weight / tagCount
+ }
+ }
+
+ if len(tagWeightedCounts) == 0 || totalWeightSum <= 0 {
+ return nil
+ }
+
+ // 4. Score calculation: Rank-decay gradient model ensuring healthy [8/9, 7, 6, 5, 4] distribution.
+ // topK acts as a normalization denominator for average coverage across top matched slots.
+ avgCov := totalWeightSum / float64(topK)
+ type tagCandidate struct {
+ name string
+ weightedC float64
+ lift float64
+ combined float64
+ }
+ var candidates []tagCandidate
+ var maxWeightedC float64
+ for t, weightedC := range tagWeightedCounts {
+ if weightedC > maxWeightedC {
+ maxWeightedC = weightedC
+ }
+ bg := idx.allTags[t]
+ if bg <= 0 {
+ bg = 0.0001
+ }
+ pMatch := weightedC / totalWeightSum
+ lift := pMatch / bg
+ combined := weightedC * math.Sqrt(max(0.1, lift))
+ candidates = append(candidates, tagCandidate{
+ name: t,
+ weightedC: weightedC,
+ lift: lift,
+ combined: combined,
+ })
+ }
+ if maxWeightedC <= 0 {
+ maxWeightedC = 1.0
+ }
+
+ sort.Slice(candidates, func(i, j int) bool {
+ if candidates[i].combined != candidates[j].combined {
+ return candidates[i].combined > candidates[j].combined
+ }
+ if candidates[i].weightedC != candidates[j].weightedC {
+ return candidates[i].weightedC > candidates[j].weightedC
+ }
+ return candidates[i].name < candidates[j].name
+ })
+
+ type tagScore struct {
+ name string
+ score int
+ }
+ var scored []tagScore
+ effectiveRank := 0
+ covFactor := min(1.0, math.Sqrt(max(0.0, avgCov)))
+ for i, c := range candidates {
+ if i > 0 {
+ prev := candidates[i-1].combined
+ if prev > 0 && (prev-c.combined)/prev > rankDecayTierDropRelativeThreshold {
+ effectiveRank++
+ }
+ }
+ rankBase := max(rankDecayBaseMin, rankDecayBaseMax-rankDecayStep*float64(effectiveRank))
+ relWeight := c.weightedC / maxWeightedC
+ raw := rankBase * covFactor * math.Pow(relWeight, rankDecayWeightPower)
+ // Upper bounded by 10 as a defensive clamp; maximum expected rank score is 9.
+ s := min(10, max(1, roundInt(raw)))
+ scored = append(scored, tagScore{name: c.name, score: s})
+ }
+
+ if len(scored) == 0 {
+ return nil
+ }
+
+ sort.Slice(scored, func(i, j int) bool {
+ if scored[i].score != scored[j].score {
+ return scored[i].score > scored[j].score
+ }
+ return scored[i].name < scored[j].name
+ })
+ if len(scored) > topN {
+ scored = scored[:topN]
+ }
+
+ tagWeights := make(OrderedTagWeights, len(scored))
+ matchedTags := make([]string, 0, len(scored))
+ for _, ts := range scored {
+ tagWeights[ts.name] = ts.score
+ matchedTags = append(matchedTags, ts.name)
+ }
+
+ chunk[common.TAG_FLD] = tagWeights
+ chunk["tag_kwd"] = matchedTags
+
+ return &schema.TaggedChunk{
+ Content: text,
+ Tags: matchedTags,
+ TagWeights: tagWeights,
+ }
}
+// ----------------------------------------------------------------------
+// Cache & scheduling pipeline with language passthrough and few-shot fallback.
+// ----------------------------------------------------------------------
+
const tagSourceCacheMax = 128
type boundedTagCache struct {
mu sync.Mutex
cap int
- items map[string]*indexedTagSource
+ items map[string]*MemoryTagIndex
recent []string
}
func newBoundedTagCache(cap int) *boundedTagCache {
return &boundedTagCache{
cap: cap,
- items: make(map[string]*indexedTagSource, cap),
+ items: make(map[string]*MemoryTagIndex, cap),
}
}
-func (c *boundedTagCache) load(key string) (*indexedTagSource, bool) {
+func (c *boundedTagCache) load(key string) (*MemoryTagIndex, bool) {
c.mu.Lock()
defer c.mu.Unlock()
v, ok := c.items[key]
@@ -107,7 +516,7 @@ func (c *boundedTagCache) load(key string) (*indexedTagSource, bool) {
return v, true
}
-func (c *boundedTagCache) store(key string, val *indexedTagSource) *indexedTagSource {
+func (c *boundedTagCache) store(key string, val *MemoryTagIndex) *MemoryTagIndex {
c.mu.Lock()
defer c.mu.Unlock()
if existing, ok := c.items[key]; ok {
@@ -135,8 +544,25 @@ func (c *boundedTagCache) markRecentLocked(key string) {
var tagSourceFileIndexCache = newBoundedTagCache(tagSourceCacheMax)
+func isHighConfidenceMatch(matched *schema.TaggedChunk) bool {
+ if matched == nil {
+ return false
+ }
+ for _, score := range matched.TagWeights {
+ if score >= 6 {
+ return true
+ }
+ }
+ return false
+}
+
func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in extractorInputs) ([]map[string]any, error) {
- indexed, ok := c.resolveTagSource(ctx)
+ lang := in.lang
+ if lang == "" {
+ lang = detectTextLanguage(in.chunks)
+ }
+
+ indexed, ok := c.resolveTagSource(ctx, lang)
if !ok || len(in.chunks) == 0 {
common.Info("extractor tags: skipped",
zap.Int("chunk_count", len(in.chunks)),
@@ -145,17 +571,9 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
)
return in.chunks, nil
}
- if len(indexed.examples) == 0 || len(indexed.allTags) == 0 {
- common.Info("extractor tags: empty indexed source",
- zap.Int("chunk_count", len(in.chunks)),
- zap.Int("example_count", len(indexed.examples)),
- zap.Int("all_tag_count", len(indexed.allTags)),
- zap.String("llm_id", in.llmID),
- )
- return in.chunks, nil
- }
topN := c.Param.Tags.TopN
+ tok := tokenizer.New(lang)
var examples []schema.TaggedChunk
var docsToTag []map[string]any
@@ -163,9 +581,11 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
if ctx.Err() != nil {
break
}
- matched := matchAndTagChunk(d, indexed.examples, indexed.tagTokens, indexed.allTags, topN)
+ matched := matchAndTagChunk(d, indexed, tok, topN)
if matched != nil {
- examples = append(examples, *matched)
+ if isHighConfidenceMatch(matched) {
+ examples = append(examples, *matched)
+ }
} else {
docsToTag = append(docsToTag, d)
}
@@ -191,7 +611,7 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
case <-ctx.Done():
return
}
- llmTagChunk(ctx, db, inv, docsToTag[idx], indexed.allTags, examples, in.llmID, driver, model, apiKey, baseURL, topN)
+ llmTagChunk(ctx, db, inv, docsToTag[idx], indexed.allTags, examples, in.llmID, driver, model, apiKey, baseURL, topN, indexed)
}(i)
}
wg.Wait()
@@ -217,20 +637,20 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
return in.chunks, nil
}
-func (c *ExtractorComponent) resolveTagSource(ctx context.Context) (*indexedTagSource, bool) {
+func (c *ExtractorComponent) resolveTagSource(ctx context.Context, lang string) (*MemoryTagIndex, bool) {
if c.Param.Tags.TagFileID == "" {
return nil, false
}
- return c.loadTagFileIndexed(ctx)
+ return c.loadTagFileIndexed(ctx, lang)
}
-func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context) (*indexedTagSource, bool) {
+func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context, lang string) (*MemoryTagIndex, bool) {
f, err := dao.NewFileDAO().GetByID(ctx, dao.DB, c.Param.Tags.TagFileID)
if err != nil || f == nil || f.Location == nil || *f.Location == "" {
common.Warn(fmt.Sprintf("extractor tags: resolve tag_file_id %q: %v", c.Param.Tags.TagFileID, err))
return nil, false
}
- cacheKey := tagSourceFileCacheKey(f)
+ cacheKey := tagSourceFileCacheKey(f, lang)
if cached, ok := tagSourceFileIndexCache.load(cacheKey); ok {
common.Info("extractor tags: reused tag source file index",
zap.String("file_id", c.Param.Tags.TagFileID),
@@ -250,9 +670,9 @@ func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context) (*indexedTa
common.Warn(fmt.Sprintf("extractor tags: load tag source %q/%q: %v", f.ParentID, *f.Location, err))
return nil, false
}
- indexed, ok := buildIndexedTagSourceFromBytes(data, f.Name)
- if !ok {
- return nil, false
+ indexed, ok := buildIndexedTagSourceFromBytes(data, f.Name, lang)
+ if !ok || indexed == nil {
+ return nil, false // Guard against caching nil index in LRU.
}
indexed = tagSourceFileIndexCache.store(cacheKey, indexed)
common.Info("extractor tags: loaded tag source file",
@@ -265,13 +685,32 @@ func (c *ExtractorComponent) loadTagFileIndexed(ctx context.Context) (*indexedTa
return indexed, true
}
-func buildIndexedTagSourceFromBytes(data []byte, filename string) (*indexedTagSource, bool) {
- examples, err := parseTagSourceByFilename(data, filename)
- if err != nil {
- common.Warn(fmt.Sprintf("extractor tags: %v", err))
+func buildIndexedTagSourceFromBytes(data []byte, filename, lang string) (*MemoryTagIndex, bool) {
+ rawExamples, err := parseTagSourceByFilename(data, filename)
+ if err != nil || len(rawExamples) == 0 {
+ if err != nil {
+ common.Warn(fmt.Sprintf("extractor tags: %v", err))
+ }
return nil, false
}
- return buildIndexedTagSourceFromExamples(examples), true
+ tok := tokenizer.New(lang)
+ indexed := buildMemoryTagIndex(rawExamples, tok)
+ if indexed == nil {
+ return nil, false
+ }
+ return indexed, true
+}
+
+func tagSourceFileCacheKey(f *entity.File, lang string) string {
+ location := ""
+ if f.Location != nil {
+ location = *f.Location
+ }
+ updateTime := int64(0)
+ if f.UpdateTime != nil {
+ updateTime = *f.UpdateTime
+ }
+ return fmt.Sprintf("tag-file:%s:%s:%s:%d:%d:%s", f.ID, f.ParentID, location, f.Size, updateTime, lang)
}
// parseTagSourceByFilename mirrors rag/app/tag.py chunk(): the format is chosen
@@ -294,26 +733,6 @@ func parseTagSourceByFilename(data []byte, filename string) ([]schema.TagLabel,
}
}
-func buildIndexedTagSourceFromExamples(examples []schema.TagLabel) *indexedTagSource {
- return &indexedTagSource{
- examples: examples,
- allTags: buildAllTagsProportions(examples),
- tagTokens: preTokenizeExamples(examples),
- }
-}
-
-func tagSourceFileCacheKey(f *entity.File) string {
- location := ""
- if f.Location != nil {
- location = *f.Location
- }
- updateTime := int64(0)
- if f.UpdateTime != nil {
- updateTime = *f.UpdateTime
- }
- return fmt.Sprintf("tag-file:%s:%s:%s:%d:%d", f.ID, f.ParentID, location, f.Size, updateTime)
-}
-
func parseCSVTagSource(text string) []schema.TagLabel {
return parseCSVTagSourceBytes([]byte(text))
}
@@ -395,7 +814,9 @@ func parseCSVQuoteAwareReader(r io.Reader) []schema.TagLabel {
// parseXLSXTagSource mirrors rag/app/tag.py's .xlsx path: every sheet is read
// with no header, and each row contributes a (content, tags) pair from its
-// first and second non-empty cells. The second cell holds comma-separated tags.
+// first and second non-empty cells. Single-cell rows are accumulated as body text
+// and prepended to the next tagged row, matching CSV parsing behavior.
+// The second cell holds comma-separated tags.
func parseXLSXTagSource(data []byte) []schema.TagLabel {
f, err := excelize.OpenReader(bytes.NewReader(data))
if err != nil {
@@ -411,6 +832,14 @@ func parseXLSXTagSource(data []byte) []schema.TagLabel {
common.Warn(fmt.Sprintf("extractor tags: read xlsx sheet %q: %v", sheet, err))
continue
}
+ content := ""
+ appendLine := func(s string) {
+ if content == "" {
+ content = s
+ } else {
+ content += "\n" + s
+ }
+ }
for _, row := range rows {
var cells []string
for _, c := range row {
@@ -418,11 +847,17 @@ func parseXLSXTagSource(data []byte) []schema.TagLabel {
cells = append(cells, c)
}
}
- if len(cells) < 2 {
+ if len(cells) == 0 {
continue
}
+ if len(cells) == 1 {
+ appendLine(cells[0])
+ continue
+ }
+ appendLine(cells[0])
tags := splitAndTrim(cells[1], ",")
- result = append(result, schema.TagLabel{Content: cells[0], Tags: tags})
+ result = append(result, schema.TagLabel{Content: content, Tags: tags})
+ content = ""
}
}
return result
@@ -445,18 +880,36 @@ func detectCSVDelimiterBytes(data []byte) string {
scanner := newTagSourceScanner(bytes.NewReader(data), len(data))
for scanner.Scan() {
line := scanner.Text()
- if len(strings.Split(line, ",")) == 2 {
+ if strings.TrimSpace(line) == "" {
+ continue
+ }
+ // Try quote-aware parse for comma
+ rComma := csv.NewReader(strings.NewReader(line))
+ rComma.Comma = ','
+ if rec, err := rComma.Read(); err == nil {
+ if len(rec) == 2 {
+ comma++
+ }
+ } else if len(strings.Split(line, ",")) == 2 {
comma++
}
- if len(strings.Split(line, " ")) == 2 {
+
+ // Try quote-aware parse for tab
+ rTab := csv.NewReader(strings.NewReader(line))
+ rTab.Comma = '\t'
+ if rec, err := rTab.Read(); err == nil {
+ if len(rec) == 2 {
+ tab++
+ }
+ } else if len(strings.Split(line, "\t")) == 2 {
tab++
}
}
if scanner.Err() != nil {
common.Warn(fmt.Sprintf("extractor tags: delimiter scan: %v", scanner.Err()))
}
- if tab >= comma {
- return " "
+ if tab > 0 && tab >= comma {
+ return "\t"
}
return ","
}
@@ -494,153 +947,151 @@ func splitAndTrim(s, sep string) []string {
return out
}
-func buildAllTagsProportions(tagSource []schema.TagLabel) map[string]float64 {
- tagCount := make(map[string]int)
- total := 0
- for _, ex := range tagSource {
- for _, t := range ex.Tags {
- t = strings.TrimSpace(t)
- if t == "" {
- continue
- }
- tagCount[t]++
- total++
- }
+func isAlphaExt(ext string) bool {
+ if len(ext) < 2 || ext[0] != '.' {
+ return false
}
- S := 1000.0
- proportions := make(map[string]float64, len(tagCount))
- for tag, c := range tagCount {
- proportions[tag] = float64(c+1) / (float64(total) + S)
+ for _, r := range ext[1:] {
+ if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') {
+ return false
+ }
}
- return proportions
+ return true
}
-func preTokenizeExamples(tagSource []schema.TagLabel) [][]string {
- out := make([][]string, len(tagSource))
- for i := range tagSource {
- tokens, err := tokenizer.Tokenize(tagSource[i].Content)
- if err == nil && tokens != "" {
- out[i] = strings.Fields(tokens)
- }
+func cleanTitle(title string) string {
+ title = strings.TrimSpace(title)
+ if ext := filepath.Ext(title); ext != "" && len(ext) <= 6 && isAlphaExt(ext) {
+ title = strings.TrimSuffix(title, ext)
}
- return out
+ return strings.TrimSpace(title)
}
-func matchAndTagChunk(
- chunk map[string]any,
- tagSource []schema.TagLabel,
- tagTokens [][]string,
- allTags map[string]float64,
- topN int,
-) *schema.TaggedChunk {
- text := getChunkText(chunk)
- if text == "" {
- return nil
+func getChunkText(chunk map[string]any) string {
+ if chunk == nil {
+ return ""
}
- tokens, err := tokenizer.Tokenize(text)
- if err != nil || tokens == "" {
- return nil
+
+ var parts []string
+
+ // 1. Extract main chunk content (prioritize content_with_weight, then text)
+ if v, ok := chunk["content_with_weight"].(string); ok && strings.TrimSpace(v) != "" {
+ parts = append(parts, strings.TrimSpace(v))
+ } else if v, ok := chunk["text"].(string); ok && strings.TrimSpace(v) != "" {
+ parts = append(parts, strings.TrimSpace(v))
}
- chunkWords := strings.Fields(tokens)
- var best *schema.TagLabel
- var bestScore float64
- for i := range tagSource {
- exWords := tagTokens[i]
- if exWords == nil {
- continue
- }
- score := jaccardOverlap(chunkWords, exWords)
- if score > bestScore && score >= matchOverlapThreshold {
- bestScore = score
- ex := tagSource[i]
- best = &ex
+ // 2. Extract important keywords if present (supports []string, []any, string)
+ if kwdRaw, exists := chunk["important_kwd"]; exists && kwdRaw != nil {
+ switch kwds := kwdRaw.(type) {
+ case []string:
+ for _, k := range kwds {
+ if k = strings.TrimSpace(k); k != "" {
+ parts = append(parts, k)
+ }
+ }
+ case []any:
+ for _, item := range kwds {
+ if s, ok := item.(string); ok {
+ if s = strings.TrimSpace(s); s != "" {
+ parts = append(parts, s)
+ }
+ }
+ }
+ case string:
+ if k := strings.TrimSpace(kwds); k != "" {
+ parts = append(parts, k)
+ }
}
}
- if best == nil {
- return nil
- }
- S := 1000.0
- cnt := float64(len(best.Tags))
- type tagScore struct {
- name string
- score int
- }
- var scored []tagScore
- for _, t := range best.Tags {
- t = strings.TrimSpace(t)
- if t == "" {
- continue
- }
- bg := allTags[t]
- if bg <= 0 {
- bg = 0.0001
- }
- s := roundInt(0.1 * 2.0 / (cnt + S) / max(1e-6, bg))
- if s > 0 {
- scored = append(scored, tagScore{name: t, score: s})
+ // 3. Fallback: only extract title/name if chunk has NO text content or keywords
+ if len(parts) == 0 {
+ for _, titleKey := range []string{"docnm_kwd", "title_tks", "title"} {
+ if v, ok := chunk[titleKey].(string); ok && strings.TrimSpace(v) != "" {
+ if cleaned := cleanTitle(v); cleaned != "" {
+ parts = append(parts, cleaned)
+ break
+ }
+ }
}
}
- if len(scored) == 0 {
- return nil
+ if len(parts) == 0 {
+ return ""
}
+ return strings.Join(parts, " ")
+}
- sort.Slice(scored, func(i, j int) bool { return scored[i].score > scored[j].score })
- if len(scored) > topN {
- scored = scored[:topN]
+// detectTextLanguage detects language from chunk samples across the document.
+// Priority order:
+// 1. Japanese Kana (\u3040-\u309F, \u30A0-\u30FF) -> "Japanese" (Kana is exclusively Japanese)
+// 2. Korean Hangul (\uAC00-\uD7AF, \u1100-\u11FF) -> "Korean" (Hangul is exclusively Korean)
+// 3. CJK Unified Ideographs (\u4E00-\u9FFF, \u3400-\u4DBF) -> "Chinese"
+// 4. Default -> "English"
+func detectTextLanguage(chunks []map[string]any) string {
+ if len(chunks) == 0 {
+ return "English"
+ }
+ sampleIndices := make([]int, 0, 10)
+ if len(chunks) <= 10 {
+ for i := range chunks {
+ sampleIndices = append(sampleIndices, i)
+ }
+ } else {
+ step := float64(len(chunks)-1) / 9.0
+ for i := 0; i < 10; i++ {
+ idx := int(float64(i) * step)
+ sampleIndices = append(sampleIndices, idx)
+ }
}
- tagWeights := make(map[string]int, len(scored))
- for _, ts := range scored {
- tagWeights[strings.ReplaceAll(ts.name, ".", "_")] = ts.score
- }
- // Store as an object, not a JSON string: the ES index maps tag_feas as
- // type "rank_features", which requires an object of numeric values. A
- // string would make every bulk insert fail and drop the whole chunk.
- chunk[common.TAG_FLD] = tagWeights
+ kanaCount := 0
+ hangulCount := 0
+ cjkCount := 0
- return &schema.TaggedChunk{
- Content: text,
- Tags: best.Tags,
- TagWeights: tagWeights,
+ for _, idx := range sampleIndices {
+ if idx >= len(chunks) {
+ continue
+ }
+ txt := getChunkText(chunks[idx])
+ if txt == "" {
+ continue
+ }
+ runes := []rune(txt)
+ if len(runes) > 1000 {
+ runes = runes[:1000]
+ }
+ for _, r := range runes {
+ if (r >= 0x3040 && r <= 0x309F) || (r >= 0x30A0 && r <= 0x30FF) {
+ kanaCount++
+ } else if (r >= 0xAC00 && r <= 0xD7AF) || (r >= 0x1100 && r <= 0x11FF) {
+ hangulCount++
+ } else if (r >= 0x4E00 && r <= 0x9FFF) || (r >= 0x3400 && r <= 0x4DBF) {
+ cjkCount++
+ }
+ }
}
-}
-func jaccardOverlap(a, b []string) float64 {
- if len(a) == 0 || len(b) == 0 {
- return 0
- }
- setA := make(map[string]struct{}, len(a))
- for _, w := range a {
- setA[w] = struct{}{}
+ if kanaCount > 0 {
+ return "Japanese"
}
- setB := make(map[string]struct{}, len(b))
- for _, w := range b {
- setB[w] = struct{}{}
+ if hangulCount > 0 {
+ return "Korean"
}
- intersection := 0
- for w := range setA {
- if _, ok := setB[w]; ok {
- intersection++
- }
+ if cjkCount > 0 {
+ return "Chinese"
}
- union := len(setA) + len(setB) - intersection
- if union == 0 {
- return 0
- }
- return float64(intersection) / float64(union)
+ return "English"
}
-func getChunkText(chunk map[string]any) string {
- if v, ok := chunk["content_with_weight"].(string); ok && v != "" {
- return v
- }
- if v, ok := chunk["text"].(string); ok && v != "" {
- return v
+func containsCJK(s string) bool {
+ for _, r := range s {
+ if (r >= 0x4e00 && r <= 0x9fff) || (r >= 0x3400 && r <= 0x4dbf) {
+ return true
+ }
}
- return ""
+ return false
}
func roundInt(f float64) int {
@@ -650,6 +1101,10 @@ func roundInt(f float64) int {
return int(f + 0.5)
}
+// ----------------------------------------------------------------------
+// Phase 2 LLM tagging with real example cold-start and dot sanitization
+// ----------------------------------------------------------------------
+
func llmTagChunk(
ctx context.Context,
db *gorm.DB,
@@ -659,25 +1114,43 @@ func llmTagChunk(
examples []schema.TaggedChunk,
llmID, driver, model, apiKey, baseURL string,
topN int,
+ idx *MemoryTagIndex,
) {
text := getChunkText(chunk)
if text == "" {
return
}
- if cached := getTaggerLLMCache(ctx, llmID, text, allTags, topN); cached != nil {
- chunk[common.TAG_FLD] = cached
- return
- }
-
+ textHash := int64(xxhash.Sum64String(text))
var picked []schema.TaggedChunk
if len(examples) > 2 {
- picked = randomChoices(examples, 2)
+ picked = sampleWithoutReplacement(examples, 2, textHash)
} else if len(examples) > 0 {
picked = examples
+ } else if idx != nil && len(idx.examples) > 0 {
+ // Cold-start fallback: sample real examples deterministically using text hash seed
+ sampleCount := min(2, len(idx.examples))
+ r := rand.New(rand.NewPCG(uint64(textHash), uint64(textHash^0x5bf03635e0689dd2)))
+ perm := r.Perm(len(idx.examples))[:sampleCount]
+ for _, idxDoc := range perm {
+ ex := idx.examples[idxDoc]
+ weights := make(OrderedTagWeights, len(ex.Tags))
+ for tagRank, t := range ex.Tags {
+ score := max(3, 9-tagRank*3)
+ weights[t] = score
+ }
+ picked = append(picked, schema.TaggedChunk{
+ Content: ex.Content,
+ Tags: ex.Tags,
+ TagWeights: weights,
+ })
+ }
}
- if len(picked) == 0 {
- picked = []schema.TaggedChunk{{Content: "This is an example", TagWeights: map[string]int{"example": 1}}}
+
+ if cached := getTaggerLLMCache(ctx, llmID, text, allTags, picked, topN); cached != nil {
+ chunk[common.TAG_FLD] = cached
+ chunk["tag_kwd"] = sortedTagWeightsKeys(cached)
+ return
}
tagNames := sortedTagNames(allTags)
@@ -701,7 +1174,7 @@ func llmTagChunk(
msgs = fitted
temperature := 0.5
- var result map[string]int
+ var result OrderedTagWeights
timeoutErr := runtime.WithTimeout(ctx, taggerTimeout, func(timeoutCtx context.Context) error {
resp, err := inv.Chat(timeoutCtx, extractorChatRequest{
Driver: driver,
@@ -720,12 +1193,12 @@ func llmTagChunk(
})
if timeoutErr != nil {
common.Error("extractor tags: LLM timeout", timeoutErr)
-
}
if len(result) > 0 {
chunk[common.TAG_FLD] = result
- setTaggerLLMCache(ctx, llmID, text, allTags, topN, result)
+ chunk["tag_kwd"] = sortedTagWeightsKeys(result)
+ setTaggerLLMCache(ctx, llmID, text, allTags, picked, topN, result)
}
}
@@ -738,7 +1211,7 @@ func buildTaggerPrompt(topN int, tagSetStr string, examples []schema.TaggedChunk
return fmt.Sprintf(taggerPromptTmpl, topN, tagSetStr, examplesBlock.String(), text)
}
-func parseTaggerResponse(raw string, topN int) map[string]int {
+func parseTaggerResponse(raw string, topN int) OrderedTagWeights {
raw = strings.TrimSpace(common.StripThinkTrailing(raw))
if strings.Contains(raw, "**ERROR**") {
common.Warn("extractor tags: LLM returned **ERROR**")
@@ -753,7 +1226,7 @@ func parseTaggerResponse(raw string, topN int) map[string]int {
}
}
- result := make(map[string]int, len(obj))
+ result := make(OrderedTagWeights, len(obj))
for k, v := range obj {
score := 0
switch n := v.(type) {
@@ -762,8 +1235,11 @@ func parseTaggerResponse(raw string, topN int) map[string]int {
case int:
score = n
}
- if score > 0 {
- result[k] = score
+ cleanKey := strings.ReplaceAll(strings.TrimSpace(k), ".", "_")
+ if score > 0 && cleanKey != "" {
+ if existing, ok := result[cleanKey]; !ok || score > existing {
+ result[cleanKey] = score
+ }
}
}
@@ -776,8 +1252,13 @@ func parseTaggerResponse(raw string, topN int) map[string]int {
for k, v := range result {
sorted = append(sorted, kv{k, v})
}
- sort.Slice(sorted, func(i, j int) bool { return sorted[i].v > sorted[j].v })
- result = make(map[string]int, topN)
+ sort.Slice(sorted, func(i, j int) bool {
+ if sorted[i].v != sorted[j].v {
+ return sorted[i].v > sorted[j].v
+ }
+ return sorted[i].k < sorted[j].k
+ })
+ result = make(OrderedTagWeights, topN)
for i := 0; i < topN && i < len(sorted); i++ {
result[sorted[i].k] = sorted[i].v
}
@@ -799,7 +1280,7 @@ func jsonRepairExtract(raw string) map[string]any {
return obj
}
-func taggerCacheKey(llmID, text string, allTags map[string]float64, topN int) string {
+func taggerCacheKey(llmID, text string, allTags map[string]float64, examples []schema.TaggedChunk, topN int) string {
hasher := xxhash.New()
hasher.Write([]byte(llmID))
hasher.Write([]byte("\x00"))
@@ -808,16 +1289,23 @@ func taggerCacheKey(llmID, text string, allTags map[string]float64, topN int) st
tagNames := sortedTagNames(allTags)
hasher.Write([]byte(strings.Join(tagNames, ",")))
hasher.Write([]byte("\x00"))
+ for _, ex := range examples {
+ hasher.Write([]byte(ex.Content))
+ hasher.Write([]byte("\x00"))
+ tagsJSON, _ := json.Marshal(ex.TagWeights)
+ hasher.Write(tagsJSON)
+ hasher.Write([]byte("\x00"))
+ }
hasher.Write([]byte(fmt.Sprintf("%d", topN)))
return fmt.Sprintf("tagger:%x", hasher.Sum64())
}
-func getTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[string]float64, topN int) map[string]int {
+func getTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[string]float64, examples []schema.TaggedChunk, topN int) map[string]int {
client := redis.Get()
if client == nil {
return nil
}
- key := taggerCacheKey(llmID, text, allTags, topN)
+ key := taggerCacheKey(llmID, text, allTags, examples, topN)
data, err := client.Get(ctx, key)
if err != nil || data == "" {
return nil
@@ -829,7 +1317,7 @@ func getTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[stri
return result
}
-func setTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[string]float64, topN int, result map[string]int) {
+func setTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[string]float64, examples []schema.TaggedChunk, topN int, result map[string]int) {
if result == nil {
return
}
@@ -837,7 +1325,7 @@ func setTaggerLLMCache(ctx context.Context, llmID, text string, allTags map[stri
if client == nil {
return
}
- key := taggerCacheKey(llmID, text, allTags, topN)
+ key := taggerCacheKey(llmID, text, allTags, examples, topN)
data, err := json.Marshal(result)
if err != nil {
return
@@ -854,13 +1342,45 @@ func sortedTagNames(allTags map[string]float64) []string {
return out
}
-func randomChoices(slice []schema.TaggedChunk, k int) []schema.TaggedChunk {
- if len(slice) == 0 {
+func sortedTagWeightsKeys(m map[string]int) []string {
+ if len(m) == 0 {
return nil
}
+ type kv struct {
+ k string
+ v int
+ }
+ kvs := make([]kv, 0, len(m))
+ for k, v := range m {
+ kvs = append(kvs, kv{k, v})
+ }
+ sort.Slice(kvs, func(i, j int) bool {
+ if kvs[i].v != kvs[j].v {
+ return kvs[i].v > kvs[j].v
+ }
+ return kvs[i].k < kvs[j].k
+ })
+ keys := make([]string, len(kvs))
+ for i, item := range kvs {
+ keys[i] = item.k
+ }
+ return keys
+}
+
+func sampleWithoutReplacement(slice []schema.TaggedChunk, k int, seed int64) []schema.TaggedChunk {
+ if len(slice) == 0 || k <= 0 {
+ return nil
+ }
+ if len(slice) <= k {
+ out := make([]schema.TaggedChunk, len(slice))
+ copy(out, slice)
+ return out
+ }
+ r := rand.New(rand.NewPCG(uint64(seed), uint64(seed^0x5bf03635e0689dd2)))
+ perm := r.Perm(len(slice))
out := make([]schema.TaggedChunk, k)
- for i := range k {
- out[i] = slice[rand.IntN(len(slice))]
+ for i := 0; i < k; i++ {
+ out[i] = slice[perm[i]]
}
return out
}
diff --git a/internal/ingestion/component/extractor_tag_integration_test.go b/internal/ingestion/component/extractor_tag_integration_test.go
new file mode 100644
index 0000000000..748e768c3c
--- /dev/null
+++ b/internal/ingestion/component/extractor_tag_integration_test.go
@@ -0,0 +1,246 @@
+//go:build integration
+// +build integration
+
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/tokenizer"
+)
+
+func TestBuildMemoryTagIndex(t *testing.T) {
+ requireTokenizerPool(t)
+ tok := tokenizer.New("english")
+ raw := []schema.TagLabel{
+ {Content: "machine learning models", Tags: []string{"ML.v1", "AI"}},
+ {Content: "deep learning neural nets", Tags: []string{"AI", "DL"}},
+ {Content: " ", Tags: []string{"Ignored"}},
+ }
+ idx := buildMemoryTagIndex(raw, tok)
+ if idx == nil {
+ t.Fatal("expected non-nil MemoryTagIndex")
+ }
+ if len(idx.examples) != 2 {
+ t.Fatalf("expected 2 clean examples, got %d", len(idx.examples))
+ }
+ if _, ok := idx.allTags["ML_v1"]; !ok {
+ t.Fatalf("expected tag ML_v1 after dot sanitization, got %v", idx.allTags)
+ }
+ if _, ok := idx.allTags["ML.v1"]; ok {
+ t.Fatal("expected ML.v1 with dot to be replaced")
+ }
+
+ var sumProb float64
+ for _, p := range idx.allTags {
+ sumProb += p
+ }
+ if sumProb < 0.9999 || sumProb > 1.0001 {
+ t.Fatalf("expected background probabilities to sum to 1.0, got %f", sumProb)
+ }
+}
+
+func TestMatchAndTagChunk_AsymmetricLength(t *testing.T) {
+ requireTokenizerPool(t)
+ tok := tokenizer.New("english")
+ // Reference set: 10-word rare example
+ rawEx := []schema.TagLabel{
+ {Content: "RAGFlow vector database retrieval architecture engine", Tags: []string{"RAG", "VectorDB"}},
+ {Content: "general culinary cooking recipe baking", Tags: []string{"Cooking"}},
+ }
+ idx := buildMemoryTagIndex(rawEx, tok)
+ if idx == nil {
+ t.Fatal("expected non-nil index")
+ }
+
+ // 800-word long technical chunk containing the reference example words
+ longBody := strings.Repeat("RAGFlow is an advanced system that integrates vector database and retrieval architecture engine into scalable workflows. ", 40)
+ chunk := map[string]any{"content_with_weight": longBody}
+
+ matched := matchAndTagChunk(chunk, idx, tok, 5)
+ if matched == nil {
+ t.Fatal("expected non-nil matched chunk for asymmetric length matching")
+ }
+ if len(matched.Tags) == 0 {
+ t.Fatal("expected non-empty matched tags")
+ }
+ if matched.TagWeights["RAG"] <= 0 || matched.TagWeights["VectorDB"] <= 0 {
+ t.Fatalf("expected positive scores for RAG and VectorDB, got: %v", matched.TagWeights)
+ }
+ for tag, score := range matched.TagWeights {
+ if score < 1 || score > 10 {
+ t.Fatalf("tag %s score %d is outside [1, 10]", tag, score)
+ }
+ }
+}
+
+func TestMatchAndTagChunk_TopKWeighted(t *testing.T) {
+ requireTokenizerPool(t)
+ tok := tokenizer.New("english")
+ rawEx := []schema.TagLabel{
+ {Content: "machine learning artificial intelligence deep neural networks", Tags: []string{"AI"}},
+ {Content: "financial market stocks bonds banking economy", Tags: []string{"Finance"}},
+ }
+ idx := buildMemoryTagIndex(rawEx, tok)
+ if idx == nil {
+ t.Fatal("expected non-nil index")
+ }
+
+ // Chunk has high overlap with AI example, slight overlap with Finance
+ chunk := map[string]any{
+ "content_with_weight": "machine learning artificial intelligence deep neural networks banking financial",
+ }
+
+ matched := matchAndTagChunk(chunk, idx, tok, 5)
+ if matched == nil {
+ t.Fatal("expected non-nil matched chunk")
+ }
+ aiScore := matched.TagWeights["AI"]
+ finScore := matched.TagWeights["Finance"]
+ if aiScore <= finScore {
+ t.Fatalf("expected AI score (%d) > Finance score (%d)", aiScore, finScore)
+ }
+}
+
+func TestMatchAndTagChunk_DuplicateTagsDedup(t *testing.T) {
+ requireTokenizerPool(t)
+ tok := tokenizer.New("english")
+ rawEx := []schema.TagLabel{
+ {Content: "natural language processing text analysis", Tags: []string{"NLP", "NLP", "AI"}},
+ }
+ idx := buildMemoryTagIndex(rawEx, tok)
+ if idx == nil {
+ t.Fatal("expected non-nil index")
+ }
+ if len(idx.examples[0].Tags) != 2 {
+ t.Fatalf("expected 2 unique tags in example, got %v", idx.examples[0].Tags)
+ }
+
+ chunk := map[string]any{"content_with_weight": "natural language processing text analysis"}
+ matched := matchAndTagChunk(chunk, idx, tok, 5)
+ if matched == nil {
+ t.Fatal("expected non-nil match")
+ }
+ if matched.TagWeights["NLP"] < 1 || matched.TagWeights["NLP"] > 10 {
+ t.Fatalf("invalid score for NLP: %d", matched.TagWeights["NLP"])
+ }
+}
+
+func TestMatchAndTagChunk_AllEmptyTags(t *testing.T) {
+ requireTokenizerPool(t)
+ tok := tokenizer.New("english")
+ rawEx := []schema.TagLabel{
+ {Content: "some text without tags", Tags: []string{}},
+ {Content: "another text with empty tag", Tags: []string{"", " "}},
+ }
+ idx := buildMemoryTagIndex(rawEx, tok)
+ if idx != nil {
+ t.Fatalf("expected nil index when all tags are empty, got %v", idx)
+ }
+
+ chunk := map[string]any{"content_with_weight": "some text without tags"}
+ matched := matchAndTagChunk(chunk, idx, tok, 5)
+ if matched != nil {
+ t.Fatalf("expected nil match when index is nil, got %v", matched)
+ }
+}
+
+func TestMatchAndTagChunk_DeterministicTieBreak(t *testing.T) {
+ requireTokenizerPool(t)
+ tok := tokenizer.New("english")
+ rawEx := []schema.TagLabel{
+ {Content: "shared keyword match document", Tags: []string{"tag_b", "tag_a", "tag_c"}},
+ }
+ idx := buildMemoryTagIndex(rawEx, tok)
+ if idx == nil {
+ t.Fatal("expected non-nil index")
+ }
+
+ chunk := map[string]any{"content_with_weight": "shared keyword match document"}
+ // Ask for top 2 out of 3 equal-scoring tags
+ matched := matchAndTagChunk(chunk, idx, tok, 2)
+ if matched == nil {
+ t.Fatal("expected non-nil matched chunk")
+ }
+ if len(matched.Tags) != 2 {
+ t.Fatalf("expected exactly 2 tags, got %d", len(matched.Tags))
+ }
+ // Alphabetical tie-break: tag_a, tag_b
+ if matched.Tags[0] != "tag_a" || matched.Tags[1] != "tag_b" {
+ t.Fatalf("expected deterministic alphabetical tie-break ['tag_a', 'tag_b'], got %v", matched.Tags)
+ }
+}
+
+func TestMatchAndTagChunk_TitleFallback(t *testing.T) {
+ requireTokenizerPool(t)
+ tok := tokenizer.New("english")
+ rawEx := []schema.TagLabel{
+ {Content: "engineering bidding specifications procurement procedure", Tags: []string{"BiddingDoc", "Engineering"}},
+ {Content: "culinary recipe cooking", Tags: []string{"Cooking"}},
+ }
+ idx := buildMemoryTagIndex(rawEx, tok)
+ if idx == nil {
+ t.Fatal("expected non-nil index")
+ }
+
+ // Chunk has no body content, so it falls back to docnm_kwd title
+ chunk := map[string]any{
+ "docnm_kwd": "engineering_bidding_specifications_procurement_procedure.pdf",
+ }
+
+ matched := matchAndTagChunk(chunk, idx, tok, 5)
+ if matched == nil {
+ t.Fatal("expected non-nil matched chunk triggered by title fallback")
+ }
+ if matched.TagWeights["BiddingDoc"] <= 0 || matched.TagWeights["Engineering"] <= 0 {
+ t.Fatalf("expected BiddingDoc and Engineering tags matched, got: %v", matched.TagWeights)
+ }
+ if chunk["tag_kwd"] == nil {
+ t.Fatal("expected tag_kwd to be populated on chunk")
+ }
+}
+
+func BenchmarkMatchAndTagChunk_5000Examples(b *testing.B) {
+ requireTokenizerPool(b)
+ tok := tokenizer.New("english")
+ rawEx := make([]schema.TagLabel, 5000)
+ for i := 0; i < 5000; i++ {
+ rawEx[i] = schema.TagLabel{
+ Content: fmt.Sprintf("sample content document %d with keywords topic%d and subtopic%d for domain categorization", i, i%50, i%100),
+ Tags: []string{fmt.Sprintf("topic_%d", i%50), fmt.Sprintf("subtopic_%d", i%100)},
+ }
+ }
+ idx := buildMemoryTagIndex(rawEx, tok)
+ if idx == nil {
+ b.Fatal("failed to build index")
+ }
+
+ chunk := map[string]any{
+ "content_with_weight": "This is a detailed chunk talking about sample content document 42 with keywords topic42 and subtopic42 for domain categorization in practice.",
+ }
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ matchAndTagChunk(chunk, idx, tok, 3)
+ }
+}
diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.go
index a5260bb77e..20229b7d23 100644
--- a/internal/ingestion/component/extractor_tag_test.go
+++ b/internal/ingestion/component/extractor_tag_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
+ "fmt"
"strings"
"testing"
@@ -61,6 +62,9 @@ func TestExtractorTags_NoTagFileID(t *testing.T) {
if _, ok := chunks[0][common.TAG_FLD]; ok {
t.Fatal("tag_feas should not be set when tag_file_id is absent")
}
+ if _, ok := chunks[0]["tag_kwd"]; ok {
+ t.Fatal("tag_kwd should not be set when tag_file_id is absent")
+ }
}
func TestExtractorTags_NoLLMID(t *testing.T) {
@@ -78,6 +82,9 @@ func TestExtractorTags_NoLLMID(t *testing.T) {
if _, ok := chunks[0][common.TAG_FLD]; ok {
t.Fatal("tag_feas should not be set without tag_file_id")
}
+ if _, ok := chunks[0]["tag_kwd"]; ok {
+ t.Fatal("tag_kwd should not be set without tag_file_id")
+ }
}
func TestExtractorTags_WithKeywords(t *testing.T) {
@@ -105,6 +112,9 @@ func TestExtractorTags_WithKeywords(t *testing.T) {
if chunks[0][common.TAG_FLD] != nil {
t.Fatal("tag_feas should not be set without tag_file_id")
}
+ if chunks[0]["tag_kwd"] != nil {
+ t.Fatal("tag_kwd should not be set without tag_file_id")
+ }
}
func TestExtractorTags_ComponentRegistration(t *testing.T) {
@@ -145,8 +155,6 @@ func TestParseTaggerResponse_JSONRepair(t *testing.T) {
}
func TestParseCSVTagSource_Comma(t *testing.T) {
- // A comma-delimited file maps to exactly two columns, so each line can
- // carry only a single tag (tags are comma-separated within the 2nd col).
text := "RAGFlow tutorial,RAG\nsome text,LLM"
result := parseCSVTagSource(text)
if len(result) != 2 {
@@ -200,9 +208,6 @@ func TestParseCSVTagSource_Tab(t *testing.T) {
}
func TestParseCSVTagSource_Accumulation(t *testing.T) {
- // Mirrors rag/app/tag.py txt semantics: lines that do not split into two
- // columns are accumulated as body text and prepended to the next tagged
- // line's content.
text := "intro paragraph\nmore body\nRAGFlow tutorial\tRAG,tutorial"
result := parseCSVTagSource(text)
if len(result) != 1 {
@@ -241,8 +246,6 @@ func stubXLSXBytes(t *testing.T) []byte {
return buf.Bytes()
}
-// capturingExtractorTagChat records the request it was given so tests can
-// assert on the exact messages sent to the LLM.
type capturingExtractorTagChat struct {
req extractorChatRequest
}
@@ -261,14 +264,9 @@ func pushCapturingTagChat(t *testing.T) *capturingExtractorTagChat {
}
func longChunkText() string {
- return strings.Repeat("RAGFlow is an open source retrieval augmented generation engine. ", 10)
+ return strings.TrimSpace(strings.Repeat("RAGFlow is an open source retrieval augmented generation engine. ", 10))
}
-// TestLlmtagChunk_MessageFit verifies that llmTagChunk trims the prompt to the
-// model's context window before sending, mirroring Python's message_fit_in in
-// content_tagging (generator.py:331). The tagger system prompt embeds the full
-// chunk text plus the whole tag set, so an oversized chunk must be trimmed
-// rather than rejected by the provider with "context length exceeded".
func TestLlmtagChunk_MessageFit(t *testing.T) {
const budget = 20
SetExtractorContextLengthOverride(func(_ context.Context, _ string) int { return budget })
@@ -281,28 +279,21 @@ func TestLlmtagChunk_MessageFit(t *testing.T) {
allTags := map[string]float64{"RAG": 1, "database": 1, "AI": 1}
examples := []schema.TaggedChunk{{Content: "example one", TagWeights: map[string]int{"AI": 5}}}
- llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, "test@test", "test_driver", "test_model", "test_key", "", 3)
+ llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, "test@test", "test_driver", "test_model", "test_key", "", 3, nil)
if len(capt.req.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))
}
- // The full chunk text must have been trimmed away from the system prompt.
if strings.Contains(capt.req.Messages[0].Content, longText) {
t.Fatal("system prompt was not trimmed to the context budget")
}
total := tokenizer.NumTokensFromString(capt.req.Messages[0].Content) +
tokenizer.NumTokensFromString(capt.req.Messages[1].Content)
- // The budget is 97% of the resolved content_length (the override value),
- // mirroring the agent's contextFitBudget; the margin keeps a fitted
- // prompt inside the provider's real context limit.
if total > extractorContextFitBudget(budget) {
t.Fatalf("fitted messages total %d tokens exceeds the margin-adjusted budget %d", total, extractorContextFitBudget(budget))
}
}
-// TestLlmtagChunk_NoContextLength_SkipsFit verifies the guard: when the model's
-// context length is unknown (extractorContextLength returns 0), the tagger
-// passes the prompt through untrimmed instead of erroring.
func TestLlmtagChunk_NoContextLength_SkipsFit(t *testing.T) {
capt := pushCapturingTagChat(t)
@@ -311,7 +302,7 @@ func TestLlmtagChunk_NoContextLength_SkipsFit(t *testing.T) {
allTags := map[string]float64{"RAG": 1}
examples := []schema.TaggedChunk{{Content: "example", TagWeights: map[string]int{"AI": 5}}}
- llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, "test@test", "test_driver", "test_model", "test_key", "", 3)
+ llmTagChunk(t.Context(), nil, capt, chunk, allTags, examples, "test@test", "test_driver", "test_model", "test_key", "", 3, nil)
if len(capt.req.Messages) != 2 {
t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))
@@ -321,22 +312,46 @@ func TestLlmtagChunk_NoContextLength_SkipsFit(t *testing.T) {
}
}
+func TestLlmtagChunk_ColdStartFallback(t *testing.T) {
+ capt := pushCapturingTagChat(t)
+
+ idx := &MemoryTagIndex{
+ examples: []schema.TagLabel{
+ {Content: "real sample text one", Tags: []string{"NLP", "AI"}},
+ {Content: "real sample text two", Tags: []string{"Search"}},
+ },
+ allTags: map[string]float64{"NLP": 0.33, "AI": 0.33, "Search": 0.33},
+ }
+
+ chunk := map[string]any{"content_with_weight": "some content"}
+ llmTagChunk(t.Context(), nil, capt, chunk, idx.allTags, nil, "test@test", "test_driver", "test_model", "test_key", "", 3, idx)
+
+ if len(capt.req.Messages) != 2 {
+ t.Fatalf("expected 2 messages, got %d", len(capt.req.Messages))
+ }
+ // Verify that real sample content is present in prompt
+ promptContent := capt.req.Messages[0].Content
+ if !strings.Contains(promptContent, "real sample text one") && !strings.Contains(promptContent, "real sample text two") {
+ t.Fatalf("cold start fallback did not include real sample text: %s", promptContent)
+ }
+ // Verify fake mock example is NOT present
+ if strings.Contains(promptContent, "This is an example") {
+ t.Fatal("prompt contains fake example 'This is an example'")
+ }
+}
+
func TestParseCSVQuoteAwareReader(t *testing.T) {
- // A quoted content field containing a comma must not be split into extra
- // columns: "RAGFlow, the guide",RAG is two fields, not three.
text := "\"RAGFlow, the guide\",RAG\nplain line,LLM"
result := parseCSVQuoteAwareReader(strings.NewReader(text))
if len(result) != 2 {
t.Fatalf("expected 2 examples, got %d", len(result))
}
- // First line: quoted content keeps the comma -> content "RAGFlow, the guide", tags [RAG]
if result[0].Content != "RAGFlow, the guide" {
t.Errorf("content[0] = %q", result[0].Content)
}
if len(result[0].Tags) != 1 || result[0].Tags[0] != "RAG" {
t.Errorf("tags[0] = %v", result[0].Tags)
}
- // Second line: plain comma split -> content "plain line", tags [LLM]
if result[1].Content != "plain line" {
t.Errorf("content[1] = %q", result[1].Content)
}
@@ -351,7 +366,6 @@ func TestParseXLSXTagSource(t *testing.T) {
if len(result) != 3 {
t.Fatalf("expected 3 examples (multi-sheet), got %d", len(result))
}
- // Each row's first cell is content, second is comma-separated tags.
if result[0].Content != "RAGFlow guide" {
t.Errorf("content[0] = %q", result[0].Content)
}
@@ -367,8 +381,6 @@ func TestParseXLSXTagSource(t *testing.T) {
}
func TestParseCSVTagSource_EmptyTags(t *testing.T) {
- // Mirrors rag/app/tag.py: a row with exactly two columns is appended even
- // when the tag column is empty (beAdoc always appends for a 2-column row).
text := "content one\tRAG,tutorial\nsolo line\t"
result := parseCSVTagSource(text)
if len(result) != 2 {
@@ -377,9 +389,8 @@ func TestParseCSVTagSource_EmptyTags(t *testing.T) {
if len(result[0].Tags) != 2 {
t.Errorf("tags[0] = %v", result[0].Tags)
}
- // Second row has an empty tag column -> still an example, with no tags.
if len(result[1].Tags) != 0 {
- t.Errorf("tags[1] = %v, want empty (matches Python)", result[1].Tags)
+ t.Errorf("tags[1] = %v, want empty", result[1].Tags)
}
if result[1].Content != "solo line" {
t.Errorf("content[1] = %q", result[1].Content)
@@ -387,11 +398,9 @@ func TestParseCSVTagSource_EmptyTags(t *testing.T) {
}
func TestParseTagSourceByFilename(t *testing.T) {
- // .xlsx -> multi-sheet, 2 columns per row.
if got, err := parseTagSourceByFilename(stubXLSXBytes(t), "tags.xlsx"); err != nil || len(got) != 3 {
t.Errorf("xlsx: expected 3 examples, got %d (err=%v)", len(got), err)
}
- // .csv -> quote-aware comma parsing.
csvData := []byte("\"a, b\",RAG\nsimple,LLM")
if got, err := parseTagSourceByFilename(csvData, "tags.csv"); err != nil || len(got) != 2 {
t.Errorf("csv: expected 2 examples, got %d (err=%v)", len(got), err)
@@ -399,13 +408,10 @@ func TestParseTagSourceByFilename(t *testing.T) {
if got, err := parseTagSourceByFilename(csvData, "tags.CSV"); err != nil || len(got) != 2 {
t.Errorf("csv (uppercase ext): expected 2 examples, got %d (err=%v)", len(got), err)
}
- // .txt -> delimiter-detecting txt reader.
txtData := []byte("content one\tRAG,tutorial")
if got, err := parseTagSourceByFilename(txtData, "tags.txt"); err != nil || len(got) != 1 {
t.Errorf("txt: expected 1 example, got %d (err=%v)", len(got), err)
}
- // Unsupported / no extension mirrors Python's NotImplementedError: the tag
- // source is not parsed and an error is returned.
if _, err := parseTagSourceByFilename(txtData, "noextension"); err == nil {
t.Error("no extension: expected an error (unsupported extension)")
}
@@ -414,42 +420,35 @@ func TestParseTagSourceByFilename(t *testing.T) {
}
}
-func TestJaccardOverlap(t *testing.T) {
- tests := []struct {
- name string
- a, b []string
- expected float64
- }{
- {"identical", []string{"a", "b"}, []string{"a", "b"}, 1.0},
- {"half overlap", []string{"a", "b"}, []string{"b", "c"}, 1.0 / 3.0},
- {"no overlap", []string{"a", "b"}, []string{"c", "d"}, 0.0},
- {"empty a", []string{}, []string{"a"}, 0.0},
- {"empty b", []string{"a"}, []string{}, 0.0},
+func TestParseTaggerResponse_DotSanitization(t *testing.T) {
+ raw := `{"model.v1.0": 8, "rag_tech": 6, "rag.db.v2": 4}`
+ result := parseTaggerResponse(raw, 5)
+ if result == nil {
+ t.Fatal("expected non-nil result")
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := jaccardOverlap(tt.a, tt.b)
- if got != tt.expected {
- t.Fatalf("jaccard(%v, %v) = %.4f, want %.4f", tt.a, tt.b, got, tt.expected)
- }
- })
+ if result["model_v1_0"] != 8 {
+ t.Fatalf("expected model_v1_0=8, got %d", result["model_v1_0"])
}
-}
-
-func TestBuildAllTagsProportions(t *testing.T) {
- ts := []schema.TagLabel{
- {Content: "a", Tags: []string{"RAG", "LLM"}},
- {Content: "b", Tags: []string{"RAG", "open"}},
+ if result["rag_tech"] != 6 {
+ t.Fatalf("expected rag_tech=6, got %d", result["rag_tech"])
}
- result := buildAllTagsProportions(ts)
- if len(result) != 3 {
- t.Fatalf("expected 3 tags, got %d", len(result))
+ if result["rag_db_v2"] != 4 {
+ t.Fatalf("expected rag_db_v2=4, got %d", result["rag_db_v2"])
}
- if result["RAG"] <= result["LLM"] {
- t.Fatalf("expected RAG proportion > LLM, got RAG=%.6f LLM=%.6f", result["RAG"], result["LLM"])
+ for k := range result {
+ if strings.Contains(k, ".") {
+ t.Fatalf("found unescaped dot in tag key: %s", k)
+ }
}
- if result["LLM"] <= 0 {
- t.Fatal("expected LLM proportion > 0")
+
+ // Test collision preservation of higher score
+ collisionRaw := `{"model.v1.0": 3, "model_v1_0": 9, "tag.a": 8, "tag_a": 4}`
+ collisionResult := parseTaggerResponse(collisionRaw, 5)
+ if collisionResult["model_v1_0"] != 9 {
+ t.Fatalf("expected highest score 9 for model_v1_0 collision, got %d", collisionResult["model_v1_0"])
+ }
+ if collisionResult["tag_a"] != 8 {
+ t.Fatalf("expected highest score 8 for tag_a collision, got %d", collisionResult["tag_a"])
}
}
@@ -487,10 +486,6 @@ func TestJsonRepairExtract(t *testing.T) {
}
}
-// TestParseTaggerResponse_LastThinkTag verifies that when the LLM
-// response contains multiple </think> tags, parseTaggerResponse strips
-// up to the LAST one (greedy, matching Python's re.sub), not just the
-// first (which would leave a residual think block).
func TestParseTaggerResponse_LastThinkTag(t *testing.T) {
tests := []struct {
name string
@@ -534,3 +529,944 @@ func TestParseTaggerResponse_LastThinkTag(t *testing.T) {
})
}
}
+
+func TestGetChunkText_Enrichment(t *testing.T) {
+ tests := []struct {
+ name string
+ chunk map[string]any
+ want string
+ }{
+ {
+ name: "nil chunk",
+ chunk: nil,
+ want: "",
+ },
+ {
+ name: "content only",
+ chunk: map[string]any{"content_with_weight": "simple body text"},
+ want: "simple body text",
+ },
+ {
+ name: "text fallback",
+ chunk: map[string]any{"text": "text fallback body"},
+ want: "text fallback body",
+ },
+ {
+ name: "content present ignores docnm_kwd",
+ chunk: map[string]any{
+ "docnm_kwd": "2026_Engineering_Bidding_Doc.pdf",
+ "content_with_weight": "contract guidelines",
+ },
+ want: "contract guidelines",
+ },
+ {
+ name: "title fallback chain docnm_kwd wins when no content",
+ chunk: map[string]any{
+ "docnm_kwd": "doc_name.docx",
+ "title_tks": "ignored_title",
+ },
+ want: "doc_name",
+ },
+ {
+ name: "title fallback to title_tks when no content and docnm_kwd absent",
+ chunk: map[string]any{
+ "title_tks": "parsed_title.txt",
+ },
+ want: "parsed_title",
+ },
+ {
+ name: "important_kwd string slice",
+ chunk: map[string]any{
+ "important_kwd": []string{"Bidding", "Tender"},
+ "content_with_weight": "body",
+ },
+ want: "body Bidding Tender",
+ },
+ {
+ name: "important_kwd any slice",
+ chunk: map[string]any{
+ "important_kwd": []any{"Alpha", "Beta"},
+ "content_with_weight": "body",
+ },
+ want: "body Alpha Beta",
+ },
+ {
+ name: "important_kwd string",
+ chunk: map[string]any{
+ "important_kwd": "SingleKey",
+ "content_with_weight": "body",
+ },
+ want: "body SingleKey",
+ },
+ {
+ name: "all sources combined ignores title when content present",
+ chunk: map[string]any{
+ "docnm_kwd": "Project_Tender_Specification.pdf",
+ "important_kwd": []string{"Procurement", "Compliance"},
+ "content_with_weight": "All bidders must follow instructions.",
+ },
+ want: "All bidders must follow instructions. Procurement Compliance",
+ },
+ {
+ name: "only docnm_kwd present",
+ chunk: map[string]any{
+ "docnm_kwd": "2026_Engineering_Bidding_Doc.pdf",
+ },
+ want: "2026_Engineering_Bidding_Doc",
+ },
+ {
+ name: "all fields empty or whitespace",
+ chunk: map[string]any{
+ "docnm_kwd": " ",
+ "important_kwd": []string{" ", ""},
+ "content_with_weight": " ",
+ },
+ want: "",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := getChunkText(tt.chunk)
+ if got != tt.want {
+ t.Errorf("getChunkText() = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestOrderedTagWeights_MarshalJSON(t *testing.T) {
+ weights := OrderedTagWeights{
+ "价格咨询": 1,
+ "活动咨询": 3,
+ "物流查询": 2,
+ }
+ data, err := json.Marshal(weights)
+ if err != nil {
+ t.Fatalf("json.Marshal failed: %v", err)
+ }
+ got := string(data)
+ want := `{"活动咨询":3,"物流查询":2,"价格咨询":1}`
+ if got != want {
+ t.Fatalf("OrderedTagWeights JSON order mismatch: got %s, want %s", got, want)
+ }
+
+ // Test embedding inside a chunk map (mimicking Elasticsearch doc serialization)
+ doc := map[string]any{
+ "doc_id": "doc-1",
+ "tag_feas": weights,
+ }
+ docData, err := json.Marshal(doc)
+ if err != nil {
+ t.Fatalf("json.Marshal doc failed: %v", err)
+ }
+ docStr := string(docData)
+ if !strings.Contains(docStr, `"tag_feas":{"活动咨询":3,"物流查询":2,"价格咨询":1}`) {
+ t.Fatalf("doc JSON tag_feas order mismatch: got %s", docStr)
+ }
+}
+
+func TestContainsCJK(t *testing.T) {
+ tests := []struct {
+ input string
+ want bool
+ }{
+ {"Hello world", false},
+ {"Hello 世界", true},
+ {"这是一段中文", true},
+ {"12345!@#$", false},
+ {"\u3400\u3401", true}, // CJK Extension A
+ }
+ for _, tt := range tests {
+ if got := containsCJK(tt.input); got != tt.want {
+ t.Errorf("containsCJK(%q) = %v, want %v", tt.input, got, tt.want)
+ }
+ }
+}
+
+func TestContainsCJK_LanguageAutoDetection(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ want bool
+ }{
+ {"pure english text", "This is pure English text without any Asian characters.", false},
+ {"pure ascii numbers and symbols", "1234567890 !@#$%^&*()_+=-~`[]{}\\|;':\",.<>/?", false},
+ {"simplified chinese characters", "这是简体中文字符串", true},
+ {"traditional chinese characters", "這是繁體中文詞彙", true},
+ {"mixed english and chinese", "RAGFlow 智能知识库问答系统 version 2.0", true},
+ {"single CJK character boundary low", "\u4e00", true}, // U+4E00: '一'
+ {"single CJK character boundary high", "\u9fa5", true}, // U+9FA5
+ {"CJK Extension A boundary low", "\u3400", true}, // U+3400
+ {"CJK Extension A boundary high", "\u4dbf", true}, // U+4DBF
+ {"non-CJK unicode characters (latin accents)", "Café, résumé, naïve, señor", false},
+ {"greek unicode characters", "αβγδε", false},
+ {"cyrillic unicode characters", "Привет мир", false},
+ {"empty string", "", false},
+ {"whitespace only", " \t\r\n ", false},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := containsCJK(tt.input)
+ if got != tt.want {
+ t.Errorf("containsCJK(%q) = %v, want %v", tt.input, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestPopulateTagKwd_LLMTagChunk(t *testing.T) {
+ capt := pushCapturingTagChat(t)
+ chunk := map[string]any{"content_with_weight": "some content"}
+ allTags := map[string]float64{"RAG": 0.5, "vector database": 0.5}
+
+ llmTagChunk(t.Context(), nil, capt, chunk, allTags, nil, "test@test", "test_driver", "test_model", "test_key", "", 3, nil)
+
+ tagKwd, ok := chunk["tag_kwd"].([]string)
+ if !ok {
+ t.Fatalf("expected chunk['tag_kwd'] to be []string, got %T (%v)", chunk["tag_kwd"], chunk["tag_kwd"])
+ }
+ if len(tagKwd) != 2 {
+ t.Fatalf("expected 2 tags in tag_kwd, got %d: %v", len(tagKwd), tagKwd)
+ }
+ // Verify strict score-descending order: RAG (8) > vector database (6)
+ if tagKwd[0] != "RAG" || tagKwd[1] != "vector database" {
+ t.Fatalf("expected tag_kwd to be ordered descending by score ['RAG', 'vector database'], got: %v", tagKwd)
+ }
+ if chunk[common.TAG_FLD] == nil {
+ t.Fatal("expected tag_feas to be populated")
+ }
+}
+
+func TestSortedTagWeightsKeys(t *testing.T) {
+ weights := map[string]int{
+ "Low": 3,
+ "Highest": 9,
+ "Medium": 6,
+ "Another": 6,
+ }
+ got := sortedTagWeightsKeys(weights)
+ expected := []string{"Highest", "Another", "Medium", "Low"}
+ if len(got) != len(expected) {
+ t.Fatalf("length mismatch: got %d, want %d", len(got), len(expected))
+ }
+ if got[0] != "Highest" || got[3] != "Low" {
+ t.Fatalf("sortedTagWeightsKeys order mismatch: %v", got)
+ }
+}
+
+func TestCleanTitle_PreservesVersionNumbers(t *testing.T) {
+ tests := []struct {
+ input string
+ want string
+ }{
+ {"document.pdf", "document"},
+ {"report_v1.0.docx", "report_v1.0"},
+ {"version 2.0", "version 2.0"},
+ {"manual_v2.5.txt", "manual_v2.5"},
+ {"data.csv", "data"},
+ }
+ for _, tt := range tests {
+ got := cleanTitle(tt.input)
+ if got != tt.want {
+ t.Errorf("cleanTitle(%q) = %q, want %q", tt.input, got, tt.want)
+ }
+ }
+}
+
+func TestMatchAndTagChunk_TagKwdPopulated(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ t.Cleanup(func() { tokenizer.SetEngineType("") })
+ tok := tokenizer.New("English")
+ rawExamples := []schema.TagLabel{
+ {Content: "retrieval augmented generation vector search database", Tags: []string{"RAG", "Search"}},
+ {Content: "deep learning neural network models", Tags: []string{"AI", "DeepLearning"}},
+ }
+ idx := buildMemoryTagIndex(rawExamples, tok)
+ if idx == nil {
+ t.Fatal("buildMemoryTagIndex returned nil")
+ }
+
+ chunk := map[string]any{
+ "content_with_weight": "retrieval augmented generation system with vector search database",
+ }
+
+ res := matchAndTagChunk(chunk, idx, tok, 5)
+ if res == nil {
+ t.Fatal("expected matchAndTagChunk to succeed and return tagged chunk")
+ }
+
+ // Verify chunk["tag_feas"] (common.TAG_FLD) is populated as OrderedTagWeights
+ feas, ok := chunk[common.TAG_FLD].(OrderedTagWeights)
+ if !ok || len(feas) == 0 {
+ t.Fatalf("expected chunk[%q] to be non-empty OrderedTagWeights, got %T (%v)", common.TAG_FLD, chunk[common.TAG_FLD], chunk[common.TAG_FLD])
+ }
+
+ // Verify chunk["tag_kwd"] is populated as []string
+ kwds, ok := chunk["tag_kwd"].([]string)
+ if !ok || len(kwds) == 0 {
+ t.Fatalf("expected chunk['tag_kwd'] to be non-empty []string, got %T (%v)", chunk["tag_kwd"], chunk["tag_kwd"])
+ }
+
+ // Verify lengths match
+ if len(kwds) != len(feas) {
+ t.Fatalf("tag_kwd length (%d) != tag_feas length (%d)", len(kwds), len(feas))
+ }
+
+ // Verify every tag in tag_kwd is present in tag_feas
+ for _, k := range kwds {
+ if _, exists := feas[k]; !exists {
+ t.Errorf("tag %q in tag_kwd not found in tag_feas %v", k, feas)
+ }
+ }
+
+ // Verify res.Tags matches chunk["tag_kwd"]
+ if len(res.Tags) != len(kwds) {
+ t.Fatalf("res.Tags length (%d) != chunk['tag_kwd'] length (%d)", len(res.Tags), len(kwds))
+ }
+ for i, tag := range res.Tags {
+ if tag != kwds[i] {
+ t.Errorf("res.Tags[%d] = %q, want %q", i, tag, kwds[i])
+ }
+ }
+}
+
+func TestMatchAndTagChunk_ChunkTFSaliencyGradient(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ t.Cleanup(func() { tokenizer.SetEngineType("") })
+ tok := tokenizer.New("English")
+
+ // Construct a corpus where Topic A and Topic B have equal background prior frequency
+ var rawExamples []schema.TagLabel
+ // Topic A examples (Kubernetes / Container Orchestration)
+ rawExamples = append(rawExamples,
+ schema.TagLabel{Content: "kubernetes cluster container orchestration pod management", Tags: []string{"TopicA"}},
+ schema.TagLabel{Content: "kubernetes deployment scaling service mesh ingress traffic", Tags: []string{"TopicA"}},
+ schema.TagLabel{Content: "kubernetes persistent volume storage node scheduling workload", Tags: []string{"TopicA"}},
+ )
+ // Topic B examples (Database / SQL Optimization)
+ rawExamples = append(rawExamples,
+ schema.TagLabel{Content: "database relational sql table query indexing optimization", Tags: []string{"TopicB"}},
+ schema.TagLabel{Content: "database transaction isolation levels acid compliance locking", Tags: []string{"TopicB"}},
+ schema.TagLabel{Content: "database replication sharding clustering high availability", Tags: []string{"TopicB"}},
+ )
+ // Background corpus to establish balanced priors
+ for i := 0; i < 20; i++ {
+ rawExamples = append(rawExamples, schema.TagLabel{
+ Content: fmt.Sprintf("background general cloud computing infrastructure component %d", i),
+ Tags: []string{"CloudInfra"},
+ })
+ }
+
+ idx := buildMemoryTagIndex(rawExamples, tok)
+ if idx == nil {
+ t.Fatal("buildMemoryTagIndex returned nil")
+ }
+
+ // Chunk heavily discusses Topic A (matching multiple Topic A examples / repeated keyword facets)
+ // and only briefly mentions Topic B (matching 1 Topic B example).
+ chunkText := strings.Join([]string{
+ "Comprehensive guide to kubernetes cluster container orchestration pod management in production.",
+ "Configuring kubernetes deployment scaling service mesh ingress traffic for microservices.",
+ "Managing kubernetes persistent volume storage node scheduling workload distribution across nodes.",
+ "Kubernetes cluster container orchestration deployment scaling persistent volume storage.",
+ "Note: basic database relational sql table query indexing optimization is used for logging.",
+ }, " ")
+
+ chunk := map[string]any{
+ "content_with_weight": chunkText,
+ }
+
+ res := matchAndTagChunk(chunk, idx, tok, 5)
+ if res == nil {
+ t.Fatal("expected matchAndTagChunk to succeed")
+ }
+
+ feas, ok := chunk[common.TAG_FLD].(OrderedTagWeights)
+ if !ok {
+ t.Fatalf("expected OrderedTagWeights, got %T", chunk[common.TAG_FLD])
+ }
+
+ scoreA, hasA := feas["TopicA"]
+ scoreB, hasB := feas["TopicB"]
+ if !hasA || !hasB {
+ t.Fatalf("expected both TopicA and TopicB in tag results, got: %v", feas)
+ }
+
+ t.Logf("Chunk TF Saliency Gradient Results: TopicA=%d, TopicB=%d (all tags: %v)", scoreA, scoreB, feas)
+
+ // Topic A should receive a substantially higher score (7~9) than Topic B (4~5),
+ // confirming that chunk-side TF salience / multi-hit accumulation breaks score ties.
+ if scoreA < 7 || scoreA > 9 {
+ t.Errorf("expected TopicA score in range [7, 9], got %d", scoreA)
+ }
+ if scoreB < 4 || scoreB > 5 {
+ t.Errorf("expected TopicB score in range [4, 5], got %d", scoreB)
+ }
+ if scoreA <= scoreB {
+ t.Errorf("expected TopicA score (%d) to be substantially higher than TopicB score (%d)", scoreA, scoreB)
+ }
+}
+
+func TestMatchAndTagChunk_RankDecayScoreDistribution(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ t.Cleanup(func() { tokenizer.SetEngineType("") })
+ tok := tokenizer.New("English")
+ // Construct a corpus with high-frequency common tags, medium-frequency tags, and rare tags
+ var rawExamples []schema.TagLabel
+ for i := 0; i < 50; i++ {
+ rawExamples = append(rawExamples, schema.TagLabel{
+ Content: fmt.Sprintf("common general engineering document text number %d", i),
+ Tags: []string{"GeneralEngineering"},
+ })
+ }
+ // Distinct medium-frequency topic examples
+ rawExamples = append(rawExamples,
+ schema.TagLabel{
+ Content: "database relational sql query indexing optimization",
+ Tags: []string{"Database"},
+ },
+ schema.TagLabel{
+ Content: "database transaction acid isolation levels concurrency control",
+ Tags: []string{"Database"},
+ },
+ schema.TagLabel{
+ Content: "database replication clustering high availability failover",
+ Tags: []string{"Database"},
+ },
+ )
+ // Rare/specific topic examples
+ rawExamples = append(rawExamples,
+ schema.TagLabel{
+ Content: "retrieval augmented generation vector search algorithm indexing",
+ Tags: []string{"RareRAG"},
+ },
+ schema.TagLabel{
+ Content: "retrieval augmented generation embedding chunk retrieval ranker",
+ Tags: []string{"RareRAG"},
+ },
+ )
+
+ idx := buildMemoryTagIndex(rawExamples, tok)
+ if idx == nil {
+ t.Fatal("buildMemoryTagIndex returned nil")
+ }
+
+ // Chunk containing repeated terms that match specific/rare tag (dominant topic) and 1 mention of medium tag
+ chunk := map[string]any{
+ "content_with_weight": strings.Join([]string{
+ "retrieval augmented generation vector search algorithm indexing",
+ "retrieval augmented generation embedding chunk retrieval ranker",
+ "retrieval augmented generation vector search algorithm indexing",
+ "database relational sql query indexing optimization",
+ }, " "),
+ }
+
+ res := matchAndTagChunk(chunk, idx, tok, 10)
+ if res == nil {
+ t.Fatal("expected matchAndTagChunk to succeed")
+ }
+
+ feas, ok := chunk[common.TAG_FLD].(OrderedTagWeights)
+ if !ok || len(feas) < 1 {
+ t.Fatalf("expected at least 1 scored tag, got %d (%v)", len(feas), chunk[common.TAG_FLD])
+ }
+
+ // Verify all scores are within [1, 10]
+ scores := make(map[string]int)
+ for tag, score := range feas {
+ if score < 1 || score > 10 {
+ t.Errorf("tag %q score %d out of bounds [1, 10]", tag, score)
+ }
+ scores[tag] = score
+ }
+
+ t.Logf("RankDecayScoreDistribution Scored Tags: %v", feas)
+
+ // Rare tag with high lift and high coverage should reach top band (>= 8)
+ rareScore, hasRare := scores["RareRAG"]
+ if !hasRare {
+ t.Fatal("expected RareRAG in scored tags")
+ }
+ if rareScore < 8 || rareScore > 10 {
+ t.Errorf("expected RareRAG score in top band [8, 10], got %d", rareScore)
+ }
+
+ // Verify active score gradients across [1, 10]:
+ // Medium-frequency tag should receive a lower score than RareRAG
+ if dbScore, hasDB := scores["Database"]; hasDB {
+ if dbScore >= rareScore {
+ t.Errorf("expected Database score (%d) to be lower than RareRAG (%d)", dbScore, rareScore)
+ }
+ }
+}
+
+func TestGetChunkText_NoTitlePollution(t *testing.T) {
+ // Case 1: When content_with_weight is present, docnm_kwd should NOT be prepended
+ chunkWithContent := map[string]any{
+ "docnm_kwd": "Financial_Report_2026.pdf",
+ "content_with_weight": "Quarterly revenue increased by 15 percent.",
+ }
+ got := getChunkText(chunkWithContent)
+ want := "Quarterly revenue increased by 15 percent."
+ if got != want {
+ t.Errorf("getChunkText with content_with_weight = %q, want %q", got, want)
+ }
+
+ // Case 2: When text is present (fallback), docnm_kwd should NOT be prepended
+ chunkWithText := map[string]any{
+ "docnm_kwd": "Technical_Specification.docx",
+ "text": "System architecture overview and requirements.",
+ }
+ gotText := getChunkText(chunkWithText)
+ wantText := "System architecture overview and requirements."
+ if gotText != wantText {
+ t.Errorf("getChunkText with text = %q, want %q", gotText, wantText)
+ }
+
+ // Case 3: When important_kwd is present alongside content, keywords are appended but title is ignored
+ chunkWithKwds := map[string]any{
+ "docnm_kwd": "Internal_Guidelines.pdf",
+ "important_kwd": []string{"Security", "Compliance"},
+ "content_with_weight": "All employees must follow access protocols.",
+ }
+ gotKwds := getChunkText(chunkWithKwds)
+ wantKwds := "All employees must follow access protocols. Security Compliance"
+ if gotKwds != wantKwds {
+ t.Errorf("getChunkText with keywords = %q, want %q", gotKwds, wantKwds)
+ }
+
+ // Case 4: Title is used ONLY when both content and text are absent
+ chunkTitleOnly := map[string]any{
+ "docnm_kwd": "User_Manual_v2.pdf",
+ }
+ gotTitle := getChunkText(chunkTitleOnly)
+ wantTitle := "User_Manual_v2"
+ if gotTitle != wantTitle {
+ t.Errorf("getChunkText title only = %q, want %q", gotTitle, wantTitle)
+ }
+}
+
+func TestProbabilityMassConservation_MultiTag(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ t.Cleanup(func() { tokenizer.SetEngineType("") })
+ tok := tokenizer.New("English")
+ // Example 1 has 3 tags: TagA, TagB, TagC
+ // Example 2 has 1 tag: TagD
+ rawExamples := []schema.TagLabel{
+ {
+ Content: "quantum computing quantum algorithms superposition",
+ Tags: []string{"TagA", "TagB", "TagC"},
+ },
+ {
+ Content: "quantum computing quantum circuits entanglement",
+ Tags: []string{"TagD"},
+ },
+ }
+ idx := buildMemoryTagIndex(rawExamples, tok)
+ if idx == nil {
+ t.Fatal("buildMemoryTagIndex returned nil")
+ }
+
+ // Check that background prior totalTagCount counted all tags properly
+ if len(idx.allTags) != 4 {
+ t.Fatalf("expected 4 tags in allTags, got %d", len(idx.allTags))
+ }
+
+ // Match a chunk that matches Example 1 with 100% coverage
+ chunk := map[string]any{
+ "content_with_weight": "quantum computing quantum algorithms superposition",
+ }
+
+ res := matchAndTagChunk(chunk, idx, tok, 10)
+ if res == nil {
+ t.Fatal("expected matchAndTagChunk to succeed")
+ }
+
+ feas := chunk[common.TAG_FLD].(OrderedTagWeights)
+ // Because Example 1 has TagA, TagB, TagC, each receives 1/3 of the coverage weight.
+ // Since all 3 tags have identical background frequency in this corpus, they should receive identical scores.
+ scoreA, okA := feas["TagA"]
+ scoreB, okB := feas["TagB"]
+ scoreC, okC := feas["TagC"]
+ if !okA || !okB || !okC {
+ t.Fatalf("expected TagA, TagB, TagC to be present in %v", feas)
+ }
+ if scoreA != scoreB || scoreB != scoreC {
+ t.Errorf("expected equal scores across multi-tag example: TagA=%d, TagB=%d, TagC=%d", scoreA, scoreB, scoreC)
+ }
+}
+
+func TestSampleWithoutReplacement(t *testing.T) {
+ // Empty slice
+ if got := sampleWithoutReplacement(nil, 2, 42); got != nil {
+ t.Fatalf("expected nil for empty slice, got %v", got)
+ }
+ // k <= 0
+ items := []schema.TaggedChunk{{Content: "c1"}, {Content: "c2"}, {Content: "c3"}}
+ if got := sampleWithoutReplacement(items, 0, 42); got != nil {
+ t.Fatalf("expected nil for k=0, got %v", got)
+ }
+ // k >= len(slice)
+ if got := sampleWithoutReplacement(items, 5, 42); len(got) != 3 {
+ t.Fatalf("expected 3 items when k >= len, got %d", len(got))
+ }
+ // Verify slice length > 1 never returns duplicate elements
+ slice2 := []schema.TaggedChunk{
+ {Content: "item A"},
+ {Content: "item B"},
+ }
+ for run := 0; run < 50; run++ {
+ picked := sampleWithoutReplacement(slice2, 2, int64(run*17+3))
+ if len(picked) != 2 {
+ t.Fatalf("expected 2 items, got %d", len(picked))
+ }
+ if picked[0].Content == picked[1].Content {
+ t.Fatalf("duplicate element returned from 2-element slice: %s", picked[0].Content)
+ }
+ }
+ // Verify deterministic reproducibility: same seed ALWAYS produces identical pick
+ seedA := int64(123456789)
+ pick1 := sampleWithoutReplacement(slice2, 2, seedA)
+ pick2 := sampleWithoutReplacement(slice2, 2, seedA)
+ if pick1[0].Content != pick2[0].Content || pick1[1].Content != pick2[1].Content {
+ t.Fatalf("expected deterministic sampling for identical seed: %v vs %v", pick1, pick2)
+ }
+
+ // k < len(slice): test repeated runs never have duplicates
+ bigSlice := make([]schema.TaggedChunk, 20)
+ for i := range bigSlice {
+ bigSlice[i] = schema.TaggedChunk{Content: fmt.Sprintf("chunk-%d", i)}
+ }
+ for run := 0; run < 100; run++ {
+ picked := sampleWithoutReplacement(bigSlice, 5, int64(run*31+7))
+ if len(picked) != 5 {
+ t.Fatalf("expected 5 items, got %d", len(picked))
+ }
+ seen := make(map[string]bool)
+ for _, item := range picked {
+ if seen[item.Content] {
+ t.Fatalf("duplicate item found in sampleWithoutReplacement: %s", item.Content)
+ }
+ seen[item.Content] = true
+ }
+ }
+}
+
+func TestTaggerCacheKey_IncludesFewShot(t *testing.T) {
+ allTags := map[string]float64{"TagA": 0.5, "TagB": 0.5}
+ ex1 := []schema.TaggedChunk{
+ {Content: "sample one", TagWeights: map[string]int{"TagA": 8}},
+ }
+ ex2 := []schema.TaggedChunk{
+ {Content: "sample two", TagWeights: map[string]int{"TagB": 7}},
+ }
+ ex3 := []schema.TaggedChunk{
+ {Content: "sample one", TagWeights: map[string]int{"TagA": 5}},
+ }
+
+ k1 := taggerCacheKey("llm-1", "test text", allTags, ex1, 3)
+ k2 := taggerCacheKey("llm-1", "test text", allTags, ex2, 3)
+ k3 := taggerCacheKey("llm-1", "test text", allTags, ex3, 3)
+ kEmpty := taggerCacheKey("llm-1", "test text", allTags, nil, 3)
+
+ if k1 == k2 {
+ t.Fatalf("expected different cache keys for different few-shot examples: %s vs %s", k1, k2)
+ }
+ if k1 == k3 {
+ t.Fatalf("expected different cache keys for different tag weights: %s vs %s", k1, k3)
+ }
+ if k1 == kEmpty {
+ t.Fatalf("expected different cache keys for few-shot vs empty: %s vs %s", k1, kEmpty)
+ }
+
+ // Identical few-shot examples produce identical key
+ k1Dup := taggerCacheKey("llm-1", "test text", allTags, ex1, 3)
+ if k1 != k1Dup {
+ t.Fatalf("expected identical cache keys for same few-shot examples: %s vs %s", k1, k1Dup)
+ }
+}
+
+func TestMatchAndTagChunk_ShortExampleMultiTokenRequirement(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ t.Cleanup(func() { tokenizer.SetEngineType("") })
+ tok := tokenizer.New("English")
+
+ // Create multi-token examples with balanced background frequency
+ rawExamples := []schema.TagLabel{
+ {Content: "machine learning", Tags: []string{"ML"}},
+ {Content: "machine vision", Tags: []string{"MV"}},
+ {Content: "deep learning", Tags: []string{"DL"}},
+ {Content: "quantum machine", Tags: []string{"QM"}},
+ {Content: "active learning", Tags: []string{"AL"}},
+ }
+ idx := buildMemoryTagIndex(rawExamples, tok)
+ if idx == nil {
+ t.Fatal("buildMemoryTagIndex returned nil")
+ }
+
+ // Chunk 1: Matches only 1 token ("machine") of the 2-token example "machine learning"
+ // Should NOT match on only 1 token
+ chunkSingleMatch := map[string]any{
+ "content_with_weight": "the coffee machine is broken today",
+ }
+ res1 := matchAndTagChunk(chunkSingleMatch, idx, tok, 5)
+ if res1 != nil {
+ if _, ok := res1.TagWeights["ML"]; ok {
+ t.Fatalf("expected 1-token match on 2-token example 'machine learning' not to match, got %v", res1.Tags)
+ }
+ }
+
+ // Chunk 2: Matches both tokens ("machine learning")
+ // Should match
+ chunkFullMatch := map[string]any{
+ "content_with_weight": "practical machine learning models and training",
+ }
+ res2 := matchAndTagChunk(chunkFullMatch, idx, tok, 5)
+ if res2 == nil {
+ t.Fatal("expected matchAndTagChunk to succeed for 2-token match")
+ }
+ if _, ok := res2.TagWeights["ML"]; !ok {
+ t.Fatalf("expected ML tag for full 2-token match, got: %v", res2.TagWeights)
+ }
+}
+
+func TestParseXLSXTagSource_SingleCellAccumulation(t *testing.T) {
+ f := excelize.NewFile()
+ defer f.Close()
+ must := func(err error) {
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ must(f.SetCellValue("Sheet1", "A1", "Introductory title row"))
+ must(f.SetCellValue("Sheet1", "A2", "Second paragraph of text"))
+ must(f.SetCellValue("Sheet1", "A3", "Third paragraph body"))
+ must(f.SetCellValue("Sheet1", "B3", "TagX, TagY"))
+ buf, err := f.WriteToBuffer()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ result := parseXLSXTagSource(buf.Bytes())
+ if len(result) != 1 {
+ t.Fatalf("expected 1 accumulated example, got %d", len(result))
+ }
+ wantContent := "Introductory title row\nSecond paragraph of text\nThird paragraph body"
+ if result[0].Content != wantContent {
+ t.Errorf("content = %q, want %q", result[0].Content, wantContent)
+ }
+ if len(result[0].Tags) != 2 || result[0].Tags[0] != "TagX" || result[0].Tags[1] != "TagY" {
+ t.Errorf("tags = %v", result[0].Tags)
+ }
+}
+
+func TestDetectCSVDelimiterBytes_QuoteAware(t *testing.T) {
+ // Comma-separated with embedded quotes containing commas
+ quotedComma := []byte("\"City, Country\",Geography\n\"Name, Title\",Personnel\n")
+ delim := detectCSVDelimiterBytes(quotedComma)
+ if delim != "," {
+ t.Fatalf("expected delimiter ',' for quoted comma CSV, got %q", delim)
+ }
+
+ // Tab-separated with embedded comma in tag column
+ tabData := []byte("Document Title\tTag1,Tag2\nAnother Document\tTag3,Tag4\n")
+ delimTab := detectCSVDelimiterBytes(tabData)
+ if delimTab != "\t" {
+ t.Fatalf("expected delimiter '\\t' for TSV, got %q", delimTab)
+ }
+}
+
+func TestIsHighConfidenceMatch_Phase1Isolation(t *testing.T) {
+ lowConf := &schema.TaggedChunk{
+ Content: "low confidence sample",
+ TagWeights: OrderedTagWeights{"tag1": 3, "tag2": 5},
+ }
+ if isHighConfidenceMatch(lowConf) {
+ t.Fatal("expected isHighConfidenceMatch to be false for scores < 6")
+ }
+
+ highConf := &schema.TaggedChunk{
+ Content: "high confidence sample",
+ TagWeights: OrderedTagWeights{"tag1": 4, "tag2": 6},
+ }
+ if !isHighConfidenceMatch(highConf) {
+ t.Fatal("expected isHighConfidenceMatch to be true for score >= 6")
+ }
+
+ if isHighConfidenceMatch(nil) {
+ t.Fatal("expected false for nil chunk")
+ }
+}
+
+func TestDetectTextLanguage_CJK_Kana_Hangul(t *testing.T) {
+ tests := []struct {
+ name string
+ text string
+ want string
+ }{
+ {
+ name: "Japanese with Hiragana",
+ text: "これは日本語のテキストです。",
+ want: "Japanese",
+ },
+ {
+ name: "Japanese with Katakana and Kanji",
+ text: "ベクトルデータベースの検索システム",
+ want: "Japanese",
+ },
+ {
+ name: "Korean with Hangul",
+ text: "안녕하세요. 이것은 한국어 텍스트입니다.",
+ want: "Korean",
+ },
+ {
+ name: "Simplified Chinese",
+ text: "这是一个纯中文的测试文本。",
+ want: "Chinese",
+ },
+ {
+ name: "Traditional Chinese",
+ text: "這是一個繁體中文測試文檔。",
+ want: "Chinese",
+ },
+ {
+ name: "English",
+ text: "This is an English text without any Asian characters.",
+ want: "English",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ chunks := []map[string]any{
+ {"content_with_weight": tt.text},
+ }
+ got := detectTextLanguage(chunks)
+ if got != tt.want {
+ t.Errorf("detectTextLanguage(%q) = %q, want %q", tt.text, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestMatchAndTagChunk_RankDecaySalienceBoost(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ t.Cleanup(func() { tokenizer.SetEngineType("") })
+ tok := tokenizer.New("English")
+
+ rawExamples := []schema.TagLabel{
+ {Content: "vector database storage", Tags: []string{"Storage"}},
+ {Content: "vector database indexing", Tags: []string{"Indexing"}},
+ }
+ idx := buildMemoryTagIndex(rawExamples, tok)
+ if idx == nil {
+ t.Fatal("buildMemoryTagIndex returned nil")
+ }
+
+ // Chunk has "indexing" repeated 5 times
+ chunk := map[string]any{
+ "content_with_weight": "vector database indexing indexing indexing indexing indexing",
+ }
+
+ res := matchAndTagChunk(chunk, idx, tok, 5)
+ if res == nil {
+ t.Fatal("expected matchAndTagChunk to succeed")
+ }
+
+ weights := chunk[common.TAG_FLD].(OrderedTagWeights)
+ if weights["Indexing"] <= weights["Storage"] {
+ t.Fatalf("expected Indexing score (%d) > Storage score (%d) due to TF boost", weights["Indexing"], weights["Storage"])
+ }
+}
+
+func TestMatchAndTagChunk_RankDecayGradientScores(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ t.Cleanup(func() { tokenizer.SetEngineType("") })
+ tok := tokenizer.New("English")
+
+ var rawExamples []schema.TagLabel
+ // Background corpus
+ for i := 0; i < 50; i++ {
+ rawExamples = append(rawExamples, schema.TagLabel{
+ Content: fmt.Sprintf("general common document text number %d", i),
+ Tags: []string{"GeneralDocs"},
+ })
+ }
+ // Common peripheral topic in background
+ for i := 0; i < 8; i++ {
+ rawExamples = append(rawExamples, schema.TagLabel{
+ Content: fmt.Sprintf("unrelated cloud telemetry logging metrics cluster node %d", i),
+ Tags: []string{"CloudInfra"},
+ })
+ }
+ rawExamples = append(rawExamples,
+ schema.TagLabel{
+ Content: "cloud deployment devops kubernetes",
+ Tags: []string{"CloudInfra"},
+ },
+ )
+ // Secondary topic (moderate background)
+ for i := 0; i < 2; i++ {
+ rawExamples = append(rawExamples, schema.TagLabel{
+ Content: fmt.Sprintf("unrelated database transaction internals locking protocol %d", i),
+ Tags: []string{"Database"},
+ })
+ }
+ rawExamples = append(rawExamples,
+ schema.TagLabel{
+ Content: "database storage query index optimization postgresql",
+ Tags: []string{"Database"},
+ },
+ )
+ // Primary topic (specific domain with high prominence)
+ rawExamples = append(rawExamples,
+ schema.TagLabel{
+ Content: "neural network deep learning transformer model training",
+ Tags: []string{"DeepLearning"},
+ },
+ schema.TagLabel{
+ Content: "deep learning transformer attention mechanism architecture",
+ Tags: []string{"DeepLearning"},
+ },
+ )
+
+ idx := buildMemoryTagIndex(rawExamples, tok)
+ if idx == nil {
+ t.Fatal("buildMemoryTagIndex returned nil")
+ }
+
+ // Chunk has high prominence with repeated terms for DeepLearning, secondary support for Database, minor mention of CloudInfra
+ chunk := map[string]any{
+ "content_with_weight": "neural network deep learning transformer model training deep learning transformer attention mechanism architecture deep learning transformer neural network deep learning transformer deep learning transformer database storage query index optimization postgresql cloud deployment devops kubernetes",
+ }
+
+ res := matchAndTagChunk(chunk, idx, tok, 10)
+ if res == nil {
+ t.Fatal("expected matchAndTagChunk to succeed")
+ }
+
+ weights := chunk[common.TAG_FLD].(OrderedTagWeights)
+ dlScore := weights["DeepLearning"]
+ dbScore := weights["Database"]
+ cloudScore := weights["CloudInfra"]
+
+ t.Logf("Scores: DeepLearning=%d, Database=%d, CloudInfra=%d, All=%v", dlScore, dbScore, cloudScore, weights)
+
+ if dlScore < 8 {
+ t.Errorf("expected high-prominence DeepLearning score >= 8, got %d", dlScore)
+ }
+ if dbScore < 5 || dbScore > 7 {
+ t.Errorf("expected secondary Database score in 5~7 range, got %d", dbScore)
+ }
+ if cloudScore < 3 || cloudScore > 5 {
+ t.Errorf("expected peripheral CloudInfra score in 3~5 range, got %d", cloudScore)
+ }
+ if dlScore <= dbScore {
+ t.Errorf("expected primary DeepLearning (%d) > secondary Database (%d)", dlScore, dbScore)
+ }
+ if dbScore <= cloudScore {
+ t.Errorf("expected secondary Database (%d) > peripheral CloudInfra (%d)", dbScore, cloudScore)
+ }
+}
diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.go
index 5c7e579d2e..d0fbdbcff4 100644
--- a/internal/ingestion/component/extractor_test.go
+++ b/internal/ingestion/component/extractor_test.go
@@ -2028,3 +2028,51 @@ func TestExtractorEnabledFalseDoesNotCallLLM(t *testing.T) {
t.Fatalf("expected 1 chunk, got %v", out)
}
}
+
+func TestExtractor_KeywordsThenTagsSynergy(t *testing.T) {
+ withStubChatInvoker(t, stubResponse{Content: "Bidding, Procurement"})
+
+ params := map[string]any{
+ "llm_id": "llm-1",
+ "keywords": map[string]any{
+ "top_n": 2,
+ },
+ }
+ comp, err := NewExtractorComponent(params)
+ if err != nil {
+ t.Fatalf("NewExtractorComponent: %v", err)
+ }
+
+ in := map[string]any{
+ "chunks": []map[string]any{
+ {
+ "docnm_kwd": "Tender_Notice.pdf",
+ "content_with_weight": "General bidding notice content.",
+ },
+ },
+ }
+
+ out, err := comp.Invoke(t.Context(), nil, in)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk, got %v", out)
+ }
+
+ kwds, ok := chunks[0]["important_kwd"].([]string)
+ if !ok || len(kwds) != 2 {
+ t.Fatalf("expected important_kwd populated with 2 keywords, got %v", chunks[0]["important_kwd"])
+ }
+
+ // Verify getChunkText on the resulting chunk merges extracted keywords and content without title pollution
+ chunkText := getChunkText(chunks[0])
+ if !strings.Contains(chunkText, "Bidding") || !strings.Contains(chunkText, "Procurement") || !strings.Contains(chunkText, "General bidding notice content.") {
+ t.Fatalf("expected chunk text to contain content and extracted keywords, got %q", chunkText)
+ }
+ if strings.Contains(chunkText, "Tender_Notice") {
+ t.Fatalf("expected chunk text to NOT contain title when content is present, got %q", chunkText)
+ }
+}
diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.go
index f6ba5acb42..24c6a9968e 100644
--- a/internal/ingestion/component/tokenizer_test.go
+++ b/internal/ingestion/component/tokenizer_test.go
@@ -34,15 +34,9 @@ import (
"ragflow/internal/tokenizer"
)
-func requireTokenizerPool(t *testing.T) {
+func requireTokenizerPool(t testing.TB) {
t.Helper()
- if err := tokenizer.Init(&tokenizer.PoolConfig{
- DictPath: "/usr/share/infinity/resource",
- MinSize: 1,
- MaxSize: 2,
- IdleTimeout: 30 * time.Second,
- AcquireTimeout: 5 * time.Second,
- }); err != nil {
+ if err := tokenizer.Init(nil); err != nil {
t.Skipf("tokenizer pool unavailable: %v", err)
}
}
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index 5637323acd..e2fc00baf6 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -25,6 +25,7 @@ import (
"encoding/json"
"os"
"reflect"
+ "strconv"
"strings"
"sync/atomic"
"testing"
@@ -59,6 +60,11 @@ func (s *stubEmbedder) MaxTokens() int {
}
func (s *stubEmbedder) BatchSize() int {
+ if v := os.Getenv("TOKENIZER_EMBEDDING_BATCH_SIZE"); v != "" {
+ if n, err := strconv.Atoi(v); err == nil && n > 0 {
+ return n
+ }
+ }
return 16
}