Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(knowledge): drop generated questions when the switch is off (#4096)

AsanoReo6e91b671ee7b
diff --git a/internal/agent/tools/search_knowledge.go b/internal/agent/tools/search_knowledge.go
index 43da62aa9..3eb833d39 100644
--- a/internal/agent/tools/search_knowledge.go
+++ b/internal/agent/tools/search_knowledge.go
@@ -858,6 +858,14 @@ func (t *SearchKnowledgeTool) rerankResults(
}
if t.knowledgeBaseService != nil {
opts.LoadImage = t.knowledgeBaseService.ReadChunkImage
+ if ids := types.SearchResultKnowledgeBaseIDs(rows); len(ids) > 0 {
+ kbs, err := t.knowledgeBaseService.GetKnowledgeBasesByIDsOnly(ctx, ids)
+ if err != nil {
+ logger.Warnf(ctx, "[Tool][SearchKnowledge] Failed to read question generation config: %v", err)
+ } else {
+ types.ApplySkipGeneratedQuestions(rows, types.QuestionGenerationOffIDs(kbs))
+ }
+ }
}
res := reranking.Rerank(ctx, t.rerankModel, query, rows, opts)
if res.Diagnostics.Outcome == types.RerankOutcomeModelError {
diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.go
index cf28a9a27..9951a04f6 100644
--- a/internal/application/repository/retriever/postgres/repository.go
+++ b/internal/application/repository/retriever/postgres/repository.go
@@ -270,6 +270,9 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
SQL: "(is_enabled IS NULL OR is_enabled = ?)",
Vars: []interface{}{true},
})
+ if exclusion := generatedQuestionExclusionClause(params.ExcludeGeneratedQuestionKBIDs); exclusion != nil {
+ conds = append(conds, exclusion)
+ }
conds = append(conds, clause.OrderBy{Columns: []clause.OrderByColumn{
{Column: clause.Column{Name: "score"}, Desc: true},
}})
@@ -398,6 +401,19 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
// is_enabled filter
whereParts = append(whereParts, fmt.Sprintf("(is_enabled IS NULL OR is_enabled = $%d)", len(allVars)+1))
allVars = append(allVars, true)
+ if len(params.ExcludeGeneratedQuestionKBIDs) > 0 {
+ placeholders := make([]string, len(params.ExcludeGeneratedQuestionKBIDs))
+ paramStart := len(allVars) + 1
+ for i, id := range params.ExcludeGeneratedQuestionKBIDs {
+ placeholders[i] = fmt.Sprintf("$%d", paramStart+i)
+ allVars = append(allVars, id)
+ }
+ whereParts = append(whereParts, fmt.Sprintf(
+ "NOT (knowledge_base_id IN (%s) AND %s)",
+ strings.Join(placeholders, ", "),
+ types.GeneratedQuestionRowPredicate(""),
+ ))
+ }
// Build WHERE clause string
whereClause := ""
@@ -773,3 +789,23 @@ func (g *pgRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap ma
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk tag ID")
return nil
}
+
+func generatedQuestionExclusionClause(kbIDs []string) clause.Expression {
+ if len(kbIDs) == 0 {
+ return nil
+ }
+ placeholders := make([]string, len(kbIDs))
+ vars := make([]interface{}, len(kbIDs))
+ for i, id := range kbIDs {
+ placeholders[i] = "?"
+ vars[i] = id
+ }
+ return clause.Expr{
+ SQL: fmt.Sprintf(
+ "NOT (knowledge_base_id IN (%s) AND %s)",
+ strings.Join(placeholders, ", "),
+ types.GeneratedQuestionRowPredicate(""),
+ ),
+ Vars: vars,
+ }
+}
diff --git a/internal/application/repository/retriever/sqlite/repository.go b/internal/application/repository/retriever/sqlite/repository.go
index 5622df031..021b00bc6 100644
--- a/internal/application/repository/retriever/sqlite/repository.go
+++ b/internal/application/repository/retriever/sqlite/repository.go
@@ -889,6 +889,17 @@ func buildFilterWhere(params types.RetrieveParams, tableAlias string) []whereCla
args: toInterfaceSlice(params.TagIDs),
})
}
+ if len(params.ExcludeGeneratedQuestionKBIDs) > 0 {
+ parts = append(parts, whereClause{
+ clause: fmt.Sprintf(
+ "NOT (%s.knowledge_base_id IN (%s) AND %s)",
+ tableAlias,
+ placeholders(len(params.ExcludeGeneratedQuestionKBIDs)),
+ types.GeneratedQuestionRowPredicate(tableAlias),
+ ),
+ args: toInterfaceSlice(params.ExcludeGeneratedQuestionKBIDs),
+ })
+ }
return parts
}
diff --git a/internal/application/repository/retriever/sqlite/repository_test.go b/internal/application/repository/retriever/sqlite/repository_test.go
index 736d2bb2d..3b36304a2 100644
--- a/internal/application/repository/retriever/sqlite/repository_test.go
+++ b/internal/application/repository/retriever/sqlite/repository_test.go
@@ -115,6 +115,39 @@ func TestVectorRetrieveFiltersBeforeTopK(t *testing.T) {
}
}
+func TestVectorRetrieveDropsGeneratedQuestionsBeforeTopK(t *testing.T) {
+ repository := newSQLiteRetrieverTestRepository(t)
+ const chunkID = "chunk-body"
+ body := sqliteTestIndex(chunkID, "kb-off", "knowledge-1", "", true)
+ body.SourceID = chunkID
+ body.Content = "month end close steps"
+ question := sqliteTestIndex(chunkID, "kb-off", "knowledge-1", "", true)
+ question.SourceID = chunkID + "-q1"
+ question.Content = "how do I close the books"
+ other := sqliteTestIndex("other", "kb-on", "knowledge-2", "", true)
+ other.SourceID = "other-q1"
+ other.Content = "question in a knowledge base that still generates questions"
+
+ saveSQLiteTestVector(t, repository, question, []float32{1, 0})
+ saveSQLiteTestVector(t, repository, body, []float32{0.2, 0.8})
+ saveSQLiteTestVector(t, repository, other, []float32{0.9, 0.1})
+
+ results, err := repository.vectorRetrieve(context.Background(), types.RetrieveParams{
+ Embedding: []float32{1, 0},
+ KnowledgeBaseIDs: []string{"kb-off", "kb-on"},
+ ExcludeGeneratedQuestionKBIDs: []string{"kb-off"},
+ TopK: 2,
+ RetrieverType: types.VectorRetrieverType,
+ })
+ require.NoError(t, err)
+ require.Len(t, results, 1)
+ got := make([]string, 0, len(results[0].Results))
+ for _, hit := range results[0].Results {
+ got = append(got, hit.SourceID)
+ }
+ assert.ElementsMatch(t, []string{chunkID, "other-q1"}, got)
+}
+
func TestVectorRetrieveZeroThresholdDoesNotFilter(t *testing.T) {
repository := newSQLiteRetrieverTestRepository(t)
saveSQLiteTestVector(t, repository,
diff --git a/internal/application/service/chat_pipeline/rerank.go b/internal/application/service/chat_pipeline/rerank.go
index 4fda8f44f..c2defaa6d 100644
--- a/internal/application/service/chat_pipeline/rerank.go
+++ b/internal/application/service/chat_pipeline/rerank.go
@@ -104,6 +104,7 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
ctx = rerankCtx
logRerankInputScoreSample(ctx, chatManage.SearchResult)
+ p.skipDisabledGeneratedQuestions(ctx, chatManage.SearchResult)
opts := reranking.Options{
Threshold: chatManage.RerankThreshold,
@@ -222,6 +223,26 @@ func buildRerankSpanOutput(res *reranking.Result, chatManage *types.ChatManage)
return out
}
+// skipDisabledGeneratedQuestions applies the live question-generation switch
+// to passages that were added after hybrid search, such as graph hits.
+func (p *PluginRerank) skipDisabledGeneratedQuestions(ctx context.Context, results []*types.SearchResult) {
+ if p.kbService == nil || len(results) == 0 {
+ return
+ }
+ ids := types.SearchResultKnowledgeBaseIDs(results)
+ if len(ids) == 0 {
+ return
+ }
+ kbs, err := p.kbService.GetKnowledgeBasesByIDsOnly(ctx, ids)
+ if err != nil {
+ pipelineWarn(ctx, "Rerank", "question_generation_lookup", map[string]interface{}{
+ "error": err.Error(),
+ })
+ return
+ }
+ types.ApplySkipGeneratedQuestions(results, types.QuestionGenerationOffIDs(kbs))
+}
+
func logRerankInputScoreSample(ctx context.Context, results []*types.SearchResult) {
const maxLogRows = 8
limit := min(maxLogRows, len(results))
diff --git a/internal/application/service/chat_pipeline/rerank_test.go b/internal/application/service/chat_pipeline/rerank_test.go
index 78f4c0c6a..eaf850067 100644
--- a/internal/application/service/chat_pipeline/rerank_test.go
+++ b/internal/application/service/chat_pipeline/rerank_test.go
@@ -309,3 +309,54 @@ func TestMergeFallbackWithoutLossRecordsNothing(t *testing.T) {
t.Fatalf("fallback input = %d, truncation = %+v", len(got), cm.Truncation)
}
}
+
+type questionLookupKB struct {
+ interfaces.KnowledgeBaseService
+ calls [][]string
+ kbs []*types.KnowledgeBase
+ err error
+}
+
+func (s *questionLookupKB) GetKnowledgeBasesByIDsOnly(_ context.Context, ids []string) ([]*types.KnowledgeBase, error) {
+ s.calls = append(s.calls, append([]string(nil), ids...))
+ return s.kbs, s.err
+}
+
+func TestSkipDisabledGeneratedQuestionsLooksUpOnce(t *testing.T) {
+ kb := &questionLookupKB{kbs: []*types.KnowledgeBase{
+ {ID: "off", Type: types.KnowledgeBaseTypeDocument},
+ {
+ ID: "on", Type: types.KnowledgeBaseTypeDocument,
+ QuestionGenerationConfig: &types.QuestionGenerationConfig{Enabled: true},
+ },
+ {ID: "faq", Type: types.KnowledgeBaseTypeFAQ},
+ }}
+ results := []*types.SearchResult{
+ {ID: "a", KnowledgeBaseID: "off"},
+ {ID: "b", KnowledgeBaseID: "on"},
+ {ID: "c", KnowledgeBaseID: "faq"},
+ {ID: "d", KnowledgeBaseID: "off"},
+ }
+ (&PluginRerank{kbService: kb}).skipDisabledGeneratedQuestions(context.Background(), results)
+ if len(kb.calls) != 1 {
+ t.Fatalf("knowledge base lookups = %d, want 1", len(kb.calls))
+ }
+ if len(kb.calls[0]) != 3 {
+ t.Fatalf("looked up %v, want the three distinct knowledge bases", kb.calls[0])
+ }
+ if !results[0].SkipGeneratedQuestions || results[1].SkipGeneratedQuestions ||
+ results[2].SkipGeneratedQuestions || !results[3].SkipGeneratedQuestions {
+ t.Fatalf("skip flags = %v %v %v %v",
+ results[0].SkipGeneratedQuestions, results[1].SkipGeneratedQuestions,
+ results[2].SkipGeneratedQuestions, results[3].SkipGeneratedQuestions)
+ }
+}
+
+func TestSkipDisabledGeneratedQuestionsLeavesFlagsUnsetOnLookupError(t *testing.T) {
+ kb := &questionLookupKB{err: errors.New("db down")}
+ results := []*types.SearchResult{{ID: "a", KnowledgeBaseID: "off"}}
+ (&PluginRerank{kbService: kb}).skipDisabledGeneratedQuestions(context.Background(), results)
+ if results[0].SkipGeneratedQuestions {
+ t.Fatal("a failed knowledge base lookup marked the passage")
+ }
+}
diff --git a/internal/application/service/knowledgebase_generated_questions.go b/internal/application/service/knowledgebase_generated_questions.go
new file mode 100644
index 000000000..2c83c1e01
--- /dev/null
+++ b/internal/application/service/knowledgebase_generated_questions.go
@@ -0,0 +1,24 @@
+package service
+
+import (
+ "context"
+
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+// markSkipGeneratedQuestions sets SkipGeneratedQuestions from the live switch
+// so rerank passages omit historical questions. A lookup failure leaves the
+// flag unset.
+func (s *knowledgeBaseService) markSkipGeneratedQuestions(ctx context.Context, results []*types.SearchResult) {
+ ids := types.SearchResultKnowledgeBaseIDs(results)
+ if len(ids) == 0 {
+ return
+ }
+ kbs, err := s.repo.GetKnowledgeBaseByIDs(ctx, ids)
+ if err != nil {
+ logger.Warnf(ctx, "Failed to read question generation config for rerank: %v", err)
+ return
+ }
+ types.ApplySkipGeneratedQuestions(results, types.QuestionGenerationOffIDs(kbs))
+}
diff --git a/internal/application/service/knowledgebase_search.go b/internal/application/service/knowledgebase_search.go
index e4ef91181..2e6f79d6a 100644
--- a/internal/application/service/knowledgebase_search.go
+++ b/internal/application/service/knowledgebase_search.go
@@ -439,6 +439,15 @@ func (s *knowledgeBaseService) buildRetrievalParams(
// scope happens to include them (e.g. KBSelectionMode=all picking up a
// wiki-only KB).
var faqVectorKBIDs, docVectorKBIDs, docKeywordKBIDs []string
+ // Document KBs with question generation off must not spend TopK on
+ // historical question vectors. FAQ KBs are absent from this list: their
+ // similar-question rows use the same source-id shape.
+ var questionOffKBIDs []string
+ for _, kb := range groupKBs {
+ if types.QuestionGenerationDisabled(kb) {
+ questionOffKBIDs = append(questionOffKBIDs, kb.ID)
+ }
+ }
// The query is embedded with the primary's model, or with the first
// vector KB of the group when the primary has none (a wiki primary
// searched together with document KBs): asking the primary for its empty
@@ -478,15 +487,16 @@ func (s *knowledgeBaseService) buildRetrievalParams(
appendVectorParams := func(kbIDs []string, knowledgeType string) {
retrieveParams = append(retrieveParams, types.RetrieveParams{
- Query: params.QueryText,
- Embedding: queryEmbedding,
- KnowledgeBaseIDs: kbIDs,
- TopK: matchCount,
- Threshold: params.VectorThreshold,
- RetrieverType: types.VectorRetrieverType,
- KnowledgeIDs: params.KnowledgeIDs,
- TagIDs: params.TagIDs,
- KnowledgeType: knowledgeType,
+ Query: params.QueryText,
+ Embedding: queryEmbedding,
+ KnowledgeBaseIDs: kbIDs,
+ TopK: matchCount,
+ Threshold: params.VectorThreshold,
+ RetrieverType: types.VectorRetrieverType,
+ KnowledgeIDs: params.KnowledgeIDs,
+ TagIDs: params.TagIDs,
+ KnowledgeType: knowledgeType,
+ ExcludeGeneratedQuestionKBIDs: generatedQuestionKBIDs(kbIDs, questionOffKBIDs),
})
}
@@ -508,13 +518,14 @@ func (s *knowledgeBaseService) buildRetrievalParams(
len(docKeywordKBIDs) > 0 {
logger.Info(ctx, "Keyword retrieval supported, preparing keyword retrieval parameters")
retrieveParams = append(retrieveParams, types.RetrieveParams{
- Query: params.QueryText,
- KnowledgeBaseIDs: docKeywordKBIDs,
- TopK: matchCount,
- Threshold: params.KeywordThreshold,
- RetrieverType: types.KeywordsRetrieverType,
- KnowledgeIDs: params.KnowledgeIDs,
- TagIDs: params.TagIDs,
+ Query: params.QueryText,
+ KnowledgeBaseIDs: docKeywordKBIDs,
+ TopK: matchCount,
+ Threshold: params.KeywordThreshold,
+ RetrieverType: types.KeywordsRetrieverType,
+ KnowledgeIDs: params.KnowledgeIDs,
+ TagIDs: params.TagIDs,
+ ExcludeGeneratedQuestionKBIDs: generatedQuestionKBIDs(docKeywordKBIDs, questionOffKBIDs),
})
logger.Info(ctx, "Keyword retrieval parameters setup completed")
}
@@ -522,6 +533,25 @@ func (s *knowledgeBaseService) buildRetrievalParams(
return retrieveParams, nil
}
+// generatedQuestionKBIDs keeps the off-switch ids that this retrieval param
+// actually searches. An empty result leaves the SQL filter off.
+func generatedQuestionKBIDs(searched, disabled []string) []string {
+ if len(searched) == 0 || len(disabled) == 0 {
+ return nil
+ }
+ off := make(map[string]struct{}, len(disabled))
+ for _, id := range disabled {
+ off[id] = struct{}{}
+ }
+ var matched []string
+ for _, id := range searched {
+ if _, ok := off[id]; ok {
+ matched = append(matched, id)
+ }
+ }
+ return matched
+}
+
// resolveQueryEmbedding returns the query embedding for a store group. It
// reuses params.QueryEmbedding when the caller pre-computed it (the common
// path — HybridSearch embeds once before fan-out), otherwise it embeds the
diff --git a/internal/application/service/knowledgebase_search_embedding_kb_test.go b/internal/application/service/knowledgebase_search_embedding_kb_test.go
index f9540ff2d..3f475710d 100644
--- a/internal/application/service/knowledgebase_search_embedding_kb_test.go
+++ b/internal/application/service/knowledgebase_search_embedding_kb_test.go
@@ -44,3 +44,58 @@ func TestBuildRetrievalParamsEmbedsWithAVectorKB(t *testing.T) {
require.NotEmpty(t, params)
require.Len(t, params[0].Embedding, 768)
}
+
+// Document knowledge bases with question generation off must drop historical
+// question rows before TopK. FAQ knowledge bases use the same source-id shape
+// for their own similar questions, so they stay off that list.
+func TestBuildRetrievalParamsExcludesGeneratedQuestionsOnlyForDocumentKBs(t *testing.T) {
+ s := &knowledgeBaseService{}
+ engine := buildBoundComposite(t, &fakeRetrieveEngineService{
+ engineType: types.PostgresRetrieverEngineType,
+ support: []types.RetrieverType{types.VectorRetrieverType, types.KeywordsRetrieverType},
+ })
+ docOff := &types.KnowledgeBase{
+ ID: "doc-off", TenantID: 1, Type: types.KnowledgeBaseTypeDocument,
+ EmbeddingModelID: "embed-1",
+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true, KeywordEnabled: true},
+ }
+ docOn := &types.KnowledgeBase{
+ ID: "doc-on", TenantID: 1, Type: types.KnowledgeBaseTypeDocument,
+ EmbeddingModelID: "embed-1",
+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true, KeywordEnabled: true},
+ QuestionGenerationConfig: &types.QuestionGenerationConfig{Enabled: true},
+ }
+ faq := &types.KnowledgeBase{
+ ID: "faq", TenantID: 1, Type: types.KnowledgeBaseTypeFAQ,
+ EmbeddingModelID: "embed-1",
+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true, KeywordEnabled: true},
+ }
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
+
+ params, err := s.buildRetrievalParams(ctx, engine, docOff,
+ []*types.KnowledgeBase{docOff, docOn, faq},
+ types.SearchParams{QueryText: "q", QueryEmbedding: []float32{1}}, 5)
+ require.NoError(t, err)
+
+ var docVector, faqVector, keyword *types.RetrieveParams
+ for i := range params {
+ p := &params[i]
+ switch {
+ case p.RetrieverType == types.VectorRetrieverType && p.KnowledgeType == types.KnowledgeTypeFAQ:
+ faqVector = p
+ case p.RetrieverType == types.VectorRetrieverType:
+ docVector = p
+ case p.RetrieverType == types.KeywordsRetrieverType:
+ keyword = p
+ }
+ }
+ require.NotNil(t, docVector)
+ require.NotNil(t, faqVector)
+ require.NotNil(t, keyword)
+ require.Equal(t, []string{"doc-off"}, docVector.ExcludeGeneratedQuestionKBIDs)
+ require.Empty(t, faqVector.ExcludeGeneratedQuestionKBIDs)
+ require.Equal(t, []string{"doc-off"}, keyword.ExcludeGeneratedQuestionKBIDs)
+ require.ElementsMatch(t, []string{"doc-off", "doc-on"}, docVector.KnowledgeBaseIDs)
+ require.Equal(t, []string{"faq"}, faqVector.KnowledgeBaseIDs)
+ require.NotContains(t, keyword.KnowledgeBaseIDs, "faq")
+}
diff --git a/internal/application/service/knowledgebase_search_rerank.go b/internal/application/service/knowledgebase_search_rerank.go
index e111af5e4..45008dc32 100644
--- a/internal/application/service/knowledgebase_search_rerank.go
+++ b/internal/application/service/knowledgebase_search_rerank.go
@@ -181,6 +181,7 @@ func (s *knowledgeBaseService) rerankCandidates(
diag.Error = err.Error()
return retrievalOrder
}
+ s.markSkipGeneratedQuestions(ctx, candidates)
res := reranking.Rerank(ctx, model, query, candidates, reranking.Options{
Threshold: threshold,
diff --git a/internal/handler/initialization.go b/internal/handler/initialization.go
index 91db96aa1..8060ac5ea 100644
--- a/internal/handler/initialization.go
+++ b/internal/handler/initialization.go
@@ -417,7 +417,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
return
}
- // 更新问题生成配置
+ // 更新问题生成配置。检索是否使用问句由查询时的开关决定,保存设置不改索引。
if req.QuestionGeneration.Enabled {
questionCount := req.QuestionGeneration.QuestionCount
if questionCount <= 0 {
diff --git a/internal/reranking/passage.go b/internal/reranking/passage.go
index fd3216568..23a6632b4 100644
--- a/internal/reranking/passage.go
+++ b/internal/reranking/passage.go
@@ -166,6 +166,8 @@ func cleanPassageProse(text string) string {
// EnrichedPassage is the cleaned chunk body plus the image captions, OCR text
// and generated questions attached to it. MMR compares candidates on this
// text, so chunks of one document are not made similar by a shared title.
+// Generated questions are omitted when the knowledge base has turned question
+// generation off.
func EnrichedPassage(ctx context.Context, result *types.SearchResult) string {
combinedText := CleanPassage(result.Content)
var enrichments []string
@@ -194,8 +196,10 @@ func EnrichedPassage(ctx context.Context, result *types.SearchResult) string {
var docMeta types.DocumentChunkMetadata
if err := json.Unmarshal(result.ChunkMetadata, &docMeta); err != nil {
logger.Warnf(ctx, "[Rerank] Failed to parse chunk metadata of chunk %s: %v", result.ID, err)
- } else if questionStrings := docMeta.GetQuestionStrings(); len(questionStrings) > 0 {
- enrichments = append(enrichments, strings.Join(questionStrings, "; "))
+ } else if includeGeneratedQuestions(result) {
+ if questionStrings := docMeta.GetQuestionStrings(); len(questionStrings) > 0 {
+ enrichments = append(enrichments, strings.Join(questionStrings, "; "))
+ }
}
}
@@ -208,6 +212,10 @@ func EnrichedPassage(ctx context.Context, result *types.SearchResult) string {
return combinedText + strings.Join(enrichments, "\n")
}
+func includeGeneratedQuestions(result *types.SearchResult) bool {
+ return result == nil || !result.SkipGeneratedQuestions
+}
+
// ModelPassage is the text the rerank model scores: the document title
// followed by the enriched chunk. A chunk rarely restates what its document
// is about, so without the title a passage from "Show HN: Echo - ... using
diff --git a/internal/reranking/passage_test.go b/internal/reranking/passage_test.go
index 50fee35f2..ba0dca74c 100644
--- a/internal/reranking/passage_test.go
+++ b/internal/reranking/passage_test.go
@@ -29,6 +29,29 @@ func TestGetEnrichedPassageKeepsQuestionsFromEarlierRevision(t *testing.T) {
}
}
+func TestEnrichedPassageOmitsQuestionsWhenGenerationIsOff(t *testing.T) {
+ metadata, err := json.Marshal(types.DocumentChunkMetadata{
+ GeneratedQuestions: []types.GeneratedQuestion{{
+ ID: "q1", Question: "stale generated question",
+ }},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ result := &types.SearchResult{
+ Content: "chunk body",
+ ChunkMetadata: types.JSON(metadata),
+ SkipGeneratedQuestions: true,
+ }
+ passage := EnrichedPassage(context.Background(), result)
+ if strings.Contains(passage, "stale generated question") {
+ t.Fatalf("disabled knowledge base still put the question in the passage: %q", passage)
+ }
+ if !strings.Contains(passage, "chunk body") {
+ t.Fatalf("passage dropped the chunk body: %q", passage)
+ }
+}
+
func TestGetEnrichedPassageKeepsCodeAndMathCandidates(t *testing.T) {
tests := []struct {
name string
diff --git a/internal/types/generated_question_retrieval.go b/internal/types/generated_question_retrieval.go
new file mode 100644
index 000000000..2d6acdd77
--- /dev/null
+++ b/internal/types/generated_question_retrieval.go
@@ -0,0 +1,98 @@
+package types
+
+import "fmt"
+
+// QuestionGenerationActive reports whether a document knowledge base still
+// wants generated questions in retrieval. FAQ knowledge bases index their own
+// similar questions under the same source-id shape and are never treated as
+// this switch.
+func QuestionGenerationActive(kb *KnowledgeBase) bool {
+ return kb != nil &&
+ kb.Type != KnowledgeBaseTypeFAQ &&
+ kb.QuestionGenerationConfig != nil &&
+ kb.QuestionGenerationConfig.Enabled
+}
+
+// QuestionGenerationDisabled reports whether generated-question rows for a
+// document knowledge base must stay out of recall and rerank. A nil config is
+// off: question generation only runs when the switch is explicitly enabled.
+func QuestionGenerationDisabled(kb *KnowledgeBase) bool {
+ return kb != nil && kb.Type != KnowledgeBaseTypeFAQ && !QuestionGenerationActive(kb)
+}
+
+// IsGeneratedQuestionSource reports whether an index row is a generated
+// question for its parent chunk. Rows are written as
+// source_id = chunk_id + "-" + question id (or the hashed "-q" form) with
+// source_type chunk. The chunk body itself uses source_id = chunk_id.
+// Identification does not use the length of source_id.
+func IsGeneratedQuestionSource(chunkID, sourceID string, sourceType SourceType) bool {
+ if sourceType != ChunkSourceType || chunkID == "" || sourceID == "" || sourceID == chunkID {
+ return false
+ }
+ return len(sourceID) > len(chunkID)+1 && sourceID[:len(chunkID)+1] == chunkID+"-"
+}
+
+// GeneratedQuestionRowPredicate is the SQL form of IsGeneratedQuestionSource.
+// alias is the table qualifier without a dot, or empty for an unqualified
+// column. source_type is inlined; chunk ids are ASCII so length and substr
+// agree on PostgreSQL and SQLite.
+func GeneratedQuestionRowPredicate(alias string) string {
+ col := func(name string) string {
+ if alias == "" {
+ return name
+ }
+ return alias + "." + name
+ }
+ return fmt.Sprintf(
+ "%s = %d AND %s <> '' AND %s <> %s AND substr(%s, 1, length(%s) + 1) = %s || '-'",
+ col("source_type"), int(ChunkSourceType),
+ col("chunk_id"),
+ col("source_id"), col("chunk_id"),
+ col("source_id"), col("chunk_id"), col("chunk_id"),
+ )
+}
+
+// SearchResultKnowledgeBaseIDs returns the distinct knowledge base IDs on results.
+func SearchResultKnowledgeBaseIDs(results []*SearchResult) []string {
+ seen := make(map[string]struct{}, len(results))
+ ids := make([]string, 0, len(results))
+ for _, result := range results {
+ if result == nil || result.KnowledgeBaseID == "" {
+ continue
+ }
+ if _, ok := seen[result.KnowledgeBaseID]; ok {
+ continue
+ }
+ seen[result.KnowledgeBaseID] = struct{}{}
+ ids = append(ids, result.KnowledgeBaseID)
+ }
+ return ids
+}
+
+// QuestionGenerationOffIDs returns document knowledge bases whose generated
+// questions must stay out of rerank passages.
+func QuestionGenerationOffIDs(kbs []*KnowledgeBase) map[string]struct{} {
+ disabled := make(map[string]struct{})
+ for _, kb := range kbs {
+ if QuestionGenerationDisabled(kb) {
+ disabled[kb.ID] = struct{}{}
+ }
+ }
+ return disabled
+}
+
+// ApplySkipGeneratedQuestions marks rerank passages whose knowledge base has
+// question generation turned off. disabled maps knowledge base IDs.
+func ApplySkipGeneratedQuestions(results []*SearchResult, disabled map[string]struct{}) {
+ if len(disabled) == 0 {
+ return
+ }
+ for _, result := range results {
+ if result == nil {
+ continue
+ }
+ if _, ok := disabled[result.KnowledgeBaseID]; ok {
+ result.SkipGeneratedQuestions = true
+ }
+ }
+}
diff --git a/internal/types/generated_question_retrieval_test.go b/internal/types/generated_question_retrieval_test.go
new file mode 100644
index 000000000..99b79b104
--- /dev/null
+++ b/internal/types/generated_question_retrieval_test.go
@@ -0,0 +1,69 @@
+package types
+
+import "testing"
+
+func TestIsGeneratedQuestionSource(t *testing.T) {
+ const chunkID = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"
+ cases := []struct {
+ name string
+ chunkID string
+ sourceID string
+ sourceType SourceType
+ want bool
+ }{
+ {name: "body row", chunkID: chunkID, sourceID: chunkID, sourceType: ChunkSourceType},
+ {
+ name: "question row", chunkID: chunkID, sourceID: chunkID + "-q123",
+ sourceType: ChunkSourceType, want: true,
+ },
+ {
+ name: "hashed question row", chunkID: chunkID, sourceID: chunkID + "-q0123456789abcdef01234567",
+ sourceType: ChunkSourceType, want: true,
+ },
+ {
+ name: "image row with a suffix", chunkID: chunkID, sourceID: chunkID + "-image",
+ sourceType: ImageSourceType,
+ },
+ {name: "empty chunk", sourceID: "-q1", sourceType: ChunkSourceType},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := IsGeneratedQuestionSource(tc.chunkID, tc.sourceID, tc.sourceType); got != tc.want {
+ t.Fatalf("IsGeneratedQuestionSource() = %v, want %v", got, tc.want)
+ }
+ })
+ }
+}
+
+func TestQuestionGenerationOffIDs(t *testing.T) {
+ off := &KnowledgeBase{ID: "off", Type: KnowledgeBaseTypeDocument}
+ on := &KnowledgeBase{
+ ID: "on", Type: KnowledgeBaseTypeDocument,
+ QuestionGenerationConfig: &QuestionGenerationConfig{Enabled: true},
+ }
+ got := QuestionGenerationOffIDs([]*KnowledgeBase{off, on, nil})
+ if _, ok := got["off"]; !ok {
+ t.Fatal("closed document knowledge base was not marked")
+ }
+ if _, ok := got["on"]; ok {
+ t.Fatal("enabled knowledge base was marked off")
+ }
+}
+
+func TestQuestionGenerationDisabledLeavesFAQAlone(t *testing.T) {
+ faq := &KnowledgeBase{Type: KnowledgeBaseTypeFAQ}
+ if QuestionGenerationDisabled(faq) {
+ t.Fatal("FAQ similar-question rows must not be treated as generated questions")
+ }
+ off := &KnowledgeBase{Type: KnowledgeBaseTypeDocument}
+ if !QuestionGenerationDisabled(off) {
+ t.Fatal("a document knowledge base with no question config is off")
+ }
+ on := &KnowledgeBase{
+ Type: KnowledgeBaseTypeDocument,
+ QuestionGenerationConfig: &QuestionGenerationConfig{Enabled: true},
+ }
+ if QuestionGenerationDisabled(on) || !QuestionGenerationActive(on) {
+ t.Fatal("an enabled document knowledge base should keep generated questions")
+ }
+}
diff --git a/internal/types/retriever.go b/internal/types/retriever.go
index 1bbadd6c2..6ebaa26fd 100644
--- a/internal/types/retriever.go
+++ b/internal/types/retriever.go
@@ -48,6 +48,12 @@ type RetrieveParams struct {
KnowledgeIDs []string
// Tag IDs for filtering (used for FAQ priority filtering)
TagIDs []string
+ // ExcludeGeneratedQuestionKBIDs drops generated-question index rows for
+ // these knowledge bases before TopK is applied. Document knowledge bases
+ // with question generation turned off are listed here so historical
+ // question vectors stop occupying recall. FAQ similar-question rows share
+ // the source-id shape and must not be listed.
+ ExcludeGeneratedQuestionKBIDs []string
// Excluded knowledge IDs
ExcludeKnowledgeIDs []string
// Excluded chunk IDs
diff --git a/internal/types/search.go b/internal/types/search.go
index 29fe8b5bd..725127730 100644
--- a/internal/types/search.go
+++ b/internal/types/search.go
@@ -211,6 +211,11 @@ type SearchResult struct {
// ChunkMetadata stores chunk-level metadata (e.g., generated questions)
ChunkMetadata JSON `json:"chunk_metadata,omitempty"`
+ // SkipGeneratedQuestions keeps generated questions out of the rerank
+ // passage. Search sets it when the knowledge base's question generation
+ // switch is off, including bases that were turned off before the index
+ // rows were disabled.
+ SkipGeneratedQuestions bool `json:"-" gorm:"-"`
// ContextHeader is the chunk's heading breadcrumb, part of what was
// embedded (Chunk.EmbeddingContent) but not of Content. Internal only.
ContextHeader string `json:"-"`