Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(knowledge): drop generated questions when the switch is off (#4096)
AsanoReo6e91b671ee7b
diff --git a/internal/agent/tools/search_knowledge.go b/internal/agent/tools/search_knowledge.goindex 43da62aa9..3eb833d39 100644--- a/internal/agent/tools/search_knowledge.go+++ b/internal/agent/tools/search_knowledge.go@@ -858,6 +858,14 @@ func (t *SearchKnowledgeTool) rerankResults(}if t.knowledgeBaseService != nil {opts.LoadImage = t.knowledgeBaseService.ReadChunkImage+ if ids := types.SearchResultKnowledgeBaseIDs(rows); len(ids) > 0 {+ kbs, err := t.knowledgeBaseService.GetKnowledgeBasesByIDsOnly(ctx, ids)+ if err != nil {+ logger.Warnf(ctx, "[Tool][SearchKnowledge] Failed to read question generation config: %v", err)+ } else {+ types.ApplySkipGeneratedQuestions(rows, types.QuestionGenerationOffIDs(kbs))+ }+ }}res := reranking.Rerank(ctx, t.rerankModel, query, rows, opts)if res.Diagnostics.Outcome == types.RerankOutcomeModelError {diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.goindex cf28a9a27..9951a04f6 100644--- a/internal/application/repository/retriever/postgres/repository.go+++ b/internal/application/repository/retriever/postgres/repository.go@@ -270,6 +270,9 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,SQL: "(is_enabled IS NULL OR is_enabled = ?)",Vars: []interface{}{true},})+ if exclusion := generatedQuestionExclusionClause(params.ExcludeGeneratedQuestionKBIDs); exclusion != nil {+ conds = append(conds, exclusion)+ }conds = append(conds, clause.OrderBy{Columns: []clause.OrderByColumn{{Column: clause.Column{Name: "score"}, Desc: true},}})@@ -398,6 +401,19 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,// is_enabled filterwhereParts = append(whereParts, fmt.Sprintf("(is_enabled IS NULL OR is_enabled = $%d)", len(allVars)+1))allVars = append(allVars, true)+ if len(params.ExcludeGeneratedQuestionKBIDs) > 0 {+ placeholders := make([]string, len(params.ExcludeGeneratedQuestionKBIDs))+ paramStart := len(allVars) + 1+ for i, id := range params.ExcludeGeneratedQuestionKBIDs {+ placeholders[i] = fmt.Sprintf("$%d", paramStart+i)+ allVars = append(allVars, id)+ }+ whereParts = append(whereParts, fmt.Sprintf(+ "NOT (knowledge_base_id IN (%s) AND %s)",+ strings.Join(placeholders, ", "),+ types.GeneratedQuestionRowPredicate(""),+ ))+ }// Build WHERE clause stringwhereClause := ""@@ -773,3 +789,23 @@ func (g *pgRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMap malogger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk tag ID")return nil}++func generatedQuestionExclusionClause(kbIDs []string) clause.Expression {+ if len(kbIDs) == 0 {+ return nil+ }+ placeholders := make([]string, len(kbIDs))+ vars := make([]interface{}, len(kbIDs))+ for i, id := range kbIDs {+ placeholders[i] = "?"+ vars[i] = id+ }+ return clause.Expr{+ SQL: fmt.Sprintf(+ "NOT (knowledge_base_id IN (%s) AND %s)",+ strings.Join(placeholders, ", "),+ types.GeneratedQuestionRowPredicate(""),+ ),+ Vars: vars,+ }+}diff --git a/internal/application/repository/retriever/sqlite/repository.go b/internal/application/repository/retriever/sqlite/repository.goindex 5622df031..021b00bc6 100644--- a/internal/application/repository/retriever/sqlite/repository.go+++ b/internal/application/repository/retriever/sqlite/repository.go@@ -889,6 +889,17 @@ func buildFilterWhere(params types.RetrieveParams, tableAlias string) []whereClaargs: toInterfaceSlice(params.TagIDs),})}+ if len(params.ExcludeGeneratedQuestionKBIDs) > 0 {+ parts = append(parts, whereClause{+ clause: fmt.Sprintf(+ "NOT (%s.knowledge_base_id IN (%s) AND %s)",+ tableAlias,+ placeholders(len(params.ExcludeGeneratedQuestionKBIDs)),+ types.GeneratedQuestionRowPredicate(tableAlias),+ ),+ args: toInterfaceSlice(params.ExcludeGeneratedQuestionKBIDs),+ })+ }return parts}diff --git a/internal/application/repository/retriever/sqlite/repository_test.go b/internal/application/repository/retriever/sqlite/repository_test.goindex 736d2bb2d..3b36304a2 100644--- a/internal/application/repository/retriever/sqlite/repository_test.go+++ b/internal/application/repository/retriever/sqlite/repository_test.go@@ -115,6 +115,39 @@ func TestVectorRetrieveFiltersBeforeTopK(t *testing.T) {}}+func TestVectorRetrieveDropsGeneratedQuestionsBeforeTopK(t *testing.T) {+ repository := newSQLiteRetrieverTestRepository(t)+ const chunkID = "chunk-body"+ body := sqliteTestIndex(chunkID, "kb-off", "knowledge-1", "", true)+ body.SourceID = chunkID+ body.Content = "month end close steps"+ question := sqliteTestIndex(chunkID, "kb-off", "knowledge-1", "", true)+ question.SourceID = chunkID + "-q1"+ question.Content = "how do I close the books"+ other := sqliteTestIndex("other", "kb-on", "knowledge-2", "", true)+ other.SourceID = "other-q1"+ other.Content = "question in a knowledge base that still generates questions"++ saveSQLiteTestVector(t, repository, question, []float32{1, 0})+ saveSQLiteTestVector(t, repository, body, []float32{0.2, 0.8})+ saveSQLiteTestVector(t, repository, other, []float32{0.9, 0.1})++ results, err := repository.vectorRetrieve(context.Background(), types.RetrieveParams{+ Embedding: []float32{1, 0},+ KnowledgeBaseIDs: []string{"kb-off", "kb-on"},+ ExcludeGeneratedQuestionKBIDs: []string{"kb-off"},+ TopK: 2,+ RetrieverType: types.VectorRetrieverType,+ })+ require.NoError(t, err)+ require.Len(t, results, 1)+ got := make([]string, 0, len(results[0].Results))+ for _, hit := range results[0].Results {+ got = append(got, hit.SourceID)+ }+ assert.ElementsMatch(t, []string{chunkID, "other-q1"}, got)+}+func TestVectorRetrieveZeroThresholdDoesNotFilter(t *testing.T) {repository := newSQLiteRetrieverTestRepository(t)saveSQLiteTestVector(t, repository,diff --git a/internal/application/service/chat_pipeline/rerank.go b/internal/application/service/chat_pipeline/rerank.goindex 4fda8f44f..c2defaa6d 100644--- a/internal/application/service/chat_pipeline/rerank.go+++ b/internal/application/service/chat_pipeline/rerank.go@@ -104,6 +104,7 @@ func (p *PluginRerank) OnEvent(ctx context.Context,ctx = rerankCtxlogRerankInputScoreSample(ctx, chatManage.SearchResult)+ p.skipDisabledGeneratedQuestions(ctx, chatManage.SearchResult)opts := reranking.Options{Threshold: chatManage.RerankThreshold,@@ -222,6 +223,26 @@ func buildRerankSpanOutput(res *reranking.Result, chatManage *types.ChatManage)return out}+// skipDisabledGeneratedQuestions applies the live question-generation switch+// to passages that were added after hybrid search, such as graph hits.+func (p *PluginRerank) skipDisabledGeneratedQuestions(ctx context.Context, results []*types.SearchResult) {+ if p.kbService == nil || len(results) == 0 {+ return+ }+ ids := types.SearchResultKnowledgeBaseIDs(results)+ if len(ids) == 0 {+ return+ }+ kbs, err := p.kbService.GetKnowledgeBasesByIDsOnly(ctx, ids)+ if err != nil {+ pipelineWarn(ctx, "Rerank", "question_generation_lookup", map[string]interface{}{+ "error": err.Error(),+ })+ return+ }+ types.ApplySkipGeneratedQuestions(results, types.QuestionGenerationOffIDs(kbs))+}+func logRerankInputScoreSample(ctx context.Context, results []*types.SearchResult) {const maxLogRows = 8limit := min(maxLogRows, len(results))diff --git a/internal/application/service/chat_pipeline/rerank_test.go b/internal/application/service/chat_pipeline/rerank_test.goindex 78f4c0c6a..eaf850067 100644--- a/internal/application/service/chat_pipeline/rerank_test.go+++ b/internal/application/service/chat_pipeline/rerank_test.go@@ -309,3 +309,54 @@ func TestMergeFallbackWithoutLossRecordsNothing(t *testing.T) {t.Fatalf("fallback input = %d, truncation = %+v", len(got), cm.Truncation)}}++type questionLookupKB struct {+ interfaces.KnowledgeBaseService+ calls [][]string+ kbs []*types.KnowledgeBase+ err error+}++func (s *questionLookupKB) GetKnowledgeBasesByIDsOnly(_ context.Context, ids []string) ([]*types.KnowledgeBase, error) {+ s.calls = append(s.calls, append([]string(nil), ids...))+ return s.kbs, s.err+}++func TestSkipDisabledGeneratedQuestionsLooksUpOnce(t *testing.T) {+ kb := &questionLookupKB{kbs: []*types.KnowledgeBase{+ {ID: "off", Type: types.KnowledgeBaseTypeDocument},+ {+ ID: "on", Type: types.KnowledgeBaseTypeDocument,+ QuestionGenerationConfig: &types.QuestionGenerationConfig{Enabled: true},+ },+ {ID: "faq", Type: types.KnowledgeBaseTypeFAQ},+ }}+ results := []*types.SearchResult{+ {ID: "a", KnowledgeBaseID: "off"},+ {ID: "b", KnowledgeBaseID: "on"},+ {ID: "c", KnowledgeBaseID: "faq"},+ {ID: "d", KnowledgeBaseID: "off"},+ }+ (&PluginRerank{kbService: kb}).skipDisabledGeneratedQuestions(context.Background(), results)+ if len(kb.calls) != 1 {+ t.Fatalf("knowledge base lookups = %d, want 1", len(kb.calls))+ }+ if len(kb.calls[0]) != 3 {+ t.Fatalf("looked up %v, want the three distinct knowledge bases", kb.calls[0])+ }+ if !results[0].SkipGeneratedQuestions || results[1].SkipGeneratedQuestions ||+ results[2].SkipGeneratedQuestions || !results[3].SkipGeneratedQuestions {+ t.Fatalf("skip flags = %v %v %v %v",+ results[0].SkipGeneratedQuestions, results[1].SkipGeneratedQuestions,+ results[2].SkipGeneratedQuestions, results[3].SkipGeneratedQuestions)+ }+}++func TestSkipDisabledGeneratedQuestionsLeavesFlagsUnsetOnLookupError(t *testing.T) {+ kb := &questionLookupKB{err: errors.New("db down")}+ results := []*types.SearchResult{{ID: "a", KnowledgeBaseID: "off"}}+ (&PluginRerank{kbService: kb}).skipDisabledGeneratedQuestions(context.Background(), results)+ if results[0].SkipGeneratedQuestions {+ t.Fatal("a failed knowledge base lookup marked the passage")+ }+}diff --git a/internal/application/service/knowledgebase_generated_questions.go b/internal/application/service/knowledgebase_generated_questions.gonew file mode 100644index 000000000..2c83c1e01--- /dev/null+++ b/internal/application/service/knowledgebase_generated_questions.go@@ -0,0 +1,24 @@+package service++import (+ "context"++ "github.com/Tencent/WeKnora/internal/logger"+ "github.com/Tencent/WeKnora/internal/types"+)++// markSkipGeneratedQuestions sets SkipGeneratedQuestions from the live switch+// so rerank passages omit historical questions. A lookup failure leaves the+// flag unset.+func (s *knowledgeBaseService) markSkipGeneratedQuestions(ctx context.Context, results []*types.SearchResult) {+ ids := types.SearchResultKnowledgeBaseIDs(results)+ if len(ids) == 0 {+ return+ }+ kbs, err := s.repo.GetKnowledgeBaseByIDs(ctx, ids)+ if err != nil {+ logger.Warnf(ctx, "Failed to read question generation config for rerank: %v", err)+ return+ }+ types.ApplySkipGeneratedQuestions(results, types.QuestionGenerationOffIDs(kbs))+}diff --git a/internal/application/service/knowledgebase_search.go b/internal/application/service/knowledgebase_search.goindex e4ef91181..2e6f79d6a 100644--- a/internal/application/service/knowledgebase_search.go+++ b/internal/application/service/knowledgebase_search.go@@ -439,6 +439,15 @@ func (s *knowledgeBaseService) buildRetrievalParams(// scope happens to include them (e.g. KBSelectionMode=all picking up a// wiki-only KB).var faqVectorKBIDs, docVectorKBIDs, docKeywordKBIDs []string+ // Document KBs with question generation off must not spend TopK on+ // historical question vectors. FAQ KBs are absent from this list: their+ // similar-question rows use the same source-id shape.+ var questionOffKBIDs []string+ for _, kb := range groupKBs {+ if types.QuestionGenerationDisabled(kb) {+ questionOffKBIDs = append(questionOffKBIDs, kb.ID)+ }+ }// The query is embedded with the primary's model, or with the first// vector KB of the group when the primary has none (a wiki primary// searched together with document KBs): asking the primary for its empty@@ -478,15 +487,16 @@ func (s *knowledgeBaseService) buildRetrievalParams(appendVectorParams := func(kbIDs []string, knowledgeType string) {retrieveParams = append(retrieveParams, types.RetrieveParams{- Query: params.QueryText,- Embedding: queryEmbedding,- KnowledgeBaseIDs: kbIDs,- TopK: matchCount,- Threshold: params.VectorThreshold,- RetrieverType: types.VectorRetrieverType,- KnowledgeIDs: params.KnowledgeIDs,- TagIDs: params.TagIDs,- KnowledgeType: knowledgeType,+ Query: params.QueryText,+ Embedding: queryEmbedding,+ KnowledgeBaseIDs: kbIDs,+ TopK: matchCount,+ Threshold: params.VectorThreshold,+ RetrieverType: types.VectorRetrieverType,+ KnowledgeIDs: params.KnowledgeIDs,+ TagIDs: params.TagIDs,+ KnowledgeType: knowledgeType,+ ExcludeGeneratedQuestionKBIDs: generatedQuestionKBIDs(kbIDs, questionOffKBIDs),})}@@ -508,13 +518,14 @@ func (s *knowledgeBaseService) buildRetrievalParams(len(docKeywordKBIDs) > 0 {logger.Info(ctx, "Keyword retrieval supported, preparing keyword retrieval parameters")retrieveParams = append(retrieveParams, types.RetrieveParams{- Query: params.QueryText,- KnowledgeBaseIDs: docKeywordKBIDs,- TopK: matchCount,- Threshold: params.KeywordThreshold,- RetrieverType: types.KeywordsRetrieverType,- KnowledgeIDs: params.KnowledgeIDs,- TagIDs: params.TagIDs,+ Query: params.QueryText,+ KnowledgeBaseIDs: docKeywordKBIDs,+ TopK: matchCount,+ Threshold: params.KeywordThreshold,+ RetrieverType: types.KeywordsRetrieverType,+ KnowledgeIDs: params.KnowledgeIDs,+ TagIDs: params.TagIDs,+ ExcludeGeneratedQuestionKBIDs: generatedQuestionKBIDs(docKeywordKBIDs, questionOffKBIDs),})logger.Info(ctx, "Keyword retrieval parameters setup completed")}@@ -522,6 +533,25 @@ func (s *knowledgeBaseService) buildRetrievalParams(return retrieveParams, nil}+// generatedQuestionKBIDs keeps the off-switch ids that this retrieval param+// actually searches. An empty result leaves the SQL filter off.+func generatedQuestionKBIDs(searched, disabled []string) []string {+ if len(searched) == 0 || len(disabled) == 0 {+ return nil+ }+ off := make(map[string]struct{}, len(disabled))+ for _, id := range disabled {+ off[id] = struct{}{}+ }+ var matched []string+ for _, id := range searched {+ if _, ok := off[id]; ok {+ matched = append(matched, id)+ }+ }+ return matched+}+// resolveQueryEmbedding returns the query embedding for a store group. It// reuses params.QueryEmbedding when the caller pre-computed it (the common// path — HybridSearch embeds once before fan-out), otherwise it embeds thediff --git a/internal/application/service/knowledgebase_search_embedding_kb_test.go b/internal/application/service/knowledgebase_search_embedding_kb_test.goindex f9540ff2d..3f475710d 100644--- a/internal/application/service/knowledgebase_search_embedding_kb_test.go+++ b/internal/application/service/knowledgebase_search_embedding_kb_test.go@@ -44,3 +44,58 @@ func TestBuildRetrievalParamsEmbedsWithAVectorKB(t *testing.T) {require.NotEmpty(t, params)require.Len(t, params[0].Embedding, 768)}++// Document knowledge bases with question generation off must drop historical+// question rows before TopK. FAQ knowledge bases use the same source-id shape+// for their own similar questions, so they stay off that list.+func TestBuildRetrievalParamsExcludesGeneratedQuestionsOnlyForDocumentKBs(t *testing.T) {+ s := &knowledgeBaseService{}+ engine := buildBoundComposite(t, &fakeRetrieveEngineService{+ engineType: types.PostgresRetrieverEngineType,+ support: []types.RetrieverType{types.VectorRetrieverType, types.KeywordsRetrieverType},+ })+ docOff := &types.KnowledgeBase{+ ID: "doc-off", TenantID: 1, Type: types.KnowledgeBaseTypeDocument,+ EmbeddingModelID: "embed-1",+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true, KeywordEnabled: true},+ }+ docOn := &types.KnowledgeBase{+ ID: "doc-on", TenantID: 1, Type: types.KnowledgeBaseTypeDocument,+ EmbeddingModelID: "embed-1",+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true, KeywordEnabled: true},+ QuestionGenerationConfig: &types.QuestionGenerationConfig{Enabled: true},+ }+ faq := &types.KnowledgeBase{+ ID: "faq", TenantID: 1, Type: types.KnowledgeBaseTypeFAQ,+ EmbeddingModelID: "embed-1",+ IndexingStrategy: types.IndexingStrategy{VectorEnabled: true, KeywordEnabled: true},+ }+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))++ params, err := s.buildRetrievalParams(ctx, engine, docOff,+ []*types.KnowledgeBase{docOff, docOn, faq},+ types.SearchParams{QueryText: "q", QueryEmbedding: []float32{1}}, 5)+ require.NoError(t, err)++ var docVector, faqVector, keyword *types.RetrieveParams+ for i := range params {+ p := ¶ms[i]+ switch {+ case p.RetrieverType == types.VectorRetrieverType && p.KnowledgeType == types.KnowledgeTypeFAQ:+ faqVector = p+ case p.RetrieverType == types.VectorRetrieverType:+ docVector = p+ case p.RetrieverType == types.KeywordsRetrieverType:+ keyword = p+ }+ }+ require.NotNil(t, docVector)+ require.NotNil(t, faqVector)+ require.NotNil(t, keyword)+ require.Equal(t, []string{"doc-off"}, docVector.ExcludeGeneratedQuestionKBIDs)+ require.Empty(t, faqVector.ExcludeGeneratedQuestionKBIDs)+ require.Equal(t, []string{"doc-off"}, keyword.ExcludeGeneratedQuestionKBIDs)+ require.ElementsMatch(t, []string{"doc-off", "doc-on"}, docVector.KnowledgeBaseIDs)+ require.Equal(t, []string{"faq"}, faqVector.KnowledgeBaseIDs)+ require.NotContains(t, keyword.KnowledgeBaseIDs, "faq")+}diff --git a/internal/application/service/knowledgebase_search_rerank.go b/internal/application/service/knowledgebase_search_rerank.goindex e111af5e4..45008dc32 100644--- a/internal/application/service/knowledgebase_search_rerank.go+++ b/internal/application/service/knowledgebase_search_rerank.go@@ -181,6 +181,7 @@ func (s *knowledgeBaseService) rerankCandidates(diag.Error = err.Error()return retrievalOrder}+ s.markSkipGeneratedQuestions(ctx, candidates)res := reranking.Rerank(ctx, model, query, candidates, reranking.Options{Threshold: threshold,diff --git a/internal/handler/initialization.go b/internal/handler/initialization.goindex 91db96aa1..8060ac5ea 100644--- a/internal/handler/initialization.go+++ b/internal/handler/initialization.go@@ -417,7 +417,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {return}- // 更新问题生成配置+ // 更新问题生成配置。检索是否使用问句由查询时的开关决定,保存设置不改索引。if req.QuestionGeneration.Enabled {questionCount := req.QuestionGeneration.QuestionCountif questionCount <= 0 {diff --git a/internal/reranking/passage.go b/internal/reranking/passage.goindex fd3216568..23a6632b4 100644--- a/internal/reranking/passage.go+++ b/internal/reranking/passage.go@@ -166,6 +166,8 @@ func cleanPassageProse(text string) string {// EnrichedPassage is the cleaned chunk body plus the image captions, OCR text// and generated questions attached to it. MMR compares candidates on this// text, so chunks of one document are not made similar by a shared title.+// Generated questions are omitted when the knowledge base has turned question+// generation off.func EnrichedPassage(ctx context.Context, result *types.SearchResult) string {combinedText := CleanPassage(result.Content)var enrichments []string@@ -194,8 +196,10 @@ func EnrichedPassage(ctx context.Context, result *types.SearchResult) string {var docMeta types.DocumentChunkMetadataif err := json.Unmarshal(result.ChunkMetadata, &docMeta); err != nil {logger.Warnf(ctx, "[Rerank] Failed to parse chunk metadata of chunk %s: %v", result.ID, err)- } else if questionStrings := docMeta.GetQuestionStrings(); len(questionStrings) > 0 {- enrichments = append(enrichments, strings.Join(questionStrings, "; "))+ } else if includeGeneratedQuestions(result) {+ if questionStrings := docMeta.GetQuestionStrings(); len(questionStrings) > 0 {+ enrichments = append(enrichments, strings.Join(questionStrings, "; "))+ }}}@@ -208,6 +212,10 @@ func EnrichedPassage(ctx context.Context, result *types.SearchResult) string {return combinedText + strings.Join(enrichments, "\n")}+func includeGeneratedQuestions(result *types.SearchResult) bool {+ return result == nil || !result.SkipGeneratedQuestions+}+// ModelPassage is the text the rerank model scores: the document title// followed by the enriched chunk. A chunk rarely restates what its document// is about, so without the title a passage from "Show HN: Echo - ... usingdiff --git a/internal/reranking/passage_test.go b/internal/reranking/passage_test.goindex 50fee35f2..ba0dca74c 100644--- a/internal/reranking/passage_test.go+++ b/internal/reranking/passage_test.go@@ -29,6 +29,29 @@ func TestGetEnrichedPassageKeepsQuestionsFromEarlierRevision(t *testing.T) {}}+func TestEnrichedPassageOmitsQuestionsWhenGenerationIsOff(t *testing.T) {+ metadata, err := json.Marshal(types.DocumentChunkMetadata{+ GeneratedQuestions: []types.GeneratedQuestion{{+ ID: "q1", Question: "stale generated question",+ }},+ })+ if err != nil {+ t.Fatal(err)+ }+ result := &types.SearchResult{+ Content: "chunk body",+ ChunkMetadata: types.JSON(metadata),+ SkipGeneratedQuestions: true,+ }+ passage := EnrichedPassage(context.Background(), result)+ if strings.Contains(passage, "stale generated question") {+ t.Fatalf("disabled knowledge base still put the question in the passage: %q", passage)+ }+ if !strings.Contains(passage, "chunk body") {+ t.Fatalf("passage dropped the chunk body: %q", passage)+ }+}+func TestGetEnrichedPassageKeepsCodeAndMathCandidates(t *testing.T) {tests := []struct {name stringdiff --git a/internal/types/generated_question_retrieval.go b/internal/types/generated_question_retrieval.gonew file mode 100644index 000000000..2d6acdd77--- /dev/null+++ b/internal/types/generated_question_retrieval.go@@ -0,0 +1,98 @@+package types++import "fmt"++// QuestionGenerationActive reports whether a document knowledge base still+// wants generated questions in retrieval. FAQ knowledge bases index their own+// similar questions under the same source-id shape and are never treated as+// this switch.+func QuestionGenerationActive(kb *KnowledgeBase) bool {+ return kb != nil &&+ kb.Type != KnowledgeBaseTypeFAQ &&+ kb.QuestionGenerationConfig != nil &&+ kb.QuestionGenerationConfig.Enabled+}++// QuestionGenerationDisabled reports whether generated-question rows for a+// document knowledge base must stay out of recall and rerank. A nil config is+// off: question generation only runs when the switch is explicitly enabled.+func QuestionGenerationDisabled(kb *KnowledgeBase) bool {+ return kb != nil && kb.Type != KnowledgeBaseTypeFAQ && !QuestionGenerationActive(kb)+}++// IsGeneratedQuestionSource reports whether an index row is a generated+// question for its parent chunk. Rows are written as+// source_id = chunk_id + "-" + question id (or the hashed "-q" form) with+// source_type chunk. The chunk body itself uses source_id = chunk_id.+// Identification does not use the length of source_id.+func IsGeneratedQuestionSource(chunkID, sourceID string, sourceType SourceType) bool {+ if sourceType != ChunkSourceType || chunkID == "" || sourceID == "" || sourceID == chunkID {+ return false+ }+ return len(sourceID) > len(chunkID)+1 && sourceID[:len(chunkID)+1] == chunkID+"-"+}++// GeneratedQuestionRowPredicate is the SQL form of IsGeneratedQuestionSource.+// alias is the table qualifier without a dot, or empty for an unqualified+// column. source_type is inlined; chunk ids are ASCII so length and substr+// agree on PostgreSQL and SQLite.+func GeneratedQuestionRowPredicate(alias string) string {+ col := func(name string) string {+ if alias == "" {+ return name+ }+ return alias + "." + name+ }+ return fmt.Sprintf(+ "%s = %d AND %s <> '' AND %s <> %s AND substr(%s, 1, length(%s) + 1) = %s || '-'",+ col("source_type"), int(ChunkSourceType),+ col("chunk_id"),+ col("source_id"), col("chunk_id"),+ col("source_id"), col("chunk_id"), col("chunk_id"),+ )+}++// SearchResultKnowledgeBaseIDs returns the distinct knowledge base IDs on results.+func SearchResultKnowledgeBaseIDs(results []*SearchResult) []string {+ seen := make(map[string]struct{}, len(results))+ ids := make([]string, 0, len(results))+ for _, result := range results {+ if result == nil || result.KnowledgeBaseID == "" {+ continue+ }+ if _, ok := seen[result.KnowledgeBaseID]; ok {+ continue+ }+ seen[result.KnowledgeBaseID] = struct{}{}+ ids = append(ids, result.KnowledgeBaseID)+ }+ return ids+}++// QuestionGenerationOffIDs returns document knowledge bases whose generated+// questions must stay out of rerank passages.+func QuestionGenerationOffIDs(kbs []*KnowledgeBase) map[string]struct{} {+ disabled := make(map[string]struct{})+ for _, kb := range kbs {+ if QuestionGenerationDisabled(kb) {+ disabled[kb.ID] = struct{}{}+ }+ }+ return disabled+}++// ApplySkipGeneratedQuestions marks rerank passages whose knowledge base has+// question generation turned off. disabled maps knowledge base IDs.+func ApplySkipGeneratedQuestions(results []*SearchResult, disabled map[string]struct{}) {+ if len(disabled) == 0 {+ return+ }+ for _, result := range results {+ if result == nil {+ continue+ }+ if _, ok := disabled[result.KnowledgeBaseID]; ok {+ result.SkipGeneratedQuestions = true+ }+ }+}diff --git a/internal/types/generated_question_retrieval_test.go b/internal/types/generated_question_retrieval_test.gonew file mode 100644index 000000000..99b79b104--- /dev/null+++ b/internal/types/generated_question_retrieval_test.go@@ -0,0 +1,69 @@+package types++import "testing"++func TestIsGeneratedQuestionSource(t *testing.T) {+ const chunkID = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"+ cases := []struct {+ name string+ chunkID string+ sourceID string+ sourceType SourceType+ want bool+ }{+ {name: "body row", chunkID: chunkID, sourceID: chunkID, sourceType: ChunkSourceType},+ {+ name: "question row", chunkID: chunkID, sourceID: chunkID + "-q123",+ sourceType: ChunkSourceType, want: true,+ },+ {+ name: "hashed question row", chunkID: chunkID, sourceID: chunkID + "-q0123456789abcdef01234567",+ sourceType: ChunkSourceType, want: true,+ },+ {+ name: "image row with a suffix", chunkID: chunkID, sourceID: chunkID + "-image",+ sourceType: ImageSourceType,+ },+ {name: "empty chunk", sourceID: "-q1", sourceType: ChunkSourceType},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := IsGeneratedQuestionSource(tc.chunkID, tc.sourceID, tc.sourceType); got != tc.want {+ t.Fatalf("IsGeneratedQuestionSource() = %v, want %v", got, tc.want)+ }+ })+ }+}++func TestQuestionGenerationOffIDs(t *testing.T) {+ off := &KnowledgeBase{ID: "off", Type: KnowledgeBaseTypeDocument}+ on := &KnowledgeBase{+ ID: "on", Type: KnowledgeBaseTypeDocument,+ QuestionGenerationConfig: &QuestionGenerationConfig{Enabled: true},+ }+ got := QuestionGenerationOffIDs([]*KnowledgeBase{off, on, nil})+ if _, ok := got["off"]; !ok {+ t.Fatal("closed document knowledge base was not marked")+ }+ if _, ok := got["on"]; ok {+ t.Fatal("enabled knowledge base was marked off")+ }+}++func TestQuestionGenerationDisabledLeavesFAQAlone(t *testing.T) {+ faq := &KnowledgeBase{Type: KnowledgeBaseTypeFAQ}+ if QuestionGenerationDisabled(faq) {+ t.Fatal("FAQ similar-question rows must not be treated as generated questions")+ }+ off := &KnowledgeBase{Type: KnowledgeBaseTypeDocument}+ if !QuestionGenerationDisabled(off) {+ t.Fatal("a document knowledge base with no question config is off")+ }+ on := &KnowledgeBase{+ Type: KnowledgeBaseTypeDocument,+ QuestionGenerationConfig: &QuestionGenerationConfig{Enabled: true},+ }+ if QuestionGenerationDisabled(on) || !QuestionGenerationActive(on) {+ t.Fatal("an enabled document knowledge base should keep generated questions")+ }+}diff --git a/internal/types/retriever.go b/internal/types/retriever.goindex 1bbadd6c2..6ebaa26fd 100644--- a/internal/types/retriever.go+++ b/internal/types/retriever.go@@ -48,6 +48,12 @@ type RetrieveParams struct {KnowledgeIDs []string// Tag IDs for filtering (used for FAQ priority filtering)TagIDs []string+ // ExcludeGeneratedQuestionKBIDs drops generated-question index rows for+ // these knowledge bases before TopK is applied. Document knowledge bases+ // with question generation turned off are listed here so historical+ // question vectors stop occupying recall. FAQ similar-question rows share+ // the source-id shape and must not be listed.+ ExcludeGeneratedQuestionKBIDs []string// Excluded knowledge IDsExcludeKnowledgeIDs []string// Excluded chunk IDsdiff --git a/internal/types/search.go b/internal/types/search.goindex 29fe8b5bd..725127730 100644--- a/internal/types/search.go+++ b/internal/types/search.go@@ -211,6 +211,11 @@ type SearchResult struct {// ChunkMetadata stores chunk-level metadata (e.g., generated questions)ChunkMetadata JSON `json:"chunk_metadata,omitempty"`+ // SkipGeneratedQuestions keeps generated questions out of the rerank+ // passage. Search sets it when the knowledge base's question generation+ // switch is off, including bases that were turned off before the index+ // rows were disabled.+ SkipGeneratedQuestions bool `json:"-" gorm:"-"`// ContextHeader is the chunk's heading breadcrumb, part of what was// embedded (Chunk.EmbeddingContent) but not of Content. Internal only.ContextHeader string `json:"-"`