Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Feature: add knowledge_compilation_scope (#20917)

Wang Qi5d25a3f85930
diff --git a/docker/.env b/docker/.env
index 0d6aa56d37..bd4a8abae4 100644
--- a/docker/.env
+++ b/docker/.env
@@ -445,7 +445,7 @@ ENABLE_REGISTER=1
# RAGFlow. Current Sandbox source rejects /run without configured authentication;
# verify the behavior of the published image version you deploy.
# Generate token with: openssl rand -hex 32
-# SANDBOX_EXECUTOR_MANAGER_ALLOW_UNAUTHENTICATED=true
+# SANDBOX_EXECUTOR_MANAGER_ALLOW_UNAUTHENTICATED=false
# SANDBOX_EXECUTOR_MANAGER_API_TOKEN=
# Docker network used for sandbox runner containers. "none" (default) means
diff --git a/docs/references/http_api_reference.md b/docs/references/http_api_reference.md
index 937a029d49..7ef2186778 100644
--- a/docs/references/http_api_reference.md
+++ b/docs/references/http_api_reference.md
@@ -2486,6 +2486,7 @@ Retrieves chunks from specified datasets.
- `"metadata_condition"`: `object`
- `"use_kg"`: `boolean`
- `"include_knowledge_compilation"`: `boolean`
+ - `"knowledge_compilation_scope"`: `string`
##### Request example
@@ -2549,6 +2550,9 @@ curl --request POST \
The number of initial retrieval candidates to rank. It must be at least `"page"` multiplied by `"page_size"`. Defaults to `64`.
- `"include_knowledge_compilation"`: `boolean`
Whether to include knowledge-compilation chunks in the results. Defaults to `true`.
+- `"knowledge_compilation_scope"`: `string`
+ Selects which searchable knowledge-compilation chunks to include: `"dataset"` for dataset artifacts, `"document"` for document artifacts, or `"all"` for both. Defaults to `"dataset"` when omitted, `null`, or an empty string. Ordinary source chunks remain eligible. When `"include_knowledge_compilation"` is `false`, all compiled chunks are excluded regardless of scope. Unsupported scope values and non-string values return an argument error. Intermediate wiki rows and disabled chunks remain excluded.
+ With `"document_ids"` or metadata filtering, document artifacts and source chunks are limited to the selected documents; dataset artifacts must reference a selected document in their `source_doc_ids`. This parameter is also supported by `POST /api/v1/datasets/search` and `POST /api/v1/datasets/{dataset_id}/search`.
- `"use_kg"`: `boolean`
Whether to search chunks related to the generated knowledge graph for multi-hop queries. Defaults to `False`. Before enabling this, ensure you have successfully constructed a knowledge graph for the specified datasets. See [here](../guides/knowledge_compilation/built_in_templates_and_dedicated_configuration.md#graph) for details.
- `"rerank_id"`: `string`
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index fa5a82d1da..3693e56231 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -302,6 +302,9 @@ func (e *Engine) insertChunks(ctx context.Context, chunks []map[string]interface
// Document line: work with a copy to avoid mutating the original
docCopy := copyFields(doc)
docCopy["kb_id"] = datasetID
+ if !isMemoryIndex && docCopy["available_int"] == nil {
+ docCopy["available_int"] = 1
+ }
if raw, ok := formatOrderedTagFeas(docCopy["tag_feas"]); ok {
docCopy["tag_feas"] = raw
}
diff --git a/internal/engine/elasticsearch/chunk_readback_test.go b/internal/engine/elasticsearch/chunk_readback_test.go
index 6c52510c02..a7b3540ef9 100644
--- a/internal/engine/elasticsearch/chunk_readback_test.go
+++ b/internal/engine/elasticsearch/chunk_readback_test.go
@@ -88,7 +88,10 @@ func TestInsertChunks_WritesIngestionShape(t *testing.T) {
"available_int": int(0),
}
- if _, err := engine.InsertChunks(t.Context(), []map[string]interface{}{chunk}, "ragflow_chunk_readback_test", "kb-1"); err != nil {
+ visible := map[string]interface{}{"id": "visible", "doc_id": "doc-1", "content_with_weight": "visible source"}
+ nullAvailability := map[string]interface{}{"id": "null", "doc_id": "doc-1", "available_int": nil}
+ enabled := map[string]interface{}{"id": "enabled", "doc_id": "doc-1", "available_int": 1}
+ if _, err := engine.InsertChunks(t.Context(), []map[string]interface{}{chunk, visible, nullAvailability, enabled}, "ragflow_chunk_readback_test", "kb-1"); err != nil {
t.Fatalf("InsertChunks: %v", err)
}
@@ -112,4 +115,17 @@ func TestInsertChunks_WritesIngestionShape(t *testing.T) {
assertStoredField(t, doc, "available_int", float64(0))
// InsertChunks overrides kb_id with datasetID on write.
assertStoredField(t, doc, "kb_id", "kb-1")
+ if len(lines) != 8 {
+ t.Fatalf("bulk body has %d lines, want 8", len(lines))
+ }
+ for _, line := range []string{lines[3], lines[5], lines[7]} {
+ var stored map[string]interface{}
+ if err := json.Unmarshal([]byte(line), &stored); err != nil {
+ t.Fatal(err)
+ }
+ assertStoredField(t, stored, "available_int", float64(1))
+ }
+ if _, ok := visible["available_int"]; ok || nullAvailability["available_int"] != nil {
+ t.Fatal("InsertChunks must not modify the caller's availability fields")
+ }
}
diff --git a/internal/handler/dataset.go b/internal/handler/dataset.go
index d4e3408128..0b24593664 100644
--- a/internal/handler/dataset.go
+++ b/internal/handler/dataset.go
@@ -1130,12 +1130,12 @@ func (h *DatasetsHandler) SearchDatasets(c *gin.Context) {
return
}
if req.DatasetIDs == nil {
- common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "kb_id is required")
+ common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "dataset_ids is required")
return
}
if len(req.DatasetIDs) == 0 {
- common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "kb_id array cannot be empty")
+ common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "dataset_ids array cannot be empty")
return
}
if err := validateSearchDatasetsRequest(&req); err != nil {
@@ -1226,6 +1226,9 @@ func (h *DatasetsHandler) SearchDataset(c *gin.Context) {
}
func validateSearchDatasetsRequest(req *service.SearchDatasetsRequest) error {
+ if err := service.ValidateKnowledgeCompilationScope(req.KnowledgeCompilationScope); err != nil {
+ return err
+ }
vectorSimilarityWeight, err := service.ResolveVectorSimilarityWeight(req.KeywordsSimilarityWeight, req.VectorSimilarityWeight)
if err != nil {
return err
@@ -1235,6 +1238,9 @@ func validateSearchDatasetsRequest(req *service.SearchDatasetsRequest) error {
}
func validateSearchDatasetRequest(req *service.SearchDatasetRequest) error {
+ if err := service.ValidateKnowledgeCompilationScope(req.KnowledgeCompilationScope); err != nil {
+ return err
+ }
vectorSimilarityWeight, err := service.ResolveVectorSimilarityWeight(req.KeywordsSimilarityWeight, req.VectorSimilarityWeight)
if err != nil {
return err
diff --git a/internal/handler/dataset_search_test.go b/internal/handler/dataset_search_test.go
index 6e73ef92ba..ea64af1a6a 100644
--- a/internal/handler/dataset_search_test.go
+++ b/internal/handler/dataset_search_test.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"math"
"net/http"
"net/http/httptest"
@@ -214,6 +215,9 @@ func TestDatasetsHandlerSearchDatasetsValidationErrorsUseArgumentEnvelope(t *tes
{name: "knn candidates below default top k", body: `{"question":"hello","dataset_ids":["ds-1"],"knn_num_candidates":10}`},
{name: "knn candidates below top k", body: `{"question":"hello","dataset_ids":["ds-1"],"knn_top_k":8,"knn_num_candidates":7}`},
{name: "invalid threshold", body: `{"question":"hello","dataset_ids":["ds-1"],"similarity_threshold":1.1}`},
+ {name: "invalid compilation scope", body: `{"question":"hello","dataset_ids":["ds-1"],"knowledge_compilation_scope":"doc"}`},
+ {name: "compilation scope is an array", body: `{"question":"hello","dataset_ids":["ds-1"],"knowledge_compilation_scope":["dataset"]}`},
+ {name: "compilation scope is non-string", body: `{"question":"hello","dataset_ids":["ds-1"],"knowledge_compilation_scope":1}`},
}
for _, tt := range tests {
@@ -254,6 +258,74 @@ func TestValidateSearchParamsKNNBounds(t *testing.T) {
}
}
+func TestDatasetsHandlerKnowledgeCompilationScope(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ for _, single := range []bool{false, true} {
+ for _, tt := range []struct {
+ scope string
+ invalid bool
+ }{
+ {scope: `"dataset"`},
+ {scope: `"document"`},
+ {scope: `"all"`},
+ {scope: `""`},
+ {scope: `null`},
+ {scope: ""},
+ {scope: `"invalid"`, invalid: true},
+ {scope: `["dataset"]`, invalid: true},
+ {scope: `[]`, invalid: true},
+ {scope: `1`, invalid: true},
+ {scope: `{}`, invalid: true},
+ } {
+ t.Run(fmt.Sprintf("single=%v/scope=%s", single, tt.scope), func(t *testing.T) {
+ fakeSingle := &fakeSearchDatasetService{resp: &service.SearchDatasetsResponse{}}
+ fakeMulti := &fakeSearchDatasetsService{resp: &service.SearchDatasetsResponse{}}
+ h := &DatasetsHandler{searchDatasetService: fakeSingle, searchDatasetsService: fakeMulti}
+ rec := httptest.NewRecorder()
+ body := `{"question":"hello","dataset_ids":["ds-1"],"include_knowledge_compilation":true`
+ if tt.scope != "" {
+ body += `,"knowledge_compilation_scope":` + tt.scope
+ }
+ body += `}`
+ req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(body))
+ req.Header.Set("Content-Type", "application/json")
+ c, _ := gin.CreateTestContext(rec)
+ c.Request = req
+ c.Set("user", &entity.User{ID: "user-1"})
+ if single {
+ c.Params = gin.Params{{Key: "dataset_id", Value: "ds-1"}}
+ h.SearchDataset(c)
+ } else {
+ h.SearchDatasets(c)
+ }
+ response := decodeSearchResponse(t, rec)
+ if tt.invalid {
+ if response["code"] != float64(common.CodeArgumentError) || fakeSingle.req != nil || fakeMulti.req != nil {
+ t.Fatalf("invalid scope reached service or returned wrong response: %v", response)
+ }
+ return
+ }
+ if response["code"] != float64(common.CodeSuccess) {
+ t.Fatalf("response=%v", response)
+ }
+ got := fakeMulti.req
+ if single {
+ got = fakeSingle.req.ToSearchDatasetsRequest("ds-1")
+ }
+ var want string
+ if tt.scope != "" {
+ if err := json.Unmarshal([]byte(tt.scope), &want); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if got == nil || got.KnowledgeCompilationScope != want || got.IncludeCompiledChunks == nil || !*got.IncludeCompiledChunks {
+ t.Fatalf("request=%#v want scope=%q and compilation enabled", got, want)
+ }
+ })
+ }
+ }
+}
+
func TestDatasetsHandlerSearchDatasetsPropagatesServiceError(t *testing.T) {
gin.SetMode(gin.TestMode)
fake := &fakeSearchDatasetsService{err: errors.New("boom")}
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go
index 3657cdc569..c5ccf566a6 100644
--- a/internal/ingestion/component/knowledge_compiler/component_test.go
+++ b/internal/ingestion/component/knowledge_compiler/component_test.go
@@ -338,6 +338,9 @@ func runVariant(t *testing.T, variant string, extra map[string]any) []map[string
}
if ck, _ := cm["compile_kwd"].(string); ck != "" {
seenCompiled = true
+ if cm["scope_kwd"] != "doc" {
+ t.Fatalf("Invoke(%s): scope_kwd = %v, want doc", variant, cm["scope_kwd"])
+ }
if _, ok := cm["q_8_vec"]; !ok {
t.Fatalf("Invoke(%s): compiled chunk missing vector: %v", variant, cm)
}
diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.go
index 3bbf07826b..3637b06ad2 100644
--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go
@@ -113,6 +113,11 @@ func TestProjectWikiGraphRowsCrossBatchEdgeSurvives(t *testing.T) {
if findRelation(rows, "entity/alpha", "entity/beta") == nil {
t.Fatalf("cross-page edge entity/alpha -> entity/beta not materialized")
}
+ for _, row := range rows {
+ if row["scope_kwd"] != "dataset" {
+ t.Fatalf("scope_kwd = %v, want dataset for wiki graph row", row["scope_kwd"])
+ }
+ }
}
// TestProjectWikiGraphRowsDropAfterReprojection verifies a page removed from the
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 60dabda8cc..577c99b00b 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -529,6 +529,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
return err
}
payload := string(payloadBytes)
+ contentLTKS, contentSMLTKS := tokenizeWikiGraphContent("", kccommon.PayloadDescription(payloadMap, nil))
row := map[string]interface{}{
// Stable dataset-level id keyed on the template-scoped entity name or
// relation identity plus the raw compile kind.
@@ -539,6 +540,8 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
"compile_kwd": types.CanonicalCompilationKind(kccommon.FirstNonEmpty(b.TemplateKind, ckwd)),
"scope_kwd": "dataset",
"content_with_weight": payload,
+ "content_ltks": contentLTKS,
+ "content_sm_ltks": contentSMLTKS,
"source_doc_ids": b.SourceDocIDs,
"source_chunk_ids": b.SourceChunkIDs,
"create_time": now.Format("2006-01-02 15:04:05"),
@@ -1791,6 +1794,7 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,
"available_int": 1,
"compile_kwd": "wiki",
"type_kwd": compileKwdWikiEntity,
+ "scope_kwd": "dataset",
"entity_type_kwd": p.PageType,
"slug_kwd": p.Slug,
"title_kwd": p.Title,
@@ -1847,6 +1851,7 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,
"available_int": 1,
"compile_kwd": "wiki",
"type_kwd": compileKwdWikiRelation,
+ "scope_kwd": "dataset",
"from_kwd": p.Slug,
"to_kwd": tgt,
"source_doc_ids": srcDocs,
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index 49373befbe..142cd5c5d9 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -256,6 +256,12 @@ func TestWriteMergedStructureKeepsEntityKeywordLowercase(t *testing.T) {
if !strings.Contains(payload, `"name":"Acme Corp"`) {
t.Errorf("entity payload = %s, want extracted spelling Acme Corp", payload)
}
+ for _, field := range []string{"content_ltks", "content_sm_ltks"} {
+ text, _ := row[field].(string)
+ if !strings.Contains(text, "company") || !strings.Contains(strings.ToLower(text), "acme corp") || strings.Contains(text, `"name":`) {
+ t.Errorf("%s = %q, want searchable entity payload text", field, text)
+ }
+ }
return
}
t.Fatal("inserted rows contain no entity")
@@ -276,6 +282,12 @@ func TestWriteMergedStructurePreservesDistinctRelationTypes(t *testing.T) {
if row["type_kwd"] != "relation" {
continue
}
+ for _, field := range []string{"content_ltks", "content_sm_ltks"} {
+ text, _ := row[field].(string)
+ if !strings.Contains(strings.ToLower(text), "acme corp") || !strings.Contains(strings.ToLower(text), "beta ltd") || strings.Contains(text, `"from":`) {
+ t.Errorf("%s = %q, want searchable relation payload text", field, text)
+ }
+ }
payload, _ := row["content_with_weight"].(string)
for _, relationType := range []string{"supplies", "owns"} {
if strings.Contains(payload, `"type":"`+relationType+`"`) {
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 236f05fdb1..f6534c323a 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -302,12 +302,10 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
}
}
- // Per-document compiled knowledge products (those emitted by the
- // KnowledgeCompiler component and stamped with `compile_kwd`) are persisted
- // as available_int=0: invisible to the normal retriever until the dataset-level
- // post-processing consumer (§11) merges them into available_int=1 products.
- // Ordinary source chunks stay available_int=1 (the index default) unless the
- // document itself is disabled (status=0).
+ // Per-document wiki products are staged as available_int=0 until the
+ // dataset-level consumer merges them into available_int=1 products.
+ // Final per-document structures and ordinary source chunks are written with
+ // available_int=1 unless the document itself is disabled (status=0).
markCompiledProductsHidden(chunks)
// Reload status immediately before write: LoadFromIngestionTask copied a
// snapshot that may be stale if BatchUpdateDocumentStatus ran mid-pipeline.
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index c48c57c2c3..c4075160c6 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -544,10 +544,7 @@ func (s *ChunkService) Get(ctx context.Context, req *service.GetChunkRequest, us
return nil, fmt.Errorf("chunk not found")
}
- // Return the stored row with only the tokenized/vector runtime fields
- // dropped, mirroring Python's _strip_chunk_runtime_fields. The frontend
- // normalizes both this shape and the renamed one (mapChunkToLegacy), so
- // returning the row verbatim is what Python does and costs no UI change.
+ // Return the stored row without tokenized/vector runtime fields.
result := make(map[string]interface{}, len(chunk))
for k, v := range chunk {
if isChunkRuntimeField(k) {
@@ -558,9 +555,8 @@ func (s *ChunkService) Get(ctx context.Context, req *service.GetChunkRequest, us
return &service.GetChunkResponse{Chunk: result}, nil
}
-// isChunkRuntimeField reports the fields Python's _strip_chunk_runtime_fields
-// removes before returning a chunk: the regex (_vec$|_sm_|_tks|_ltks) — _vec is
-// anchored at the end, the other three are substring matches.
+// isChunkRuntimeField identifies internal fields omitted from chunk responses.
+// The _vec suffix is anchored at the end; token field names use substring matches.
func isChunkRuntimeField(name string) bool {
return strings.HasSuffix(name, "_vec") ||
strings.Contains(name, "_sm_") ||
@@ -847,7 +843,6 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,
"to_entity_kwd",
"toc_kwd",
"doc_type_kwd",
- "available_int",
},
Filter: map[string]interface{}{
"doc_id": req.DocID,
@@ -874,7 +869,7 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,
// Inline formatChunkForList
result := make(map[string]interface{})
skipFields := map[string]bool{
- "_id": true, "authors": true, "_score": true, "SCORE": true,
+ "_id": true, "authors": true, "_score": true, "SCORE": true, "available_int": true,
"important_kwd_empty_count": true, "kb_id": true, "mom_id": true, "page_num_int": true,
}
for k, v := range chunk {
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 89a990a74b..7eebb0f24b 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -324,14 +324,21 @@ func TestListSortsChunksByDocumentPosition(t *testing.T) {
ctx := t.Context()
page := 1
size := 30
- if _, err := svc.List(ctx, &service.ListChunksRequest{
+ resp, err := svc.List(ctx, &service.ListChunksRequest{
DatasetID: datasetID,
DocID: documentID,
Page: &page,
Size: &size,
- }, userID); err != nil {
+ }, userID)
+ if err != nil {
t.Fatalf("List() error = %v", err)
}
+ if len(resp.Chunks) != 1 {
+ t.Fatalf("chunks = %v, want one chunk", resp.Chunks)
+ }
+ if _, ok := resp.Chunks[0]["available_int"]; ok {
+ t.Fatal("available_int should not be returned in chunk lists")
+ }
if engine.searchReq == nil {
t.Fatal("expected Search to be called")
@@ -424,6 +431,7 @@ func TestGetScopesChunkLookupToDocumentDataset(t *testing.T) {
"docnm_kwd": "doc.txt",
"content_with_weight": "invoice body",
"content_ltks": []string{"invoice", "body"},
+ "available_int": 0,
}}
svc := &ChunkService{
docEngine: engine,
@@ -461,6 +469,12 @@ func TestGetScopesChunkLookupToDocumentDataset(t *testing.T) {
if _, ok := resp.Chunk["content_ltks"]; ok {
t.Fatal("content_ltks should be stripped like Python's _strip_chunk_runtime_fields")
}
+ if got := resp.Chunk["available_int"]; got != 0 {
+ t.Fatalf("available_int = %v, want stored availability 0", got)
+ }
+ if engine.chunk["available_int"] != 0 {
+ t.Fatal("stored availability should remain unchanged")
+ }
}
// TestGetRejectsChunkOfAnotherDocument pins the doc_id guard: holding a chunk id
@@ -540,9 +554,8 @@ func TestGetRequiresDocumentInRouteDataset(t *testing.T) {
}
}
-// TestIsChunkRuntimeField pins the strip rule's Python-parity semantics
-// (_vec$|_sm_|_tks|_ltks — only _vec is end-anchored): tokenized/vector runtime
-// fields are dropped, stored fields survive.
+// TestIsChunkRuntimeField checks that tokenized/vector fields are dropped while
+// other stored fields survive. Only _vec is end-anchored.
func TestIsChunkRuntimeField(t *testing.T) {
cases := []struct {
name string
@@ -2096,6 +2109,7 @@ func (e *listChunksSearchEngine) Search(_ context.Context, req *types.SearchRequ
"id": "chunk-1",
"content": "plain invoice terms body",
"content_with_weight": "weighted invoice terms body",
+ "available_int": 0,
"doc_id": "doc-1",
},
},
diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.go
index ec6a895a69..4c8351921a 100644
--- a/internal/service/dataset/search.go
+++ b/internal/service/dataset/search.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"ragflow/internal/dao"
+ "slices"
"go.uber.org/zap"
@@ -30,6 +31,9 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
if len(req.DatasetIDs) == 0 {
return nil, fmt.Errorf("dataset_ids is required")
}
+ if err := service.ValidateKnowledgeCompilationScope(req.KnowledgeCompilationScope); err != nil {
+ return nil, err
+ }
common.Info("SearchDatasets started", zap.String("userID", userID), zap.Any("datasets", req.DatasetIDs), zap.String("question", req.Question))
page := 1
@@ -306,8 +310,10 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
EmbeddingModel: embeddingModel,
Highlight: req.Highlight,
}
- if req.IncludeCompiledChunks != nil && !*req.IncludeCompiledChunks {
- retrievalReq.Filter = map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}}
+ retrievalReq.Filter = knowledgeCompilationFilter(req.IncludeCompiledChunks, req.KnowledgeCompilationScope)
+ if len(docIDs) > 0 && (req.IncludeCompiledChunks == nil || *req.IncludeCompiledChunks) && req.KnowledgeCompilationScope != "document" {
+ // Dataset artifacts use dataset IDs instead of source document IDs.
+ retrievalReq.DocIDs = append(slices.Clone(docIDs), datasetIDs...)
}
retrievalResult, err := nlp.NewRetrievalService(d.docEngine, d.documentDAO).Retrieval(ctx, retrievalReq)
@@ -354,6 +360,22 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
}, nil
}
+func knowledgeCompilationFilter(include *bool, scope string) map[string]interface{} {
+ ordinary := map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}}
+ if include != nil && !*include {
+ return ordinary
+ }
+ switch scope {
+ case "all":
+ return nil
+ case "document":
+ scope = "doc"
+ default:
+ scope = "dataset"
+ }
+ return map[string]interface{}{"or": []map[string]interface{}{ordinary, {"scope_kwd": scope}}}
+}
+
func selectMetadataFilteredDocIDs(currentDocIDs, filteredDocIDs []string, hasMetadataCondition, filterReturnedEmpty bool) []string {
if hasMetadataCondition || !filterReturnedEmpty {
return filteredDocIDs
diff --git a/internal/service/dataset/search_test.go b/internal/service/dataset/search_test.go
index 0b4790bc11..e17fea8e9f 100644
--- a/internal/service/dataset/search_test.go
+++ b/internal/service/dataset/search_test.go
@@ -1,9 +1,13 @@
package dataset
import (
+ "reflect"
+ "slices"
"testing"
+ "ragflow/internal/engine/types"
"ragflow/internal/service"
+ "ragflow/internal/service/nlp"
)
func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {
@@ -34,6 +38,8 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {
SimilarityThreshold: &similarityThreshold,
VectorSimilarityWeight: &vectorSimilarityWeight,
IncludeCompiledChunks: &includeKnowledgeCompilation,
+
+ KnowledgeCompilationScope: "all",
}
converted := req.ToSearchDatasetsRequest("dataset-1")
@@ -58,6 +64,103 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {
if converted.IncludeCompiledChunks != req.IncludeCompiledChunks {
t.Fatalf("converted request did not preserve include_knowledge_compilation: %#v", converted)
}
+ if converted.KnowledgeCompilationScope != req.KnowledgeCompilationScope {
+ t.Fatalf("knowledge_compilation_scope=%v want %v", converted.KnowledgeCompilationScope, req.KnowledgeCompilationScope)
+ }
+}
+
+func TestKnowledgeCompilationFilter(t *testing.T) {
+ rows := []map[string]interface{}{
+ {"id": "source-1", "doc_id": "doc-1"},
+ {"id": "source-2", "doc_id": "doc-2"},
+ {"id": "document-1", "compile_kwd": "graph", "scope_kwd": "doc", "doc_id": "doc-1"},
+ {"id": "document-2", "compile_kwd": "graph", "scope_kwd": "doc", "doc_id": "doc-2"},
+ {"id": "dataset-1", "compile_kwd": "graph", "scope_kwd": "dataset", "doc_id": "kb-1", "source_doc_ids": []string{"doc-1", "doc-2"}},
+ {"id": "dataset-2", "compile_kwd": "graph", "scope_kwd": "dataset", "doc_id": "kb-1", "source_doc_ids": []string{"doc-2"}},
+ {"id": "unscoped", "compile_kwd": "graph", "doc_id": "doc-2"},
+ }
+ for _, tt := range []struct {
+ name string
+ include *bool
+ scope string
+ docIDs, want []string
+ }{
+ {name: "omitted", want: []string{"source-1", "source-2", "dataset-1", "dataset-2"}},
+ {name: "omitted with compilation enabled", include: new(true), want: []string{"source-1", "source-2", "dataset-1", "dataset-2"}},
+ {name: "omitted with compilation disabled", include: new(false), want: []string{"source-1", "source-2"}},
+ {name: "omitted with selected document", docIDs: []string{"doc-1"}, want: []string{"source-1", "dataset-1", "dataset-2"}},
+ {name: "dataset", include: new(true), scope: "dataset", want: []string{"source-1", "source-2", "dataset-1", "dataset-2"}},
+ {name: "document", include: new(true), scope: "document", want: []string{"source-1", "source-2", "document-1", "document-2"}},
+ {name: "all", include: new(true), scope: "all", want: []string{"source-1", "source-2", "document-1", "document-2", "dataset-1", "dataset-2", "unscoped"}},
+ {name: "default include", scope: "document", want: []string{"source-1", "source-2", "document-1", "document-2"}},
+ {name: "disabled overrides scope", include: new(false), scope: "all", want: []string{"source-1", "source-2"}},
+ {name: "all with selected document", scope: "all", docIDs: []string{"doc-1"}, want: []string{"source-1", "document-1", "dataset-1", "dataset-2"}},
+ {name: "dataset with selected document", scope: "dataset", docIDs: []string{"doc-1"}, want: []string{"source-1", "dataset-1", "dataset-2"}},
+ {name: "document with selected document", scope: "document", docIDs: []string{"doc-1"}, want: []string{"source-1", "document-1"}},
+ {name: "all with unscoped compilation", scope: "all", docIDs: []string{"doc-2"}, want: []string{"source-2", "document-2", "dataset-1", "dataset-2", "unscoped"}},
+ {name: "dataset with unrelated document", scope: "dataset", docIDs: []string{"doc-3"}, want: []string{"dataset-1", "dataset-2"}},
+ {name: "document with unrelated document", scope: "document", docIDs: []string{"doc-3"}},
+ {name: "all with unrelated document", scope: "all", docIDs: []string{"doc-3"}, want: []string{"dataset-1", "dataset-2"}},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ req := nlp.RetrievalSearchRequest{DocIDs: tt.docIDs, Filter: knowledgeCompilationFilter(tt.include, tt.scope)}
+ if len(tt.docIDs) > 0 && (tt.include == nil || *tt.include) && tt.scope != "document" {
+ req.DocIDs = append(slices.Clone(tt.docIDs), "kb-1")
+ }
+ filter := req.GetFilters()
+ var got []string
+ for _, row := range rows {
+ if matchesKnowledgeCompilationFilter(filter, row) {
+ got = append(got, row["id"].(string))
+ }
+ }
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Fatalf("matched chunks=%v want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func matchesKnowledgeCompilationFilter(filter, row map[string]interface{}) bool {
+ for field, value := range filter {
+ switch field {
+ case "or":
+ matched := false
+ for _, branch := range types.FilterClauses(value) {
+ matched = matched || matchesKnowledgeCompilationFilter(branch, row)
+ }
+ if !matched {
+ return false
+ }
+ case "must_not":
+ if matchesKnowledgeCompilationFilter(value.(map[string]interface{}), row) {
+ return false
+ }
+ case "exists":
+ if row[value.(string)] == nil {
+ return false
+ }
+ default:
+ if values, ok := value.([]string); ok {
+ matched := false
+ for _, expected := range values {
+ if actual, ok := row[field].([]string); ok {
+ for _, item := range actual {
+ matched = matched || item == expected
+ }
+ } else {
+ matched = matched || row[field] == expected
+ }
+ }
+ if !matched {
+ return false
+ }
+ } else if row[field] != value {
+ return false
+ }
+ }
+ }
+ return true
}
func TestSelectMetadataFilteredDocIDs(t *testing.T) {
diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.go
index 21ad9c1f76..4b79d7c648 100644
--- a/internal/service/dataset_types.go
+++ b/internal/service/dataset_types.go
@@ -1,5 +1,7 @@
package service
+import "fmt"
+
// CheckEmbeddingRequest is the request structure for checking embedding compatibility.
type CheckEmbeddingRequest struct {
EmbeddingID string `json:"embd_id" binding:"required"`
@@ -61,6 +63,8 @@ type SearchDatasetsRequest struct {
VectorSimilarityWeight *float64 `json:"vector_similarity_weight,omitempty"`
IncludeCompiledChunks *bool `json:"include_knowledge_compilation,omitempty"`
ForceRefresh bool `json:"force_refresh"`
+
+ KnowledgeCompilationScope string `json:"knowledge_compilation_scope,omitempty"`
}
// SearchDatasetsResponse is the response structure for dataset search results.
@@ -94,6 +98,8 @@ type SearchDatasetRequest struct {
KeywordsSimilarityWeight *float64 `json:"keywords_similarity_weight,omitempty"`
VectorSimilarityWeight *float64 `json:"vector_similarity_weight,omitempty"`
IncludeCompiledChunks *bool `json:"include_knowledge_compilation,omitempty"`
+
+ KnowledgeCompilationScope string `json:"knowledge_compilation_scope,omitempty"`
}
// ToSearchDatasetsRequest converts a single-dataset search request into the multi-dataset form.
@@ -124,7 +130,17 @@ func (req *SearchDatasetRequest) ToSearchDatasetsRequest(datasetID string) *Sear
KeywordsSimilarityWeight: req.KeywordsSimilarityWeight,
VectorSimilarityWeight: req.VectorSimilarityWeight,
IncludeCompiledChunks: req.IncludeCompiledChunks,
+
+ KnowledgeCompilationScope: req.KnowledgeCompilationScope,
+ }
+}
+
+// ValidateKnowledgeCompilationScope checks the supported search scopes.
+func ValidateKnowledgeCompilationScope(scope string) error {
+ if scope != "" && scope != "dataset" && scope != "document" && scope != "all" {
+ return fmt.Errorf("knowledge_compilation_scope must be dataset, document, or all")
}
+ return nil
}
// MetadataConfigField mirrors one field in the dataset metadata config API.
diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.go
index 06a5fb4e74..04b51e18ee 100644
--- a/internal/service/nlp/datasetnav.go
+++ b/internal/service/nlp/datasetnav.go
@@ -745,7 +745,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"type_kwd": []string{nav.TypeNavDoc},
"doc_id": []string{in.DocID},
}),
- []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd", "docnm_kwd", "compile_kwd"}, 0, 1, nil)
+ []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd", "docnm_kwd", "compile_kwd", "scope_kwd"}, 0, 1, nil)
if err != nil {
return err
}
@@ -758,16 +758,20 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
title := navDocTitle(in.Summary)
titleStale := navRowLabel(existing[0]) != title
kindStale := firstStringValue(existing[0]["compile_kwd"]) != in.CompileKind
+ scopeStale := firstStringValue(existing[0]["scope_kwd"]) != "doc"
ltksStale := firstStringValue(existing[0]["content_ltks"]) == "" ||
firstStringValue(existing[0]["content_sm_ltks"]) == ""
if payload, ok := existing[0]["content_with_weight"].(string); ok {
var m map[string]interface{}
if err := json.Unmarshal([]byte(payload), &m); err == nil {
if d, _ := m["description"].(string); d == in.Summary {
- if !titleStale && !ltksStale && !kindStale {
+ if !titleStale && !ltksStale && !kindStale && !scopeStale {
return nil // unchanged
}
updates := map[string]interface{}{}
+ if scopeStale {
+ updates["scope_kwd"] = "doc"
+ }
if kindStale {
updates["compile_kwd"] = in.CompileKind
}
@@ -832,6 +836,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
row := map[string]interface{}{
"id": navDocID(in.TenantID, in.KbID, in.DocID),
"compile_kwd": in.CompileKind,
+ "scope_kwd": "doc",
"available_int": 0,
"type_kwd": "nav_doc",
// The nav_doc's label is a readable title derived from the summary
@@ -892,6 +897,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"id": navClusterID(in.TenantID, in.KbID, name),
"doc_id": in.KbID, // cluster rows carry the kb as doc_id (Python _build_nav_cluster_row), so ES InsertChunks does not skip them
"compile_kwd": in.CompileKind,
+ "scope_kwd": "dataset",
"available_int": 0,
"type_kwd": "nav_cluster",
"title_kwd": name,
@@ -916,6 +922,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
docRow := map[string]interface{}{
"id": navDocID(in.TenantID, in.KbID, in.DocID),
"compile_kwd": in.CompileKind,
+ "scope_kwd": "doc",
"available_int": 0,
"type_kwd": "nav_doc",
"title_kwd": navDocTitle(in.Summary),
@@ -1283,6 +1290,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus
row := map[string]interface{}{
"id": navClusterID(tenantID, kbID, spl.name),
"compile_kwd": splitCompileKind,
+ "scope_kwd": "dataset",
"available_int": 0,
"type_kwd": "nav_cluster",
"title_kwd": spl.name,
@@ -1559,7 +1567,7 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine
// this one. The read-modify-write here is expected to run under a per-dataset
// lock held by the UpsertDoc caller; without it, concurrent appends to the
// same cluster can lose updates.
- updates := map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count}
+ updates := map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count, "scope_kwd": "dataset"}
if kind := types.CompilationKind(chunks[0]); kind != "tree" && kind != "page_index" {
updates["compile_kwd"] = compileKind
}
diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.go
index 48a024b1eb..d484e54ced 100644
--- a/internal/service/nlp/datasetnav_test.go
+++ b/internal/service/nlp/datasetnav_test.go
@@ -322,6 +322,13 @@ func TestNavServicePageIndexRowsAndCleanup(t *testing.T) {
if row["compile_kwd"] != "page_index" {
t.Fatalf("compile_kwd = %v, want page_index", row["compile_kwd"])
}
+ wantScope := "doc"
+ if row["type_kwd"] == "nav_cluster" {
+ wantScope = "dataset"
+ }
+ if row["scope_kwd"] != wantScope {
+ t.Fatalf("scope_kwd = %v, want %s for %s", row["scope_kwd"], wantScope, row["type_kwd"])
+ }
if row["type_kwd"] == "nav_cluster" {
row["compile_kwd"] = "dataset_nav"
}
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go
index f32b620130..e1a3f45696 100644
--- a/internal/service/nlp/retrieval.go
+++ b/internal/service/nlp/retrieval.go
@@ -403,6 +403,9 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)
if v, ok := chunk["doc_id"].(string); ok {
docID = v
}
+ if kbID, _ := chunk["kb_id"].(string); docID != "" && docID == kbID {
+ continue
+ }
if entry, exists := docAggsMap[docName]; exists {
entry.count++
docAggsMap[docName] = entry
@@ -711,6 +714,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque
// displays, doc_type_kwd marks image/table chunks, mom_id is what promotes
// child fragments to their parent, row_id() the table row identity.
"content_with_weight", "doc_type_kwd", "mom_id", "row_id()",
+ "compile_kwd", "scope_kwd",
"_score",
}
@@ -1197,6 +1201,9 @@ func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *Retri
// Collect all doc_ids from chunks
chunkDocIDs := make([]string, 0, len(result.Field))
for _, chunk := range result.Field {
+ if isDatasetCompilationChunk(chunk) {
+ continue
+ }
if docID, ok := chunk["doc_id"].(string); ok && docID != "" {
chunkDocIDs = append(chunkDocIDs, docID)
}
@@ -1245,8 +1252,8 @@ func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *Retri
continue
}
docID, ok := chunk["doc_id"].(string)
- if !ok || docID == "" {
- // Keep chunks without doc_id
+ if !ok || docID == "" || isDatasetCompilationChunk(chunk) {
+ // Dataset artifacts use the dataset ID instead of a source document ID.
filteredIDs = append(filteredIDs, chunkID)
filteredChunks = append(filteredChunks, chunk)
filteredField[chunkID] = chunk
@@ -1288,6 +1295,13 @@ func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *Retri
}, nil
}
+func isDatasetCompilationChunk(chunk map[string]interface{}) bool {
+ docID, _ := chunk["doc_id"].(string)
+ kbID, _ := chunk["kb_id"].(string)
+ compileKind, _ := chunk["compile_kwd"].(string)
+ return chunk["scope_kwd"] == "dataset" && compileKind != "" && docID != "" && docID == kbID
+}
+
// buildIndexNames creates one index name per distinct tenant ID.
// Each tenantID may be a comma-separated list.
func buildIndexNames(tenantIDs []string) []string {
diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.go
index 94229d9687..0421ef86f6 100644
--- a/internal/service/nlp/retrieval_test.go
+++ b/internal/service/nlp/retrieval_test.go
@@ -12,8 +12,10 @@ import (
"ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
+ "ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
+ "github.com/glebarez/sqlite"
"gorm.io/gorm"
)
@@ -117,6 +119,85 @@ func TestRetrievalReturnsRegexHighlightWithoutFallbackOrCleanup(t *testing.T) {
}
}
+func TestRetrievalPreservesDatasetCompilationChunks(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer sqlDB.Close()
+ if err := db.AutoMigrate(&entity.Document{}); err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Create(&entity.Document{ID: "doc-1", KbID: "kb-1", ParserConfig: entity.JSONMap{}}).Error; err != nil {
+ t.Fatal(err)
+ }
+ oldDB, oldQueryBuilder := dao.DB, globalQueryBuilder
+ dao.DB, globalQueryBuilder = db, NewQueryBuilder()
+ defer func() { dao.DB, globalQueryBuilder = oldDB, oldQueryBuilder }()
+
+ artifact := map[string]interface{}{"id": "artifact", "doc_id": "kb-1", "kb_id": "kb-1", "compile_kwd": "graph", "scope_kwd": "dataset", "available_int": 1, "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9}
+ for _, tt := range []struct {
+ name string
+ rows []map[string]interface{}
+ want []string
+ }{
+ {name: "dataset only", rows: []map[string]interface{}{artifact}, want: []string{"artifact"}},
+ {name: "mixed with deleted documents", rows: []map[string]interface{}{
+ artifact,
+ {"id": "document", "doc_id": "doc-1", "kb_id": "kb-1", "compile_kwd": "graph", "scope_kwd": "doc", "content_ltks": "alpha", "_score": 0.8},
+ {"id": "deleted", "doc_id": "deleted-doc", "kb_id": "kb-1", "compile_kwd": "graph", "scope_kwd": "doc", "content_ltks": "alpha", "_score": 0.7},
+ {"id": "wrong identity", "doc_id": "deleted-doc", "kb_id": "kb-1", "compile_kwd": "graph", "scope_kwd": "dataset", "content_ltks": "alpha", "_score": 0.6},
+ }, want: []string{"artifact", "document"}},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ eng := &projectedRetrievalEngine{retrievalCountEngine: retrievalCountEngine{rows: tt.rows}}
+ result, err := NewRetrievalService(eng, &dao.DocumentDAO{}).Retrieval(t.Context(), &RetrievalRequest{
+ Question: "alpha", TenantIDs: []string{"tenant-1"}, KbIDs: []string{"kb-1"}, Page: 1, PageSize: 10,
+ VectorSimilarityWeight: new(1.0), SimilarityThreshold: new(0.0), Aggs: new(true),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got []string
+ for _, chunk := range result.Chunks {
+ got = append(got, chunk["chunk_id"].(string))
+ if _, ok := chunk["available_int"]; ok {
+ t.Fatal("available_int should not be returned in retrieval results")
+ }
+ }
+ if !slices.Equal(got, tt.want) || result.Total != int64(len(tt.want)) {
+ t.Fatalf("chunks=%v total=%d want %v", got, result.Total, tt.want)
+ }
+ if len(result.DocAggs) != len(tt.want)-1 {
+ t.Fatalf("doc_aggs=%v, want only source documents", result.DocAggs)
+ }
+ if len(result.DocAggs) > 0 && (result.DocAggs[0]["doc_id"] != "doc-1" || result.DocAggs[0]["count"] != 1) {
+ t.Fatalf("doc_aggs=%v, want doc-1 with count 1", result.DocAggs)
+ }
+ })
+ }
+}
+
+type projectedRetrievalEngine struct{ retrievalCountEngine }
+
+func (e *projectedRetrievalEngine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} {
+ result := make(map[string]map[string]interface{}, len(chunks))
+ for _, chunk := range chunks {
+ projected := make(map[string]interface{}, len(fields))
+ for _, field := range fields {
+ if value, ok := chunk[field]; ok {
+ projected[field] = value
+ }
+ }
+ result[chunk["id"].(string)] = projected
+ }
+ return result
+}
+
type retrievalCountEngine struct {
rows []map[string]interface{}
searchLimits []int
@@ -252,10 +333,14 @@ func (parentChunkMissingEngine) Search(context.Context, *types.SearchRequest) (*
type parentChunkScopeEngine struct {
engine.DocEngine
parentSearch *types.SearchRequest
+ parentChunk map[string]interface{}
}
func (e *parentChunkScopeEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
e.parentSearch = req
+ if e.parentChunk != nil {
+ return &types.SearchResult{Chunks: []map[string]interface{}{e.parentChunk}}, nil
+ }
return &types.SearchResult{}, nil
}
@@ -290,6 +375,20 @@ func TestRetrievalByChildrenScopesParentLookupToChildDocument(t *testing.T) {
if !engine.parentSearch.IncludeUnavailable {
t.Fatal("parent lookup must include hidden parent chunks")
}
+ engine.parentChunk = map[string]interface{}{
+ "id": "legacy-shared-parent-id", "doc_id": "doc-a", "kb_id": "kb-1",
+ "available_int": 0, "content_with_weight": "parent text",
+ }
+ got = RetrievalByChildren([]map[string]interface{}{child}, []string{"tenant-1", "tenant-2"}, engine, t.Context())
+ if len(got) != 1 || got[0]["chunk_id"] != "legacy-shared-parent-id" || got[0]["content_with_weight"] != "parent text" {
+ t.Fatalf("retrieval result = %#v, want aggregated parent", got)
+ }
+ if _, ok := got[0]["available_int"]; ok {
+ t.Fatal("available_int should not be returned in aggregated retrieval results")
+ }
+ if engine.parentChunk["available_int"] != 0 {
+ t.Fatal("stored parent availability should remain unchanged")
+ }
}
// TestRetrievalByChildrenKeepsChildWhenParentIsMissing verifies a partial
diff --git a/web/src/pages/next-search/markdown-content/index.tsx b/web/src/pages/next-search/markdown-content/index.tsx
index ac260a1253..95298b0fb7 100644
--- a/web/src/pages/next-search/markdown-content/index.tsx
+++ b/web/src/pages/next-search/markdown-content/index.tsx
@@ -209,7 +209,7 @@ const MarkdownContent = ({
}}
className={classNames(styles.chunkContentText)}
></div>
- {documentId && (
+ {documentId && documentId !== chunkItem?.dataset_id && (
<div className="flex gap-2">
{fileThumbnail ? (
<AuthenticatedImg
diff --git a/web/src/pages/next-search/search-view.tsx b/web/src/pages/next-search/search-view.tsx
index c7aeae5f50..8d278f347e 100644
--- a/web/src/pages/next-search/search-view.tsx
+++ b/web/src/pages/next-search/search-view.tsx
@@ -290,18 +290,22 @@ export default function SearchingView({
)}
</div>
)}
- <div
- className="flex gap-2 items-center text-xs text-text-secondary border p-1 rounded-lg w-fit mt-3"
- onClick={() =>
- clickDocumentButton(
- chunk.document_id,
- chunk as any,
- )
- }
- >
- <FileIcon name={chunk.document_keyword}></FileIcon>
- {chunk.document_keyword}
- </div>
+ {chunk.document_id !== chunk.dataset_id && (
+ <div
+ className="flex gap-2 items-center text-xs text-text-secondary border p-1 rounded-lg w-fit mt-3"
+ onClick={() =>
+ clickDocumentButton(
+ chunk.document_id,
+ chunk as any,
+ )
+ }
+ >
+ <FileIcon
+ name={chunk.document_keyword}
+ ></FileIcon>
+ {chunk.document_keyword}
+ </div>
+ )}
</div>
{index < chunks.length - 1 && (
<div className="w-full border-b border-border-default/80 mt-6 mb-2"></div>