Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Feature: add highlight for retrieval testing and search (#18981)

Wang Qi66113709ccd7
diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.py
index 69e5cc6075..c2b5f000e5 100644
--- a/api/apps/restful_apis/dataset_api.py
+++ b/api/apps/restful_apis/dataset_api.py
@@ -514,7 +514,7 @@ async def search_datasets(tenant_id):
POST /api/v1/datasets/search
JSON body: {"dataset_ids": list[str] (required), "question": str (required), "doc_ids": list[str], "knn_top_k": int (default 1024), "knn_num_candidates": int (default 2048), "page": int, "page_size": int, "size": int (fallback),
- "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool,
+ "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool, "highlight": bool,
"cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)}
The legacy "top_k" parameter is accepted as an alias for "knn_top_k".
"knn_num_candidates" currently applies only to Elasticsearch.
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index a279ac9823..57fa627362 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -1572,6 +1572,7 @@ async def search_datasets(tenant_id: str, req: dict):
knn_top_k=knn_top_k,
knn_num_candidates=knn_num_candidates,
rerank_mdl=rerank_mdl,
+ highlight=req.get("highlight", False),
rank_feature=labels,
trace_id=search_id,
must_not=None if req.get("include_knowledge_compilation", True) else {"exists": "compile_kwd"},
diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.py
index 7e52e4726f..8c816833c8 100644
--- a/api/utils/validation_utils.py
+++ b/api/utils/validation_utils.py
@@ -1023,6 +1023,7 @@ class SearchDatasetsReq(BaseModel):
use_kg: Annotated[bool, Field(default=False)]
cross_languages: Annotated[list[str], Field(default=[])]
keyword: Annotated[bool, Field(default=False)]
+ highlight: Annotated[bool, Field(default=False)]
search_id: Annotated[str | None, Field(default=None)]
rerank_id: Annotated[str | None, Field(default=None)]
tenant_rerank_id: Annotated[str | None, Field(default=None)]
diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.go
index 1c0f4a79e2..9f8827bc63 100644
--- a/internal/service/dataset/search.go
+++ b/internal/service/dataset/search.go
@@ -286,6 +286,7 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
RerankModel: rerankModel,
RankFeature: &labels,
EmbeddingModel: embeddingModel,
+ Highlight: req.Highlight,
}
if req.IncludeCompiledChunks != nil && !*req.IncludeCompiledChunks {
retrievalReq.Filter = map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}}
diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.go
index 45833eefa9..e9080b0905 100644
--- a/internal/service/dataset_types.go
+++ b/internal/service/dataset_types.go
@@ -52,6 +52,7 @@ type SearchDatasetsRequest struct {
MetadataFilter map[string]interface{} `json:"meta_data_filter,omitempty"`
RerankID *string `json:"rerank_id,omitempty"`
Keyword *bool `json:"keyword,omitempty"`
+ Highlight *bool `json:"highlight,omitempty"`
SimilarityThreshold *float64 `json:"similarity_threshold,omitempty"`
VectorSimilarityWeight *float64 `json:"vector_similarity_weight,omitempty"`
IncludeCompiledChunks *bool `json:"include_knowledge_compilation,omitempty"`
diff --git a/web/src/pages/dataset/testing/testing-result.tsx b/web/src/pages/dataset/testing/testing-result.tsx
index 9fc16742e1..c15022ecb3 100644
--- a/web/src/pages/dataset/testing/testing-result.tsx
+++ b/web/src/pages/dataset/testing/testing-result.tsx
@@ -7,6 +7,7 @@ import { Card } from '@/components/ui/card';
import { useTranslate } from '@/hooks/common-hooks';
import { useTestRetrieval } from '@/hooks/use-knowledge-request';
import { ITestingChunk } from '@/interfaces/database/dataset';
+import { sanitizeHtmlWithImagesAsText } from '@/utils/dom-util';
import { t } from 'i18next';
import camelCase from 'lodash/camelCase';
import { useMemo } from 'react';
@@ -84,9 +85,14 @@ export function TestingResult({
<article key={x.chunk_id}>
<Card className="px-5 py-2.5 bg-transparent shadow-none">
<ChunkTitle item={x}></ChunkTitle>
- <p className="!mt-2.5 whitespace-pre-wrap">
- {x.content_with_weight}
- </p>
+ <div
+ className="!mt-2.5 whitespace-pre-wrap [&_em]:text-accent-primary [&_em]:not-italic"
+ dangerouslySetInnerHTML={{
+ __html: sanitizeHtmlWithImagesAsText(
+ x.highlight || x.content_with_weight,
+ ),
+ }}
+ />
<div className="mt-2.5 text-right text-xs text-text-sub-title-invert">
{x.doc_name || x.docnm_kwd}
</div>
diff --git a/web/src/pages/next-search/search-view.tsx b/web/src/pages/next-search/search-view.tsx
index 83483e3004..336816ea11 100644
--- a/web/src/pages/next-search/search-view.tsx
+++ b/web/src/pages/next-search/search-view.tsx
@@ -259,7 +259,7 @@ export default function SearchingView({
<div
dangerouslySetInnerHTML={{
__html: sanitizeHtmlWithImagesAsText(
- chunk.content_with_weight,
+ chunk.highlight || chunk.content_with_weight,
).trim(),
}}
className={classNames(