RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Feature: add highlight for retrieval testing and search (#18981)
Wang Qi66113709ccd7
diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.pyindex 69e5cc6075..c2b5f000e5 100644--- a/api/apps/restful_apis/dataset_api.py+++ b/api/apps/restful_apis/dataset_api.py@@ -514,7 +514,7 @@ async def search_datasets(tenant_id):POST /api/v1/datasets/searchJSON body: {"dataset_ids": list[str] (required), "question": str (required), "doc_ids": list[str], "knn_top_k": int (default 1024), "knn_num_candidates": int (default 2048), "page": int, "page_size": int, "size": int (fallback),- "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool,+ "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool, "highlight": bool,"cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)}The legacy "top_k" parameter is accepted as an alias for "knn_top_k"."knn_num_candidates" currently applies only to Elasticsearch.diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex a279ac9823..57fa627362 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -1572,6 +1572,7 @@ async def search_datasets(tenant_id: str, req: dict):knn_top_k=knn_top_k,knn_num_candidates=knn_num_candidates,rerank_mdl=rerank_mdl,+ highlight=req.get("highlight", False),rank_feature=labels,trace_id=search_id,must_not=None if req.get("include_knowledge_compilation", True) else {"exists": "compile_kwd"},diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.pyindex 7e52e4726f..8c816833c8 100644--- a/api/utils/validation_utils.py+++ b/api/utils/validation_utils.py@@ -1023,6 +1023,7 @@ class SearchDatasetsReq(BaseModel):use_kg: Annotated[bool, Field(default=False)]cross_languages: Annotated[list[str], Field(default=[])]keyword: Annotated[bool, Field(default=False)]+ highlight: Annotated[bool, Field(default=False)]search_id: Annotated[str | None, Field(default=None)]rerank_id: Annotated[str | None, Field(default=None)]tenant_rerank_id: Annotated[str | None, Field(default=None)]diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.goindex 1c0f4a79e2..9f8827bc63 100644--- a/internal/service/dataset/search.go+++ b/internal/service/dataset/search.go@@ -286,6 +286,7 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.SearchRerankModel: rerankModel,RankFeature: &labels,EmbeddingModel: embeddingModel,+ Highlight: req.Highlight,}if req.IncludeCompiledChunks != nil && !*req.IncludeCompiledChunks {retrievalReq.Filter = map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}}diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.goindex 45833eefa9..e9080b0905 100644--- a/internal/service/dataset_types.go+++ b/internal/service/dataset_types.go@@ -52,6 +52,7 @@ type SearchDatasetsRequest struct {MetadataFilter map[string]interface{} `json:"meta_data_filter,omitempty"`RerankID *string `json:"rerank_id,omitempty"`Keyword *bool `json:"keyword,omitempty"`+ Highlight *bool `json:"highlight,omitempty"`SimilarityThreshold *float64 `json:"similarity_threshold,omitempty"`VectorSimilarityWeight *float64 `json:"vector_similarity_weight,omitempty"`IncludeCompiledChunks *bool `json:"include_knowledge_compilation,omitempty"`diff --git a/web/src/pages/dataset/testing/testing-result.tsx b/web/src/pages/dataset/testing/testing-result.tsxindex 9fc16742e1..c15022ecb3 100644--- a/web/src/pages/dataset/testing/testing-result.tsx+++ b/web/src/pages/dataset/testing/testing-result.tsx@@ -7,6 +7,7 @@ import { Card } from '@/components/ui/card';import { useTranslate } from '@/hooks/common-hooks';import { useTestRetrieval } from '@/hooks/use-knowledge-request';import { ITestingChunk } from '@/interfaces/database/dataset';+import { sanitizeHtmlWithImagesAsText } from '@/utils/dom-util';import { t } from 'i18next';import camelCase from 'lodash/camelCase';import { useMemo } from 'react';@@ -84,9 +85,14 @@ export function TestingResult({<article key={x.chunk_id}><Card className="px-5 py-2.5 bg-transparent shadow-none"><ChunkTitle item={x}></ChunkTitle>- <p className="!mt-2.5 whitespace-pre-wrap">- {x.content_with_weight}- </p>+ <div+ className="!mt-2.5 whitespace-pre-wrap [&_em]:text-accent-primary [&_em]:not-italic"+ dangerouslySetInnerHTML={{+ __html: sanitizeHtmlWithImagesAsText(+ x.highlight || x.content_with_weight,+ ),+ }}+ /><div className="mt-2.5 text-right text-xs text-text-sub-title-invert">{x.doc_name || x.docnm_kwd}</div>diff --git a/web/src/pages/next-search/search-view.tsx b/web/src/pages/next-search/search-view.tsxindex 83483e3004..336816ea11 100644--- a/web/src/pages/next-search/search-view.tsx+++ b/web/src/pages/next-search/search-view.tsx@@ -259,7 +259,7 @@ export default function SearchingView({<divdangerouslySetInnerHTML={{__html: sanitizeHtmlWithImagesAsText(- chunk.content_with_weight,+ chunk.highlight || chunk.content_with_weight,).trim(),}}className={classNames(