Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: tune search relevance for partial and unique queries

Olfa Maslahdb88ef82d9be
diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.ts
index 88bb9d45e..91504da78 100644
--- a/frontend/app/api/queries/useGetSearchQuery.ts
+++ b/frontend/app/api/queries/useGetSearchQuery.ts
@@ -70,7 +70,6 @@ export const useGetSearchQuery = (
options?: Omit<UseQueryOptions, "queryKey" | "queryFn">,
) => {
const queryClient = useQueryClient();
-
const getFileIdentity = (chunk: ChunkResult): string => {
const normalizedFilename = chunk.filename?.trim();
if (normalizedFilename) {
@@ -87,6 +86,7 @@ export const useGetSearchQuery = (
// Normalize the query to match what will actually be searched
const effectiveQuery = query || queryData?.query || "*";
+ const normalizedQuery = effectiveQuery.trim();
async function getFiles(): Promise<File[]> {
try {
@@ -98,10 +98,21 @@ export const useGetSearchQuery = (
? SEARCH_CONSTANTS.WILDCARD_QUERY_LIMIT
: queryData?.limit || 100;
+ const baseScoreThreshold =
+ queryData?.scoreThreshold ?? SEARCH_CONSTANTS.DEFAULT_SCORE_THRESHOLD;
+ const isShortSingleTokenQuery =
+ normalizedQuery !== "*" &&
+ normalizedQuery.length > 0 &&
+ normalizedQuery.length <= 4 &&
+ !normalizedQuery.includes(" ");
+ const dynamicScoreThreshold = isShortSingleTokenQuery
+ ? Math.min(baseScoreThreshold, 1.0)
+ : baseScoreThreshold;
+
const searchPayload: SearchPayload = {
query: effectiveQuery,
limit: searchLimit,
- scoreThreshold: queryData?.scoreThreshold || 0,
+ scoreThreshold: dynamicScoreThreshold,
};
if (queryData?.filters) {
searchPayload.filters =
diff --git a/frontend/lib/constants.ts b/frontend/lib/constants.ts
index f0e1a70eb..cdc7b2ec3 100644
--- a/frontend/lib/constants.ts
+++ b/frontend/lib/constants.ts
@@ -30,6 +30,7 @@ export const UI_CONSTANTS = {
*/
export const SEARCH_CONSTANTS = {
WILDCARD_QUERY_LIMIT: 10000, // Maximum allowed limit for wildcard searches
+ DEFAULT_SCORE_THRESHOLD: 1.25, // Default relevance threshold for knowledge search
} as const;
export const ANIMATION_DURATION = 0.4;
diff --git a/src/services/search_service.py b/src/services/search_service.py
index b1992efe0..0ac9c3e1b 100644
--- a/src/services/search_service.py
+++ b/src/services/search_service.py
@@ -315,10 +315,21 @@ class SearchService:
"query": query,
"fields": ["text^2", "filename^1.5"],
"type": "best_fields",
- "fuzziness": "AUTO",
+ "operator": "and",
+ "fuzziness": "AUTO:4,7",
"boost": 0.3, # 30% weight for keyword search
}
},
+ {
+ # Help prefix/partial inputs (e.g. "vita" -> "vitamin")
+ # without making the whole query fully fuzzy.
+ "multi_match": {
+ "query": query,
+ "fields": ["text^1.5", "filename^2"],
+ "type": "bool_prefix",
+ "boost": 0.25,
+ }
+ },
],
"minimum_should_match": 1,
"filter": all_filters,
@@ -479,15 +490,35 @@ class SearchService:
}
)
+ # If query text appears verbatim in one subset of files, prefer those files
+ # to avoid broad semantic spillover for unique lookups.
+ normalized_query = query.strip().lower()
+ if (
+ normalized_query
+ and not is_wildcard_match_all
+ and len(normalized_query) >= 4
+ ):
+ exact_files = {
+ filename
+ for chunk in chunks
+ for filename in [chunk.get("filename")]
+ if isinstance(filename, str)
+ and (
+ normalized_query in filename.lower()
+ or (
+ isinstance(chunk.get("text"), str)
+ and normalized_query in chunk.get("text", "").lower()
+ )
+ )
+ }
+ if exact_files:
+ chunks = [chunk for chunk in chunks if chunk.get("filename") in exact_files]
+
# Return both transformed results and aggregations
return {
"results": chunks,
"aggregations": results.get("aggregations", {}),
- "total": (
- results.get("hits", {}).get("total", {}).get("value")
- if isinstance(results.get("hits", {}).get("total"), dict)
- else results.get("hits", {}).get("total")
- ),
+ "total": len(chunks),
}
async def search(