Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: Knowledge table filtering (#2242)

Jason Dong152225a148f1
diff --git a/frontend/app/api/queries/useGetSearchAggregations.ts b/frontend/app/api/queries/useGetSearchAggregations.ts
index 0f7fa9914..b18ae85b2 100644
--- a/frontend/app/api/queries/useGetSearchAggregations.ts
+++ b/frontend/app/api/queries/useGetSearchAggregations.ts
@@ -6,7 +6,9 @@ import {
export interface FacetBucket {
key: string;
+ label?: string;
count?: number;
+ doc_count?: number;
}
export interface SearchAggregations {
diff --git a/frontend/app/api/queries/useListFiles.ts b/frontend/app/api/queries/useListFiles.ts
index c9513c591..396975022 100644
--- a/frontend/app/api/queries/useListFiles.ts
+++ b/frontend/app/api/queries/useListFiles.ts
@@ -10,9 +10,10 @@ export interface ListFilesParams {
pageSize?: number;
sortBy?: string;
sortOrder?: "asc" | "desc";
- connectorType?: string;
- mimetype?: string;
- owner?: string;
+ connectorType?: string[];
+ mimetype?: string[];
+ owner?: string[];
+ dataSources?: string[];
search?: string;
afterKey?: Record<string, unknown> | null; //composite pagination cursor, could be undefined in the beginning
}
@@ -39,10 +40,12 @@ export const useListFiles = (
if (params.pageSize) searchParams.set("page_size", String(params.pageSize));
if (params.sortBy) searchParams.set("sort_by", params.sortBy);
if (params.sortOrder) searchParams.set("sort_order", params.sortOrder);
- if (params.connectorType)
- searchParams.set("connector_type", params.connectorType);
- if (params.mimetype) searchParams.set("mimetype", params.mimetype);
- if (params.owner) searchParams.set("owner", params.owner);
+ for (const v of params.connectorType ?? [])
+ searchParams.append("connector_type", v);
+ for (const v of params.mimetype ?? []) searchParams.append("mimetype", v);
+ for (const v of params.owner ?? []) searchParams.append("owner", v);
+ for (const v of params.dataSources ?? [])
+ searchParams.append("data_sources", v);
if (params.search) searchParams.set("search", params.search);
if (params.afterKey)
searchParams.set("after_key", JSON.stringify(params.afterKey));
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index 648f456cd..1fa210a97 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -102,15 +102,6 @@ function pruneNonDeletableGridSelection(
return pruned;
}
-/** List-files uses term filters; "*" means "any" in the UI — do not send it literally. */
-function listFilesFilterParam(values?: string[]): string | undefined {
- const raw = values?.[0]?.trim();
- if (!raw || raw === "*") {
- return undefined;
- }
- return raw;
-}
-
// Function to get the appropriate icon for a connector type
function getSourceIcon(connectorType?: string) {
if (connectorType) {
@@ -141,6 +132,27 @@ const AG_FIELD_TO_SORT_BY: Record<string, string> = {
status: "status",
};
+function listFilesFilterValues(values?: string[]) {
+ const filtered = values?.filter((value) => value !== "*");
+ return filtered && filtered.length > 0 ? filtered : undefined;
+}
+
+function buildFilterPageResetKey(
+ parsedFilterData: ReturnType<typeof useKnowledgeFilter>["parsedFilterData"],
+) {
+ if (!parsedFilterData) {
+ return "";
+ }
+
+ return JSON.stringify({
+ query: parsedFilterData.query,
+ connector_types: parsedFilterData.filters.connector_types,
+ document_types: parsedFilterData.filters.document_types,
+ owners: parsedFilterData.filters.owners,
+ data_sources: parsedFilterData.filters.data_sources,
+ });
+}
+
function SearchPage() {
const isCloudBrand = useIsCloudBrand();
const queryClient = useQueryClient();
@@ -341,6 +353,7 @@ function SearchPage() {
const isWildcardQuery =
(effectiveSearchText === "" || effectiveSearchText === "*") &&
!hasActiveFilters;
+ const filterPageResetKey = buildFilterPageResetKey(parsedFilterData);
const {
data: listFilesData,
@@ -355,11 +368,16 @@ function SearchPage() {
sortBy,
sortOrder,
afterKey: cursorCacheRef.current.get(currentPage) ?? null,
- connectorType: listFilesFilterParam(
+ connectorType: listFilesFilterValues(
parsedFilterData?.filters?.connector_types,
),
- mimetype: listFilesFilterParam(parsedFilterData?.filters?.document_types),
- owner: listFilesFilterParam(parsedFilterData?.filters?.owners),
+ mimetype: listFilesFilterValues(
+ parsedFilterData?.filters?.document_types,
+ ),
+ owner: listFilesFilterValues(parsedFilterData?.filters?.owners),
+ dataSources: listFilesFilterValues(
+ parsedFilterData?.filters?.data_sources,
+ ),
},
{
refetchInterval: 5000,
@@ -493,7 +511,7 @@ function SearchPage() {
useEffect(() => {
cursorCacheRef.current = new Map();
setCurrentPage(1);
- }, [effectiveSearchText]);
+ }, [effectiveSearchText, filterPageResetKey]);
// when the server responds with an after_key for page N, cache it as the cursor for page N+1
useEffect(() => {
diff --git a/frontend/components/knowledge-filter-panel.tsx b/frontend/components/knowledge-filter-panel.tsx
index 774d69610..7ad90b4ae 100644
--- a/frontend/components/knowledge-filter-panel.tsx
+++ b/frontend/components/knowledge-filter-panel.tsx
@@ -5,11 +5,10 @@ import { useEffect, useRef, useState } from "react";
import { useCreateFilter } from "@/app/api/mutations/useCreateFilter";
import { useDeleteFilter } from "@/app/api/mutations/useDeleteFilter";
import { useUpdateFilter } from "@/app/api/mutations/useUpdateFilter";
-import { useGetSearchAggregations } from "@/app/api/queries/useGetSearchAggregations";
import {
- EMPTY_SEARCH_RESULT,
- useGetSearchQuery,
-} from "@/app/api/queries/useGetSearchQuery";
+ type FacetBucket,
+ useGetSearchAggregations,
+} from "@/app/api/queries/useGetSearchAggregations";
import { FilterIconPopover } from "@/components/filter-icon-popover";
import { Button } from "@/components/ui/button";
import {
@@ -25,19 +24,9 @@ import { MultiSelect } from "@/components/ui/multi-select";
import { Slider } from "@/components/ui/slider";
import { Textarea } from "@/components/ui/textarea";
import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";
-import { useTask } from "@/contexts/task-context";
import { usePermissions } from "@/hooks/use-permissions";
import { trackButton } from "@/lib/analytics";
import type { FilterColor, IconKey } from "@/lib/filter-constants";
-import {
- buildActiveSourceOptions,
- buildKnowledgeTableRows,
-} from "@/lib/knowledge-table-state";
-
-interface FacetBucket {
- key: string;
- count?: number;
-}
interface AvailableFacets {
data_sources: FacetBucket[];
@@ -66,6 +55,13 @@ const formatDate = (dateString: string) => {
});
};
+const formatFacetLabel = (bucket: FacetBucket) => {
+ const displayValue = bucket.label ?? bucket.key;
+ return typeof bucket.count === "number"
+ ? `${displayValue} (${bucket.count})`
+ : displayValue;
+};
+
export function KnowledgeFilterPanel() {
const {
queryOverride,
@@ -77,7 +73,6 @@ export function KnowledgeFilterPanel() {
createMode,
endCreateMode,
} = useKnowledgeFilter();
- const { files: taskFiles } = useTask();
const deleteFilterMutation = useDeleteFilter();
const updateFilterMutation = useUpdateFilter();
const createFilterMutation = useCreateFilter();
@@ -171,26 +166,27 @@ export function KnowledgeFilterPanel() {
gcTime: 5 * 60_000,
});
- const { data = EMPTY_SEARCH_RESULT } = useGetSearchQuery("*", null, {
- enabled: isPanelOpen,
- });
- const allSearchData = data.files;
-
useEffect(() => {
if (!aggregations) return;
- const extractKeys = (buckets: { key: string }[] = []): FacetBucket[] =>
- buckets.map((b) => ({ key: b.key }));
+ const extractBuckets = (buckets: FacetBucket[] = []): FacetBucket[] =>
+ buckets.map((bucket) => ({
+ key: bucket.key,
+ label: bucket.label,
+ count: bucket.doc_count ?? bucket.count,
+ }));
const facets = {
- data_sources: extractKeys(aggregations.data_sources?.buckets),
- document_types: extractKeys(aggregations.document_types?.buckets),
- owners: extractKeys(aggregations.owners?.buckets),
- connector_types: extractKeys(aggregations.connector_types?.buckets),
+ data_sources: extractBuckets(aggregations.data_sources?.buckets),
+ document_types: extractBuckets(aggregations.document_types?.buckets),
+ owners: extractBuckets(aggregations.owners?.buckets),
+ connector_types: extractBuckets(aggregations.connector_types?.buckets),
};
setAvailableFacets(facets);
}, [aggregations]);
- const tableRows = buildKnowledgeTableRows(allSearchData, taskFiles);
- const sourceOptions = buildActiveSourceOptions(tableRows);
+ const sourceOptions = (availableFacets.data_sources || []).map((bucket) => ({
+ value: bucket.key,
+ label: formatFacetLabel(bucket),
+ }));
const availableSourceValues = new Set(sourceOptions.map((o) => o.value));
// Don't render if panel is closed or we don't have any data
@@ -393,7 +389,7 @@ export function KnowledgeFilterPanel() {
options={(availableFacets.document_types || []).map(
(bucket) => ({
value: bucket.key,
- label: bucket.key,
+ label: formatFacetLabel(bucket),
}),
)}
value={selectedFilters.document_types}
@@ -409,7 +405,7 @@ export function KnowledgeFilterPanel() {
<MultiSelect
options={(availableFacets.owners || []).map((bucket) => ({
value: bucket.key,
- label: bucket.key,
+ label: formatFacetLabel(bucket),
}))}
value={selectedFilters.owners}
onValueChange={(values) => handleFilterChange("owners", values)}
@@ -423,7 +419,7 @@ export function KnowledgeFilterPanel() {
options={(availableFacets.connector_types || []).map(
(bucket) => ({
value: bucket.key,
- label: bucket.key,
+ label: formatFacetLabel(bucket),
}),
)}
value={selectedFilters.connector_types}
diff --git a/src/api/files.py b/src/api/files.py
index cc4b16c9c..c5ea92167 100644
--- a/src/api/files.py
+++ b/src/api/files.py
@@ -44,11 +44,14 @@ async def list_files(
page_size: int = Query(25, ge=1, le=500, description="Items per page"),
sort_by: str = Query("filename", description="Sort field"),
sort_order: str = Query("asc", pattern="^(asc|desc)$", description="Sort order"),
- connector_type: str | None = Query(None, description="Filter by connector type"),
- mimetype: str | None = Query(None, description="Filter by MIME type"),
- owner: str | None = Query(None, description="Filter by owner"),
+ connector_type: list[str] | None = Query(
+ None, description="Filter by connector type (repeatable)"
+ ),
+ mimetype: list[str] | None = Query(None, description="Filter by MIME type (repeatable)"),
+ owner: list[str] | None = Query(None, description="Filter by owner (repeatable)"),
search: str | None = Query(None, description="Search filename"),
after_key: str | None = Query(None, description="Composite pagination cursor (JSON-encoded)"),
+ data_sources: list[str] | None = Query(None, description="Filename whitelist (repeatable)"),
file_service=Depends(get_file_service_v2),
user: User = Depends(get_current_user),
):
@@ -68,6 +71,7 @@ async def list_files(
owner=owner,
search=search,
after_key=parsed_after_key,
+ data_sources=data_sources,
)
return JSONResponse(result)
except Exception as e:
@@ -86,10 +90,13 @@ async def search_files(
q: str = Query(..., min_length=1, description="Search query"),
page: int = Query(1, ge=1, description="Page number"),
page_size: int = Query(25, ge=1, le=500, description="Items per page"),
- connector_type: str | None = Query(None, description="Filter by connector type"),
- mimetype: str | None = Query(None, description="Filter by MIME type"),
- owner: str | None = Query(None, description="Filter by owner"),
+ connector_type: list[str] | None = Query(
+ None, description="Filter by connector type (repeatable)"
+ ),
+ mimetype: list[str] | None = Query(None, description="Filter by MIME type (repeatable)"),
+ owner: list[str] | None = Query(None, description="Filter by owner (repeatable)"),
after_key: str | None = Query(None, description="Composite pagination cursor (JSON-encoded)"),
+ data_sources: list[str] | None = Query(None, description="Filename whitelist (repeatable)"),
file_service=Depends(get_file_service_v2),
user: User = Depends(get_current_user),
):
@@ -107,6 +114,7 @@ async def search_files(
mimetype=mimetype,
owner=owner,
after_key=parsed_after_key,
+ data_sources=data_sources,
)
return JSONResponse(result)
except Exception as e:
diff --git a/src/api/v2/files.py b/src/api/v2/files.py
index 1691ffab9..cec3b70d7 100644
--- a/src/api/v2/files.py
+++ b/src/api/v2/files.py
@@ -22,11 +22,14 @@ async def list_files(
page_size: int = Query(25, ge=1, le=500, description="Items per page"),
sort_by: str = Query("filename", description="Sort field"),
sort_order: str = Query("asc", pattern="^(asc|desc)$", description="Sort order"),
- connector_type: str | None = Query(None, description="Filter by connector type"),
- mimetype: str | None = Query(None, description="Filter by MIME type"),
- owner: str | None = Query(None, description="Filter by owner"),
+ connector_type: list[str] | None = Query(
+ None, description="Filter by connector type (repeatable)"
+ ),
+ mimetype: list[str] | None = Query(None, description="Filter by MIME type (repeatable)"),
+ owner: list[str] | None = Query(None, description="Filter by owner (repeatable)"),
search: str | None = Query(None, description="Search filename"),
after_key: str | None = Query(None, description="Composite pagination cursor (JSON-encoded)"),
+ data_sources: list[str] | None = Query(None, description="Filename whitelist (repeatable)"),
file_service=Depends(get_file_service_v2),
user: User = Depends(require_api_key_permission("knowledge:read:own")),
):
@@ -41,6 +44,7 @@ async def list_files(
owner=owner,
search=search,
after_key=after_key,
+ data_sources=data_sources,
file_service=file_service,
user=user,
)
@@ -50,10 +54,13 @@ async def search_files(
q: str = Query(..., min_length=1, description="Search query"),
page: int = Query(1, ge=1, description="Page number"),
page_size: int = Query(25, ge=1, le=500, description="Items per page"),
- connector_type: str | None = Query(None, description="Filter by connector type"),
- mimetype: str | None = Query(None, description="Filter by MIME type"),
- owner: str | None = Query(None, description="Filter by owner"),
+ connector_type: list[str] | None = Query(
+ None, description="Filter by connector type (repeatable)"
+ ),
+ mimetype: list[str] | None = Query(None, description="Filter by MIME type (repeatable)"),
+ owner: list[str] | None = Query(None, description="Filter by owner (repeatable)"),
after_key: str | None = Query(None, description="Composite pagination cursor (JSON-encoded)"),
+ data_sources: list[str] | None = Query(None, description="Filename whitelist (repeatable)"),
file_service=Depends(get_file_service_v2),
user: User = Depends(require_api_key_permission("knowledge:read:own")),
):
@@ -66,6 +73,7 @@ async def search_files(
mimetype=mimetype,
owner=owner,
after_key=after_key,
+ data_sources=data_sources,
file_service=file_service,
user=user,
)
diff --git a/src/services/file_service.py b/src/services/file_service.py
index da4a98f18..1e581dadb 100644
--- a/src/services/file_service.py
+++ b/src/services/file_service.py
@@ -117,8 +117,6 @@ class FileService:
filter_clauses.append({"term": {"owner": owner}})
if search:
- # Combine wildcard (partial) and prefix for flexible matching.
- # case_insensitive=True so "jason" matches "JASON_RESUME.pdf".
must.append(
{
"bool": {
diff --git a/src/services/file_service_v2.py b/src/services/file_service_v2.py
index fc78e24d6..6f882a2a7 100644
--- a/src/services/file_service_v2.py
+++ b/src/services/file_service_v2.py
@@ -49,11 +49,12 @@ class FileServiceV2:
page_size: int = 25,
sort_by: str = "filename",
sort_order: str = "asc",
- connector_type: str | None = None,
- mimetype: str | None = None,
- owner: str | None = None,
+ connector_type: list[str] | None = None,
+ mimetype: list[str] | None = None,
+ owner: list[str] | None = None,
search: str | None = None,
after_key: dict | None = None,
+ data_sources: list[str] | None = None,
) -> dict[str, Any]:
"""
List files with server-side pagination via composite aggregation.
@@ -69,7 +70,9 @@ class FileServiceV2:
"""
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
- query = self._build_filter_query(user_id, connector_type, mimetype, owner, search)
+ query = self._build_filter_query(
+ user_id, connector_type, mimetype, owner, search, data_sources
+ )
total, is_approximate = await self._get_file_count(opensearch_client, query)
if sort_by == "chunk_count":
@@ -199,10 +202,11 @@ class FileServiceV2:
query: str = "",
page: int = 1,
page_size: int = 25,
- connector_type: str | None = None,
- mimetype: str | None = None,
- owner: str | None = None,
+ connector_type: list[str] | None = None,
+ mimetype: list[str] | None = None,
+ owner: list[str] | None = None,
after_key: dict | None = None,
+ data_sources: list[str] | None = None,
) -> dict[str, Any]:
"""Search files by name with fuzzy/prefix matching."""
return await self.list_files(
@@ -217,25 +221,63 @@ class FileServiceV2:
owner=owner,
search=query,
after_key=after_key,
+ data_sources=data_sources,
)
def _build_filter_query(
self,
user_id: str,
- connector_type: str | None = None,
- mimetype: str | None = None,
- owner: str | None = None,
+ connector_type: list[str] | None = None,
+ mimetype: list[str] | None = None,
+ owner: list[str] | None = None,
search: str | None = None,
+ data_sources: list[str] | None = None,
) -> dict[str, Any]:
+ def _effective_values(values: str | list[str] | None) -> list[str]:
+ if values is None:
+ return []
+ if isinstance(values, str):
+ values = [values]
+ return [value for value in values if value != "*"]
+
must = []
filter_clauses = []
- if connector_type:
- filter_clauses.append({"term": {"connector_type": connector_type}})
- if mimetype:
- filter_clauses.append({"term": {"mimetype": mimetype}})
- if owner:
- filter_clauses.append({"term": {"owner": owner}})
+ effective_connector_types = _effective_values(connector_type)
+ if effective_connector_types:
+ clause = (
+ {"term": {"connector_type": effective_connector_types[0]}}
+ if len(effective_connector_types) == 1
+ else {"terms": {"connector_type": effective_connector_types}}
+ )
+ filter_clauses.append(clause)
+
+ effective_mimetypes = _effective_values(mimetype)
+ if effective_mimetypes:
+ clause = (
+ {"term": {"mimetype": effective_mimetypes[0]}}
+ if len(effective_mimetypes) == 1
+ else {"terms": {"mimetype": effective_mimetypes}}
+ )
+ filter_clauses.append(clause)
+
+ effective_owners = _effective_values(owner)
+ if effective_owners:
+ clause = (
+ {"term": {"owner": effective_owners[0]}}
+ if len(effective_owners) == 1
+ else {"terms": {"owner": effective_owners}}
+ )
+ filter_clauses.append(clause)
+
+ effective_sources = _effective_values(data_sources)
+ if effective_sources:
+ clause = (
+ {"term": {"filename": effective_sources[0]}}
+ if len(effective_sources) == 1
+ else {"terms": {"filename": effective_sources}}
+ )
+ filter_clauses.append(clause)
if search:
must.append(
diff --git a/src/services/search_service.py b/src/services/search_service.py
index 105cd697a..51bb82a2c 100644
--- a/src/services/search_service.py
+++ b/src/services/search_service.py
@@ -37,6 +37,87 @@ def _is_exact_token_query(query: str) -> bool:
return bool(re.search(r"[a-zA-Z]", query) and re.search(r"\d", query))
+def _build_file_facet_aggregations(size_overrides: dict[str, int] | None = None) -> dict[str, Any]:
+ size_overrides = size_overrides or {}
+ facet_fields = {
+ "data_sources": "filename",
+ "document_types": "mimetype",
+ "owners": "owner",
+ "connector_types": "connector_type",
+ "embedding_models": "embedding_model",
+ }
+
+ aggregations: dict[str, Any] = {}
+ for facet_name, field_name in facet_fields.items():
+ agg: dict[str, Any] = {
+ "terms": {
+ "field": field_name,
+ "size": size_overrides.get(facet_name, 1000),
+ },
+ }
+ if facet_name == "owners":
+ agg["aggs"] = {
+ "files": {"cardinality": {"field": "filename"}},
+ "owner_metadata": {
+ "top_hits": {
+ "size": 1,
+ "_source": {
+ "includes": ["owner_name", "owner_email", "owner"],
+ },
+ }
+ },
+ }
+ elif facet_name != "data_sources":
+ agg["aggs"] = {"files": {"cardinality": {"field": "filename"}}}
+ aggregations[facet_name] = agg
+ return aggregations
+
+
+def _normalize_file_facet_aggregations(aggregations: dict[str, Any]) -> dict[str, Any]:
+ normalized = dict(aggregations)
+ for facet_name in (
+ "data_sources",
+ "document_types",
+ "owners",
+ "connector_types",
+ "embedding_models",
+ ):
+ facet = aggregations.get(facet_name)
+ if not isinstance(facet, dict):
+ normalized[facet_name] = {"buckets": []}
+ continue
+
+ raw_buckets = facet.get("buckets", [])
+ buckets = raw_buckets if isinstance(raw_buckets, list) else []
+ normalized_buckets: list[dict[str, Any]] = []
+ for bucket in buckets:
+ if not isinstance(bucket, dict) or not bucket.get("key"):
+ continue
+
+ normalized_bucket = {"key": bucket.get("key")}
+ if facet_name == "owners":
+ owner_hits = bucket.get("owner_metadata", {}).get("hits", {}).get("hits", [])
+ owner_source = owner_hits[0].get("_source", {}) if owner_hits else {}
+ normalized_bucket["label"] = (
+ owner_source.get("owner_name")
+ or owner_source.get("owner_email")
+ or bucket.get("key")
+ )
+
+ if facet_name != "data_sources":
+ normalized_bucket["doc_count"] = bucket.get("files", {}).get(
+ "value", bucket.get("doc_count", 0)
+ )
+
+ normalized_buckets.append(normalized_bucket)
+
+ normalized[facet_name] = {
+ **facet,
+ "buckets": normalized_buckets,
+ }
+ return normalized
+
+
def _apply_exact_match_file_filter(
query: str,
chunks: list[dict[str, Any]],
@@ -81,17 +162,34 @@ def _apply_exact_match_file_filter(
# Filter to only chunks from files with exact matches
chunks = [chunk for chunk in chunks if chunk.get("filename") in exact_files]
- def _build_terms_agg(field: str) -> dict[str, Any]:
- counts = Counter(
- value
- for chunk in chunks
- for value in [chunk.get(field)]
- if isinstance(value, str) and value
- )
+ def _build_terms_agg(field: str, label_field: str | None = None) -> dict[str, Any]:
+ file_counts: Counter[Any] = Counter()
+ labels_by_value: dict[str, str] = {}
+ for chunk in chunks:
+ value = chunk.get(field)
+ filename = chunk.get("filename")
+ if not isinstance(value, str) or not value:
+ continue
+ if not isinstance(filename, str) or not filename:
+ continue
+ file_counts[(value, filename)] += 1
+ if label_field and value not in labels_by_value:
+ label = chunk.get(label_field)
+ if isinstance(label, str) and label:
+ labels_by_value[value] = label
+
+ counts = Counter(value for value, _filename in file_counts)
return {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
- "buckets": [{"key": key, "doc_count": count} for key, count in counts.most_common()],
+ "buckets": [
+ {
+ "key": key,
+ "doc_count": count,
+ **({"label": labels_by_value.get(key, key)} if label_field else {}),
+ }
+ for key, count in counts.most_common()
+ ],
}
# Keep aggregations consistent with the post-filtered result set.
@@ -99,7 +197,7 @@ def _apply_exact_match_file_filter(
**aggregations,
"data_sources": _build_terms_agg("filename"),
"document_types": _build_terms_agg("mimetype"),
- "owners": _build_terms_agg("owner"),
+ "owners": _build_terms_agg("owner", label_field="owner_name"),
"connector_types": _build_terms_agg("connector_type"),
"embedding_models": _build_terms_agg("embedding_model"),
}
@@ -496,13 +594,7 @@ class SearchService:
search_body: dict[str, Any] = {
"query": query_block,
- "aggs": {
- "data_sources": {"terms": {"field": "filename", "size": 20}},
- "document_types": {"terms": {"field": "mimetype", "size": 10}},
- "owners": {"terms": {"field": "owner", "size": 10}},
- "connector_types": {"terms": {"field": "connector_type", "size": 10}},
- "embedding_models": {"terms": {"field": "embedding_model", "size": 10}},
- },
+ "aggs": _build_file_facet_aggregations(),
"_source": [
"filename",
"mimetype",
@@ -665,7 +757,7 @@ class SearchService:
chunks, aggregations = _apply_exact_match_file_filter(
query,
chunks,
- results.get("aggregations", {}),
+ _normalize_file_facet_aggregations(results.get("aggregations", {})),
is_wildcard_match_all=is_wildcard_match_all,
)
diff --git a/tests/unit/test_query_acl_filtering.py b/tests/unit/test_query_acl_filtering.py
index df7833287..aa4f98393 100644
--- a/tests/unit/test_query_acl_filtering.py
+++ b/tests/unit/test_query_acl_filtering.py
@@ -46,6 +46,65 @@ def test_file_service_query_omits_application_acl_filter():
]
+def test_file_service_v2_data_sources_single_emits_term():
+ from src.services.file_service_v2 import FileServiceV2
+
+ query = FileServiceV2()._build_filter_query(
+ user_id="user-123",
+ data_sources=["report.pdf"],
+ )
+ assert {"term": {"filename": "report.pdf"}} in query["bool"]["filter"]
+
+
+def test_file_service_v2_data_sources_multi_emits_terms():
+ from src.services.file_service_v2 import FileServiceV2
+
+ query = FileServiceV2()._build_filter_query(
+ user_id="user-123",
+ data_sources=["report.pdf", "roadmap.docx"],
+ )
+ assert {"terms": {"filename": ["report.pdf", "roadmap.docx"]}} in query["bool"]["filter"]
+
+
+def test_file_service_v2_multi_connector_and_data_sources():
+ from src.services.file_service_v2 import FileServiceV2
+
+ query = FileServiceV2()._build_filter_query(
+ user_id="user-123",
+ connector_type=["google_drive", "confluence"],
+ data_sources=["doc1.pdf", "doc2.pdf"],
+ )
+ filters = query["bool"]["filter"]
+ assert {"terms": {"connector_type": ["google_drive", "confluence"]}} in filters
+ assert {"terms": {"filename": ["doc1.pdf", "doc2.pdf"]}} in filters
+
+
+def test_file_service_v2_data_sources_wildcard_sentinel_produces_no_filter():
+ from src.services.file_service_v2 import FileServiceV2
+
+ # ["*"] means "all files" — must not emit a filename term
+ query = FileServiceV2()._build_filter_query(
+ user_id="user-123",
+ data_sources=["*"],
+ )
+ filters = query["bool"]["filter"]
+ assert not any(
+ "filename" in list(c.get("term", {})) or "filename" in list(c.get("terms", {}))
+ for c in filters
+ )
+
+
+def test_file_service_v2_data_sources_mixed_wildcard_strips_sentinel():
+ from src.services.file_service_v2 import FileServiceV2
+
+ # ["*", "report.pdf"] — strip "*", keep "report.pdf"
+ query = FileServiceV2()._build_filter_query(
+ user_id="user-123",
+ data_sources=["*", "report.pdf"],
+ )
+ assert {"term": {"filename": "report.pdf"}} in query["bool"]["filter"]
+
+
def test_service_query_paths_do_not_apply_document_visibility_filters():
repo_root = Path(__file__).resolve().parents[2]
helper_name = "build" + "_acl_filter"