OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: more filter_id issues (#1730)
Sebastián Estévezdb9ae7f72a8a
diff --git a/sdks/python/openrag_sdk/documents.py b/sdks/python/openrag_sdk/documents.pyindex d2ee795a7..414144c3d 100644--- a/sdks/python/openrag_sdk/documents.py+++ b/sdks/python/openrag_sdk/documents.py@@ -163,7 +163,7 @@ class DocumentsClient:except NotFoundError as e:# Keep delete idempotent for SDK callers: a missing document is not# an exception.- # (Filter-not-found 404s do raise — that's a caller error, not idempotency.)+ # Filter-not-found 404s still raise because the filter_id is caller input.if filename is not None and getattr(e, "status_code", None) == 404:return DeleteDocumentResponse(success=False,diff --git a/src/api/v1/_filter_resolution.py b/src/api/v1/_filter_resolution.pyindex 488cb095a..417e306ca 100644--- a/src/api/v1/_filter_resolution.py+++ b/src/api/v1/_filter_resolution.py@@ -4,8 +4,9 @@ API consumers expect `filter_id` on /v1/chat, /v1/search, /v1/documents to "justwithout first GETting the filter, parsing its `query_data`, and resending the parts asinline `filters`. This helper performs that lookup + normalization server-side.-Wildcard handling mirrors `frontend/lib/filter-normalization.ts::buildSearchPayloadFilters`:-a dimension like `data_sources: ["*"]` collapses to `[]` (i.e. "no filter on this field").+Wildcard handling mirrors `frontend/lib/filter-normalization.ts::buildSearchPayloadFilters`.+Each filter dimension is a list; if that list contains the wildcard value `"*"`+(for example, `data_sources: ["*"]`), the dimension is treated as unscoped."""import json@@ -17,7 +18,7 @@ _FILTER_DIMENSIONS = ("data_sources", "document_types", "owners", "connector_typdef _strip_wildcards(filters: dict[str, Any] | None) -> dict[str, list[str]]:- """Drop `["*"]` and empty lists from each filter dimension."""+ """Keep only filter dimensions that contain concrete values."""if not filters:return {}cleaned: dict[str, list[str]] = {}@@ -66,3 +67,34 @@ async def resolve_filter_id("limit": query_data.get("limit", 10),"score_threshold": query_data.get("scoreThreshold", 0),}+++def merge_filter_overrides(+ resolved: dict[str, Any],+ request_body: Any,+) -> tuple[dict[str, Any] | None, int, float]:+ """Merge resolved filter values with explicitly provided request fields.++ Inline request fields override saved filter values by field presence, not by+ truthiness. This lets callers intentionally set values such as `limit=10`,+ `score_threshold=0`, or `filters={}`.+ """+ provided_fields: set[str] = getattr(request_body, "model_fields_set", set())++ filters: dict[str, Any] | None = resolved["filters"]+ if "filters" in provided_fields:+ inline_filters = request_body.filters+ if inline_filters:+ filters = {**resolved["filters"], **inline_filters}+ else:+ filters = inline_filters++ limit = request_body.limit+ if "limit" not in provided_fields:+ limit = resolved["limit"]++ score_threshold = request_body.score_threshold+ if "score_threshold" not in provided_fields:+ score_threshold = resolved["score_threshold"]++ return filters, limit, score_thresholddiff --git a/src/api/v1/chat.py b/src/api/v1/chat.pyindex 99bc30c07..b3d6cb97f 100644--- a/src/api/v1/chat.py+++ b/src/api/v1/chat.py@@ -13,7 +13,7 @@ from fastapi import Depends, HTTPException, Requestfrom fastapi.responses import JSONResponse, StreamingResponsefrom pydantic import BaseModel-from api.v1._filter_resolution import resolve_filter_id+from api.v1._filter_resolution import merge_filter_overrides, resolve_filter_idfrom auth_context import set_auth_context, set_score_threshold, set_search_filters, set_search_limitfrom dependencies import (get_chat_service,@@ -147,13 +147,9 @@ async def chat_create_endpoint(user_id=user.user_id,jwt_token=jwt_token,)- # Inline values override per-field; defaults (10 / 0) fall back to the filter.- if not body.filters:- resolved_filters = resolved["filters"]- if body.limit == 10:- resolved_limit = resolved["limit"]- if body.score_threshold == 0:- resolved_score_threshold = resolved["score_threshold"]+ resolved_filters, resolved_limit, resolved_score_threshold = merge_filter_overrides(+ resolved, body+ )if resolved_filters:set_search_filters(resolved_filters)diff --git a/src/api/v1/documents.py b/src/api/v1/documents.pyindex 7e5900386..dd108b259 100644--- a/src/api/v1/documents.py+++ b/src/api/v1/documents.py@@ -50,12 +50,10 @@ async def ingest_endpoint(POST /v1/documents/ingestRequest: multipart/form-data with "file" field- NOTE: `create_filter` is kept here for response-shape compatibility — the- non-v1 onboarding flow consumes the `create_filter` field echoed back in- the response. v1 SDK consumers do not currently have a workflow that uses- it, and the field is never forwarded to the actual ingest task. It should- be removed in a future major version of the v1 API once we are willing to- take the breaking change (response no longer contains `create_filter`).+ NOTE: `create_filter` is upload-router/onboarding metadata. The router+ echoes it in the upload response so onboarding can create a saved knowledge+ filter after ingestion completes. It does not affect task creation or+ ingestion behavior, and the v1 SDK does not expose this option."""return await upload_ingest_router(file=file,@@ -139,8 +137,8 @@ async def delete_document_endpoint(Provide exactly one of:- `filename`: delete all chunks for that filename.- `filter_id`: resolve the filter's `data_sources` and delete chunks for- each of those filenames. Wildcard (`["*"]`) or empty `data_sources`- is rejected to prevent mass deletion.+ each concrete filename. Empty `data_sources` or a list containing the+ wildcard value `"*"` is rejected to prevent mass deletion."""if bool(body.filename) == bool(body.filter_id):return JSONResponse(@@ -153,7 +151,7 @@ async def delete_document_endpoint(body.filter_id,knowledge_filter_service,user_id=user.user_id,- jwt_token=None,+ jwt_token=user.jwt_token,)filenames = resolved["filters"].get("data_sources") or []if not filenames:@@ -164,19 +162,21 @@ async def delete_document_endpoint(results = []total_deleted = 0+ statuses = []for fname in filenames:payload, _status = await delete_documents_by_filename_core(filename=fname,session_manager=session_manager,user_id=user.user_id,- jwt_token=None,+ jwt_token=user.jwt_token,)results.append(payload)+ statuses.append(_status)total_deleted += payload.get("deleted_chunks", 0) or 0return JSONResponse({- "success": True,+ "success": all(200 <= status < 300 for status in statuses),"deleted_chunks": total_deleted,"filenames": filenames,"filter_id": body.filter_id,@@ -188,6 +188,6 @@ async def delete_document_endpoint(filename=body.filename,session_manager=session_manager,user_id=user.user_id,- jwt_token=None,+ jwt_token=user.jwt_token,)return JSONResponse(payload, status_code=status_code)diff --git a/src/api/v1/search.py b/src/api/v1/search.pyindex c9c08da7d..a8ac58624 100644--- a/src/api/v1/search.py+++ b/src/api/v1/search.py@@ -11,7 +11,7 @@ from fastapi import Dependsfrom fastapi.responses import JSONResponsefrom pydantic import BaseModel-from api.v1._filter_resolution import resolve_filter_id+from api.v1._filter_resolution import merge_filter_overrides, resolve_filter_idfrom auth_context import set_auth_contextfrom dependencies import (get_knowledge_filter_service,@@ -44,9 +44,8 @@ async def search_endpoint(if not query:return JSONResponse({"error": "Query is required"}, status_code=400)- # API-key auth has no JWT; the gate inside search_service.search()- # skips set_auth_context() in that case, leaving search_tool() unable to- # resolve the user. Set it explicitly here, mirroring v1 chat.+ # API-key requests can arrive without a JWT. Set the auth context before+ # resolving filters so search_tool() can still identify the caller.set_auth_context(user.user_id, user.jwt_token)resolved_filters = body.filters@@ -57,14 +56,11 @@ async def search_endpoint(body.filter_id,knowledge_filter_service,user_id=user.user_id,- jwt_token=None,+ jwt_token=user.jwt_token,+ )+ resolved_filters, resolved_limit, resolved_score_threshold = merge_filter_overrides(+ resolved, body)- if not body.filters:- resolved_filters = resolved["filters"]- if body.limit == 10:- resolved_limit = resolved["limit"]- if body.score_threshold == 0:- resolved_score_threshold = resolved["score_threshold"]logger.debug("Public API search request",diff --git a/tests/integration/sdk/test_documents.py b/tests/integration/sdk/test_documents.pyindex aac8fb59c..f46ad25fd 100644--- a/tests/integration/sdk/test_documents.py+++ b/tests/integration/sdk/test_documents.py@@ -155,40 +155,50 @@ class TestDeleteByFilterId:})assert result.success is True, f"Failed to create filter: {result.error}"+ assert isinstance(result.id, str) and result.id, (+ f"Filter creation returned no id: {result.error}"+ )return result.id@pytest.mark.asyncioasync def test_delete_documents_by_filter_id(self, client, tmp_path):"""Deleting by filter_id removes only the filenames in the filter's data_sources."""alpha, beta = await self._ingest_two(client, tmp_path)- filter_id = await self._create_filter(client, [alpha.name])+ filter_id = Nonetry:+ filter_id = await self._create_filter(client, [alpha.name])result = await client.documents.delete(filter_id=filter_id)assert result.success is Trueassert result.filter_id == filter_idassert alpha.name in (result.filenames or [])assert beta.name not in (result.filenames or [])- # Beta still searchable- still_there = await client.search.query("tigers")+ # Beta still exists; scope the verification to beta's filename so+ # the assertion does not depend on semantic ranking.+ still_there = await client.search.query(+ "*", filters={"data_sources": [beta.name]}, limit=5+ )assert any(r.filename == beta.name for r in still_there.results), ("Beta should still be present after filter-id delete of alpha")finally:- await client.knowledge_filters.delete(filter_id)+ if filter_id is not None:+ await client.knowledge_filters.delete(filter_id)# Best-effort cleanupawait client.documents.delete(alpha.name)await client.documents.delete(beta.name)@pytest.mark.asyncioasync def test_delete_by_filter_id_with_wildcard_rejects(self, client):- """A filter with `["*"]` data_sources must NOT be allowed to mass-delete."""- filter_id = await self._create_filter(client, ["*"])+ """A filter whose data_sources list contains `"*"` is rejected."""+ filter_id = Nonetry:+ filter_id = await self._create_filter(client, ["*"])with pytest.raises(OpenRAGError):await client.documents.delete(filter_id=filter_id)finally:- await client.knowledge_filters.delete(filter_id)+ if filter_id is not None:+ await client.knowledge_filters.delete(filter_id)@pytest.mark.asyncioasync def test_delete_with_both_filename_and_filter_id_rejects(self, client):diff --git a/tests/integration/sdk/test_filters.py b/tests/integration/sdk/test_filters.pyindex 252801f08..b11fc7426 100644--- a/tests/integration/sdk/test_filters.py+++ b/tests/integration/sdk/test_filters.py@@ -59,6 +59,9 @@ async def _create_filter_for(client, name: str, data_sources: list[str]) -> str:})assert result.success is True, f"Failed to create filter: {result.error}"+ assert isinstance(result.id, str) and result.id, (+ f"Filter creation returned no id: {result.error}"+ )return result.id@@ -138,7 +141,7 @@ class TestFilterIdInChat:@pytest.mark.asyncioasync def test_filter_id_in_chat_inline_overrides(self, client, tmp_path):- """Inline `filters` win over filter_id per the v1 override contract."""+ """Inline filters override the resolved filter_id per field."""alpha, beta = await _ingest_pair(client, tmp_path)filter_id = await _create_filter_for(client, "SDK chat inline-override", [alpha.name])@@ -215,7 +218,7 @@ class TestFilterIdInSearch:@pytest.mark.asyncioasync def test_filter_id_in_search_inline_overrides(self, client, tmp_path):- """Inline filters override the resolved filter_id per-field."""+ """Inline filters override the resolved filter_id per field."""alpha, beta = await _ingest_pair(client, tmp_path)filter_id = await _create_filter_for(client, "SDK search inline-override", [alpha.name])diff --git a/tests/unit/api/test_v1_filter_resolution.py b/tests/unit/api/test_v1_filter_resolution.pynew file mode 100644index 000000000..649c0d7ef--- /dev/null+++ b/tests/unit/api/test_v1_filter_resolution.py@@ -0,0 +1,55 @@+from typing import Any++from pydantic import BaseModel++from api.v1._filter_resolution import merge_filter_overrides+++class _RequestBody(BaseModel):+ filters: dict[str, Any] | None = None+ limit: int = 10+ score_threshold: float = 0+++def test_merge_filter_overrides_uses_resolved_values_when_fields_absent():+ resolved = {+ "filters": {"data_sources": ["alpha.md"], "owners": ["alice"]},+ "limit": 5,+ "score_threshold": 0.4,+ }++ filters, limit, score_threshold = merge_filter_overrides(resolved, _RequestBody())++ assert filters == resolved["filters"]+ assert limit == 5+ assert score_threshold == 0.4+++def test_merge_filter_overrides_respects_explicit_defaults_and_empty_filters():+ resolved = {+ "filters": {"data_sources": ["alpha.md"]},+ "limit": 5,+ "score_threshold": 0.4,+ }+ body = _RequestBody(filters={}, limit=10, score_threshold=0)++ filters, limit, score_threshold = merge_filter_overrides(resolved, body)++ assert filters == {}+ assert limit == 10+ assert score_threshold == 0+++def test_merge_filter_overrides_merges_partial_inline_filters_per_field():+ resolved = {+ "filters": {"data_sources": ["alpha.md"], "owners": ["alice"]},+ "limit": 5,+ "score_threshold": 0.4,+ }+ body = _RequestBody(filters={"data_sources": ["beta.md"]})++ filters, limit, score_threshold = merge_filter_overrides(resolved, body)++ assert filters == {"data_sources": ["beta.md"], "owners": ["alice"]}+ assert limit == 5+ assert score_threshold == 0.4