Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Add filter_id support and v1 filter resolution (#1666)

Edwin Joseeda6aaf9370c
diff --git a/sdks/python/openrag_sdk/documents.py b/sdks/python/openrag_sdk/documents.py
index f5e483ddc..d2ee795a7 100644
--- a/sdks/python/openrag_sdk/documents.py
+++ b/sdks/python/openrag_sdk/documents.py
@@ -125,27 +125,46 @@ class DocumentsClient:
await asyncio.sleep(poll_interval)
elapsed += poll_interval
- raise TimeoutError(f"Ingestion task {task_id} did not complete within {timeout}s")
+ raise TimeoutError(
+ f"Ingestion task {task_id} did not complete within {timeout}s"
+ )
- async def delete(self, filename: str) -> DeleteDocumentResponse:
+ async def delete(
+ self,
+ filename: str | None = None,
+ *,
+ filter_id: str | None = None,
+ ) -> DeleteDocumentResponse:
"""
- Delete a document from the knowledge base.
+ Delete document(s) from the knowledge base.
- Args:
- filename: Name of the file to delete.
+ Provide exactly one of:
+ filename: delete all chunks for that filename.
+ filter_id: delete chunks for each filename in the filter's data_sources.
Returns:
DeleteDocumentResponse with deleted chunk count.
"""
+ if bool(filename) == bool(filter_id):
+ raise ValueError("Provide exactly one of `filename` or `filter_id`")
+
+ body: dict[str, str] = {}
+ if filename is not None:
+ body["filename"] = filename
+ if filter_id is not None:
+ body["filter_id"] = filter_id
+
try:
response = await self._client._request(
"DELETE",
"/api/v1/documents",
- json={"filename": filename},
+ json=body,
)
except NotFoundError as e:
- # Keep delete idempotent for SDK callers: a missing document is not an exception.
- if getattr(e, "status_code", None) == 404:
+ # Keep delete idempotent for SDK callers: a missing document is not
+ # an exception.
+ # (Filter-not-found 404s do raise — that's a caller error, not idempotency.)
+ if filename is not None and getattr(e, "status_code", None) == 404:
return DeleteDocumentResponse(
success=False,
deleted_chunks=0,
diff --git a/sdks/python/openrag_sdk/models.py b/sdks/python/openrag_sdk/models.py
index 46dfe7878..b0eee0864 100644
--- a/sdks/python/openrag_sdk/models.py
+++ b/sdks/python/openrag_sdk/models.py
@@ -1,7 +1,6 @@
"""OpenRAG SDK data models."""
-from datetime import datetime
-from typing import Any, Literal
+from typing import Literal
from pydantic import BaseModel, Field
@@ -98,6 +97,10 @@ class DeleteDocumentResponse(BaseModel):
filename: str | None = None
message: str | None = None
error: str | None = None
+ # Populated when deleting by filter_id — one entry per resolved data_source.
+ filenames: list[str] | None = None
+ filter_id: str | None = None
+ per_file: list[dict] | None = None
# Chat history models
diff --git a/sdks/typescript/src/documents.ts b/sdks/typescript/src/documents.ts
index 2ec4a555b..d7f771aa1 100644
--- a/sdks/typescript/src/documents.ts
+++ b/sdks/typescript/src/documents.ts
@@ -4,6 +4,7 @@
import type { OpenRAGClient } from "./client";
import type {
+ DeleteDocumentOptions,
DeleteDocumentResponse,
IngestResponse,
IngestTaskStatus,
@@ -144,33 +145,57 @@ export class DocumentsClient {
}
/**
- * Delete a document from the knowledge base.
+ * Delete document(s) from the knowledge base.
+ *
+ * Provide exactly one of:
+ * - filename: a single filename, or
+ * - { filename } / { filterId }: an options object.
*
- * @param filename - Name of the file to delete.
* @returns DeleteDocumentResponse with deleted chunk count.
*/
- async delete(filename: string): Promise<DeleteDocumentResponse> {
+ async delete(
+ arg: string | DeleteDocumentOptions
+ ): Promise<DeleteDocumentResponse> {
+ const opts: DeleteDocumentOptions =
+ typeof arg === "string" ? { filename: arg } : arg;
+
+ if (!opts.filename === !opts.filterId) {
+ throw new Error(
+ "Provide exactly one of `filename` or `filterId`"
+ );
+ }
+
+ const body: Record<string, string> = {};
+ if (opts.filename) body.filename = opts.filename;
+ if (opts.filterId) body.filter_id = opts.filterId;
+
try {
const response = await this.client._request("DELETE", "/api/v1/documents", {
- body: JSON.stringify({ filename }),
+ body: JSON.stringify(body),
});
const data = await response.json();
return {
success: data.success ?? false,
deleted_chunks: data.deleted_chunks ?? 0,
- filename: data.filename ?? filename,
+ filename: data.filename ?? opts.filename ?? null,
message: data.message ?? null,
error: data.error ?? null,
+ filenames: data.filenames ?? null,
+ filter_id: data.filter_id ?? null,
+ per_file: data.per_file ?? null,
};
} catch (error) {
- // Delete is idempotent: if no chunks match, backend may return 404.
- // Surface this as a non-throwing "nothing deleted" response.
- if ((error as NotFoundError)?.statusCode === 404) {
+ // Filename delete stays idempotent (404 -> success:false). Filter-id 404s
+ // are caller errors (bad filter id) and should propagate.
+ if (
+ opts.filename &&
+ (error as NotFoundError)?.statusCode === 404
+ ) {
return {
success: false,
deleted_chunks: 0,
- filename,
+ filename: opts.filename,
message: null,
error: (error as Error)?.message ?? "Resource not found",
};
diff --git a/sdks/typescript/src/types.ts b/sdks/typescript/src/types.ts
index 395f0a36a..2d2336ee8 100644
--- a/sdks/typescript/src/types.ts
+++ b/sdks/typescript/src/types.ts
@@ -77,6 +77,15 @@ export interface DeleteDocumentResponse {
filename?: string | null;
message?: string | null;
error?: string | null;
+ // Populated when deleting by filter_id — one entry per resolved data_source.
+ filenames?: string[] | null;
+ filter_id?: string | null;
+ per_file?: Array<Record<string, unknown>> | null;
+}
+
+export interface DeleteDocumentOptions {
+ filename?: string;
+ filterId?: string;
}
// Chat history types
diff --git a/sdks/typescript/tests/integration.test.ts b/sdks/typescript/tests/integration.test.ts
index 5677ba809..871dc47e4 100644
--- a/sdks/typescript/tests/integration.test.ts
+++ b/sdks/typescript/tests/integration.test.ts
@@ -190,55 +190,147 @@ describe.skipIf(SKIP_TESTS)("OpenRAG TypeScript SDK Integration", () => {
expect(filter).toBeNull();
});
- it("should use filterId in chat", async () => {
- // Create a filter first
+ it("filterId in chat actually scopes retrieval to data_sources", async () => {
+ // Ingest two distinguishable docs.
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sdk-filter-"));
+ const alphaName = `alpha_${Date.now()}.md`;
+ const betaName = `beta_${Date.now()}.md`;
+ const alphaPath = path.join(tmpDir, alphaName);
+ const betaPath = path.join(tmpDir, betaName);
+ fs.writeFileSync(alphaPath, "# Alpha\n\nPurple elephants live here.\n");
+ fs.writeFileSync(betaPath, "# Beta\n\nYellow tigers live here.\n");
+ await client.documents.ingest({ filePath: alphaPath });
+ await client.documents.ingest({ filePath: betaPath });
+
const createResult = await client.knowledgeFilters.create({
- name: "Chat Test Filter",
- description: "Filter for testing chat with filterId",
+ name: `TS chat filter scope ${Date.now()}`,
+ description: "Filter scoped to alpha only",
queryData: {
- query: "test",
- limit: 5,
+ query: "",
+ filters: {
+ data_sources: [alphaName],
+ document_types: ["*"],
+ owners: ["*"],
+ connector_types: ["*"],
+ },
+ limit: 10,
+ scoreThreshold: 0,
},
});
expect(createResult.success).toBe(true);
const filterId = createResult.id!;
try {
- // Use filter in chat
const response = await client.chat.create({
- message: "Hello with filter",
+ message: "What animals appear in these documents?",
filterId,
});
+ expect(response.sources).toBeDefined();
+ const names = (response.sources ?? []).map((s) => s.filename);
+ // Beta must NOT leak through the filter.
+ expect(names).not.toContain(betaName);
+ } finally {
+ await client.knowledgeFilters.delete(filterId);
+ await client.documents.delete(alphaName);
+ await client.documents.delete(betaName);
+ }
+ }, 60_000);
+
+ it("filterId in search actually scopes results to data_sources", async () => {
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sdk-filter-"));
+ const alphaName = `alpha_${Date.now()}.md`;
+ const betaName = `beta_${Date.now()}.md`;
+ const alphaPath = path.join(tmpDir, alphaName);
+ const betaPath = path.join(tmpDir, betaName);
+ fs.writeFileSync(alphaPath, "# Alpha\n\nPurple elephants live here.\n");
+ fs.writeFileSync(betaPath, "# Beta\n\nYellow tigers live here.\n");
+ await client.documents.ingest({ filePath: alphaPath });
+ await client.documents.ingest({ filePath: betaPath });
- expect(response.response).toBeDefined();
+ const createResult = await client.knowledgeFilters.create({
+ name: `TS search filter scope ${Date.now()}`,
+ description: "Filter scoped to alpha only",
+ queryData: {
+ query: "",
+ filters: {
+ data_sources: [alphaName],
+ document_types: ["*"],
+ owners: ["*"],
+ connector_types: ["*"],
+ },
+ limit: 10,
+ scoreThreshold: 0,
+ },
+ });
+ expect(createResult.success).toBe(true);
+ const filterId = createResult.id!;
+
+ try {
+ const results = await client.search.query("animals", { filterId });
+ for (const r of results.results) {
+ expect(r.filename).not.toBe(betaName);
+ }
} finally {
- // Cleanup
await client.knowledgeFilters.delete(filterId);
+ await client.documents.delete(alphaName);
+ await client.documents.delete(betaName);
}
- });
+ }, 60_000);
+
+ it("documents.delete(filterId) only removes filenames in the filter", async () => {
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "sdk-filter-"));
+ const alphaName = `alpha_${Date.now()}.md`;
+ const betaName = `beta_${Date.now()}.md`;
+ const alphaPath = path.join(tmpDir, alphaName);
+ const betaPath = path.join(tmpDir, betaName);
+ fs.writeFileSync(alphaPath, "# Alpha\n\nPurple elephants.\n");
+ fs.writeFileSync(betaPath, "# Beta\n\nYellow tigers.\n");
+ await client.documents.ingest({ filePath: alphaPath });
+ await client.documents.ingest({ filePath: betaPath });
- it("should use filterId in search", async () => {
- // Create a filter first
const createResult = await client.knowledgeFilters.create({
- name: "Search Test Filter",
- description: "Filter for testing search with filterId",
+ name: `TS delete-by-filter ${Date.now()}`,
+ description: "Filter scoped to alpha only",
queryData: {
- query: "test",
- limit: 5,
+ query: "",
+ filters: {
+ data_sources: [alphaName],
+ document_types: ["*"],
+ owners: ["*"],
+ connector_types: ["*"],
+ },
+ limit: 10,
+ scoreThreshold: 0,
},
});
expect(createResult.success).toBe(true);
const filterId = createResult.id!;
try {
- // Use filter in search
- const results = await client.search.query("test query", { filterId });
+ const result = await client.documents.delete({ filterId });
+ expect(result.success).toBe(true);
+ expect(result.filenames).toContain(alphaName);
+ expect(result.filenames ?? []).not.toContain(betaName);
- expect(results.results).toBeDefined();
+ // Beta still searchable
+ const remaining = await client.search.query("tigers");
+ const remainingNames = remaining.results.map((r) => r.filename);
+ expect(remainingNames).toContain(betaName);
} finally {
- // Cleanup
await client.knowledgeFilters.delete(filterId);
+ await client.documents.delete(alphaName);
+ await client.documents.delete(betaName);
}
+ }, 60_000);
+
+ it("documents.delete rejects both filename and filterId together", async () => {
+ await expect(
+ client.documents.delete({ filename: "x.pdf", filterId: "y" })
+ ).rejects.toThrow();
+ });
+
+ it("documents.delete rejects when neither arg is set", async () => {
+ await expect(client.documents.delete({})).rejects.toThrow();
});
});
diff --git a/src/api/v1/_filter_resolution.py b/src/api/v1/_filter_resolution.py
new file mode 100644
index 000000000..488cb095a
--- /dev/null
+++ b/src/api/v1/_filter_resolution.py
@@ -0,0 +1,68 @@
+"""Shared helper to resolve a `filter_id` into concrete filter values for v1 endpoints.
+
+API consumers expect `filter_id` on /v1/chat, /v1/search, /v1/documents to "just work"
+without first GETting the filter, parsing its `query_data`, and resending the parts as
+inline `filters`. This helper performs that lookup + normalization server-side.
+
+Wildcard handling mirrors `frontend/lib/filter-normalization.ts::buildSearchPayloadFilters`:
+a dimension like `data_sources: ["*"]` collapses to `[]` (i.e. "no filter on this field").
+"""
+
+import json
+from typing import Any
+
+from fastapi import HTTPException
+
+_FILTER_DIMENSIONS = ("data_sources", "document_types", "owners", "connector_types")
+
+
+def _strip_wildcards(filters: dict[str, Any] | None) -> dict[str, list[str]]:
+ """Drop `["*"]` and empty lists from each filter dimension."""
+ if not filters:
+ return {}
+ cleaned: dict[str, list[str]] = {}
+ for key in _FILTER_DIMENSIONS:
+ values = filters.get(key)
+ if not values or not isinstance(values, list):
+ continue
+ if "*" in values:
+ continue
+ cleaned[key] = values
+ return cleaned
+
+
+async def resolve_filter_id(
+ filter_id: str,
+ knowledge_filter_service,
+ user_id: str,
+ jwt_token: str | None,
+) -> dict[str, Any]:
+ """Resolve `filter_id` -> `{"filters": {...}, "limit": int, "score_threshold": float}`.
+
+ Raises HTTPException(404) if the filter does not exist or is not accessible to
+ the calling user.
+ """
+ result = await knowledge_filter_service.get_knowledge_filter(
+ filter_id, user_id=user_id, jwt_token=jwt_token
+ )
+ if not result.get("success"):
+ raise HTTPException(
+ status_code=404,
+ detail={"error": f"Filter {filter_id} not found"},
+ )
+
+ filter_doc = result["filter"]
+ query_data_raw = filter_doc.get("query_data") or "{}"
+ if isinstance(query_data_raw, str):
+ try:
+ query_data = json.loads(query_data_raw)
+ except json.JSONDecodeError:
+ query_data = {}
+ else:
+ query_data = query_data_raw or {}
+
+ return {
+ "filters": _strip_wildcards(query_data.get("filters")),
+ "limit": query_data.get("limit", 10),
+ "score_threshold": query_data.get("scoreThreshold", 0),
+ }
diff --git a/src/api/v1/chat.py b/src/api/v1/chat.py
index 50c4da162..99bc30c07 100644
--- a/src/api/v1/chat.py
+++ b/src/api/v1/chat.py
@@ -13,8 +13,14 @@ from fastapi import Depends, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
from pydantic import BaseModel
+from api.v1._filter_resolution import resolve_filter_id
from auth_context import set_auth_context, set_score_threshold, set_search_filters, set_search_limit
-from dependencies import get_chat_service, get_session_manager, require_api_key_permission
+from dependencies import (
+ get_chat_service,
+ get_knowledge_filter_service,
+ get_session_manager,
+ require_api_key_permission,
+)
from session_manager import User
from utils.logging_config import get_logger
@@ -115,6 +121,7 @@ async def chat_create_endpoint(
chat_service=Depends(get_chat_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_api_key_permission("chat:use")),
+ knowledge_filter_service=Depends(get_knowledge_filter_service),
):
"""Send a chat message via Langflow. POST /v1/chat"""
message = body.message.strip()
@@ -130,10 +137,28 @@ async def chat_create_endpoint(
await _assert_owns(body.chat_id, storage_user_id)
- if body.filters:
- set_search_filters(body.filters)
- set_search_limit(body.limit)
- set_score_threshold(body.score_threshold)
+ resolved_filters = body.filters
+ resolved_limit = body.limit
+ resolved_score_threshold = body.score_threshold
+ if body.filter_id:
+ resolved = await resolve_filter_id(
+ body.filter_id,
+ knowledge_filter_service,
+ user_id=user.user_id,
+ jwt_token=jwt_token,
+ )
+ # Inline values override per-field; defaults (10 / 0) fall back to the filter.
+ if not body.filters:
+ resolved_filters = resolved["filters"]
+ if body.limit == 10:
+ resolved_limit = resolved["limit"]
+ if body.score_threshold == 0:
+ resolved_score_threshold = resolved["score_threshold"]
+
+ if resolved_filters:
+ set_search_filters(resolved_filters)
+ set_search_limit(resolved_limit)
+ set_score_threshold(resolved_score_threshold)
set_auth_context(user_id, jwt_token)
start = time.perf_counter()
diff --git a/src/api/v1/documents.py b/src/api/v1/documents.py
index 4860ff98f..7e5900386 100644
--- a/src/api/v1/documents.py
+++ b/src/api/v1/documents.py
@@ -11,8 +11,10 @@ from pydantic import BaseModel
from api.documents import delete_documents_by_filename_core
from api.router import upload_ingest_router
+from api.v1._filter_resolution import resolve_filter_id
from dependencies import (
get_document_service,
+ get_knowledge_filter_service,
get_langflow_file_service,
get_session_manager,
get_task_service,
@@ -25,7 +27,8 @@ logger = get_logger(__name__)
class DeleteDocV1Body(BaseModel):
- filename: str
+ filename: str | None = None
+ filter_id: str | None = None
async def ingest_endpoint(
@@ -46,8 +49,14 @@ async def ingest_endpoint(
POST /v1/documents/ingest
Request: multipart/form-data with "file" field
+
+ NOTE: `create_filter` is kept here for response-shape compatibility — the
+ non-v1 onboarding flow consumes the `create_filter` field echoed back in
+ the response. v1 SDK consumers do not currently have a workflow that uses
+ it, and the field is never forwarded to the actual ingest task. It should
+ be removed in a future major version of the v1 API once we are willing to
+ take the breaking change (response no longer contains `create_filter`).
"""
- # Delegate to the router which handles both Langflow and traditional paths
return await upload_ingest_router(
file=file,
session_id=session_id,
@@ -123,8 +132,58 @@ async def delete_document_endpoint(
body: DeleteDocV1Body,
session_manager=Depends(get_session_manager),
user: User = Depends(require_api_key_permission("knowledge:delete:own")),
+ knowledge_filter_service=Depends(get_knowledge_filter_service),
):
- """Delete a document from the knowledge base. DELETE /v1/documents"""
+ """Delete document(s) from the knowledge base. DELETE /v1/documents
+
+ Provide exactly one of:
+ - `filename`: delete all chunks for that filename.
+ - `filter_id`: resolve the filter's `data_sources` and delete chunks for
+ each of those filenames. Wildcard (`["*"]`) or empty `data_sources`
+ is rejected to prevent mass deletion.
+ """
+ if bool(body.filename) == bool(body.filter_id):
+ return JSONResponse(
+ {"error": "Provide exactly one of `filename` or `filter_id`"},
+ status_code=400,
+ )
+
+ if body.filter_id:
+ resolved = await resolve_filter_id(
+ body.filter_id,
+ knowledge_filter_service,
+ user_id=user.user_id,
+ jwt_token=None,
+ )
+ filenames = resolved["filters"].get("data_sources") or []
+ if not filenames:
+ return JSONResponse(
+ {"error": "Filter has no specific data_sources to delete"},
+ status_code=400,
+ )
+
+ results = []
+ total_deleted = 0
+ for fname in filenames:
+ payload, _status = await delete_documents_by_filename_core(
+ filename=fname,
+ session_manager=session_manager,
+ user_id=user.user_id,
+ jwt_token=None,
+ )
+ results.append(payload)
+ total_deleted += payload.get("deleted_chunks", 0) or 0
+
+ return JSONResponse(
+ {
+ "success": True,
+ "deleted_chunks": total_deleted,
+ "filenames": filenames,
+ "filter_id": body.filter_id,
+ "per_file": results,
+ }
+ )
+
payload, status_code = await delete_documents_by_filename_core(
filename=body.filename,
session_manager=session_manager,
diff --git a/src/api/v1/search.py b/src/api/v1/search.py
index 4ed35c514..c9c08da7d 100644
--- a/src/api/v1/search.py
+++ b/src/api/v1/search.py
@@ -11,7 +11,13 @@ from fastapi import Depends
from fastapi.responses import JSONResponse
from pydantic import BaseModel
-from dependencies import get_search_service, require_api_key_permission
+from api.v1._filter_resolution import resolve_filter_id
+from auth_context import set_auth_context
+from dependencies import (
+ get_knowledge_filter_service,
+ get_search_service,
+ require_api_key_permission,
+)
from session_manager import User
from utils.logging_config import get_logger
from utils.opensearch_utils import DISK_SPACE_ERROR_MESSAGE, OpenSearchDiskSpaceError
@@ -24,35 +30,60 @@ class SearchV1Body(BaseModel):
filters: dict[str, Any] | None = None
limit: int = 10
score_threshold: float = 0
+ filter_id: str | None = None
async def search_endpoint(
body: SearchV1Body,
search_service=Depends(get_search_service),
user: User = Depends(require_api_key_permission("search:use")),
+ knowledge_filter_service=Depends(get_knowledge_filter_service),
):
"""Perform semantic search on documents. POST /v1/search"""
query = body.query.strip()
if not query:
return JSONResponse({"error": "Query is required"}, status_code=400)
+ # API-key auth has no JWT; the gate inside search_service.search()
+ # skips set_auth_context() in that case, leaving search_tool() unable to
+ # resolve the user. Set it explicitly here, mirroring v1 chat.
+ set_auth_context(user.user_id, user.jwt_token)
+
+ resolved_filters = body.filters
+ resolved_limit = body.limit
+ resolved_score_threshold = body.score_threshold
+ if body.filter_id:
+ resolved = await resolve_filter_id(
+ body.filter_id,
+ knowledge_filter_service,
+ user_id=user.user_id,
+ jwt_token=None,
+ )
+ if not body.filters:
+ resolved_filters = resolved["filters"]
+ if body.limit == 10:
+ resolved_limit = resolved["limit"]
+ if body.score_threshold == 0:
+ resolved_score_threshold = resolved["score_threshold"]
+
logger.debug(
"Public API search request",
user_id=user.user_id,
query=query,
- filters=body.filters,
- limit=body.limit,
- score_threshold=body.score_threshold,
+ filters=resolved_filters,
+ limit=resolved_limit,
+ score_threshold=resolved_score_threshold,
+ filter_id=body.filter_id,
)
try:
result = await search_service.search(
query,
user_id=user.user_id,
- jwt_token=None, # API key auth has no JWT
- filters=body.filters or {},
- limit=body.limit,
- score_threshold=body.score_threshold,
+ jwt_token=user.jwt_token,
+ filters=resolved_filters or {},
+ limit=resolved_limit,
+ score_threshold=resolved_score_threshold,
)
results = [
diff --git a/src/mcp_http/server.py b/src/mcp_http/server.py
index 15b6be6ee..66f2cf124 100644
--- a/src/mcp_http/server.py
+++ b/src/mcp_http/server.py
@@ -40,6 +40,7 @@ Usage (MCP client config):
}
}
"""
+
from fastapi import FastAPI
from fastmcp import FastMCP
from fastmcp.server.providers.openapi import (
@@ -50,6 +51,7 @@ from fastmcp.server.providers.openapi import (
RouteMap,
)
from fastmcp.server.providers.openapi.routing import HTTPRoute
+
from utils.logging_config import get_logger
logger = get_logger(__name__)
@@ -84,28 +86,49 @@ COMPONENT_CUSTOMIZATIONS: dict[tuple[str, str], dict[str, str]] = {
"description": (
"Search the OpenRAG knowledge base using semantic search. "
"Returns matching document chunks with relevance scores. "
- "Optionally filter by data sources or document types."
+ "Optionally pass `filter_id` to scope results to a knowledge "
+ "filter's data_sources, or inline `filters` (data_sources, "
+ "document_types, owners, connector_types) for a per-call scope. "
+ "If both are provided, inline filters override per-field."
),
},
# Documents endpoints
- ("/v1/documents/ingest", "POST"): {
- "name": "openrag_ingest",
+ # NOTE: /v1/documents/ingest is intentionally NOT customized here because
+ # it is excluded from MCP exposure entirely (see route_maps below).
+ # Multipart file uploads are not supported through FastMCP's from_fastapi
+ # auto-conversion; use the HTTP API or SDK directly to ingest documents.
+ ("/v1/tasks/enhanced", "GET"): {
+ "name": "openrag_list_tasks_enhanced",
"description": (
- "Ingest documents into the OpenRAG knowledge base. "
- "Supports file uploads, URLs, and text content. "
- "Returns a task_id for tracking ingestion progress."
+ "List all ingestion tasks with structured failure metadata "
+ "(component, failure_phase, user_facing_message, actionable_by) "
+ "on any failed file. Completed files are omitted to reduce payload "
+ "size; use openrag_get_task_status_enhanced for a task's full file list."
),
},
("/v1/tasks/{task_id}", "GET"): {
"name": "openrag_get_task_status",
"description": (
- "Check the status of an ingestion task. "
- "Use the task_id returned from openrag_ingest."
+ "Check the status of an ingestion task. Use the task_id returned from openrag_ingest."
+ ),
+ },
+ ("/v1/tasks/{task_id}/enhanced", "GET"): {
+ "name": "openrag_get_task_status_enhanced",
+ "description": (
+ "Check the status of an ingestion task with structured failure "
+ "metadata (component, failure_phase, user_facing_message, "
+ "actionable_by) on any failed file. Includes completed files in "
+ "the task's file list. Use the task_id returned from openrag_ingest."
),
},
("/v1/documents", "DELETE"): {
"name": "openrag_delete_document",
- "description": "Delete a document from the OpenRAG knowledge base by filename.",
+ "description": (
+ "Delete document(s) from the OpenRAG knowledge base. "
+ "Provide exactly one of: `filename` to delete a single file, "
+ "or `filter_id` to delete every filename listed in that "
+ "knowledge filter's `data_sources` (wildcards rejected for safety)."
+ ),
},
# Settings endpoints
("/v1/settings", "GET"): {
@@ -188,27 +211,31 @@ def create_mcp_server(app: FastAPI) -> FastMCP:
FastMCP.from_fastapi() can discover them.
Route mapping:
- - /v1/* routes → MCP tools (POST, PUT, DELETE, PATCH)
- - /v1/* routes → MCP resource templates (GET with path params)
- - /v1/* routes → MCP resources (GET without path params)
+ - /v1/* routes → MCP tools (GET, POST, PUT, DELETE, PATCH)
- All other routes → excluded
+
+ Note: GET endpoints are exposed as TOOLS, not resources/resource templates.
+ The MCP convention is "GET = resource," but most LLM clients in agent mode
+ only invoke tools — resources require a separate read protocol that many
+ clients don't surface to the model. Exposing GETs as tools makes
+ operations like `openrag_get_knowledge_filter` callable in agent loops.
"""
route_maps = [
- # Expose all /v1/ GET routes with path params as resource templates
- RouteMap(
- methods=["GET"],
- pattern=r"^/v1/",
- mcp_type=MCPType.RESOURCE_TEMPLATE,
- ),
- # Expose all /v1/ GET routes without path params as resources
+ # Exclude /v1/documents/ingest: multipart/form-data file uploads are
+ # not supported through FastMCP's from_fastapi proxy (the LLM-facing
+ # base64-array schema does not get marshaled back into multipart on
+ # the way to the FastAPI handler, so the endpoint always sees the
+ # `file` field as missing). Clients should ingest via the HTTP API
+ # or SDK directly. This RouteMap must come before the catch-all
+ # patterns below.
RouteMap(
- methods=["GET"],
- pattern=r"^/v1/",
- mcp_type=MCPType.RESOURCE,
+ methods=["POST"],
+ pattern=r"^/v1/documents/ingest$",
+ mcp_type=MCPType.EXCLUDE,
),
- # Expose all /v1/ mutating routes as tools
+ # Expose all /v1/ routes (read + write) as MCP tools.
RouteMap(
- methods=["POST", "PUT", "DELETE", "PATCH"],
+ methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
pattern=r"^/v1/",
mcp_type=MCPType.TOOL,
),
diff --git a/tests/integration/sdk/test_documents.py b/tests/integration/sdk/test_documents.py
index 9fcab83ab..aac8fb59c 100644
--- a/tests/integration/sdk/test_documents.py
+++ b/tests/integration/sdk/test_documents.py
@@ -6,6 +6,7 @@ import uuid
from pathlib import Path
import pytest
+from openrag_sdk.exceptions import OpenRAGError
pytestmark = pytest.mark.skipif(
os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
@@ -119,3 +120,84 @@ class TestDocumentsExtended:
assert final.status in ("completed", "failed")
await client.documents.delete(file_path.name)
+
+
+class TestDeleteByFilterId:
+ """Verify DELETE /v1/documents resolves filter_id to data_sources and deletes those files only."""
+
+ async def _ingest_two(self, client, tmp_path):
+ """Helper: ingest two docs and return their Paths."""
+ token = uuid.uuid4().hex[:8]
+ alpha = tmp_path / f"alpha_{token}.md"
+ beta = tmp_path / f"beta_{token}.md"
+ alpha.write_text("# Alpha\n\nUnique content about purple elephants.\n")
+ beta.write_text("# Beta\n\nUnique content about yellow tigers.\n")
+ await client.documents.ingest(file_path=str(alpha))
+ await client.documents.ingest(file_path=str(beta))
+ return alpha, beta
+
+ async def _create_filter(self, client, data_sources: list[str]) -> str:
+ result = await client.knowledge_filters.create(
+ {
+ "name": f"SDK delete-filter {uuid.uuid4().hex[:6]}",
+ "description": "Auto-created by SDK delete-by-filter test",
+ "queryData": {
+ "query": "",
+ "filters": {
+ "data_sources": data_sources,
+ "document_types": ["*"],
+ "owners": ["*"],
+ "connector_types": ["*"],
+ },
+ "limit": 10,
+ "scoreThreshold": 0,
+ },
+ }
+ )
+ assert result.success is True, f"Failed to create filter: {result.error}"
+ return result.id
+
+ @pytest.mark.asyncio
+ async def test_delete_documents_by_filter_id(self, client, tmp_path):
+ """Deleting by filter_id removes only the filenames in the filter's data_sources."""
+ alpha, beta = await self._ingest_two(client, tmp_path)
+ filter_id = await self._create_filter(client, [alpha.name])
+
+ try:
+ result = await client.documents.delete(filter_id=filter_id)
+ assert result.success is True
+ assert result.filter_id == filter_id
+ assert alpha.name in (result.filenames or [])
+ assert beta.name not in (result.filenames or [])
+ # Beta still searchable
+ still_there = await client.search.query("tigers")
+ assert any(r.filename == beta.name for r in still_there.results), (
+ "Beta should still be present after filter-id delete of alpha"
+ )
+ finally:
+ await client.knowledge_filters.delete(filter_id)
+ # Best-effort cleanup
+ await client.documents.delete(alpha.name)
+ await client.documents.delete(beta.name)
+
+ @pytest.mark.asyncio
+ async def test_delete_by_filter_id_with_wildcard_rejects(self, client):
+ """A filter with `["*"]` data_sources must NOT be allowed to mass-delete."""
+ filter_id = await self._create_filter(client, ["*"])
+ try:
+ with pytest.raises(OpenRAGError):
+ await client.documents.delete(filter_id=filter_id)
+ finally:
+ await client.knowledge_filters.delete(filter_id)
+
+ @pytest.mark.asyncio
+ async def test_delete_with_both_filename_and_filter_id_rejects(self, client):
+ """Passing both filename and filter_id must be rejected by the SDK."""
+ with pytest.raises(ValueError):
+ await client.documents.delete("foo.pdf", filter_id="something")
+
+ @pytest.mark.asyncio
+ async def test_delete_with_neither_filename_nor_filter_id_rejects(self, client):
+ """Passing neither filename nor filter_id must be rejected by the SDK."""
+ with pytest.raises(ValueError):
+ await client.documents.delete()
diff --git a/tests/integration/sdk/test_filters.py b/tests/integration/sdk/test_filters.py
index 5cab4ba6a..252801f08 100644
--- a/tests/integration/sdk/test_filters.py
+++ b/tests/integration/sdk/test_filters.py
@@ -1,8 +1,16 @@
-"""Tests for knowledge filter CRUD and usage in chat/search."""
+"""Tests for knowledge filter CRUD and usage in chat/search.
+
+The filter_id usage tests (everything below TestKnowledgeFilters) verify that
+a filter actually constrains the search to the filenames in its `data_sources`,
+not just that the parameter is accepted without error.
+"""
import os
+import uuid
+from pathlib import Path
import pytest
+from openrag_sdk.exceptions import OpenRAGError
pytestmark = pytest.mark.skipif(
os.environ.get("SKIP_SDK_INTEGRATION_TESTS") == "true",
@@ -10,21 +18,67 @@ pytestmark = pytest.mark.skipif(
)
+def _make_doc(tmp_path: Path, label: str, animal: str) -> Path:
+ """Write a unique markdown file mentioning `animal`. Returns the path."""
+ token = uuid.uuid4().hex[:8]
+ path = tmp_path / f"{label}_{token}.md"
+ path.write_text(
+ f"# {label.title()} doc {token}\n\n"
+ f"This document discusses {animal}.\n"
+ "It is used only for SDK filter integration tests.\n"
+ )
+ return path
+
+
+async def _ingest_pair(client, tmp_path: Path) -> tuple[Path, Path]:
+ """Ingest two distinguishable documents and return their paths."""
+ alpha = _make_doc(tmp_path, "alpha", "purple elephants")
+ beta = _make_doc(tmp_path, "beta", "yellow tigers")
+ await client.documents.ingest(file_path=str(alpha))
+ await client.documents.ingest(file_path=str(beta))
+ return alpha, beta
+
+
+async def _create_filter_for(client, name: str, data_sources: list[str]) -> str:
+ """Create a knowledge filter scoped to the given filenames. Returns filter_id."""
+ result = await client.knowledge_filters.create(
+ {
+ "name": name,
+ "description": f"Auto-created by SDK test ({uuid.uuid4().hex[:6]})",
+ "queryData": {
+ "query": "",
+ "filters": {
+ "data_sources": data_sources,
+ "document_types": ["*"],
+ "owners": ["*"],
+ "connector_types": ["*"],
+ },
+ "limit": 10,
+ "scoreThreshold": 0,
+ },
+ }
+ )
+ assert result.success is True, f"Failed to create filter: {result.error}"
+ return result.id
+
+
class TestKnowledgeFilters:
"""Test knowledge filter create, read, update, delete and usage."""
@pytest.mark.asyncio
async def test_knowledge_filter_crud(self, client):
"""Full CRUD lifecycle for a knowledge filter."""
- create_result = await client.knowledge_filters.create({
- "name": "Python SDK Test Filter",
- "description": "Filter created by Python SDK integration tests",
- "queryData": {
- "query": "test documents",
- "limit": 10,
- "scoreThreshold": 0.5,
- },
- })
+ create_result = await client.knowledge_filters.create(
+ {
+ "name": "Python SDK Test Filter",
+ "description": "Filter created by Python SDK integration tests",
+ "queryData": {
+ "query": "test documents",
+ "limit": 10,
+ "scoreThreshold": 0.5,
+ },
+ }
+ )
assert create_result.success is True
assert create_result.id is not None
filter_id = create_result.id
@@ -57,39 +111,134 @@ class TestKnowledgeFilters:
deleted_filter = await client.knowledge_filters.get(filter_id)
assert deleted_filter is None
+
+class TestFilterIdInChat:
+ """Verify filter_id actually constrains chat retrieval, not just that it's accepted."""
+
@pytest.mark.asyncio
- async def test_filter_id_in_chat(self, client):
- """A filter_id can be passed to chat without error."""
- create_result = await client.knowledge_filters.create({
- "name": "Chat Test Filter Python",
- "description": "Filter for testing chat with filter_id",
- "queryData": {"query": "test", "limit": 5},
- })
- assert create_result.success is True
- filter_id = create_result.id
+ async def test_filter_id_in_chat_actually_filters(self, client, tmp_path):
+ """Sources returned must only include the file in the filter's data_sources."""
+ alpha, beta = await _ingest_pair(client, tmp_path)
+ filter_id = await _create_filter_for(client, "SDK chat filter scope", [alpha.name])
try:
response = await client.chat.create(
- message="Hello with filter",
+ message="What animals appear in these documents?",
filter_id=filter_id,
)
- assert response.response is not None
+ assert response.sources is not None
+ source_names = [s.filename for s in response.sources]
+ # Beta must NOT appear; alpha may or may not (RAG can return empty),
+ # but anything that does come back must be alpha.
+ assert beta.name not in source_names, f"Filter leaked: beta in sources {source_names}"
finally:
await client.knowledge_filters.delete(filter_id)
+ await client.documents.delete(alpha.name)
+ await client.documents.delete(beta.name)
@pytest.mark.asyncio
- async def test_filter_id_in_search(self, client):
- """A filter_id can be passed to search without error."""
- create_result = await client.knowledge_filters.create({
- "name": "Search Test Filter Python",
- "description": "Filter for testing search with filter_id",
- "queryData": {"query": "test", "limit": 5},
- })
- assert create_result.success is True
- filter_id = create_result.id
+ async def test_filter_id_in_chat_inline_overrides(self, client, tmp_path):
+ """Inline `filters` win over filter_id per the v1 override contract."""
+ alpha, beta = await _ingest_pair(client, tmp_path)
+ filter_id = await _create_filter_for(client, "SDK chat inline-override", [alpha.name])
+
+ try:
+ response = await client.chat.create(
+ message="What animals appear in these documents?",
+ filter_id=filter_id,
+ filters={"data_sources": [beta.name]},
+ )
+ assert response.sources is not None
+ source_names = [s.filename for s in response.sources]
+ assert alpha.name not in source_names, (
+ f"Inline override didn't win: alpha in sources {source_names}"
+ )
+ finally:
+ await client.knowledge_filters.delete(filter_id)
+ await client.documents.delete(alpha.name)
+ await client.documents.delete(beta.name)
+
+ @pytest.mark.asyncio
+ async def test_filter_id_in_chat_streaming_also_filters(self, client, tmp_path):
+ """Streaming path must apply the resolved filter just like non-streaming."""
+ alpha, beta = await _ingest_pair(client, tmp_path)
+ filter_id = await _create_filter_for(client, "SDK chat stream filter", [alpha.name])
try:
- results = await client.search.query("test query", filter_id=filter_id)
+ collected_sources: list[str] = []
+ async for event in await client.chat.create(
+ message="What animals appear in these documents?",
+ filter_id=filter_id,
+ stream=True,
+ ):
+ if event.type == "sources":
+ collected_sources.extend(s.filename for s in event.sources)
+
+ assert beta.name not in collected_sources, (
+ f"Streaming filter leaked: beta in {collected_sources}"
+ )
+ finally:
+ await client.knowledge_filters.delete(filter_id)
+ await client.documents.delete(alpha.name)
+ await client.documents.delete(beta.name)
+
+ @pytest.mark.asyncio
+ async def test_filter_id_not_found_chat(self, client):
+ """A non-existent filter_id surfaces as a 404-class error."""
+ with pytest.raises(OpenRAGError):
+ await client.chat.create(
+ message="hi",
+ filter_id=f"does-not-exist-{uuid.uuid4().hex}",
+ )
+
+
+class TestFilterIdInSearch:
+ """Verify filter_id actually constrains search results."""
+
+ @pytest.mark.asyncio
+ async def test_filter_id_in_search_actually_filters(self, client, tmp_path):
+ """All search results must come from the filter's data_sources only."""
+ alpha, beta = await _ingest_pair(client, tmp_path)
+ filter_id = await _create_filter_for(client, "SDK search filter scope", [alpha.name])
+
+ try:
+ results = await client.search.query("animals", filter_id=filter_id)
assert results.results is not None
+ for r in results.results:
+ assert r.filename != beta.name, (
+ f"Filter leaked: search returned beta ({r.filename})"
+ )
+ finally:
+ await client.knowledge_filters.delete(filter_id)
+ await client.documents.delete(alpha.name)
+ await client.documents.delete(beta.name)
+
+ @pytest.mark.asyncio
+ async def test_filter_id_in_search_inline_overrides(self, client, tmp_path):
+ """Inline filters override the resolved filter_id per-field."""
+ alpha, beta = await _ingest_pair(client, tmp_path)
+ filter_id = await _create_filter_for(client, "SDK search inline-override", [alpha.name])
+
+ try:
+ results = await client.search.query(
+ "animals",
+ filter_id=filter_id,
+ filters={"data_sources": [beta.name]},
+ )
+ for r in results.results:
+ assert r.filename != alpha.name, (
+ f"Inline override didn't win: search returned alpha ({r.filename})"
+ )
finally:
await client.knowledge_filters.delete(filter_id)
+ await client.documents.delete(alpha.name)
+ await client.documents.delete(beta.name)
+
+ @pytest.mark.asyncio
+ async def test_filter_id_not_found_search(self, client):
+ """A non-existent filter_id on search surfaces as an error."""
+ with pytest.raises(OpenRAGError):
+ await client.search.query(
+ "anything",
+ filter_id=f"does-not-exist-{uuid.uuid4().hex}",
+ )