OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: backend bug fixes for dates/OpenSearch (#2578)
VisveshJ265077261568
diff --git a/src/api/files.py b/src/api/files.pyindex 0e0091130..58163d3c8 100644--- a/src/api/files.py+++ b/src/api/files.py@@ -16,7 +16,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundErrorfrom fastapi import Depends, HTTPException, Queryfrom fastapi.responses import JSONResponse-from dependencies import get_current_user, get_file_service_v2, require_permission+from dependencies import get_file_service_v2, require_permissionfrom session_manager import Userfrom utils.logging_config import get_logger@@ -26,8 +26,7 @@ logger = get_logger(__name__)# strict_date_optional_time mapping on indexed_time rejects._EXTENDED_ISO_RE = re.compile(r"^\d{4}-\d{2}-\d{2}"- r"(T\d{2}:\d{2}(:\d{2}(\.\d{1,6})?)?)?"- r"(Z|[+-]\d{2}:\d{2})?$"+ r"(T\d{2}:\d{2}(:\d{2}(\.\d{1,6})?)?(Z|[+-]\d{2}:\d{2})?)?$")@@ -128,7 +127,7 @@ async def list_files(created_after: str | None = Query(None, description=CREATED_AFTER_DESCRIPTION),created_before: str | None = Query(None, description=CREATED_BEFORE_DESCRIPTION),file_service=Depends(get_file_service_v2),- user: User = Depends(get_current_user),+ user: User = Depends(require_permission("knowledge:read:own")),):"""List ingested files with composite-aggregation pagination, filtering, and sorting."""_validate_date_range(created_after, created_before)@@ -178,7 +177,7 @@ async def search_files(created_after: str | None = Query(None, description=CREATED_AFTER_DESCRIPTION),created_before: str | None = Query(None, description=CREATED_BEFORE_DESCRIPTION),file_service=Depends(get_file_service_v2),- user: User = Depends(get_current_user),+ user: User = Depends(require_permission("knowledge:read:own")),):"""Search files by name with fuzzy/partial matching (composite pagination)."""_validate_date_range(created_after, created_before)diff --git a/src/services/file_service_v2.py b/src/services/file_service_v2.pyindex 8e9d6266a..d526f6c2e 100644--- a/src/services/file_service_v2.py+++ b/src/services/file_service_v2.py@@ -19,12 +19,24 @@ Sort-field notesfrom typing import Any+from opensearchpy.exceptions import NotFoundError+from config.settings import get_index_namefrom utils.logging_config import get_loggerfrom utils.opensearch_queries import build_date_range_querylogger = get_logger(__name__)++def _is_index_not_found(exc: Exception) -> bool:+ """True for a missing index (fresh install, nothing ingested yet)."""+ return (+ isinstance(exc, NotFoundError)+ and exc.status_code == 404+ and exc.error == "index_not_found_exception"+ )++# A file row is a (filename, document_id) pair: document_id is a content hash,# so identical content under two names is two rows. Missing values count as ""._FILE_IDENTITY_SCRIPT = (@@ -34,6 +46,7 @@ _FILE_IDENTITY_SCRIPT = (def _file_identity_script() -> dict[str, Any]:+ """Painless script building the (filename, document_id) identity key."""return {"lang": "painless","source": _FILE_IDENTITY_SCRIPT,@@ -43,6 +56,7 @@ def _file_identity_script() -> dict[str, Any]:def _distinct_files_agg() -> dict[str, Any]:+ """Cardinality aggregation counting distinct files by identity."""return {"cardinality": {"script": _file_identity_script(), "precision_threshold": 3000}}@@ -161,6 +175,8 @@ class FileServiceV2:if is_opensearch_auth_error(e):raise+ if not _is_index_not_found(e):+ raisereturn {"files": [],"total": 0,@@ -273,6 +289,8 @@ class FileServiceV2:if is_opensearch_auth_error(e):raise+ if not _is_index_not_found(e):+ raisereturn {"files": [],"total": 0,@@ -336,6 +354,8 @@ class FileServiceV2:created_after: str | None = None,created_before: str | None = None,) -> dict[str, Any]:+ """Build the bool query (filters, filename search, date range) for file views."""+def _effective_values(values: str | list[str] | None) -> list[str]:if values is None:return []@@ -445,6 +465,7 @@ class FileServiceV2:sort_order: str,after_key: dict | None,) -> dict[str, Any]:+ """Build the composite aggregation request, with filename/document_id tiebreakers."""composite: dict[str, Any] = {"size": page_size,"sources": [@@ -629,9 +650,10 @@ class FileServiceV2:) -> tuple[int, bool]:"""Approximate unique-file count (filename + document_id) via cardinality aggregation (O(1)).- Returns (count, is_approximate). is_approximate is always True on- success (cardinality agg is inherently approximate) and True when the- aggregation fails and 0 is returned as a fallback.+ Returns (count, is_approximate). is_approximate is always True+ (cardinality agg is inherently approximate). A missing index (fresh+ install) yields (0, True); every other failure is re-raised so the API+ returns an error instead of a wrong total or an empty page."""body = {"size": 0,@@ -642,9 +664,10 @@ class FileServiceV2:result = await opensearch_client.search(index=get_index_name(), body=body)return result.get("aggregations", {}).get("file_count", {}).get("value", 0), Trueexcept Exception as e:- logger.warning(- "Failed to retrieve file count; pagination total will show 0", error=str(e)- )+ if not _is_index_not_found(e):+ logger.error("Failed to retrieve file count (v2)", error=str(e))+ raise+ logger.warning("File count skipped; index not found", error=str(e))return 0, Truedef _parse_composite_buckets(diff --git a/src/utils/opensearch_queries.py b/src/utils/opensearch_queries.pyindex a3689b00c..85db8bae6 100644--- a/src/utils/opensearch_queries.py+++ b/src/utils/opensearch_queries.py@@ -121,22 +121,25 @@ def build_replace_filename_query(filename: str, owner: str) -> dict:_PARTIAL_TIMESTAMP_RE = re.compile(- r"^(?P<date>\d{4}-\d{2}-\d{2})(?:T(?P<hm>\d{2}:\d{2}))?(?P<tz>Z|[+-]\d{2}:\d{2})?$"+ r"^(?P<date>\d{4}-\d{2}-\d{2})"+ r"(?:T(?P<hm>\d{2}:\d{2})(?::(?P<s>\d{2})(?P<frac>\.\d{1,6})?)?)?"+ r"(?P<tz>Z|[+-]\d{2}:\d{2})?$")def normalize_date_bound(value: str) -> str:- """Spell out the seconds a bound omits.+ """Spell out the seconds and fraction a bound omits.API validation reads ``2026-10-10`` as midnight and ``…T10:30`` as 10:30:00,but OpenSearch expands the missing parts of an ``lte`` bound to the end of- that day or minute. Sending the explicit instant keeps both in agreement.- Full timestamps are returned unchanged.+ that day, minute or second. Sending the explicit instant, with a zero+ fraction, keeps both in agreement. A caller-supplied fraction is preserved."""match = _PARTIAL_TIMESTAMP_RE.match(value)- if match is None:+ if match is None or match["frac"]:return value- return f"{match['date']}T{match['hm'] or '00:00'}:00{match['tz'] or 'Z'}"+ hm = match["hm"] or "00:00"+ return f"{match['date']}T{hm}:{match['s'] or '00'}.000000{match['tz'] or 'Z'}"def build_date_range_query(diff --git a/tests/unit/api/test_files_timestamp_validation.py b/tests/unit/api/test_files_timestamp_validation.pyindex d821edfd3..95150ddeb 100644--- a/tests/unit/api/test_files_timestamp_validation.py+++ b/tests/unit/api/test_files_timestamp_validation.py@@ -32,6 +32,10 @@ def test_accepts_extended_iso_forms(value):"20240101T000000","2024-01-01T000000","2024-13-01",+ # Offset/Z without a time: fromisoformat misreads it as an hour offset.+ "2024-01-01+05:00",+ "2024-01-01-05:00",+ "2024-01-01Z","not-a-date","",],diff --git a/tests/unit/api/test_files_upload_histogram.py b/tests/unit/api/test_files_upload_histogram.pyindex 99a14d45f..0e66a941d 100644--- a/tests/unit/api/test_files_upload_histogram.py+++ b/tests/unit/api/test_files_upload_histogram.py@@ -12,8 +12,8 @@ USER = SimpleNamespace(user_id="u1", jwt_token="jwt")def _call(file_service, **overrides):params = {- "start": "2024-05-01T00:00:00Z",- "end": "2024-06-20T23:59:59Z",+ "start": "2024-05-01T00:00:00.000000Z",+ "end": "2024-06-20T23:59:59.000000Z","tz": "America/New_York","connector_type": None,"mimetype": None,@@ -36,7 +36,7 @@ async def test_passes_window_timezone_and_filters_to_service():assert response.status_code == 200kwargs = service.upload_histogram.await_args.kwargsassert kwargs["time_zone"] == "America/New_York"- assert kwargs["start"] == "2024-05-01T00:00:00Z"+ assert kwargs["start"] == "2024-05-01T00:00:00.000000Z"assert kwargs["owner"] == ["u1"]@@ -90,8 +90,8 @@ async def test_service_buckets_by_local_day_and_keeps_all_time_total():result = await FileServiceV2(session_manager).upload_histogram(user_id="u1",- start="2024-05-01T00:00:00Z",- end="2024-06-20T23:59:59Z",+ start="2024-05-01T00:00:00.000000Z",+ end="2024-06-20T23:59:59.000000Z",time_zone="Asia/Kolkata",connector_type=["local"],)@@ -103,7 +103,12 @@ async def test_service_buckets_by_local_day_and_keeps_all_time_total():assert {"term": {"connector_type": "local"}} in body["query"]["bool"]["filter"]window = body["aggs"]["window"]assert window["filter"] == {- "range": {"indexed_time": {"gte": "2024-05-01T00:00:00Z", "lte": "2024-06-20T23:59:59Z"}}+ "range": {+ "indexed_time": {+ "gte": "2024-05-01T00:00:00.000000Z",+ "lte": "2024-06-20T23:59:59.000000Z",+ }+ }}histogram = window["aggs"]["days"]["date_histogram"]assert histogram["time_zone"] == "Asia/Kolkata"diff --git a/tests/unit/api/test_search_website_pages.py b/tests/unit/api/test_search_website_pages.pyindex 222a50024..80a6aa6ae 100644--- a/tests/unit/api/test_search_website_pages.py+++ b/tests/unit/api/test_search_website_pages.py@@ -347,7 +347,7 @@ async def test_page_manifest_query_filters_indexed_time(monkeypatch):dated, undated = (call.kwargs["body"]["query"]["bool"]["filter"] for call in client.search.await_args_list)- assert {"range": {"indexed_time": {"gte": "2026-10-07T00:00:00Z"}}} in dated+ assert {"range": {"indexed_time": {"gte": "2026-10-07T00:00:00.000000Z"}}} in datedassert not any("range" in clause for clause in undated)diff --git a/tests/unit/dependencies/test_require_permission.py b/tests/unit/dependencies/test_require_permission.pyindex 0712e081b..5df2846d7 100644--- a/tests/unit/dependencies/test_require_permission.py+++ b/tests/unit/dependencies/test_require_permission.py@@ -207,6 +207,8 @@ async def app(monkeypatch):):return JSONResponse({"user_id": user.user_id})+ from api.files import list_files as real_list_files+ from api.files import search_files as real_search_filesfrom api.files import upload_histogram as real_upload_histogramfrom dependencies import get_file_service_v2@@ -214,6 +216,15 @@ async def app(monkeypatch):async def upload_histogram(self, **_kwargs):return {"total": 0, "days": {}}+ async def list_files(self, **_kwargs):+ return {"files": [], "total": 0}++ async def search_files(self, **_kwargs):+ return {"files": [], "total": 0}++ app.add_api_route("/test/real-list-files", real_list_files, methods=["GET"])+ app.add_api_route("/test/real-search-files", real_search_files, methods=["GET"])+app.add_api_route("/test/real-upload-histogram", real_upload_histogram, methods=["GET"])app.dependency_overrides[get_file_service_v2] = lambda: _Histogram()@@ -331,3 +342,19 @@ async def test_real_upload_histogram_route_requires_knowledge_read(app, persona,assert r.status_code == expected, f"{persona} -> {r.status_code} {r.text}"if expected == 403:assert r.json()["detail"]["error"] == "permission_denied"+++ "path, params",+ [("/test/real-list-files", {}), ("/test/real-search-files", {"q": "x"})],+)+ "persona, expected",+ [("admin", 200), ("developer", 200), ("user", 200), ("viewer", 403)],+)+async def test_real_file_list_routes_require_knowledge_read(app, path, params, persona, expected):+ transport = httpx.ASGITransport(app=app)+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:+ r = await c.get(path, params=params, headers={"X-Test-Persona": persona})+ assert r.status_code == expected, f"{persona} -> {r.status_code} {r.text}"diff --git a/tests/unit/services/test_chat_service_date_filters.py b/tests/unit/services/test_chat_service_date_filters.pyindex f2e5e2523..1bd62d768 100644--- a/tests/unit/services/test_chat_service_date_filters.py+++ b/tests/unit/services/test_chat_service_date_filters.py@@ -12,7 +12,9 @@ from services.chat_service import ChatServiceHEADER = "X-LANGFLOW-GLOBAL-VAR-OPENRAG-QUERY-FILTER"RANGE = {- "range": {"indexed_time": {"gte": "2026-10-07T07:00:00Z", "lte": "2026-10-11T06:59:59.999Z"}}+ "range": {+ "indexed_time": {"gte": "2026-10-07T07:00:00.000000Z", "lte": "2026-10-11T06:59:59.999Z"}+ }}PDF = {"term": {"mimetype": "application/pdf"}}SAMPLE_EXCLUSION = {"bool": {"must_not": [{"term": {"is_sample_data": "true"}}]}}diff --git a/tests/unit/test_file_service_v2_search_errors.py b/tests/unit/test_file_service_v2_search_errors.pynew file mode 100644index 000000000..6db4827cc--- /dev/null+++ b/tests/unit/test_file_service_v2_search_errors.py@@ -0,0 +1,68 @@+from unittest.mock import AsyncMock, MagicMock++import pytest+from opensearchpy.exceptions import ConnectionError as OSConnectionError+from opensearchpy.exceptions import NotFoundError++from services.file_service_v2 import FileServiceV2+++def _service(search_side_effect):+ client = MagicMock()+ client.search = AsyncMock(side_effect=search_side_effect)+ sm = MagicMock()+ sm.get_user_opensearch_client.return_value = client+ svc = FileServiceV2(sm)+ svc._get_file_count = AsyncMock(return_value=(0, False))+ return svc++[email protected]("sort_by", ["filename", "chunk_count"])+async def test_opensearch_failure_propagates(sort_by):+ svc = _service(OSConnectionError("N/A", "boom", Exception("down")))+ with pytest.raises(OSConnectionError):+ await svc.list_files(user_id="u", sort_by=sort_by)++[email protected]("sort_by", ["filename", "chunk_count"])+async def test_missing_index_returns_empty(sort_by):+ err = NotFoundError(404, "index_not_found_exception", {})+ svc = _service(err)+ result = await svc.list_files(user_id="u", sort_by=sort_by)+ assert result["files"] == []+ assert result["total"] == 0+++def _real_count_service(search_side_effect):+ client = MagicMock()+ client.search = AsyncMock(side_effect=search_side_effect)+ sm = MagicMock()+ sm.get_user_opensearch_client.return_value = client+ return FileServiceV2(sm), client+++async def test_count_failure_propagates():+ svc, _ = _real_count_service(OSConnectionError("N/A", "boom", Exception("down")))+ with pytest.raises(OSConnectionError):+ await svc.list_files(user_id="u")+++async def test_count_failure_propagates_on_deep_page_without_cursor():+ # page > 1 with no after_key returns before the listing search, so the+ # count is the only call that can surface the outage.+ svc, client = _real_count_service(OSConnectionError("N/A", "boom", Exception("down")))+ with pytest.raises(OSConnectionError):+ await svc.list_files(user_id="u", page=2)+ assert client.search.await_count == 1+++async def test_count_missing_index_returns_zero():+ svc, _ = _real_count_service(NotFoundError(404, "index_not_found_exception", {}))+ result = await svc.list_files(user_id="u", page=2)+ assert result["files"] == []+ assert result["total"] == 0diff --git a/tests/unit/test_filter_clause_builder.py b/tests/unit/test_filter_clause_builder.pyindex 1ce4c0859..8a2190477 100644--- a/tests/unit/test_filter_clause_builder.py+++ b/tests/unit/test_filter_clause_builder.py@@ -3,7 +3,9 @@ import pytestfrom utils.opensearch_queries import build_filter_clausesRANGE = {- "range": {"indexed_time": {"gte": "2026-10-07T00:00:00Z", "lte": "2026-10-10T23:59:59.999Z"}}+ "range": {+ "indexed_time": {"gte": "2026-10-07T00:00:00.000000Z", "lte": "2026-10-10T23:59:59.999Z"}+ }}DATES = {"created_after": "2026-10-07T00:00:00Z", "created_before": "2026-10-10T23:59:59.999Z"}@@ -42,7 +44,7 @@ def test_dates_combined_with_lists_and_not_treated_as_terms():def test_single_bound_and_blank_bounds():assert build_filter_clauses({"created_after": "2026-10-07T00:00:00Z"}) == [- {"range": {"indexed_time": {"gte": "2026-10-07T00:00:00Z"}}}+ {"range": {"indexed_time": {"gte": "2026-10-07T00:00:00.000000Z"}}}]assert build_filter_clauses({"created_after": "", "created_before": None}) == []@@ -50,12 +52,13 @@ def test_single_bound_and_blank_bounds():@pytest.mark.parametrize(("raw", "expected"),[- ("2026-10-10", "2026-10-10T00:00:00Z"),- ("2026-10-10Z", "2026-10-10T00:00:00Z"),- ("2026-10-10T10:30", "2026-10-10T10:30:00Z"),- ("2026-10-10T10:30+05:30", "2026-10-10T10:30:00+05:30"),- ("2026-10-10T10:30:15Z", "2026-10-10T10:30:15Z"),+ ("2026-10-10", "2026-10-10T00:00:00.000000Z"),+ ("2026-10-10Z", "2026-10-10T00:00:00.000000Z"),+ ("2026-10-10T10:30", "2026-10-10T10:30:00.000000Z"),+ ("2026-10-10T10:30+05:30", "2026-10-10T10:30:00.000000+05:30"),+ ("2026-10-10T10:30:15Z", "2026-10-10T10:30:15.000000Z"),("2026-10-10T23:59:59.999999Z", "2026-10-10T23:59:59.999999Z"),+ ("2026-10-10T23:59:59.5", "2026-10-10T23:59:59.5"),],)def test_date_bounds_are_sent_as_explicit_instants(raw, expected):diff --git a/tests/unit/test_search_service_score_threshold.py b/tests/unit/test_search_service_score_threshold.pyindex 34c6af58e..c02e2e4f3 100644--- a/tests/unit/test_search_service_score_threshold.py+++ b/tests/unit/test_search_service_score_threshold.py@@ -186,7 +186,7 @@ async def test_filters_are_pushed_into_each_knn_clause(monkeypatch):params = _knn_params(os_client.bodies[-1]["query"])assert params- date_clause = {"range": {"indexed_time": {"gte": "2024-01-01T00:00:00Z"}}}+ date_clause = {"range": {"indexed_time": {"gte": "2024-01-01T00:00:00.000000Z"}}}for knn in params:# The date range sits beside the chunk-mode exclusion of website manifests.assert date_clause in knn["filter"]["bool"]["filter"]@@ -268,7 +268,7 @@ async def test_nmslib_scores_the_filtered_documents_exactly(monkeypatch):script = clause["script_score"]["script"]assert script["lang"] == "knn"assert script["params"]["space_type"] == "cosinesimil"- date_clause = {"range": {"indexed_time": {"gte": "2024-01-01T00:00:00Z"}}}+ date_clause = {"range": {"indexed_time": {"gte": "2024-01-01T00:00:00.000000Z"}}}assert date_clause in clause["script_score"]["query"]["bool"]["filter"]assert date_clause in hybrid["bool"]["filter"] # the outer filter is kept