Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: backend bug fixes for dates/OpenSearch (#2578)

VisveshJ265077261568
diff --git a/src/api/files.py b/src/api/files.py
index 0e0091130..58163d3c8 100644
--- a/src/api/files.py
+++ b/src/api/files.py
@@ -16,7 +16,7 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from fastapi import Depends, HTTPException, Query
from fastapi.responses import JSONResponse
-from dependencies import get_current_user, get_file_service_v2, require_permission
+from dependencies import get_file_service_v2, require_permission
from session_manager import User
from utils.logging_config import get_logger
@@ -26,8 +26,7 @@ logger = get_logger(__name__)
# strict_date_optional_time mapping on indexed_time rejects.
_EXTENDED_ISO_RE = re.compile(
r"^\d{4}-\d{2}-\d{2}"
- r"(T\d{2}:\d{2}(:\d{2}(\.\d{1,6})?)?)?"
- r"(Z|[+-]\d{2}:\d{2})?$"
+ r"(T\d{2}:\d{2}(:\d{2}(\.\d{1,6})?)?(Z|[+-]\d{2}:\d{2})?)?$"
)
@@ -128,7 +127,7 @@ async def list_files(
created_after: str | None = Query(None, description=CREATED_AFTER_DESCRIPTION),
created_before: str | None = Query(None, description=CREATED_BEFORE_DESCRIPTION),
file_service=Depends(get_file_service_v2),
- user: User = Depends(get_current_user),
+ user: User = Depends(require_permission("knowledge:read:own")),
):
"""List ingested files with composite-aggregation pagination, filtering, and sorting."""
_validate_date_range(created_after, created_before)
@@ -178,7 +177,7 @@ async def search_files(
created_after: str | None = Query(None, description=CREATED_AFTER_DESCRIPTION),
created_before: str | None = Query(None, description=CREATED_BEFORE_DESCRIPTION),
file_service=Depends(get_file_service_v2),
- user: User = Depends(get_current_user),
+ user: User = Depends(require_permission("knowledge:read:own")),
):
"""Search files by name with fuzzy/partial matching (composite pagination)."""
_validate_date_range(created_after, created_before)
diff --git a/src/services/file_service_v2.py b/src/services/file_service_v2.py
index 8e9d6266a..d526f6c2e 100644
--- a/src/services/file_service_v2.py
+++ b/src/services/file_service_v2.py
@@ -19,12 +19,24 @@ Sort-field notes
from typing import Any
+from opensearchpy.exceptions import NotFoundError
+
from config.settings import get_index_name
from utils.logging_config import get_logger
from utils.opensearch_queries import build_date_range_query
logger = get_logger(__name__)
+
+def _is_index_not_found(exc: Exception) -> bool:
+ """True for a missing index (fresh install, nothing ingested yet)."""
+ return (
+ isinstance(exc, NotFoundError)
+ and exc.status_code == 404
+ and exc.error == "index_not_found_exception"
+ )
+
+
# A file row is a (filename, document_id) pair: document_id is a content hash,
# so identical content under two names is two rows. Missing values count as "".
_FILE_IDENTITY_SCRIPT = (
@@ -34,6 +46,7 @@ _FILE_IDENTITY_SCRIPT = (
def _file_identity_script() -> dict[str, Any]:
+ """Painless script building the (filename, document_id) identity key."""
return {
"lang": "painless",
"source": _FILE_IDENTITY_SCRIPT,
@@ -43,6 +56,7 @@ def _file_identity_script() -> dict[str, Any]:
def _distinct_files_agg() -> dict[str, Any]:
+ """Cardinality aggregation counting distinct files by identity."""
return {"cardinality": {"script": _file_identity_script(), "precision_threshold": 3000}}
@@ -161,6 +175,8 @@ class FileServiceV2:
if is_opensearch_auth_error(e):
raise
+ if not _is_index_not_found(e):
+ raise
return {
"files": [],
"total": 0,
@@ -273,6 +289,8 @@ class FileServiceV2:
if is_opensearch_auth_error(e):
raise
+ if not _is_index_not_found(e):
+ raise
return {
"files": [],
"total": 0,
@@ -336,6 +354,8 @@ class FileServiceV2:
created_after: str | None = None,
created_before: str | None = None,
) -> dict[str, Any]:
+ """Build the bool query (filters, filename search, date range) for file views."""
+
def _effective_values(values: str | list[str] | None) -> list[str]:
if values is None:
return []
@@ -445,6 +465,7 @@ class FileServiceV2:
sort_order: str,
after_key: dict | None,
) -> dict[str, Any]:
+ """Build the composite aggregation request, with filename/document_id tiebreakers."""
composite: dict[str, Any] = {
"size": page_size,
"sources": [
@@ -629,9 +650,10 @@ class FileServiceV2:
) -> tuple[int, bool]:
"""Approximate unique-file count (filename + document_id) via cardinality aggregation (O(1)).
- Returns (count, is_approximate). is_approximate is always True on
- success (cardinality agg is inherently approximate) and True when the
- aggregation fails and 0 is returned as a fallback.
+ Returns (count, is_approximate). is_approximate is always True
+ (cardinality agg is inherently approximate). A missing index (fresh
+ install) yields (0, True); every other failure is re-raised so the API
+ returns an error instead of a wrong total or an empty page.
"""
body = {
"size": 0,
@@ -642,9 +664,10 @@ class FileServiceV2:
result = await opensearch_client.search(index=get_index_name(), body=body)
return result.get("aggregations", {}).get("file_count", {}).get("value", 0), True
except Exception as e:
- logger.warning(
- "Failed to retrieve file count; pagination total will show 0", error=str(e)
- )
+ if not _is_index_not_found(e):
+ logger.error("Failed to retrieve file count (v2)", error=str(e))
+ raise
+ logger.warning("File count skipped; index not found", error=str(e))
return 0, True
def _parse_composite_buckets(
diff --git a/src/utils/opensearch_queries.py b/src/utils/opensearch_queries.py
index a3689b00c..85db8bae6 100644
--- a/src/utils/opensearch_queries.py
+++ b/src/utils/opensearch_queries.py
@@ -121,22 +121,25 @@ def build_replace_filename_query(filename: str, owner: str) -> dict:
_PARTIAL_TIMESTAMP_RE = re.compile(
- r"^(?P<date>\d{4}-\d{2}-\d{2})(?:T(?P<hm>\d{2}:\d{2}))?(?P<tz>Z|[+-]\d{2}:\d{2})?$"
+ r"^(?P<date>\d{4}-\d{2}-\d{2})"
+ r"(?:T(?P<hm>\d{2}:\d{2})(?::(?P<s>\d{2})(?P<frac>\.\d{1,6})?)?)?"
+ r"(?P<tz>Z|[+-]\d{2}:\d{2})?$"
)
def normalize_date_bound(value: str) -> str:
- """Spell out the seconds a bound omits.
+ """Spell out the seconds and fraction a bound omits.
API validation reads ``2026-10-10`` as midnight and ``…T10:30`` as 10:30:00,
but OpenSearch expands the missing parts of an ``lte`` bound to the end of
- that day or minute. Sending the explicit instant keeps both in agreement.
- Full timestamps are returned unchanged.
+ that day, minute or second. Sending the explicit instant, with a zero
+ fraction, keeps both in agreement. A caller-supplied fraction is preserved.
"""
match = _PARTIAL_TIMESTAMP_RE.match(value)
- if match is None:
+ if match is None or match["frac"]:
return value
- return f"{match['date']}T{match['hm'] or '00:00'}:00{match['tz'] or 'Z'}"
+ hm = match["hm"] or "00:00"
+ return f"{match['date']}T{hm}:{match['s'] or '00'}.000000{match['tz'] or 'Z'}"
def build_date_range_query(
diff --git a/tests/unit/api/test_files_timestamp_validation.py b/tests/unit/api/test_files_timestamp_validation.py
index d821edfd3..95150ddeb 100644
--- a/tests/unit/api/test_files_timestamp_validation.py
+++ b/tests/unit/api/test_files_timestamp_validation.py
@@ -32,6 +32,10 @@ def test_accepts_extended_iso_forms(value):
"20240101T000000",
"2024-01-01T000000",
"2024-13-01",
+ # Offset/Z without a time: fromisoformat misreads it as an hour offset.
+ "2024-01-01+05:00",
+ "2024-01-01-05:00",
+ "2024-01-01Z",
"not-a-date",
"",
],
diff --git a/tests/unit/api/test_files_upload_histogram.py b/tests/unit/api/test_files_upload_histogram.py
index 99a14d45f..0e66a941d 100644
--- a/tests/unit/api/test_files_upload_histogram.py
+++ b/tests/unit/api/test_files_upload_histogram.py
@@ -12,8 +12,8 @@ USER = SimpleNamespace(user_id="u1", jwt_token="jwt")
def _call(file_service, **overrides):
params = {
- "start": "2024-05-01T00:00:00Z",
- "end": "2024-06-20T23:59:59Z",
+ "start": "2024-05-01T00:00:00.000000Z",
+ "end": "2024-06-20T23:59:59.000000Z",
"tz": "America/New_York",
"connector_type": None,
"mimetype": None,
@@ -36,7 +36,7 @@ async def test_passes_window_timezone_and_filters_to_service():
assert response.status_code == 200
kwargs = service.upload_histogram.await_args.kwargs
assert kwargs["time_zone"] == "America/New_York"
- assert kwargs["start"] == "2024-05-01T00:00:00Z"
+ assert kwargs["start"] == "2024-05-01T00:00:00.000000Z"
assert kwargs["owner"] == ["u1"]
@@ -90,8 +90,8 @@ async def test_service_buckets_by_local_day_and_keeps_all_time_total():
result = await FileServiceV2(session_manager).upload_histogram(
user_id="u1",
- start="2024-05-01T00:00:00Z",
- end="2024-06-20T23:59:59Z",
+ start="2024-05-01T00:00:00.000000Z",
+ end="2024-06-20T23:59:59.000000Z",
time_zone="Asia/Kolkata",
connector_type=["local"],
)
@@ -103,7 +103,12 @@ async def test_service_buckets_by_local_day_and_keeps_all_time_total():
assert {"term": {"connector_type": "local"}} in body["query"]["bool"]["filter"]
window = body["aggs"]["window"]
assert window["filter"] == {
- "range": {"indexed_time": {"gte": "2024-05-01T00:00:00Z", "lte": "2024-06-20T23:59:59Z"}}
+ "range": {
+ "indexed_time": {
+ "gte": "2024-05-01T00:00:00.000000Z",
+ "lte": "2024-06-20T23:59:59.000000Z",
+ }
+ }
}
histogram = window["aggs"]["days"]["date_histogram"]
assert histogram["time_zone"] == "Asia/Kolkata"
diff --git a/tests/unit/api/test_search_website_pages.py b/tests/unit/api/test_search_website_pages.py
index 222a50024..80a6aa6ae 100644
--- a/tests/unit/api/test_search_website_pages.py
+++ b/tests/unit/api/test_search_website_pages.py
@@ -347,7 +347,7 @@ async def test_page_manifest_query_filters_indexed_time(monkeypatch):
dated, undated = (
call.kwargs["body"]["query"]["bool"]["filter"] for call in client.search.await_args_list
)
- assert {"range": {"indexed_time": {"gte": "2026-10-07T00:00:00Z"}}} in dated
+ assert {"range": {"indexed_time": {"gte": "2026-10-07T00:00:00.000000Z"}}} in dated
assert not any("range" in clause for clause in undated)
diff --git a/tests/unit/dependencies/test_require_permission.py b/tests/unit/dependencies/test_require_permission.py
index 0712e081b..5df2846d7 100644
--- a/tests/unit/dependencies/test_require_permission.py
+++ b/tests/unit/dependencies/test_require_permission.py
@@ -207,6 +207,8 @@ async def app(monkeypatch):
):
return JSONResponse({"user_id": user.user_id})
+ from api.files import list_files as real_list_files
+ from api.files import search_files as real_search_files
from api.files import upload_histogram as real_upload_histogram
from dependencies import get_file_service_v2
@@ -214,6 +216,15 @@ async def app(monkeypatch):
async def upload_histogram(self, **_kwargs):
return {"total": 0, "days": {}}
+ async def list_files(self, **_kwargs):
+ return {"files": [], "total": 0}
+
+ async def search_files(self, **_kwargs):
+ return {"files": [], "total": 0}
+
+ app.add_api_route("/test/real-list-files", real_list_files, methods=["GET"])
+ app.add_api_route("/test/real-search-files", real_search_files, methods=["GET"])
+
app.add_api_route("/test/real-upload-histogram", real_upload_histogram, methods=["GET"])
app.dependency_overrides[get_file_service_v2] = lambda: _Histogram()
@@ -331,3 +342,19 @@ async def test_real_upload_histogram_route_requires_knowledge_read(app, persona,
assert r.status_code == expected, f"{persona} -> {r.status_code} {r.text}"
if expected == 403:
assert r.json()["detail"]["error"] == "permission_denied"
+
+
+ "path, params",
+ [("/test/real-list-files", {}), ("/test/real-search-files", {"q": "x"})],
+)
+ "persona, expected",
+ [("admin", 200), ("developer", 200), ("user", 200), ("viewer", 403)],
+)
+async def test_real_file_list_routes_require_knowledge_read(app, path, params, persona, expected):
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:
+ r = await c.get(path, params=params, headers={"X-Test-Persona": persona})
+ assert r.status_code == expected, f"{persona} -> {r.status_code} {r.text}"
diff --git a/tests/unit/services/test_chat_service_date_filters.py b/tests/unit/services/test_chat_service_date_filters.py
index f2e5e2523..1bd62d768 100644
--- a/tests/unit/services/test_chat_service_date_filters.py
+++ b/tests/unit/services/test_chat_service_date_filters.py
@@ -12,7 +12,9 @@ from services.chat_service import ChatService
HEADER = "X-LANGFLOW-GLOBAL-VAR-OPENRAG-QUERY-FILTER"
RANGE = {
- "range": {"indexed_time": {"gte": "2026-10-07T07:00:00Z", "lte": "2026-10-11T06:59:59.999Z"}}
+ "range": {
+ "indexed_time": {"gte": "2026-10-07T07:00:00.000000Z", "lte": "2026-10-11T06:59:59.999Z"}
+ }
}
PDF = {"term": {"mimetype": "application/pdf"}}
SAMPLE_EXCLUSION = {"bool": {"must_not": [{"term": {"is_sample_data": "true"}}]}}
diff --git a/tests/unit/test_file_service_v2_search_errors.py b/tests/unit/test_file_service_v2_search_errors.py
new file mode 100644
index 000000000..6db4827cc
--- /dev/null
+++ b/tests/unit/test_file_service_v2_search_errors.py
@@ -0,0 +1,68 @@
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from opensearchpy.exceptions import ConnectionError as OSConnectionError
+from opensearchpy.exceptions import NotFoundError
+
+from services.file_service_v2 import FileServiceV2
+
+
+def _service(search_side_effect):
+ client = MagicMock()
+ client.search = AsyncMock(side_effect=search_side_effect)
+ sm = MagicMock()
+ sm.get_user_opensearch_client.return_value = client
+ svc = FileServiceV2(sm)
+ svc._get_file_count = AsyncMock(return_value=(0, False))
+ return svc
+
+
[email protected]("sort_by", ["filename", "chunk_count"])
+async def test_opensearch_failure_propagates(sort_by):
+ svc = _service(OSConnectionError("N/A", "boom", Exception("down")))
+ with pytest.raises(OSConnectionError):
+ await svc.list_files(user_id="u", sort_by=sort_by)
+
+
[email protected]("sort_by", ["filename", "chunk_count"])
+async def test_missing_index_returns_empty(sort_by):
+ err = NotFoundError(404, "index_not_found_exception", {})
+ svc = _service(err)
+ result = await svc.list_files(user_id="u", sort_by=sort_by)
+ assert result["files"] == []
+ assert result["total"] == 0
+
+
+def _real_count_service(search_side_effect):
+ client = MagicMock()
+ client.search = AsyncMock(side_effect=search_side_effect)
+ sm = MagicMock()
+ sm.get_user_opensearch_client.return_value = client
+ return FileServiceV2(sm), client
+
+
+async def test_count_failure_propagates():
+ svc, _ = _real_count_service(OSConnectionError("N/A", "boom", Exception("down")))
+ with pytest.raises(OSConnectionError):
+ await svc.list_files(user_id="u")
+
+
+async def test_count_failure_propagates_on_deep_page_without_cursor():
+ # page > 1 with no after_key returns before the listing search, so the
+ # count is the only call that can surface the outage.
+ svc, client = _real_count_service(OSConnectionError("N/A", "boom", Exception("down")))
+ with pytest.raises(OSConnectionError):
+ await svc.list_files(user_id="u", page=2)
+ assert client.search.await_count == 1
+
+
+async def test_count_missing_index_returns_zero():
+ svc, _ = _real_count_service(NotFoundError(404, "index_not_found_exception", {}))
+ result = await svc.list_files(user_id="u", page=2)
+ assert result["files"] == []
+ assert result["total"] == 0
diff --git a/tests/unit/test_filter_clause_builder.py b/tests/unit/test_filter_clause_builder.py
index 1ce4c0859..8a2190477 100644
--- a/tests/unit/test_filter_clause_builder.py
+++ b/tests/unit/test_filter_clause_builder.py
@@ -3,7 +3,9 @@ import pytest
from utils.opensearch_queries import build_filter_clauses
RANGE = {
- "range": {"indexed_time": {"gte": "2026-10-07T00:00:00Z", "lte": "2026-10-10T23:59:59.999Z"}}
+ "range": {
+ "indexed_time": {"gte": "2026-10-07T00:00:00.000000Z", "lte": "2026-10-10T23:59:59.999Z"}
+ }
}
DATES = {"created_after": "2026-10-07T00:00:00Z", "created_before": "2026-10-10T23:59:59.999Z"}
@@ -42,7 +44,7 @@ def test_dates_combined_with_lists_and_not_treated_as_terms():
def test_single_bound_and_blank_bounds():
assert build_filter_clauses({"created_after": "2026-10-07T00:00:00Z"}) == [
- {"range": {"indexed_time": {"gte": "2026-10-07T00:00:00Z"}}}
+ {"range": {"indexed_time": {"gte": "2026-10-07T00:00:00.000000Z"}}}
]
assert build_filter_clauses({"created_after": "", "created_before": None}) == []
@@ -50,12 +52,13 @@ def test_single_bound_and_blank_bounds():
@pytest.mark.parametrize(
("raw", "expected"),
[
- ("2026-10-10", "2026-10-10T00:00:00Z"),
- ("2026-10-10Z", "2026-10-10T00:00:00Z"),
- ("2026-10-10T10:30", "2026-10-10T10:30:00Z"),
- ("2026-10-10T10:30+05:30", "2026-10-10T10:30:00+05:30"),
- ("2026-10-10T10:30:15Z", "2026-10-10T10:30:15Z"),
+ ("2026-10-10", "2026-10-10T00:00:00.000000Z"),
+ ("2026-10-10Z", "2026-10-10T00:00:00.000000Z"),
+ ("2026-10-10T10:30", "2026-10-10T10:30:00.000000Z"),
+ ("2026-10-10T10:30+05:30", "2026-10-10T10:30:00.000000+05:30"),
+ ("2026-10-10T10:30:15Z", "2026-10-10T10:30:15.000000Z"),
("2026-10-10T23:59:59.999999Z", "2026-10-10T23:59:59.999999Z"),
+ ("2026-10-10T23:59:59.5", "2026-10-10T23:59:59.5"),
],
)
def test_date_bounds_are_sent_as_explicit_instants(raw, expected):
diff --git a/tests/unit/test_search_service_score_threshold.py b/tests/unit/test_search_service_score_threshold.py
index 34c6af58e..c02e2e4f3 100644
--- a/tests/unit/test_search_service_score_threshold.py
+++ b/tests/unit/test_search_service_score_threshold.py
@@ -186,7 +186,7 @@ async def test_filters_are_pushed_into_each_knn_clause(monkeypatch):
params = _knn_params(os_client.bodies[-1]["query"])
assert params
- date_clause = {"range": {"indexed_time": {"gte": "2024-01-01T00:00:00Z"}}}
+ date_clause = {"range": {"indexed_time": {"gte": "2024-01-01T00:00:00.000000Z"}}}
for knn in params:
# The date range sits beside the chunk-mode exclusion of website manifests.
assert date_clause in knn["filter"]["bool"]["filter"]
@@ -268,7 +268,7 @@ async def test_nmslib_scores_the_filtered_documents_exactly(monkeypatch):
script = clause["script_score"]["script"]
assert script["lang"] == "knn"
assert script["params"]["space_type"] == "cosinesimil"
- date_clause = {"range": {"indexed_time": {"gte": "2024-01-01T00:00:00Z"}}}
+ date_clause = {"range": {"indexed_time": {"gte": "2024-01-01T00:00:00.000000Z"}}}
assert date_clause in clause["script_score"]["query"]["bool"]["filter"]
assert date_clause in hybrid["bool"]["filter"] # the outer filter is kept