OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: japanese file names (#2053) (#2098)
Edwin Josedd187c484773
diff --git a/src/connectors/service.py b/src/connectors/service.pyindex 72d7eede5..421cb8015 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -135,7 +135,7 @@ class ConnectorService:docling_polling_service=self.task_service.docling_polling_serviceif self.task_serviceelse None,- document_id=document.id,+ connector_file_id=document.id,source_url=document.source_url,allowed_users=allowed_users,allowed_groups=allowed_groups,@@ -215,6 +215,7 @@ class ConnectorService:file_size=len(document.content) if document.content else 0,connector_type=connector_type,acl=document.acl,+ connector_file_id=document.id,**standard_kwargs,)@@ -228,7 +229,10 @@ class ConnectorService:if result["status"] in ["indexed", "unchanged"]:# Update all chunks with connector-specific metadataawait self._update_connector_metadata(- document, owner_user_id, connector_type, jwt_token+ document,+ owner_user_id,+ connector_type,+ jwt_token,)return {@@ -243,7 +247,6 @@ class ConnectorService:owner_user_id: str,connector_type: str,jwt_token: str = None,- id_field: str = "document_id",indexed_filename: str | None = None,):"""Update indexed chunks with connector-specific metadata"""@@ -260,9 +263,11 @@ class ConnectorService:raise RuntimeError("Backend OpenSearch write client is unavailable")# Update ACL if changed (hash-based skip optimization).- # Match both document_id and connector_file_id: non-Langflow connector- # chunks store the connector id in connector_file_id (document_id holds the- # content hash), while Langflow chunks store it in document_id.+ # Match both document_id and connector_file_id: both the Langflow and+ # non-Langflow ingestion paths store the raw connector id in+ # connector_file_id (document_id holds a content/id hash), except for+ # pre-migration chunks indexed before that split existed, which only+ # have document_id set to the raw connector id.acl_result = await update_document_acl(document_id=document.id,acl=document.acl,@@ -288,14 +293,19 @@ class ConnectorService:await write_client.update_by_query(index=self.index_name,body={- # Match both fields: non-Langflow chunks carry the connector id- # in connector_file_id (document_id is the content hash),- # Langflow chunks carry it in document_id.+ # Match both fields: both ingestion paths carry the raw+ # connector id in connector_file_id (document_id is a+ # content/id hash); pre-migration chunks only have it in+ # document_id."query": {"bool": {"should": [{"term": {"document_id": document.id}},{"term": {"connector_file_id": document.id}},+ # See check_document_exists (models/processors.py):+ # some indices predate the explicit keyword+ # mapping for this field.+ {"term": {"connector_file_id.keyword": document.id}},],"minimum_should_match": 1,}diff --git a/src/models/processors.py b/src/models/processors.pyindex 88a4255b7..c8b7bee4a 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -79,6 +79,7 @@ class TaskProcessor:on_error: Literal["assume_missing", "assume_exists"] = "assume_missing",*,wait_for_visibility: bool = False,+ field: str = "document_id",) -> bool:"""Check if a document with the given hash already exists in OpenSearch.@@ -102,6 +103,26 @@ class TaskProcessor:max_retries = 3retry_delay = 1.0+ # Some deployments' indices predate connector_file_id's addition to the+ # explicit mapping (config/settings.py), so OpenSearch dynamically+ # mapped it as analyzed `text` (with a `.keyword` multi-field) instead+ # of the intended `keyword` type. A plain term query against such a+ # field tokenizes the query value and rarely matches the raw id, so+ # also match its `.keyword` multi-field. document_id has always been+ # explicitly `keyword` since index creation and never has this issue.+ query: dict[str, Any]+ if field == "connector_file_id":+ query = {+ "bool": {+ "should": [+ {"term": {field: file_hash}},+ {"term": {f"{field}.keyword": file_hash}},+ ]+ }+ }+ else:+ query = {"term": {field: file_hash}}+for attempt in range(max_retries):try:response = await opensearch_client.search(@@ -109,7 +130,7 @@ class TaskProcessor:body={"size": 1,"_source": False,- "query": {"term": {"document_id": file_hash}},+ "query": query,},)hits = response.get("hits", {}).get("hits", [])@@ -342,6 +363,11 @@ class TaskProcessor:"should": [{"term": {"document_id": file_id}},{"term": {"connector_file_id": file_id}},+ # Some deployments' indices predate this field's+ # addition to the explicit mapping, so it was+ # dynamically mapped as analyzed text with a+ # `.keyword` multi-field instead of `keyword`.+ {"term": {"connector_file_id.keyword": file_id}},],"minimum_should_match": 1,}@@ -1091,10 +1117,25 @@ class ConnectorFileProcessor(TaskProcessor):delete_document_ids,)+ # Match both fields: bucket-connector chunks carry the+ # raw connector id in connector_file_id (document_id is+ # a hash), while pre-migration chunks only have it in+ # document_id.chunk_ids = await collect_visible_document_ids(opensearch_client,index=get_index_name(),- query={"term": {"document_id": document.id}},+ query={+ "bool": {+ "should": [+ {"term": {"document_id": document.id}},+ {"term": {"connector_file_id": document.id}},+ # See check_document_exists: some indices+ # predate the explicit keyword mapping for+ # this field.+ {"term": {"connector_file_id.keyword": document.id}},+ ]+ }+ },)deleted_count = await delete_document_ids(opensearch_client,@@ -1173,7 +1214,7 @@ class ConnectorFileProcessor(TaskProcessor):if self.connector_service.task_serviceelse None,file_task=file_task,- document_id=document.id,+ connector_file_id=document.id,source_url=document.source_url,allowed_users=allowed_users,allowed_groups=allowed_groups,@@ -1193,6 +1234,7 @@ class ConnectorFileProcessor(TaskProcessor):_verification_client(opensearch_client),on_error="assume_exists",wait_for_visibility=True,+ field="connector_file_id",):result = {"status": "error",@@ -1209,7 +1251,6 @@ class ConnectorFileProcessor(TaskProcessor):self.user_id,connector_type,self.jwt_token,- id_field="document_id",indexed_filename=file_task.filename,)else:@@ -1257,7 +1298,6 @@ class ConnectorFileProcessor(TaskProcessor):self.user_id,connector_type,self.jwt_token,- id_field="connector_file_id",indexed_filename=file_task.filename,)diff --git a/src/services/document_index_writer.py b/src/services/document_index_writer.pyindex ba64e3d7b..a1eff9767 100644--- a/src/services/document_index_writer.py+++ b/src/services/document_index_writer.py@@ -32,6 +32,7 @@ class DocumentIndexContext:file_size: int | None = Noneconnector_type: str | None = Nonesource_url: str | None = None+ connector_file_id: str | None = Noneallowed_users: list[str] = field(default_factory=list)allowed_groups: list[str] = field(default_factory=list)allowed_principals: list[str] = field(default_factory=list)@@ -244,7 +245,9 @@ class DocumentIndexWriter:doc["owner_email"] = context.owner_emailif context.ingest_run_id:doc["ingest_run_id"] = context.ingest_run_id- if metadata.get("connector_file_id"):+ if context.connector_file_id:+ doc["connector_file_id"] = context.connector_file_id+ elif metadata.get("connector_file_id"):doc["connector_file_id"] = metadata["connector_file_id"]if context.is_sample_data:doc["is_sample_data"] = "true"diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex 1dc488b12..3be3c48dd 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -168,7 +168,16 @@ class LangflowFileService:self,file_tuples: list[tuple[str, Any, str]] | None,document_id: str | None,+ connector_file_id: str | None = None,) -> str:+ # Bucket-style connectors (COS/Azure/S3) pass their raw, potentially+ # non-ASCII and unbounded "bucket::key" as connector_file_id. Hash it+ # into a stable ASCII document_id instead of using it verbatim — the+ # raw value still travels downstream as connector_file_id, but+ # document_id must stay safe for HTTP headers and OpenSearch's chunk+ # _id (mirrors the content-hash document_id used by manual upload).+ if connector_file_id:+ return hash_id(io.BytesIO(connector_file_id.encode("utf-8")))if document_id:return document_idif file_tuples and len(file_tuples[0]) > 1:@@ -199,6 +208,7 @@ class LangflowFileService:parser: str | None = None,chunk_size: int | None = None,chunk_overlap: int | None = None,+ connector_file_id: str | None = None,) -> tuple[str | None, str | None]:if self.ingest_token_service is None:logger.warning(@@ -234,6 +244,7 @@ class LangflowFileService:parser=parser,chunk_size=chunk_size,chunk_overlap=chunk_overlap,+ connector_file_id=connector_file_id,)token = self.ingest_token_service.create_token(context)logger.info(@@ -353,6 +364,7 @@ class LangflowFileService:owner_email: str | None = None,connector_type: str | None = None,document_id: str | None = None,+ connector_file_id: str | None = None,source_url: str | None = None,allowed_users: list[str] | None = None,allowed_groups: list[str] | None = None,@@ -419,7 +431,9 @@ class LangflowFileService:if file_tuples and len(file_tuples) > 0 and len(file_tuples[0]) > 2else "")- resolved_document_id = self._resolve_document_id(file_tuples, document_id)+ resolved_document_id = self._resolve_document_id(+ file_tuples, document_id, connector_file_id+ )# Get the current embedding model and provider credentials from configfrom config.settings import get_openrag_config@@ -480,6 +494,7 @@ class LangflowFileService:parser="Docling Serve 1.20.0",chunk_size=chunk_size,chunk_overlap=chunk_overlap,+ connector_file_id=connector_file_id,)headers.update(self._ingest_callback_global_var_headers(@@ -892,6 +907,7 @@ class LangflowFileService:docling_polling_service: Any | None = None,file_task: Any | None = None,document_id: str | None = None,+ connector_file_id: str | None = None,source_url: str | None = None,allowed_users: list[str] | None = None,allowed_groups: list[str] | None = None,@@ -1036,6 +1052,7 @@ class LangflowFileService:connector_type=connector_type,docling_task_id=task_id,document_id=document_id,+ connector_file_id=connector_file_id,source_url=source_url,selected_embedding_model=selected_embedding,allowed_users=allowed_users,diff --git a/src/services/langflow_ingest_token_service.py b/src/services/langflow_ingest_token_service.pyindex d50cb4279..0f4eccf96 100644--- a/src/services/langflow_ingest_token_service.py+++ b/src/services/langflow_ingest_token_service.py@@ -167,6 +167,7 @@ class LangflowIngestTokenService:"parser": context.parser,"chunk_size": context.chunk_size,"chunk_overlap": context.chunk_overlap,+ "connector_file_id": context.connector_file_id,}@staticmethod@@ -198,4 +199,5 @@ class LangflowIngestTokenService:parser=payload.get("parser"),chunk_size=payload.get("chunk_size"),chunk_overlap=payload.get("chunk_overlap"),+ connector_file_id=payload.get("connector_file_id"),)diff --git a/src/utils/acl_utils.py b/src/utils/acl_utils.pyindex 5375dae3f..9def041b0 100644--- a/src/utils/acl_utils.py+++ b/src/utils/acl_utils.py@@ -41,12 +41,22 @@ def _build_id_query(document_id: str, id_fields: tuple[str, ...]) -> dict:``connector_file_id`` while ``document_id`` holds the content hash, whereasLangflow chunks and local uploads store the id in ``document_id``. Matchingmultiple fields lets a single id reliably target chunks from either pipeline.++ Also matches ``connector_file_id.keyword``: some deployments' indices+ predate that field's addition to the explicit mapping, so OpenSearch+ dynamically mapped it as analyzed text (with a ``.keyword`` multi-field)+ instead of the intended ``keyword`` type, and a plain term query against+ it tokenizes the query value instead of matching the raw id. This clause+ is a no-op on indices where the field is already ``keyword``."""- if len(id_fields) == 1:- return {"term": {id_fields[0]: document_id}}+ clauses = [{"term": {field: document_id}} for field in id_fields]+ if "connector_file_id" in id_fields:+ clauses.append({"term": {"connector_file_id.keyword": document_id}})+ if len(clauses) == 1:+ return clauses[0]return {"bool": {- "should": [{"term": {field: document_id}} for field in id_fields],+ "should": clauses,"minimum_should_match": 1,}}diff --git a/tests/unit/connectors/test_bucket_connector_unicode_file_ids.py b/tests/unit/connectors/test_bucket_connector_unicode_file_ids.pynew file mode 100644index 000000000..15bf1f21b--- /dev/null+++ b/tests/unit/connectors/test_bucket_connector_unicode_file_ids.py@@ -0,0 +1,61 @@+"""Regression coverage: bucket-style connectors (IBM COS, Azure Blob) must+preserve non-ASCII (e.g. Japanese) object keys unchanged through their+composite file-id round trip.++`document.id` (built from `_make_file_id`) is what downstream ingestion now+passes as `connector_file_id` — it must stay the raw, unmangled key so the+bucket/key can still be split back out for delete/status lookups+(`enhancements/connectors/*/api.py`), even though `document_id` itself is now+derived from a hash of this value (see+`tests/unit/test_resolve_document_id_connector_file_id.py`).+"""++import sys+from pathlib import Path++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++from enhancements.connectors.azure_blob.connector import ( # noqa: E402+ _make_file_id as azure_make_file_id,+)+from enhancements.connectors.azure_blob.connector import ( # noqa: E402+ _split_file_id as azure_split_file_id,+)+from enhancements.connectors.ibm_cos.connector import ( # noqa: E402+ _make_file_id as cos_make_file_id,+)+from enhancements.connectors.ibm_cos.connector import ( # noqa: E402+ _split_file_id as cos_split_file_id,+)++JAPANESE_KEY = "報告書.pdf"+JAPANESE_NESTED_KEY = "フォルダ/報告書_最終版.pdf"+++def test_cos_file_id_round_trips_japanese_key():+ file_id = cos_make_file_id("my-bucket", JAPANESE_KEY)++ assert file_id == f"my-bucket::{JAPANESE_KEY}"+ bucket, key = cos_split_file_id(file_id)+ assert bucket == "my-bucket"+ assert key == JAPANESE_KEY+++def test_cos_file_id_round_trips_nested_japanese_key():+ file_id = cos_make_file_id("my-bucket", JAPANESE_NESTED_KEY)++ bucket, key = cos_split_file_id(file_id)+ assert bucket == "my-bucket"+ assert key == JAPANESE_NESTED_KEY+++def test_azure_file_id_round_trips_japanese_blob_name():+ file_id = azure_make_file_id("my-container", JAPANESE_KEY)++ assert file_id == f"my-container::{JAPANESE_KEY}"+ container, blob = azure_split_file_id(file_id)+ assert container == "my-container"+ assert blob == JAPANESE_KEYdiff --git a/tests/unit/connectors/test_update_connector_metadata_filename.py b/tests/unit/connectors/test_update_connector_metadata_filename.pyindex 20c71d687..0807693a7 100644--- a/tests/unit/connectors/test_update_connector_metadata_filename.py+++ b/tests/unit/connectors/test_update_connector_metadata_filename.py@@ -92,6 +92,9 @@ async def test_update_includes_filename_in_script_and_params(monkeypatch):"should": [{"term": {"document_id": "graph-item-id-stable"}},{"term": {"connector_file_id": "graph-item-id-stable"}},+ # Some indices predate the explicit keyword mapping for this+ # field and dynamically mapped it as analyzed text instead.+ {"term": {"connector_file_id.keyword": "graph-item-id-stable"}},],"minimum_should_match": 1,}diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.pyindex 1ddd23ced..a40bffd8a 100644--- a/tests/unit/test_connector_processor_filename_dedupe.py+++ b/tests/unit/test_connector_processor_filename_dedupe.py@@ -582,7 +582,7 @@ async def test_rename_cleanup_matches_both_id_fields(monkeypatch, backend_write_shoulds = captured["query"]["bool"]["filter"][0]["bool"]["should"]fields = {next(iter(c["term"])) for c in shoulds}- assert fields == {"document_id", "connector_file_id"}+ assert fields == {"document_id", "connector_file_id", "connector_file_id.keyword"}excluded = captured["query"]["bool"]["must_not"][0]["terms"]["filename"]assert set(get_filename_aliases("Renamed.pdf")).issubset(set(excluded))diff --git a/tests/unit/test_document_index_writer_connector_file_id.py b/tests/unit/test_document_index_writer_connector_file_id.pynew file mode 100644index 000000000..1618147bc--- /dev/null+++ b/tests/unit/test_document_index_writer_connector_file_id.py@@ -0,0 +1,74 @@+"""`_build_chunk_document` must carry `connector_file_id` from the context onto+the indexed document when present, so bucket-connector documents (COS/Azure/S3)+keep their raw, human-meaningful id available for dedupe/orphan lookups even+though `document_id` is now a stable hash (see+`tests/unit/test_resolve_document_id_connector_file_id.py`).+"""++from services.document_index_writer import (+ DocumentIndexChunk,+ DocumentIndexContext,+ DocumentIndexWriter,+)+++def _context(**overrides):+ defaults = dict(+ document_id="hashed-id",+ filename="報告書.pdf",+ mimetype="application/pdf",+ embedding_model="test-model",+ )+ defaults.update(overrides)+ return DocumentIndexContext(**defaults)+++def test_connector_file_id_from_context_is_set_on_document():+ writer = DocumentIndexWriter()+ context = _context(connector_file_id="my-bucket::報告書.pdf")+ chunk = DocumentIndexChunk(chunk_id="c1", text="hello", vector=[0.1, 0.2])++ doc = writer._build_chunk_document(+ context=context,+ chunk=chunk,+ embedding_field="embedding",+ indexed_time="2026-01-01T00:00:00Z",+ )++ assert doc["connector_file_id"] == "my-bucket::報告書.pdf"+ assert doc["document_id"] == "hashed-id"+++def test_missing_connector_file_id_falls_back_to_chunk_metadata():+ writer = DocumentIndexWriter()+ context = _context(connector_file_id=None)+ chunk = DocumentIndexChunk(+ chunk_id="c1",+ text="hello",+ vector=[0.1, 0.2],+ metadata={"connector_file_id": "legacy-id"},+ )++ doc = writer._build_chunk_document(+ context=context,+ chunk=chunk,+ embedding_field="embedding",+ indexed_time="2026-01-01T00:00:00Z",+ )++ assert doc["connector_file_id"] == "legacy-id"+++def test_no_connector_file_id_anywhere_omits_field():+ writer = DocumentIndexWriter()+ context = _context()+ chunk = DocumentIndexChunk(chunk_id="c1", text="hello", vector=[0.1, 0.2])++ doc = writer._build_chunk_document(+ context=context,+ chunk=chunk,+ embedding_field="embedding",+ indexed_time="2026-01-01T00:00:00Z",+ )++ assert "connector_file_id" not in docdiff --git a/tests/unit/test_langflow_ingest_callback.py b/tests/unit/test_langflow_ingest_callback.pyindex 0f6236583..1405fe282 100644--- a/tests/unit/test_langflow_ingest_callback.py+++ b/tests/unit/test_langflow_ingest_callback.py@@ -103,6 +103,28 @@ async def test_langflow_ingest_callback_indexes_authoritative_token_context():)+def test_ingest_token_round_trips_connector_file_id():+ """Bucket-connector chunks (COS/Azure/S3) rely on connector_file_id+ surviving the JWT round trip so post-ingest verification (which queries+ by connector_file_id) and dedupe/ACL sync can find them. Regression guard+ for the field being dropped by _context_to_payload/_payload_to_context."""+ token_service = LangflowIngestTokenService(secret="test-secret" * 4, ttl_seconds=60)+ context = DocumentIndexContext(+ document_id="hashed-id",+ filename="report.pdf",+ mimetype="application/pdf",+ embedding_model="text-embedding-3-small",+ owner="user-1",+ ingest_run_id="run-1",+ connector_file_id="my-bucket::報告書.pdf",+ )+ token = token_service.create_token(context)++ restored_context, _jti = token_service.validate_token(token)++ assert restored_context.connector_file_id == "my-bucket::報告書.pdf"++@pytest.mark.asyncioasync def test_langflow_ingest_callback_rewrites_langflow_chunk_ids():token_service = LangflowIngestTokenService(secret="test-secret" * 4, ttl_seconds=60)diff --git a/tests/unit/test_resolve_document_id_connector_file_id.py b/tests/unit/test_resolve_document_id_connector_file_id.pynew file mode 100644index 000000000..96bd90d85--- /dev/null+++ b/tests/unit/test_resolve_document_id_connector_file_id.py@@ -0,0 +1,81 @@+"""Unit tests for `LangflowFileService._resolve_document_id`'s connector_file_id path.++Bucket-style connectors (IBM COS, Azure Blob, AWS S3) build their document+identity from the raw, potentially non-ASCII object key (e.g. "bucket::報告書.pdf").+That raw value must never be used verbatim as `document_id` — it has to travel+through ASCII-only HTTP headers (X-Langflow-Global-Var-DOCUMENT_ID) and is used+as the OpenSearch chunk `_id`. These tests pin the fix: when a `connector_file_id`+is supplied, `document_id` is a stable ASCII hash of it, never the raw value.+"""++from services.langflow_file_service import LangflowFileService+++def _service():+ return LangflowFileService(docling_service=None)+++def test_connector_file_id_hashed_into_ascii_document_id():+ svc = _service()+ raw_id = "my-bucket::報告書.pdf"++ resolved = svc._resolve_document_id(None, None, raw_id)++ assert resolved != raw_id+ resolved.encode("ascii") # must not raise+++def test_connector_file_id_hash_is_deterministic():+ svc = _service()+ raw_id = "my-bucket::報告書.pdf"++ first = svc._resolve_document_id(None, None, raw_id)+ second = svc._resolve_document_id(None, None, raw_id)++ assert first == second+++def test_different_connector_file_ids_hash_differently():+ svc = _service()++ a = svc._resolve_document_id(None, None, "bucket::a.pdf")+ b = svc._resolve_document_id(None, None, "bucket::b.pdf")++ assert a != b+++def test_connector_file_id_takes_precedence_over_explicit_document_id():+ svc = _service()++ resolved = svc._resolve_document_id(None, "explicit-id", "bucket::報告書.pdf")++ assert resolved != "explicit-id"+++def test_ascii_connector_file_id_still_gets_hashed_for_consistency():+ """Even an already-ASCII connector_file_id (e.g. Google Drive's opaque file+ id) is hashed, so document_id stays a stable, bounded-length identifier+ regardless of connector type — not just a non-ASCII special case."""+ svc = _service()++ resolved = svc._resolve_document_id(None, None, "1a2B3cD4E5f")++ assert resolved != "1a2B3cD4E5f"+++def test_no_connector_file_id_falls_back_to_explicit_document_id():+ svc = _service()++ resolved = svc._resolve_document_id(None, "explicit-id", None)++ assert resolved == "explicit-id"+++def test_no_ids_falls_back_to_content_hash():+ svc = _service()+ file_tuples = [("報告書.pdf", b"file content", "application/pdf")]++ resolved = svc._resolve_document_id(file_tuples, None, None)++ assert resolved+ resolved.encode("ascii") # must not raise