OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: Implement support for a new Azure Blob connector (#1974)
Mike Pawlowski77bd3d692239
diff --git a/.env.example b/.env.exampleindex 97a3f47a4..6917262d0 100644--- a/.env.example+++ b/.env.example@@ -163,6 +163,35 @@ MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET=AWS_ACCESS_KEY_ID=AWS_SECRET_ACCESS_KEY=+# --- Azure Blob Storage connector (Enterprise/SaaS) ------------------------+# Enterprise/SaaS-only bucket connector, gated by IBM_AUTH_ENABLED (like the+# AWS S3 / IBM COS connectors). Credentials are normally entered per-connection+# in the UI; these env vars are OPTIONAL fallbacks / defaults for the config+# dialog. Two auth modes are supported: a connection string, OR account name ++# key (+ optional custom blob endpoint).+#+# Local dev / Azurite testing: set OPENRAG_DEV_AZURE_BLOB=true to enable the+# connector without IBM_AUTH_ENABLED. NEVER use in production.+# OPENRAG_DEV_AZURE_BLOB=false+#+# Connection-string mode (covers real Azure accounts and the Azurite emulator):+# - Real account: copy from Azure Portal -> Storage account -> Access keys+# - Azurite (local dev): AZURE_STORAGE_CONNECTION_STRING="UseDevelopmentStorage=true"+# AZURE_STORAGE_CONNECTION_STRING=+#+# Account-key mode (set account name + key; endpoint optional):+# - Azurite (host-side backend):+# - AZURE_STORAGE_ACCOUNT_NAME=devstoreaccount1+# - AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==+# - AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1+# - Azurite (in-compose backend):+# - AZURE_STORAGE_ACCOUNT_NAME=devstoreaccount1+# - AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==+# - AZURE_STORAGE_ENDPOINT=http://azurite:10000/devstoreaccount1+# AZURE_STORAGE_ACCOUNT_NAME=+# AZURE_STORAGE_ACCOUNT_KEY=+# AZURE_STORAGE_ENDPOINT=+# OPTIONAL: dns routable from google (etc.) to handle continous ingest (something like ngrok works). This enables continous ingestionWEBHOOK_BASE_URL=diff --git a/Makefile b/Makefileindex 3d702e4c5..2eca35150 100644--- a/Makefile+++ b/Makefile@@ -96,6 +96,7 @@ endef######################.PHONY: help check_tools help_docker help_dev help_test help_local help_utils help_operator \dev dev-cpu dev-local dev-local-cpu dev-local-build-lf dev-local-build-lf-cpu stop clean build logs \+ azurite-up azurite-down \shell-backend shell-frontend install \test test-unit test-integration test-ci test-ci-local test-ci-suite test-sdk test-os-jwt lint \ci-build-images ci-save-images \@@ -345,6 +346,8 @@ help_local: ## Show local development commands@echo " $(PURPLE)make frontend$(NC) - Run frontend locally"@echo " $(PURPLE)make docling$(NC) - Start docling-serve for document processing"@echo " $(PURPLE)make docling-stop$(NC) - Stop docling-serve"+ @echo " $(PURPLE)make azurite-up$(NC) - Start Azurite (local Azure Blob emulator) for connector testing"+ @echo " $(PURPLE)make azurite-down$(NC) - Stop Azurite emulator"@echo ''@echo "$(PURPLE)Installation:$(NC)"@echo " $(PURPLE)make install$(NC) - Install all dependencies"@@ -673,6 +676,16 @@ docling-stop: ## Stop docling-serve@uv run python scripts/docling_ctl.py stop@echo "$(PURPLE)Docling-serve stopped.$(NC)"+azurite-up: ## Start Azurite (local Azure Blob emulator) for connector testing+ @echo "$(YELLOW)Starting Azurite (Azure Blob emulator)...$(NC)"+ $(COMPOSE_CMD) --profile azurite up -d azurite+ @echo "$(PURPLE)Azurite started on http://localhost:10000 (account: devstoreaccount1).$(NC)"++azurite-down: ## Stop Azurite emulator+ @echo "$(YELLOW)Stopping Azurite...$(NC)"+ $(COMPOSE_CMD) --profile azurite stop azurite+ @echo "$(PURPLE)Azurite stopped.$(NC)"+####################### INSTALLATION######################diff --git a/docker-compose.yml b/docker-compose.ymlindex 1cc1efc74..3a22a38dc 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -219,5 +219,28 @@ services:# - DEFAULT_FOLDER_NAME=OpenRAG- HIDE_GETTING_STARTED_PROGRESS=true+ # Azurite: local Azure Storage emulator for manually testing the Azure Blob+ # connector without a paid Azure account. Opt-in only — gated behind the+ # "azurite" compose profile so it never starts with the normal stack.+ # Start it with `make azurite-up` (or `docker compose --profile azurite up -d azurite`).+ # Connect the connector with AZURE_STORAGE_CONNECTION_STRING="UseDevelopmentStorage=true"+ # (from inside the compose network use host "azurite": the well-known dev+ # endpoint http://azurite:10000/devstoreaccount1).+ azurite:+ image: mcr.microsoft.com/azure-storage/azurite:latest+ container_name: azurite+ profiles: ["azurite"]+ # --skipApiVersionCheck: the azure-storage-blob SDK may negotiate a newer+ # Storage API version than the pinned Azurite image supports; skip the check+ # so the emulator accepts requests (safe for local dev only).+ command: "azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --location /data --skipApiVersionCheck"+ ports:+ - "10000:10000" # Blob+ - "10001:10001" # Queue+ - "10002:10002" # Table+ volumes:+ - azurite-data:/data+volumes:opensearch-data:+ azurite-data:diff --git a/enhancements/__init__.py b/enhancements/__init__.pyindex b590503c4..5c86d2f15 100644--- a/enhancements/__init__.py+++ b/enhancements/__init__.py@@ -8,6 +8,7 @@ Strip the imports below to ship a bare OSS build.from connectors.base import BaseConnector+from .connectors.azure_blob import AzureBlobConnectorfrom .connectors.ibm_cos import IBMCOSConnector-ADDITIONAL_CONNECTORS: list[type[BaseConnector]] = [IBMCOSConnector]+ADDITIONAL_CONNECTORS: list[type[BaseConnector]] = [IBMCOSConnector, AzureBlobConnector]diff --git a/enhancements/connectors/azure_blob/__init__.py b/enhancements/connectors/azure_blob/__init__.pynew file mode 100644index 000000000..b2002733b--- /dev/null+++ b/enhancements/connectors/azure_blob/__init__.py@@ -0,0 +1,19 @@+from .api import (+ azure_blob_configure,+ azure_blob_container_status,+ azure_blob_defaults,+ azure_blob_list_containers,+ azure_blob_test,+)+from .connector import AzureBlobConnector+from .models import AzureBlobConfigureBody++__all__ = [+ "AzureBlobConnector",+ "AzureBlobConfigureBody",+ "azure_blob_defaults",+ "azure_blob_test",+ "azure_blob_configure",+ "azure_blob_list_containers",+ "azure_blob_container_status",+]diff --git a/enhancements/connectors/azure_blob/api.py b/enhancements/connectors/azure_blob/api.pynew file mode 100644index 000000000..37d44fd98--- /dev/null+++ b/enhancements/connectors/azure_blob/api.py@@ -0,0 +1,246 @@+"""FastAPI route handlers for Azure Blob-specific endpoints."""++import asyncio+import os++from fastapi import Depends+from fastapi.responses import JSONResponse++from config.settings import get_index_name+from dependencies import get_connector_service, get_current_user, get_session_manager+from session_manager import User+from utils.logging_config import get_logger++from .auth import create_blob_service_client+from .models import AzureBlobConfigureBody+from .support import build_azure_blob_config++logger = get_logger(__name__)+++def _list_container_names(config: dict) -> list[str]:+ """List container names for a resolved Azure Blob config (sync helper)."""+ client = create_blob_service_client(config)+ return [c.name for c in client.list_containers()]+++async def azure_blob_defaults(+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """Return current Azure Blob env-var defaults for pre-filling the config dialog.++ Sensitive values (connection string, account key) are masked — only whether+ they are set is returned, not the actual values.+ """+ connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING", "")+ account_name = os.getenv("AZURE_STORAGE_ACCOUNT_NAME", "")+ account_key = os.getenv("AZURE_STORAGE_ACCOUNT_KEY", "")+ endpoint = os.getenv("AZURE_STORAGE_ENDPOINT", "")++ connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type="azure_blob"+ )+ conn_config = connections[0].config or {} if connections else {}++ def _pick(conn_key, env_val):+ """Prefer connection config value over env var."""+ return conn_config.get(conn_key) or env_val++ has_conn_str = bool(connection_string or conn_config.get("connection_string"))+ has_account_key = bool(account_key or conn_config.get("account_key"))++ return JSONResponse(+ {+ "connection_string_set": has_conn_str,+ "account_name": _pick("account_name", account_name),+ "account_key_set": has_account_key,+ "endpoint": _pick("endpoint_url", endpoint),+ "auth_mode": conn_config.get(+ "auth_mode",+ "account_key" if (has_account_key and not has_conn_str) else "connection_string",+ ),+ "container_names": conn_config.get("container_names", []),+ "connection_id": connections[0].connection_id if connections else None,+ }+ )+++async def azure_blob_test(+ body: AzureBlobConfigureBody,+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """Validate Azure Blob credentials and list containers WITHOUT persisting.++ Backs the settings dialog's "Test Connection" action so testing or refreshing+ credentials never creates or mutates a saved connection (and so the stored+ container selection is never clobbered). Persistence happens only in+ ``azure_blob_configure`` on the final save. Credential resolution mirrors+ ``azure_blob_configure`` (request body → env → existing connection config).+ """+ existing_connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type="azure_blob"+ )+ existing_config = existing_connections[0].config if existing_connections else {}++ conn_config, error = build_azure_blob_config(body, existing_config)+ if error:+ return JSONResponse({"error": error}, status_code=400)++ # The azure-storage-blob SDK is sync; offload to keep the event loop free.+ try:+ containers = await asyncio.to_thread(_list_container_names, conn_config)+ except Exception:+ logger.exception("Failed to connect to Azure Blob during credential test.")+ return JSONResponse(+ {"error": "Could not connect to Azure Blob with the provided configuration."},+ status_code=400,+ )++ return JSONResponse({"containers": containers})+++async def azure_blob_configure(+ body: AzureBlobConfigureBody,+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """Create or update an Azure Blob connection with explicit credentials.++ Tests the credentials by listing containers, then persists the connection.+ Credentials are stored in the connection config dict (encrypted at rest) so+ the connector works even without system-level env vars.+ """+ existing_connections = await connector_service.connection_manager.list_connections(+ user_id=user.user_id, connector_type="azure_blob"+ )+ existing_config = existing_connections[0].config if existing_connections else {}++ conn_config, error = build_azure_blob_config(body, existing_config)+ if error:+ return JSONResponse({"error": error}, status_code=400)++ # Test credentials by listing containers. The azure-storage-blob SDK is sync,+ # so offload to a worker thread to keep the event loop responsive.+ try:+ await asyncio.to_thread(_list_container_names, conn_config)+ except Exception:+ logger.exception("Failed to connect to Azure Blob during credential test.")+ return JSONResponse(+ {"error": "Could not connect to Azure Blob with the provided configuration."},+ status_code=400,+ )++ # Persist: update existing connection or create a new one.+ if body.connection_id:+ existing = await connector_service.connection_manager.get_connection(body.connection_id)+ if existing and existing.user_id == user.user_id:+ await connector_service.connection_manager.update_connection(+ connection_id=body.connection_id,+ config=conn_config,+ )+ connector_service.connection_manager.active_connectors.pop(body.connection_id, None)+ return JSONResponse({"connection_id": body.connection_id, "status": "connected"})++ connection_id = await connector_service.connection_manager.create_connection(+ connector_type="azure_blob",+ name="Azure Blob Storage",+ config=conn_config,+ user_id=user.user_id,+ )+ return JSONResponse({"connection_id": connection_id, "status": "connected"})+++async def azure_blob_list_containers(+ connection_id: str,+ connector_service=Depends(get_connector_service),+ user: User = Depends(get_current_user),+):+ """List containers for an Azure Blob connection, honoring the ingestion restriction.++ When the connection has a non-empty container_names allowlist, only those+ containers are returned; otherwise all accessible containers are listed.+ """+ connection = await connector_service.connection_manager.get_connection(connection_id)+ if not connection or connection.user_id != user.user_id:+ return JSONResponse({"error": "Connection not found"}, status_code=404)+ if connection.connector_type != "azure_blob":+ return JSONResponse({"error": "Not an Azure Blob connection"}, status_code=400)++ try:+ containers = await asyncio.to_thread(_list_container_names, connection.config)+ allowed_containers = connection.config.get("container_names") or []+ if allowed_containers:+ allowed_set = set(allowed_containers)+ containers = [c for c in containers if c in allowed_set]+ return JSONResponse({"containers": containers})+ except Exception:+ logger.exception("Failed to list Azure Blob containers for connection %s", connection_id)+ return JSONResponse({"error": "Failed to list containers"}, status_code=500)+++async def azure_blob_container_status(+ connection_id: str,+ connector_service=Depends(get_connector_service),+ session_manager=Depends(get_session_manager),+ user: User = Depends(get_current_user),+):+ """Return all containers for an Azure Blob connection with ingestion status.++ Each entry includes the container name, whether it has been ingested+ (is_synced), and the count of indexed documents from that container.+ """+ connection = await connector_service.connection_manager.get_connection(connection_id)+ if not connection or connection.user_id != user.user_id:+ return JSONResponse({"error": "Connection not found"}, status_code=404)+ if connection.connector_type != "azure_blob":+ return JSONResponse({"error": "Not an Azure Blob connection"}, status_code=400)++ # 1. List containers, honoring the saved ingestion restriction. When the+ # connection has a non-empty container_names allowlist, only those+ # containers are browsable; otherwise all accessible containers are shown.+ try:+ all_containers = await asyncio.to_thread(_list_container_names, connection.config)+ except Exception:+ logger.exception("Failed to list Azure Blob containers for connection %s", connection_id)+ return JSONResponse({"error": "Failed to list containers"}, status_code=500)++ allowed_containers = connection.config.get("container_names") or []+ if allowed_containers:+ allowed_set = set(allowed_containers)+ all_containers = [c for c in all_containers if c in allowed_set]++ # 2. Count indexed documents per container from OpenSearch.+ ingested_counts: dict = {}+ try:+ opensearch_client = session_manager.get_user_opensearch_client(user.user_id, user.jwt_token)+ query_body = {+ "size": 0,+ "query": {"term": {"connector_type": "azure_blob"}},+ "aggs": {"doc_ids": {"terms": {"field": "document_id", "size": 50000}}},+ }+ index_name = get_index_name()+ os_resp = await opensearch_client.search(index=index_name, body=query_body)+ for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):+ doc_id = bucket_entry["key"]+ if "::" in doc_id:+ container_name = doc_id.split("::")[0]+ ingested_counts[container_name] = ingested_counts.get(container_name, 0) + 1+ except Exception:+ logger.warning(+ "Failed to aggregate Azure Blob ingestion counts for connection %s; "+ "container status will show zero counts",+ connection_id,+ exc_info=True,+ ) # OpenSearch unavailable / query failed — show zero counts++ result = [+ {+ "name": container,+ "ingested_count": ingested_counts.get(container, 0),+ "is_synced": ingested_counts.get(container, 0) > 0,+ }+ for container in all_containers+ ]+ return JSONResponse({"containers": result})diff --git a/enhancements/connectors/azure_blob/auth.py b/enhancements/connectors/azure_blob/auth.pynew file mode 100644index 000000000..a30918123--- /dev/null+++ b/enhancements/connectors/azure_blob/auth.py@@ -0,0 +1,144 @@+"""Azure Blob Storage authentication and client factory.++Supports two credential modes, both of which work against real Azure Storage+accounts and the local Azurite emulator:++- ``connection_string``: a full Azure Storage connection string+ (e.g. from the Azure Portal, or ``UseDevelopmentStorage=true`` for Azurite).+- ``account_key``: an account name + shared key, with an optional custom+ ``endpoint_url`` (used to point at Azurite or a sovereign cloud).++Resolution order for each credential: connector config dict → environment+variable. The sync ``azure-storage-blob`` SDK is used; callers offload blocking+operations onto a worker thread via ``asyncio.to_thread``.+"""++import os+from typing import Any++from utils.logging_config import get_logger++logger = get_logger(__name__)++# Public Azure Blob endpoint template (account_key mode, no custom endpoint).+_DEFAULT_BLOB_ENDPOINT = "https://{account}.blob.core.windows.net"+++def _resolve_credentials(config: dict[str, Any]) -> dict[str, Any]:+ """Resolve Azure Blob credentials from config dict → environment fallback.++ Returns a dict with the resolved values needed to build a BlobServiceClient.+ Does not raise — validation happens in the builder so callers get a single,+ mode-aware error message.+ """+ auth_mode = config.get("auth_mode", "connection_string")++ connection_string = config.get("connection_string") or os.getenv(+ "AZURE_STORAGE_CONNECTION_STRING"+ )+ account_name = config.get("account_name") or os.getenv("AZURE_STORAGE_ACCOUNT_NAME")+ account_key = config.get("account_key") or os.getenv("AZURE_STORAGE_ACCOUNT_KEY")+ # Optional custom blob endpoint (Azurite / sovereign clouds). For Azurite the+ # account-style endpoint is e.g. http://127.0.0.1:10000/devstoreaccount1.+ endpoint_url = config.get("endpoint_url") or os.getenv("AZURE_STORAGE_ENDPOINT") or None++ return {+ "auth_mode": auth_mode,+ "connection_string": connection_string,+ "account_name": account_name,+ "account_key": account_key,+ "endpoint_url": endpoint_url,+ }+++def _account_name_from_connection_string(connection_string: str) -> str | None:+ """Best-effort parse of the storage account name from a connection string.++ Handles the Azurite shorthand (``UseDevelopmentStorage=true`` →+ ``devstoreaccount1``), standard ``AccountName=...`` strings, and SAS-style+ strings that only carry a ``BlobEndpoint``. Returns None if it can't be+ determined. Never raises — callers fall back to a stable label.+ """+ pairs = (p.split("=", 1) for p in connection_string.split(";") if "=" in p)+ parts = {k.strip().lower(): v.strip() for k, v in pairs}++ if parts.get("usedevelopmentstorage", "").lower() == "true":+ return "devstoreaccount1"+ if parts.get("accountname"):+ return parts["accountname"]++ # SAS-style strings expose the account only via the blob endpoint host/path,+ # e.g. https://myacct.blob.core.windows.net or http://host:10000/devstoreaccount1.+ endpoint = parts.get("blobendpoint")+ if endpoint:+ from urllib.parse import urlparse++ parsed = urlparse(endpoint)+ path = parsed.path.strip("/")+ if path: # Azurite path-style: account is the first path segment+ return path.split("/")[0]+ host = parsed.hostname or ""+ if host: # production host-style: account is the first label+ return host.split(".")[0]+ return None+++def account_name_from_config(config: dict[str, Any]) -> str | None:+ """Resolve the storage account name from config (both auth modes).++ Used by the connector's ``get_client_id`` so status checks have a stable,+ non-raising identifier even in connection-string mode (where ``account_name``+ is implicit in the string rather than stored separately).+ """+ creds = _resolve_credentials(config)+ if creds["account_name"]:+ return creds["account_name"]+ if creds["connection_string"]:+ return _account_name_from_connection_string(creds["connection_string"])+ return None+++def create_blob_service_client(config: dict[str, Any]):+ """Return a sync ``BlobServiceClient`` for the configured auth mode.++ Args:+ config: Connector config dict (see module docstring for keys).++ Raises:+ ImportError: If the ``azure-storage-blob`` package is not installed.+ ValueError: If required credentials for the selected mode are missing.+ """+ try:+ from azure.storage.blob import BlobServiceClient+ except ImportError as exc:+ raise ImportError(+ "azure-storage-blob is required for the Azure Blob connector. "+ "Install it with: pip install azure-storage-blob"+ ) from exc++ creds = _resolve_credentials(config)+ auth_mode = creds["auth_mode"]++ if auth_mode == "connection_string":+ if not creds["connection_string"]:+ raise ValueError(+ "Connection string mode requires a connection string. Provide "+ "'connection_string' in the connector config or set "+ "AZURE_STORAGE_CONNECTION_STRING."+ )+ logger.debug("Creating Azure Blob client from connection string")+ return BlobServiceClient.from_connection_string(creds["connection_string"])++ # account_key mode+ if not (creds["account_name"] and creds["account_key"]):+ raise ValueError(+ "Account key mode requires account_name and account_key. Provide them "+ "in the connector config or set AZURE_STORAGE_ACCOUNT_NAME / "+ "AZURE_STORAGE_ACCOUNT_KEY."+ )++ account_url = creds["endpoint_url"] or _DEFAULT_BLOB_ENDPOINT.format(+ account=creds["account_name"]+ )+ logger.debug("Creating Azure Blob client with account key for %s", account_url)+ return BlobServiceClient(account_url=account_url, credential=creds["account_key"])diff --git a/enhancements/connectors/azure_blob/connector.py b/enhancements/connectors/azure_blob/connector.pynew file mode 100644index 000000000..c97eda53e--- /dev/null+++ b/enhancements/connectors/azure_blob/connector.py@@ -0,0 +1,369 @@+"""Azure Blob Storage connector for OpenRAG.++Enterprise/SaaS-only ``bucket``-kind connector, gated by ``IBM_AUTH_ENABLED``+(or ``OPENRAG_DEV_AZURE_BLOB=true`` for local dev/Azurite testing). Uses the+official sync ``azure-storage-blob`` SDK; all+blocking SDK calls are offloaded onto a worker thread with ``asyncio.to_thread``+so the event loop stays responsive while staying consistent with the cached+client lifecycle used by the AWS S3 / IBM COS connectors.++DLS (v1): owner-based. The connector returns an empty-principal ``DocumentACL``;+``service.process_connector_document`` assigns ``owner`` to the ingesting+OpenRAG user. Container→group principal mapping is a planned fast-follow.+"""++import asyncio+import mimetypes+import os+from datetime import UTC, datetime+from posixpath import basename+from typing import Any++from config.settings import IBM_AUTH_ENABLED, is_dev_azure_blob_enabled+from connectors.base import BaseConnector, ConnectorDocument, DocumentACL+from utils.logging_config import get_logger++from .auth import account_name_from_config, create_blob_service_client++logger = get_logger(__name__)++# Separator used in composite file IDs: "<container>::<blob>"+_ID_SEPARATOR = "::"+++def _make_file_id(container: str, blob: str) -> str:+ return f"{container}{_ID_SEPARATOR}{blob}"+++def _split_file_id(file_id: str) -> tuple[str, str]:+ """Split a composite file ID into (container, blob). Raises ValueError if invalid."""+ if _ID_SEPARATOR not in file_id:+ raise ValueError(f"Invalid Azure Blob file ID (missing separator): {file_id!r}")+ container, blob = file_id.split(_ID_SEPARATOR, 1)+ return container, blob+++class AzureBlobConnector(BaseConnector):+ """Connector for Azure Blob Storage.++ Supports connection-string and account-name/key credential modes. Both work+ against real Azure accounts and the Azurite emulator. Credentials are read+ from the connector config dict first, then from environment variables.++ Config dict keys:+ auth_mode (str): "connection_string" (default) or "account_key".+ connection_string (str): Overrides AZURE_STORAGE_CONNECTION_STRING.+ account_name (str): Overrides AZURE_STORAGE_ACCOUNT_NAME.+ account_key (str): Overrides AZURE_STORAGE_ACCOUNT_KEY.+ endpoint_url (str): Optional custom blob endpoint (Azurite / sovereign).+ container_names (list[str]): Containers to ingest from. If empty, all+ accessible containers are used.+ prefix (str): Optional blob name prefix filter.+ connection_id (str): Connection identifier used for logging.+ """++ CONNECTOR_TYPE = "azure_blob"+ CONNECTOR_KIND = "bucket"+ CONNECTOR_NAME = "Azure Blob Storage"+ CONNECTOR_DESCRIPTION = "Add knowledge from Azure Blob Storage"+ CONNECTOR_ICON = "azure-blob"+ # connection_string / account_key are Azure-specific and must be encrypted at rest.+ SECRET_CONFIG_KEYS = ("connection_string", "account_key")++ # BaseConnector uses these for the default env-availability probe; the+ # bucket-kind override below makes availability hinge on IBM_AUTH_ENABLED+ # (or OPENRAG_DEV_AZURE_BLOB for local dev).+ CLIENT_ID_ENV_VAR = "AZURE_STORAGE_ACCOUNT_NAME"+ CLIENT_SECRET_ENV_VAR = "AZURE_STORAGE_ACCOUNT_KEY"++ @classmethod+ def is_available(cls, manager, user_id=None) -> bool:+ # Gated by feature flag like the other bucket connectors (aws_s3, ibm_cos).+ # OPENRAG_DEV_AZURE_BLOB=true bypasses the IBM_AUTH_ENABLED requirement for+ # local dev testing (e.g. against Azurite). Never use in production.+ return IBM_AUTH_ENABLED or is_dev_azure_blob_enabled()++ @classmethod+ def register_routes(cls, app) -> None:+ from .api import (+ azure_blob_configure,+ azure_blob_container_status,+ azure_blob_defaults,+ azure_blob_list_containers,+ azure_blob_test,+ )++ # Registered before generic /{connector_type}/... to avoid shadowing.+ app.add_api_route(+ "/connectors/azure_blob/defaults",+ azure_blob_defaults,+ methods=["GET"],+ tags=["internal"],+ )+ # Non-persisting credential validation + container listing (Test Connection).+ app.add_api_route(+ "/connectors/azure_blob/test",+ azure_blob_test,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/azure_blob/configure",+ azure_blob_configure,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/azure_blob/{connection_id}/containers",+ azure_blob_list_containers,+ methods=["GET"],+ tags=["internal"],+ )+ app.add_api_route(+ "/connectors/azure_blob/{connection_id}/container-status",+ azure_blob_container_status,+ methods=["GET"],+ tags=["internal"],+ )++ def get_client_id(self) -> str:+ """Return the storage account name; used by status checks / availability probe.++ Must not raise in connection-string mode: the account name is implicit in+ the string rather than stored separately, so the status endpoint+ (api/connectors.py) would otherwise mark an authenticated connection as+ unauthenticated and the UI would keep showing "Connect". Falls back to a+ stable label when the account can't be parsed (e.g. SAS-only strings).+ """+ val = account_name_from_config(self.config)+ if val:+ return val+ if self.config.get("auth_mode", "connection_string") == "connection_string" and (+ self.config.get("connection_string") or os.getenv("AZURE_STORAGE_CONNECTION_STRING")+ ):+ return "azure_blob"+ raise ValueError(+ "Azure Blob credentials not set. Provide 'account_name' (account_key mode) "+ "or a 'connection_string'."+ )++ def get_client_secret(self) -> str:+ """Return account key / connection string; used by the availability probe."""+ val = (+ self.config.get("account_key")+ or self.config.get("connection_string")+ or os.getenv("AZURE_STORAGE_ACCOUNT_KEY")+ or os.getenv("AZURE_STORAGE_CONNECTION_STRING")+ )+ if val:+ return val+ raise ValueError(+ "Azure Blob credentials not set. Provide 'account_key' (account_key mode) "+ "or a 'connection_string'."+ )++ def __init__(self, config: dict[str, Any]):+ if config is None:+ config = {}+ super().__init__(config)++ self.container_names: list[str] = config.get("container_names") or []+ self.prefix: str = config.get("prefix", "")+ self.connection_id: str = config.get("connection_id", "default")++ self._client = None # Lazy-initialised, cached BlobServiceClient++ @property+ def bucket_names(self) -> list[str]:+ """Alias of ``container_names`` for the generic ``bucket``-kind sync path.++ The connector API (``api/connectors.py``) drives per-bucket sync and the+ file-picker bucket filter through ``connector.bucket_names`` (the term+ S3/IBM COS use). Azure's domain name for the same concept is a+ *container*, so we expose ``bucket_names`` as a read/write alias to+ satisfy that contract without renaming the Azure-accurate attribute.+ """+ return self.container_names++ @bucket_names.setter+ def bucket_names(self, value: list[str]) -> None:+ self.container_names = value or []++ def _get_client(self):+ """Return (and cache) the BlobServiceClient for this connection."""+ if self._client is None:+ self._client = create_blob_service_client(self.config)+ return self._client++ # ------------------------------------------------------------------+ # Internal sync helpers (run via asyncio.to_thread)+ # ------------------------------------------------------------------++ def _list_container_names_sync(self) -> list[str]:+ client = self._get_client()+ return [c.name for c in client.list_containers()]++ def _resolve_container_names_sync(self) -> list[str]:+ """Return configured container names, or auto-discover all accessible ones."""+ if self.container_names:+ return self.container_names+ try:+ containers = self._list_container_names_sync()+ logger.debug("Azure Blob auto-discovered %d container(s)", len(containers))+ return containers+ except Exception as exc:+ logger.warning("Azure Blob could not auto-discover containers: %s", exc)+ return []++ def _list_files_sync(self, max_files: int | None) -> list[dict[str, Any]]:+ client = self._get_client()+ files: list[dict[str, Any]] = []++ for container_name in self._resolve_container_names_sync():+ try:+ container_client = client.get_container_client(container_name)+ blob_iter = (+ container_client.list_blobs(name_starts_with=self.prefix)+ if self.prefix+ else container_client.list_blobs()+ )+ for blob in blob_iter:+ if blob.name.endswith("/"):+ continue # skip virtual directory markers+ last_modified = getattr(blob, "last_modified", None)+ files.append(+ {+ "id": _make_file_id(container_name, blob.name),+ "name": basename(blob.name) or blob.name,+ # "bucket" is the shared bucket-connector contract key+ # (matches aws_s3 / ibm_cos) that browse_connection_files+ # and the file browser read; Azure's domain term is+ # "container", preserved in the id and metadata.+ "bucket": container_name,+ "key": blob.name,+ "size": getattr(blob, "size", 0),+ "modified_time": last_modified.isoformat() if last_modified else None,+ }+ )+ if max_files and len(files) >= max_files:+ return files+ except Exception as exc:+ logger.error("Failed to list blobs in Azure container %s: %s", container_name, exc)+ continue++ return files++ def _download_blob_sync(self, container: str, blob: str) -> dict[str, Any]:+ client = self._get_client()+ blob_client = client.get_blob_client(container=container, blob=blob)+ downloader = blob_client.download_blob()+ content: bytes = downloader.readall()+ props = getattr(downloader, "properties", None)+ content_type = ""+ last_modified = None+ size = len(content)+ if props is not None:+ settings = getattr(props, "content_settings", None)+ content_type = getattr(settings, "content_type", "") or ""+ last_modified = getattr(props, "last_modified", None)+ size = getattr(props, "size", None) or len(content)+ return {+ "content": content,+ "content_type": content_type,+ "last_modified": last_modified,+ "size": size,+ }++ # ------------------------------------------------------------------+ # BaseConnector abstract method implementations+ # ------------------------------------------------------------------++ async def authenticate(self) -> bool:+ """Validate credentials by listing containers on the storage account."""+ try:+ await asyncio.to_thread(self._list_container_names_sync)+ self._authenticated = True+ logger.debug("Azure Blob authenticated for connection %s", self.connection_id)+ return True+ except Exception as exc:+ logger.warning("Azure Blob authentication failed: %s", exc)+ self._authenticated = False+ return False++ async def list_files(+ self,+ page_token: str | None = None,+ max_files: int | None = None,+ **kwargs,+ ) -> dict[str, Any]:+ """List blobs across all configured (or auto-discovered) containers.++ The Azure SDK paginates internally, so a single pass returns all blobs;+ ``next_page_token`` is always None.+ """+ files = await asyncio.to_thread(self._list_files_sync, max_files)+ return {"files": files, "next_page_token": None}++ async def get_file_content(self, file_id: str) -> ConnectorDocument:+ """Download a blob and return a ConnectorDocument.++ Args:+ file_id: Composite ID in the form "<container>::<blob>".+ """+ container_name, blob_name = _split_file_id(file_id)+ result = await asyncio.to_thread(self._download_blob_sync, container_name, blob_name)++ content: bytes = result["content"]+ last_modified: datetime = result["last_modified"] or datetime.now(UTC)+ size: int = result["size"]++ # Prefer the blob's stored content type, but fall back to an extension+ # guess when it is missing or the generic octet-stream default.+ raw_content_type = result["content_type"]+ if raw_content_type and raw_content_type != "application/octet-stream":+ mime_type: str = raw_content_type+ else:+ mime_type = mimetypes.guess_type(blob_name)[0] or "application/octet-stream"++ filename = basename(blob_name) or blob_name++ # Owner-based DLS (v1): no per-blob principals — ownership is assigned to+ # the ingesting OpenRAG user by service.process_connector_document.+ acl = DocumentACL(owner=None, allowed_users=[], allowed_groups=[], allowed_principals=[])++ return ConnectorDocument(+ id=file_id,+ filename=filename,+ mimetype=mime_type,+ content=content,+ source_url=f"azure://{container_name}/{blob_name}",+ acl=acl,+ modified_time=last_modified,+ created_time=last_modified, # Azure Blob does not expose a creation time+ metadata={+ "azure_container": container_name,+ "azure_blob": blob_name,+ "size": size,+ },+ )++ # ------------------------------------------------------------------+ # Webhook / subscription stubs (Azure Event Grid is out of scope here)+ # ------------------------------------------------------------------++ async def setup_subscription(self) -> str:+ """No-op: Azure Blob change notifications are out of scope for this connector."""+ return ""++ async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:+ """No-op: webhooks are not supported in this connector version."""+ return []++ def extract_webhook_channel_id(+ self, payload: dict[str, Any], headers: dict[str, str]+ ) -> str | None:+ return None++ async def cleanup_subscription(self, subscription_id: str) -> bool:+ """No-op: no subscription to clean up."""+ return Truediff --git a/enhancements/connectors/azure_blob/models.py b/enhancements/connectors/azure_blob/models.pynew file mode 100644index 000000000..042eb3152--- /dev/null+++ b/enhancements/connectors/azure_blob/models.py@@ -0,0 +1,17 @@+"""Pydantic request/response models for Azure Blob API endpoints."""++from pydantic import BaseModel+++class AzureBlobConfigureBody(BaseModel):+ auth_mode: str # "connection_string" or "account_key"+ # connection_string mode+ connection_string: str | None = None+ # account_key mode+ account_name: str | None = None+ account_key: str | None = None+ endpoint: str | None = None # optional custom blob endpoint (Azurite / sovereign)+ # Optional container selection+ container_names: list[str] | None = None+ # Optional: update an existing connection+ connection_id: str | None = Nonediff --git a/enhancements/connectors/azure_blob/support.py b/enhancements/connectors/azure_blob/support.pynew file mode 100644index 000000000..2c0a10dcb--- /dev/null+++ b/enhancements/connectors/azure_blob/support.py@@ -0,0 +1,66 @@+"""Support helpers for Azure Blob API endpoints.++Pure (non-async) business logic for credential resolution and config dict+construction, keeping the route handlers thin.+"""++import os++from .models import AzureBlobConfigureBody+++def build_azure_blob_config(+ body: AzureBlobConfigureBody,+ existing_config: dict,+) -> tuple[dict, str | None]:+ """Resolve Azure Blob credentials and build the connection config dict.++ Resolution order for each credential: request body → environment variable+ → existing connection config.++ Returns:+ (conn_config, None) on success+ ({}, error_message) on validation failure+ """+ conn_config: dict = {"auth_mode": body.auth_mode}++ if body.auth_mode == "connection_string":+ connection_string = (+ body.connection_string+ or os.getenv("AZURE_STORAGE_CONNECTION_STRING")+ or existing_config.get("connection_string")+ )+ if not connection_string:+ return {}, "Connection string mode requires a connection_string"+ conn_config["connection_string"] = connection_string+ elif body.auth_mode == "account_key":+ account_name = (+ body.account_name+ or os.getenv("AZURE_STORAGE_ACCOUNT_NAME")+ or existing_config.get("account_name")+ )+ account_key = (+ body.account_key+ or os.getenv("AZURE_STORAGE_ACCOUNT_KEY")+ or existing_config.get("account_key")+ )+ if not account_name or not account_key:+ return {}, "Account key mode requires account_name and account_key"+ conn_config["account_name"] = account_name+ conn_config["account_key"] = account_key+ endpoint = (+ body.endpoint+ or os.getenv("AZURE_STORAGE_ENDPOINT")+ or existing_config.get("endpoint_url")+ )+ if endpoint:+ conn_config["endpoint_url"] = endpoint+ else:+ return {}, f"Unknown auth_mode: {body.auth_mode!r}"++ if body.container_names is not None:+ conn_config["container_names"] = body.container_names+ elif existing_config.get("container_names"):+ conn_config["container_names"] = existing_config["container_names"]++ return conn_config, Nonediff --git a/enhancements/connectors/ibm_cos/connector.py b/enhancements/connectors/ibm_cos/connector.pyindex dd2d1279a..a6de8411a 100644--- a/enhancements/connectors/ibm_cos/connector.py+++ b/enhancements/connectors/ibm_cos/connector.py@@ -247,11 +247,11 @@ class IBMCOSConnector(BaseConnector):return {"files": files, "next_page_token": None}else:# client API: list_objects_v2 with manual pagination- kwargs: dict[str, Any] = {"Bucket": bucket_name}+ list_kwargs: dict[str, Any] = {"Bucket": bucket_name}if self.prefix:- kwargs["Prefix"] = self.prefix+ list_kwargs["Prefix"] = self.prefixwhile True:- resp = handle.list_objects_v2(**kwargs)+ resp = handle.list_objects_v2(**list_kwargs)for obj in resp.get("Contents", []):key = obj["Key"]if key.endswith("/"):@@ -271,7 +271,7 @@ class IBMCOSConnector(BaseConnector):if max_files and len(files) >= max_files:return {"files": files, "next_page_token": None}if resp.get("IsTruncated"):- kwargs["ContinuationToken"] = resp["NextContinuationToken"]+ list_kwargs["ContinuationToken"] = resp["NextContinuationToken"]else:breakdiff --git a/frontend/app/api/mutations/useDeleteDocument.ts b/frontend/app/api/mutations/useDeleteDocument.tsindex 8f92c579c..adea15b03 100644--- a/frontend/app/api/mutations/useDeleteDocument.ts+++ b/frontend/app/api/mutations/useDeleteDocument.ts@@ -43,6 +43,9 @@ export const useDeleteDocument = () => {setTimeout(() => {queryClient.invalidateQueries({ queryKey: ["search"] });queryClient.invalidateQueries({ queryKey: ["listFiles"] });+ // Connector "Browse Files" dialogs cache per-file ingestion state; drop+ // it so a deleted file no longer shows as "Ingested"/disabled there.+ queryClient.invalidateQueries({ queryKey: ["browseConnectionFiles"] });}, 1000);},});diff --git a/frontend/app/api/queries/useBrowseConnectionFiles.ts b/frontend/app/api/queries/useBrowseConnectionFiles.tsindex fa0c12d31..4df961cf7 100644--- a/frontend/app/api/queries/useBrowseConnectionFiles.ts+++ b/frontend/app/api/queries/useBrowseConnectionFiles.ts@@ -12,12 +12,27 @@ export interface RemoteFile {size: number;modified_time: string;is_ingested: boolean;+ /** Ingested, but the source version is newer than what we indexed. Such files+ * stay selectable (to re-ingest the newer version) while unchanged ingested+ * files are disabled. */+ is_stale: boolean;}export interface BrowseConnectionFilesParams {connectorType: string;connectionId: string;bucket?: string;+ /**+ * Currently unused. The FileBrowserDialog filters the fetched list in-memory+ * (client-side) instead of sending this to the backend, because the existing+ * server-side `search` is deficient: it doesn't filter directly at the+ * connector level — it fetches the capped blob list (maxFiles) and then+ * post-filters it in memory, so it can't match files beyond the cap and+ * re-fetches the whole list per query. Passing it would also remount the+ * query's loading state on every keystroke.+ * TODO: implement true server-side filtering at the connector level (push the+ * search term into the connector's list call), then reinstate this param.+ */search?: string;pageToken?: string;maxFiles?: number;@@ -68,6 +83,13 @@ export const useBrowseConnectionFiles = (queryFn: fetchFiles,retry: false,enabled: Boolean(params.connectorType && params.connectionId),+ // Always reflect live ingestion state when the dialog opens. The global+ // default staleTime (60s) would otherwise serve a cached `is_ingested`+ // snapshot — e.g. a file deleted on the Knowledge page would still render+ // as "Ingested"/disabled here until the cache went stale. Matches the+ // bucket-status queries (useS3BucketStatusQuery / azure container-status).+ staleTime: 0,+ refetchOnMount: "always",...options,},queryClient,diff --git a/frontend/app/knowledge/chunks/page.tsx b/frontend/app/knowledge/chunks/page.tsxindex 80ef7fca2..2b747db44 100644--- a/frontend/app/knowledge/chunks/page.tsx+++ b/frontend/app/knowledge/chunks/page.tsx@@ -18,6 +18,7 @@ import {} from "@/components/ui/tooltip";import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";import { trackButton } from "@/lib/analytics";+import { formatFileSize, getFileTypeLabel } from "@/lib/file-format";import {type ChunkResult,EMPTY_SEARCH_RESULT,@@ -26,13 +27,6 @@ import {useGetSearchQuery,} from "../../api/queries/useGetSearchQuery";-const getFileTypeLabel = (mimetype: string) => {- if (mimetype === "application/pdf") return "PDF";- if (mimetype === "text/plain") return "Text";- if (mimetype === "application/msword") return "Word Document";- return "Unknown";-};-function ChunksPageContent() {const router = useRouter();const searchParams = useSearchParams();@@ -304,9 +298,7 @@ function ChunksPageContent() {<div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5"><dt className="text-sm/6 text-muted-foreground">Size</dt><dd className="mt-1 text-sm/6 text-gray-800 dark:text-gray-100 sm:col-span-2 sm:mt-0">- {fileData?.size- ? `${Math.round(fileData.size / 1024)} KB`- : "Unknown"}+ {fileData?.size ? formatFileSize(fileData.size) : "Unknown"}</dd></div>{/* <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsxindex f572bee4f..2f5fcf02b 100644--- a/frontend/app/knowledge/page.tsx+++ b/frontend/app/knowledge/page.tsx@@ -42,6 +42,7 @@ import {} from "@/components/ui/tooltip";import { useIsCloudBrand } from "@/contexts/brand-context";import { getConnectorDescriptor } from "@/lib/connectors/registry";+import { formatFileSize } from "@/lib/file-format";import {buildKnowledgeTableRows,getKnowledgeFileIdentity,@@ -582,7 +583,7 @@ function SearchPage() {comparator: (valueA?: number, valueB?: number) =>(valueA || 0) - (valueB || 0),valueFormatter: (params: ValueFormatterParams<File>) =>- params.value ? `${Math.round(params.value / 1024)} KB` : "-",+ params.value ? formatFileSize(params.value) : "-",cellClass: isCloudBrand ? "text-muted-foreground" : undefined,},{diff --git a/frontend/app/settings/_components/s3-settings-dialog.tsx b/frontend/app/settings/_components/s3-settings-dialog.tsxindex 3c6a45c91..36579a307 100644--- a/frontend/app/settings/_components/s3-settings-dialog.tsx+++ b/frontend/app/settings/_components/s3-settings-dialog.tsx@@ -134,8 +134,8 @@ export default function S3SettingsDialog({toast.success("Amazon S3 configured", {description:selectedBuckets.length > 0- ? `Will ingest from: ${selectedBuckets.join(", ")}`- : "Will auto-discover and ingest all accessible buckets.",+ ? `Ingestion restricted to the following buckets: ${selectedBuckets.join(", ")}.`+ : "All accessible buckets available for ingestion.",icon: <AwsLogo className="w-4 h-4" />,});diff --git a/frontend/app/settings/_components/s3-settings-form.tsx b/frontend/app/settings/_components/s3-settings-form.tsxindex a9fe98ec6..c16bcd1e3 100644--- a/frontend/app/settings/_components/s3-settings-form.tsx+++ b/frontend/app/settings/_components/s3-settings-form.tsx@@ -168,9 +168,9 @@ export function S3SettingsForm({<div className="space-y-2"><div className="flex items-center justify-between"><Label className="text-sm font-medium">- Select Buckets to Ingest+ Restrict Ingestion to Buckets<span className="ml-1 text-muted-foreground font-normal">- (leave all unchecked to ingest everything)+ (optional)</span></Label>{buckets.length > 1 && (diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsxindex ab67d7f26..54c59c71e 100644--- a/frontend/components/connectors/shared-bucket-view.tsx+++ b/frontend/components/connectors/shared-bucket-view.tsx@@ -107,13 +107,19 @@ export function SharedBucketView({onSuccess: (result) => {invalidate();if (result.task_ids?.length) {- addTask(result.task_ids[0], {- connectorType: connector.type,- source: "connector",- });+ // The container path may return two tasks (new files + changed files);+ // track them all.+ for (const id of result.task_ids) {+ addTask(id, {+ connectorType: connector.type,+ source: "connector",+ });+ }onDone();} else {- toast.info("No files found in the selected buckets.");+ toast.info(+ result.message ?? "No files found in the selected buckets.",+ );}},onError: (err) => {diff --git a/frontend/components/file-browser-dialog.tsx b/frontend/components/file-browser-dialog.tsxindex 972714f7e..f57c15c16 100644--- a/frontend/components/file-browser-dialog.tsx+++ b/frontend/components/file-browser-dialog.tsx@@ -1,6 +1,6 @@"use client";-import { Check, Download, Loader2, Search } from "lucide-react";+import { Check, Download, Loader2, RefreshCw, Search } from "lucide-react";import { useCallback, useMemo, useState } from "react";import { toast } from "sonner";import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";@@ -8,6 +8,7 @@ import {type RemoteFile,useBrowseConnectionFiles,} from "@/app/api/queries/useBrowseConnectionFiles";+import { formatFileSize } from "@/lib/file-format";import { Badge } from "./ui/badge";import { Button } from "./ui/button";import {@@ -29,13 +30,6 @@ interface FileBrowserDialogProps {buckets?: string[];}-function formatFileSize(bytes: number): string {- if (bytes === 0) return "0 B";- const units = ["B", "KB", "MB", "GB"];- const i = Math.floor(Math.log(bytes) / Math.log(1024));- return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;-}-export function FileBrowserDialog({open,onOpenChange,@@ -53,18 +47,40 @@ export function FileBrowserDialog({const syncMutation = useSyncConnector();+ // The bucket's file set is fetched once per open (search is NOT sent to the+ // backend — its server-side filter just post-filters this same capped list,+ // so re-fetching per keystroke only adds latency and remounts the loading+ // state, making the table jump). We filter the cached list in-memory instead.const { data, isLoading, error } = useBrowseConnectionFiles({connectorType,connectionId,bucket: selectedBucket,- search: search || undefined,maxFiles: 500,},{ enabled: open },);- const files = data?.files ?? [];+ const allFiles = useMemo(() => data?.files ?? [], [data]);++ const files = useMemo(() => {+ const q = search.trim().toLowerCase();+ return q+ ? allFiles.filter((f) => f.name.toLowerCase().includes(q))+ : allFiles;+ }, [allFiles, search]);++ // Selectable files within the current (filtered) view — drives "Select all".+ // A file is selectable when it's not yet ingested, or it's ingested but stale+ // (a newer version exists at the source, so re-ingest is allowed).+ const visibleSelectable = useMemo(+ () => files.filter((f) => !f.is_ingested || f.is_stale),+ [files],+ );++ const allVisibleSelected =+ visibleSelectable.length > 0 &&+ visibleSelectable.every((f) => selectedFileIds.has(f.id));const toggleFile = useCallback((fileId: string) => {setSelectedFileIds((prev) => {@@ -79,22 +95,37 @@ export function FileBrowserDialog({}, []);const toggleAll = useCallback(() => {- const unIngestedFiles = files.filter((f) => !f.is_ingested);- if (selectedFileIds.size === unIngestedFiles.length) {- setSelectedFileIds(new Set());- } else {- setSelectedFileIds(new Set(unIngestedFiles.map((f) => f.id)));- }- }, [files, selectedFileIds.size]);+ setSelectedFileIds((prev) => {+ const next = new Set(prev);+ const allSelected =+ visibleSelectable.length > 0 &&+ visibleSelectable.every((f) => next.has(f.id));+ for (const f of visibleSelectable) {+ if (allSelected) {+ next.delete(f.id);+ } else {+ next.add(f.id);+ }+ }+ return next;+ });+ }, [visibleSelectable]);+ // Resolve selections against the full fetched set so a selection survives+ // filtering (a selected file hidden by the search box is still ingested).const selectedFiles = useMemo(- () => files.filter((f) => selectedFileIds.has(f.id)),- [files, selectedFileIds],+ () => allFiles.filter((f) => selectedFileIds.has(f.id)),+ [allFiles, selectedFileIds],);const handleIngest = useCallback(async () => {if (selectedFiles.length === 0) return;+ // If any selected file is a stale re-ingest, replace the indexed copy so the+ // newer version overwrites the old chunks (rather than being skipped as a+ // duplicate filename). New files are unaffected by this flag.+ const hasStale = selectedFiles.some((f) => f.is_stale);+try {await syncMutation.mutateAsync({connectorType,@@ -105,6 +136,7 @@ export function FileBrowserDialog({mimeType: "",size: f.size,})),+ ...(hasStale ? { replace_duplicates: true } : {}),},});@@ -121,8 +153,6 @@ export function FileBrowserDialog({}}, [selectedFiles, connectorType, syncMutation, onOpenChange]);- const unIngestedCount = files.filter((f) => !f.is_ingested).length;-return (<Dialog open={open} onOpenChange={onOpenChange}><DialogContent className="max-w-2xl max-h-[80vh] flex flex-col">@@ -183,25 +213,24 @@ export function FileBrowserDialog({</div>) : files.length === 0 ? (<div className="p-8 text-center text-muted-foreground text-sm">- No files found.+ {search.trim() && allFiles.length > 0+ ? "No files match your search."+ : "No files found."}</div>) : (<div className="divide-y">- {unIngestedCount > 0 && (+ {visibleSelectable.length > 0 && (<div className="px-3 py-2 bg-muted/50 flex items-center gap-2 sticky top-0"><inputtype="checkbox"- checked={- selectedFileIds.size === unIngestedCount &&- unIngestedCount > 0- }+ checked={allVisibleSelected}onChange={toggleAll}className="h-4 w-4 rounded border border-input"/><span className="text-xs text-muted-foreground">{selectedFileIds.size > 0? `${selectedFileIds.size} selected`- : `Select all (${unIngestedCount})`}+ : `Select all (${visibleSelectable.length})`}</span></div>)}@@ -255,21 +284,44 @@ function FileRow({selected: boolean;onToggle: () => void;}) {+ // The blob/object key is the full path within the bucket/container (e.g.+ // "invoices/2024/report.pdf"); file.name is only the basename. Surfacing the+ // directory portion disambiguates same-named files living under different+ // prefixes (e.g. 2024/report.pdf vs 2025/report.pdf), which otherwise render+ // identically. Empty for top-level/flat blobs, so flat listings are unchanged.+ const dir = useMemo(() => {+ const key = file.key ?? "";+ const idx = key.lastIndexOf("/");+ return idx >= 0 ? key.slice(0, idx + 1) : "";+ }, [file.key]);++ // Unchanged ingested files are locked; stale ones (newer version at source) stay+ // selectable so the user can re-ingest the update.+ const locked = file.is_ingested && !file.is_stale;+return (<labelclassName={`flex items-center gap-3 px-3 py-2.5 cursor-pointer hover:bg-muted/30 transition-colors ${- file.is_ingested ? "opacity-60" : ""+ locked ? "opacity-60" : ""}`}><inputtype="checkbox"- checked={selected || file.is_ingested}- disabled={file.is_ingested}+ checked={selected || locked}+ disabled={locked}onChange={onToggle}className="h-4 w-4 rounded border border-input"/><div className="flex-1 min-w-0"><div className="text-sm truncate font-medium">{file.name}</div>+ {dir && (+ <div+ className="text-xs text-muted-foreground truncate"+ title={file.key}+ >+ {dir}+ </div>+ )}<div className="text-xs text-muted-foreground flex gap-2">{file.bucket && <span>{file.bucket}</span>}<span>{formatFileSize(file.size)}</span>@@ -278,11 +330,18 @@ function FileRow({)}</div></div>- {file.is_ingested && (- <Badge variant="secondary" className="text-xs flex-shrink-0">- <Check className="h-3 w-3 mr-1" />- Ingested+ {file.is_stale ? (+ <Badge variant="outline" className="text-xs flex-shrink-0">+ <RefreshCw className="h-3 w-3 mr-1" />+ Update available</Badge>+ ) : (+ file.is_ingested && (+ <Badge variant="secondary" className="text-xs flex-shrink-0">+ <Check className="h-3 w-3 mr-1" />+ Ingested+ </Badge>+ ))}</label>);diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsxindex f6d1c9a9b..7e431554b 100644--- a/frontend/components/knowledge-dropdown.tsx+++ b/frontend/components/knowledge-dropdown.tsx@@ -31,7 +31,6 @@ import {} from "@/components/ui/dropdown-menu";import { Input } from "@/components/ui/input";import { Label } from "@/components/ui/label";-import { useAuth } from "@/contexts/auth-context";import { useIsCloudBrand } from "@/contexts/brand-context";import { useTask } from "@/contexts/task-context";import { usePermissions } from "@/hooks/use-permissions";@@ -122,7 +121,6 @@ const FolderIconWithColor = ({ className }: { className?: string }) => ();export function KnowledgeDropdown() {- const { isIbmAuthMode } = useAuth();const { can } = usePermissions();const canUpload = can("knowledge:upload");const isCloudBrand = useIsCloudBrand();@@ -142,6 +140,9 @@ export function KnowledgeDropdown() {const [bucketConnectorConfigured, setBucketConnectorConfigured] = useState<Record<string, boolean>>({});+ const [bucketConnectorAvailable, setBucketConnectorAvailable] = useState<+ Record<string, boolean>+ >({});const [pendingFile, setPendingFile] = useState<File | null>(null);const [duplicateFilename, setDuplicateFilename] = useState<string>("");const [pendingFolderUpload, setPendingFolderUpload] = useState<{@@ -215,6 +216,19 @@ export function KnowledgeDropdown() {const connectorsRes = await fetch("/api/connectors");if (connectorsRes.ok) {const connectorsResult = await connectorsRes.json();++ // Bucket connector availability mirrors the backend `is_available()`+ // gate (IBM auth, or the dev flag for Azure Blob), so the dropdown+ // surfaces a bucket entry whenever the connector is actually usable —+ // not only in IBM auth mode.+ const bucketAvailable: Record<string, boolean> = {};+ for (const d of bucketDescriptors) {+ bucketAvailable[d.connectorType] = Boolean(+ connectorsResult.connectors?.[d.connectorType]?.available,+ );+ }+ setBucketConnectorAvailable(bucketAvailable);+const cloudConnectorTypes = ["google_drive","onedrive",@@ -700,20 +714,23 @@ export function KnowledgeDropdown() {};});- const bucketConnectorItems = isIbmAuthMode- ? getConnectorDescriptors()- .filter(- (d) =>- d.kind === "bucket" &&- d.menuItem &&- bucketConnectorConfigured[d.connectorType],- )- .map((d) => ({- label: d.menuItem!.label,- icon: d.Icon,- onClick: () => router.push(d.menuItem!.route),- }))- : [];+ // Gate each bucket connector on its backend availability (IBM auth, or the+ // OPENRAG_DEV_AZURE_BLOB dev flag for Azure Blob) AND a saved connection,+ // rather than the global IBM-auth flag — this keeps S3/IBM COS hidden outside+ // IBM auth while letting Azure Blob appear in local dev once configured.+ const bucketConnectorItems = getConnectorDescriptors()+ .filter(+ (d) =>+ d.kind === "bucket" &&+ d.menuItem &&+ bucketConnectorAvailable[d.connectorType] &&+ bucketConnectorConfigured[d.connectorType],+ )+ .map((d) => ({+ label: d.menuItem!.label,+ icon: d.Icon,+ onClick: () => router.push(d.menuItem!.route),+ }));const menuItems = [{diff --git a/frontend/enhancements/connectors/azure-blob/components/bucket-view.tsx b/frontend/enhancements/connectors/azure-blob/components/bucket-view.tsxnew file mode 100644index 000000000..adfb31854--- /dev/null+++ b/frontend/enhancements/connectors/azure-blob/components/bucket-view.tsx@@ -0,0 +1,48 @@+"use client";++import type { useSyncConnector } from "@/app/api/mutations/useSyncConnector";+import type { Connector } from "@/app/api/queries/useGetConnectorsQuery";+import { SharedBucketView } from "@/components/connectors/shared-bucket-view";+import { useAzureBlobContainerStatusQuery } from "../useAzureBlobContainerStatusQuery";++export interface AzureBlobBucketViewProps {+ connector: Connector;+ syncMutation: ReturnType<typeof useSyncConnector>;+ addTask: (id: string, options?: { connectorType?: string }) => void;+ onBack: () => void;+ onDone: () => void;+}++export function AzureBlobBucketView({+ connector,+ syncMutation,+ addTask,+ onBack,+ onDone,+}: AzureBlobBucketViewProps) {+ const {+ data: containers,+ isLoading,+ error: containersError,+ refetch,+ } = useAzureBlobContainerStatusQuery(connector.connectionId, {+ enabled: true,+ });+ return (+ <SharedBucketView+ connector={connector}+ buckets={containers}+ isLoading={isLoading}+ bucketsError={containersError as Error | null}+ onRefetch={refetch}+ invalidateQueryKey={[+ "azure-blob-container-status",+ connector.connectionId,+ ]}+ syncMutation={syncMutation}+ addTask={addTask}+ onBack={onBack}+ onDone={onDone}+ />+ );+}diff --git a/frontend/enhancements/connectors/azure-blob/icon.tsx b/frontend/enhancements/connectors/azure-blob/icon.tsxnew file mode 100644index 000000000..0abc55ef7--- /dev/null+++ b/frontend/enhancements/connectors/azure-blob/icon.tsx@@ -0,0 +1,50 @@+export default function AzureBlobIcon(props: React.SVGProps<SVGSVGElement>) {+ return (+ <svg+ width="16"+ height="16"+ viewBox="0 0 32 32"+ fill="none"+ xmlns="http://www.w3.org/2000/svg"+ {...props}+ >+ <rect width="32" height="32" fill="white" fillOpacity="0.01" />+ <path+ d="M13.6 5.5h6.05l-6.28 18.6a1 1 0 0 1-.95.68H7.2a1 1 0 0 1-.95-1.32L12.65 6.18a1 1 0 0 1 .95-.68Z"+ fill="url(#azblob_a)"+ />+ <path+ d="M22.4 20.1H12.7a.46.46 0 0 0-.31.8l6.23 5.82a1 1 0 0 0 .68.27h5.58l-2.46-6.89Z"+ fill="#0078D4"+ />+ <path+ d="M13.6 5.5a1 1 0 0 0-.95.69L6.26 23.45a1 1 0 0 0 .94 1.33h5.3a1.07 1.07 0 0 0 .82-.7l1.28-3.77 4.57 4.26a1 1 0 0 0 .63.21h5.55l-2.43-6.89-7.09.02 4.34-12.62H13.6Z"+ fill="url(#azblob_b)"+ />+ <defs>+ <linearGradient+ id="azblob_a"+ x1="15.5"+ y1="6.9"+ x2="9.1"+ y2="25.8"+ gradientUnits="userSpaceOnUse"+ >+ <stop stopColor="#114A8B" />+ <stop offset="1" stopColor="#0669BC" />+ </linearGradient>+ <linearGradient+ id="azblob_b"+ x1="18.5"+ y1="6"+ x2="22"+ y2="25"+ gradientUnits="userSpaceOnUse"+ >+ <stop stopColor="#3CCBF4" />+ <stop offset="1" stopColor="#2892DF" />+ </linearGradient>+ </defs>+ </svg>+ );+}diff --git a/frontend/enhancements/connectors/azure-blob/settings-dialog.tsx b/frontend/enhancements/connectors/azure-blob/settings-dialog.tsxnew file mode 100644index 000000000..c9e6ef5e5--- /dev/null+++ b/frontend/enhancements/connectors/azure-blob/settings-dialog.tsx@@ -0,0 +1,195 @@+"use client";++import { useQueryClient } from "@tanstack/react-query";+import { useEffect, useState } from "react";+import { FormProvider, useForm } from "react-hook-form";+import { toast } from "sonner";+import { Button } from "@/components/ui/button";+import {+ Dialog,+ DialogContent,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "@/components/ui/dialog";+import AzureBlobIcon from "./icon";+import { type AzureBlobFormData, AzureBlobSettingsForm } from "./settings-form";+import { useAzureBlobConfigureMutation } from "./useAzureBlobConfigureMutation";+import { useAzureBlobDefaultsQuery } from "./useAzureBlobDefaultsQuery";++interface AzureBlobSettingsDialogProps {+ open: boolean;+ setOpen: (open: boolean) => void;+}++export default function AzureBlobSettingsDialog({+ open,+ setOpen,+}: AzureBlobSettingsDialogProps) {+ const queryClient = useQueryClient();++ const { data: defaults } = useAzureBlobDefaultsQuery({ enabled: open });++ const methods = useForm<AzureBlobFormData>({+ mode: "onSubmit",+ values: {+ auth_mode: defaults?.auth_mode ?? "connection_string",+ connection_string: "",+ account_name: defaults?.account_name ?? "",+ account_key: "",+ endpoint: defaults?.endpoint ?? "",+ },+ });++ const { handleSubmit } = methods;++ const [containers, setContainers] = useState<string[] | null>(+ defaults?.container_names?.length ? defaults.container_names : null,+ );+ const [selectedContainers, setSelectedContainers] = useState<string[]>(+ defaults?.container_names ?? [],+ );+ useEffect(() => {+ if (defaults?.container_names?.length) {+ setContainers(defaults.container_names);+ setSelectedContainers(defaults.container_names);+ }+ }, [defaults]);++ const [isFetchingContainers, setIsFetchingContainers] = useState(false);+ const [containersError, setContainersError] = useState<string | null>(null);+ const [formError, setFormError] = useState<string | null>(null);++ const configureMutation = useAzureBlobConfigureMutation();++ const handleTestConnection = handleSubmit(async (data) => {+ setIsFetchingContainers(true);+ setContainersError(null);+ setFormError(null);++ try {+ // Validate credentials + list containers WITHOUT persisting. Saving is+ // reserved for the Save button (onSubmit), so testing never creates or+ // mutates the connection or clobbers the stored container selection.+ const res = await fetch("/api/connectors/azure_blob/test", {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({+ auth_mode: data.auth_mode,+ connection_string: data.connection_string || undefined,+ account_name: data.account_name || undefined,+ account_key: data.account_key || undefined,+ endpoint: data.endpoint || undefined,+ connection_id: defaults?.connection_id ?? undefined,+ }),+ });+ const json = await res.json();+ if (!res.ok) throw new Error(json.error || "Failed to list containers");++ const fetched: string[] = json.containers;+ setContainers(fetched);+ setSelectedContainers((prev) => prev.filter((c) => fetched.includes(c)));+ } catch (err: unknown) {+ setContainersError(+ err instanceof Error ? err.message : "Connection failed",+ );+ } finally {+ setIsFetchingContainers(false);+ }+ });++ const onSubmit = handleSubmit(async (data) => {+ setFormError(null);+ if (containers === null) {+ setFormError("Test the connection first to validate credentials.");+ return;+ }++ try {+ const latestDefaults = await queryClient.fetchQuery({+ queryKey: ["azure-blob-defaults"],+ queryFn: async () => {+ const res = await fetch("/api/connectors/azure_blob/defaults");+ return res.json();+ },+ staleTime: 0,+ });++ await configureMutation.mutateAsync({+ auth_mode: data.auth_mode,+ connection_string: data.connection_string || undefined,+ account_name: data.account_name || undefined,+ account_key: data.account_key || undefined,+ endpoint: data.endpoint || undefined,+ container_names: selectedContainers,+ connection_id:+ latestDefaults?.connection_id ?? defaults?.connection_id ?? undefined,+ });++ toast.success("Azure Blob Storage configured", {+ description:+ selectedContainers.length > 0+ ? `Ingestion restricted to the following containers: ${selectedContainers.join(", ")}.`+ : "All accessible containers available for ingestion.",+ icon: <AzureBlobIcon className="w-4 h-4" />,+ });++ queryClient.invalidateQueries({ queryKey: ["connectors"] });+ setOpen(false);+ } catch (err: unknown) {+ setFormError(+ err instanceof Error ? err.message : "Failed to save configuration",+ );+ }+ });++ return (+ <Dialog open={open} onOpenChange={setOpen}>+ <DialogContent+ autoFocus={false}+ className="max-w-2xl max-h-[90vh] overflow-y-auto"+ >+ <FormProvider {...methods}>+ <form onSubmit={onSubmit} className="grid gap-4">+ <DialogHeader className="mb-2">+ <DialogTitle className="flex items-center gap-3">+ <div className="w-8 h-8 rounded flex items-center justify-center bg-white border">+ <AzureBlobIcon />+ </div>+ Azure Blob Storage Setup+ </DialogTitle>+ </DialogHeader>++ <AzureBlobSettingsForm+ containers={containers}+ selectedContainers={selectedContainers}+ onSelectedContainersChange={setSelectedContainers}+ isFetchingContainers={isFetchingContainers}+ containersError={containersError}+ onTestConnection={handleTestConnection as () => void}+ connectionStringSet={defaults?.connection_string_set}+ accountKeySet={defaults?.account_key_set}+ formError={formError}+ />++ <DialogFooter className="mt-4">+ <Button+ variant="outline"+ type="button"+ onClick={() => setOpen(false)}+ >+ Cancel+ </Button>+ <Button+ type="submit"+ disabled={configureMutation.isPending || isFetchingContainers}+ >+ {configureMutation.isPending ? "Saving…" : "Save"}+ </Button>+ </DialogFooter>+ </form>+ </FormProvider>+ </DialogContent>+ </Dialog>+ );+}diff --git a/frontend/enhancements/connectors/azure-blob/settings-form.tsx b/frontend/enhancements/connectors/azure-blob/settings-form.tsxnew file mode 100644index 000000000..872d74c65--- /dev/null+++ b/frontend/enhancements/connectors/azure-blob/settings-form.tsx@@ -0,0 +1,277 @@+"use client";++import { Loader2, RefreshCcw } from "lucide-react";+import { Controller, useFormContext } from "react-hook-form";+import { LabelWrapper } from "@/components/label-wrapper";+import { Button } from "@/components/ui/button";+import { Input } from "@/components/ui/input";+import { Label } from "@/components/ui/label";+import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";++export interface AzureBlobFormData {+ auth_mode: "connection_string" | "account_key";+ // connection_string mode+ connection_string: string;+ // account_key mode+ account_name: string;+ account_key: string;+ endpoint: string;+}++interface AzureBlobSettingsFormProps {+ /** Available containers after a successful test — null means not yet tested */+ containers: string[] | null;+ selectedContainers: string[];+ onSelectedContainersChange: (containers: string[]) => void;+ isFetchingContainers: boolean;+ containersError: string | null;+ onTestConnection: () => void;+ connectionStringSet?: boolean;+ accountKeySet?: boolean;+ formError?: string | null;+}++export function AzureBlobSettingsForm({+ containers,+ selectedContainers,+ onSelectedContainersChange,+ isFetchingContainers,+ containersError,+ onTestConnection,+ connectionStringSet,+ accountKeySet,+ formError,+}: AzureBlobSettingsFormProps) {+ const {+ register,+ control,+ formState: { errors },+ } = useFormContext<AzureBlobFormData>();++ const toggleContainer = (name: string, checked: boolean) => {+ if (checked) {+ onSelectedContainersChange([...selectedContainers, name]);+ } else {+ onSelectedContainersChange(selectedContainers.filter((c) => c !== name));+ }+ };++ const toggleAll = (checked: boolean) => {+ onSelectedContainersChange(checked ? (containers ?? []) : []);+ };++ return (+ <div className="space-y-5">+ {/* Auth mode selector using Tabs */}+ <div className="space-y-2">+ <Label className="text-sm font-medium">Authentication Method</Label>+ <Controller+ control={control}+ name="auth_mode"+ render={({ field }) => (+ <Tabs value={field.value} onValueChange={(v) => field.onChange(v)}>+ <TabsList className="w-full">+ <TabsTrigger value="connection_string">+ <span className="font-semibold text-sm">+ Connection String+ </span>+ <span className="text-xs text-muted-foreground font-normal">+ Single paste-in string+ </span>+ </TabsTrigger>+ <TabsTrigger value="account_key">+ <span className="font-semibold text-sm">Account Key</span>+ <span className="text-xs text-muted-foreground font-normal">+ Account name + key+ </span>+ </TabsTrigger>+ </TabsList>++ {/* Connection string fields — first tab */}+ <TabsContent value="connection_string">+ <div className="space-y-1">+ <LabelWrapper+ label="Connection String"+ helperText="Azure Portal → Storage account → Access keys → Connection string. For the local Azurite emulator use: UseDevelopmentStorage=true"+ id="azure-conn-str"+ required+ >+ <Input+ {...register("connection_string", {+ setValueAs: (v) => v?.trim(),+ })}+ id="azure-conn-str"+ type="password"+ placeholder={+ connectionStringSet+ ? "•••••••• (loaded from env)"+ : "DefaultEndpointsProtocol=https;AccountName=…"+ }+ autoComplete="off"+ />+ </LabelWrapper>+ </div>+ </TabsContent>++ {/* Account key fields — second tab */}+ <TabsContent value="account_key">+ <div className="space-y-4">+ <div className="space-y-1">+ <LabelWrapper+ label="Account Name"+ helperText="Your Azure Storage account name (e.g. mystorageacct). For Azurite use devstoreaccount1."+ id="azure-account-name"+ required+ >+ <Input+ {...register("account_name", {+ setValueAs: (v) => v?.trim(),+ })}+ id="azure-account-name"+ placeholder="mystorageacct"+ autoComplete="off"+ />+ </LabelWrapper>+ </div>+ <div className="space-y-1">+ <LabelWrapper+ label="Account Key"+ helperText="Azure Portal → Storage account → Access keys → key1/key2"+ id="azure-account-key"+ required+ >+ <Input+ {...register("account_key", {+ setValueAs: (v) => v?.trim(),+ })}+ id="azure-account-key"+ type="password"+ placeholder={+ accountKeySet+ ? "•••••••• (loaded from env)"+ : "base64 account key"+ }+ autoComplete="off"+ />+ </LabelWrapper>+ </div>+ <div className="space-y-1">+ <LabelWrapper+ label="Blob Endpoint (optional)"+ helperText="Leave blank for public Azure. Set for Azurite (http://127.0.0.1:10000/devstoreaccount1) or sovereign clouds."+ id="azure-endpoint"+ >+ <Input+ {...register("endpoint", {+ setValueAs: (v) => v?.trim(),+ })}+ id="azure-endpoint"+ placeholder="https://mystorageacct.blob.core.windows.net"+ />+ </LabelWrapper>+ </div>+ </div>+ </TabsContent>+ </Tabs>+ )}+ />+ </div>++ {/* Test connection */}+ <Button+ type="button"+ variant="outline"+ onClick={onTestConnection}+ disabled={isFetchingContainers}+ className="w-full"+ >+ {isFetchingContainers ? (+ <>+ <Loader2 className="h-4 w-4 mr-2 animate-spin" />+ Connecting…+ </>+ ) : containers !== null ? (+ <>+ <RefreshCcw className="h-4 w-4 mr-2" />+ Refresh Containers+ </>+ ) : (+ "Test Connection & List Containers"+ )}+ </Button>++ {containersError && (+ <p className="text-sm text-destructive rounded-lg border border-destructive/50 p-3">+ {containersError}+ </p>+ )}++ {formError && (+ <p className="text-sm text-destructive rounded-lg border border-destructive/50 p-3">+ {formError}+ </p>+ )}++ {/* Container selector */}+ {containers !== null && (+ <div className="space-y-2">+ <div className="flex items-center justify-between">+ <Label className="text-sm font-medium">+ Restrict Ingestion to Containers+ <span className="ml-1 text-muted-foreground font-normal">+ (optional)+ </span>+ </Label>+ {containers.length > 1 && (+ <button+ type="button"+ className="text-xs text-primary underline-offset-2 hover:underline"+ onClick={() =>+ toggleAll(selectedContainers.length !== containers.length)+ }+ >+ {selectedContainers.length === containers.length+ ? "Deselect all"+ : "Select all"}+ </button>+ )}+ </div>++ {containers.length === 0 ? (+ <p className="text-sm text-muted-foreground">+ No containers found for this account.+ </p>+ ) : (+ <div className="max-h-48 overflow-y-auto rounded-lg border p-3 space-y-2">+ {containers.map((container) => (+ <label+ key={container}+ htmlFor={`container-${container}`}+ className="flex items-center gap-2.5 cursor-pointer select-none"+ >+ <input+ id={`container-${container}`}+ type="checkbox"+ className="h-4 w-4 rounded border-border accent-primary"+ checked={selectedContainers.includes(container)}+ onChange={(e) =>+ toggleContainer(container, e.target.checked)+ }+ />+ <span className="text-sm">{container}</span>+ </label>+ ))}+ </div>+ )}+ </div>+ )}++ {/* Keep errors referenced so unused-var lint stays happy when fields hidden */}+ {errors.connection_string && (+ <p className="text-sm text-destructive">+ {errors.connection_string.message}+ </p>+ )}+ </div>+ );+}diff --git a/frontend/enhancements/connectors/azure-blob/useAzureBlobConfigureMutation.ts b/frontend/enhancements/connectors/azure-blob/useAzureBlobConfigureMutation.tsnew file mode 100644index 000000000..4120ded90--- /dev/null+++ b/frontend/enhancements/connectors/azure-blob/useAzureBlobConfigureMutation.ts@@ -0,0 +1,38 @@+import { useMutation, useQueryClient } from "@tanstack/react-query";++export interface AzureBlobConfigurePayload {+ auth_mode: "connection_string" | "account_key";+ // connection_string mode+ connection_string?: string;+ // account_key mode+ account_name?: string;+ account_key?: string;+ endpoint?: string;+ // Container selection+ container_names?: string[];+ // Updating an existing connection+ connection_id?: string;+}++async function configureAzureBlob(payload: AzureBlobConfigurePayload) {+ const res = await fetch("/api/connectors/azure_blob/configure", {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(payload),+ });+ const data = await res.json();+ if (!res.ok) throw new Error(data.error || "Failed to configure Azure Blob");+ return data as { connection_id: string; status: string };+}++export function useAzureBlobConfigureMutation() {+ const queryClient = useQueryClient();++ return useMutation({+ mutationFn: configureAzureBlob,+ onSuccess: () => {+ queryClient.invalidateQueries({ queryKey: ["connectors"] });+ queryClient.invalidateQueries({ queryKey: ["azure-blob-defaults"] });+ },+ });+}diff --git a/frontend/enhancements/connectors/azure-blob/useAzureBlobContainerStatusQuery.ts b/frontend/enhancements/connectors/azure-blob/useAzureBlobContainerStatusQuery.tsnew file mode 100644index 000000000..82e36b027--- /dev/null+++ b/frontend/enhancements/connectors/azure-blob/useAzureBlobContainerStatusQuery.ts@@ -0,0 +1,34 @@+import { useQuery } from "@tanstack/react-query";++export interface AzureBlobContainerStatus {+ name: string;+ ingested_count: number;+ is_synced: boolean;+}++async function fetchAzureBlobContainerStatus(+ connectionId: string,+): Promise<AzureBlobContainerStatus[]> {+ const res = await fetch(+ `/api/connectors/azure_blob/${connectionId}/container-status`,+ );+ if (!res.ok) {+ const err = await res.json().catch(() => ({}));+ throw new Error(err.error || "Failed to fetch container status");+ }+ const data = await res.json();+ return data.containers as AzureBlobContainerStatus[];+}++export function useAzureBlobContainerStatusQuery(+ connectionId: string | null | undefined,+ options?: { enabled?: boolean },+) {+ return useQuery<AzureBlobContainerStatus[]>({+ queryKey: ["azure-blob-container-status", connectionId],+ queryFn: () => fetchAzureBlobContainerStatus(connectionId!),+ enabled: (options?.enabled ?? true) && !!connectionId,+ staleTime: 0,+ refetchOnMount: "always",+ });+}diff --git a/frontend/enhancements/connectors/azure-blob/useAzureBlobDefaultsQuery.ts b/frontend/enhancements/connectors/azure-blob/useAzureBlobDefaultsQuery.tsnew file mode 100644index 000000000..eb9387f6c--- /dev/null+++ b/frontend/enhancements/connectors/azure-blob/useAzureBlobDefaultsQuery.ts@@ -0,0 +1,26 @@+import { useQuery } from "@tanstack/react-query";++export interface AzureBlobDefaults {+ connection_string_set: boolean;+ account_name: string;+ account_key_set: boolean;+ endpoint: string;+ auth_mode: "connection_string" | "account_key";+ container_names: string[];+ connection_id: string | null;+}++async function fetchAzureBlobDefaults(): Promise<AzureBlobDefaults> {+ const res = await fetch("/api/connectors/azure_blob/defaults");+ if (!res.ok) throw new Error("Failed to fetch Azure Blob defaults");+ return res.json();+}++export function useAzureBlobDefaultsQuery(options?: { enabled?: boolean }) {+ return useQuery<AzureBlobDefaults>({+ queryKey: ["azure-blob-defaults"],+ queryFn: fetchAzureBlobDefaults,+ enabled: options?.enabled ?? true,+ staleTime: 0,+ });+}diff --git a/frontend/enhancements/connectors/ibm-cos/settings-dialog.tsx b/frontend/enhancements/connectors/ibm-cos/settings-dialog.tsxindex 2dd31dcf9..dc9a90308 100644--- a/frontend/enhancements/connectors/ibm-cos/settings-dialog.tsx+++ b/frontend/enhancements/connectors/ibm-cos/settings-dialog.tsx@@ -136,8 +136,8 @@ export default function IBMCOSSettingsDialog({toast.success("IBM Cloud Object Storage configured", {description:selectedBuckets.length > 0- ? `Will ingest from: ${selectedBuckets.join(", ")}`- : "Will auto-discover and ingest all accessible buckets.",+ ? `Ingestion restricted to the following containers: ${selectedBuckets.join(", ")}.`+ : "All accessible buckets available for ingestion.",icon: <IBMCOSIcon className="w-4 h-4" />,});diff --git a/frontend/enhancements/connectors/ibm-cos/settings-form.tsx b/frontend/enhancements/connectors/ibm-cos/settings-form.tsxindex 179e4de76..c647be561 100644--- a/frontend/enhancements/connectors/ibm-cos/settings-form.tsx+++ b/frontend/enhancements/connectors/ibm-cos/settings-form.tsx@@ -275,9 +275,9 @@ export function IBMCOSSettingsForm({<div className="space-y-2"><div className="flex items-center justify-between"><Label className="text-sm font-medium">- Select Buckets to Ingest+ Restrict Ingestion to Buckets<span className="ml-1 text-muted-foreground font-normal">- (leave all unchecked to ingest everything)+ (optional)</span></Label>{buckets.length > 1 && (diff --git a/frontend/enhancements/index.ts b/frontend/enhancements/index.tsindex 80e637626..e8c68c020 100644--- a/frontend/enhancements/index.ts+++ b/frontend/enhancements/index.ts@@ -1,4 +1,8 @@import type { ConnectorUIDescriptor } from "@/lib/connectors/types";+import { AzureBlobBucketView } from "./connectors/azure-blob/components/bucket-view";+import AzureBlobIcon from "./connectors/azure-blob/icon";+import AzureBlobSettingsDialog from "./connectors/azure-blob/settings-dialog";+import { useAzureBlobDefaultsQuery } from "./connectors/azure-blob/useAzureBlobDefaultsQuery";import { IBMCOSBucketView } from "./connectors/ibm-cos/components/bucket-view";import IBMCOSIcon from "./connectors/ibm-cos/icon";import IBMCOSSettingsDialog from "./connectors/ibm-cos/settings-dialog";@@ -23,4 +27,14 @@ export const ADDITIONAL_CONNECTORS: ConnectorUIDescriptor[] = [useDefaultsQuery: useIBMCOSDefaultsQuery,menuItem: { label: "IBM Cloud Object Storage", route: "/upload/ibm_cos" },},+ {+ connectorType: "azure_blob",+ name: "Azure Blob Storage",+ Icon: AzureBlobIcon,+ kind: "bucket",+ SettingsDialog: AzureBlobSettingsDialog,+ BucketView: AzureBlobBucketView,+ useDefaultsQuery: useAzureBlobDefaultsQuery,+ menuItem: { label: "Azure Blob Storage", route: "/upload/azure_blob" },+ },];diff --git a/frontend/lib/file-format.ts b/frontend/lib/file-format.tsnew file mode 100644index 000000000..7430bae3a--- /dev/null+++ b/frontend/lib/file-format.ts@@ -0,0 +1,55 @@+/**+ * Shared formatting helpers for file metadata (size + human-readable type+ * label). Centralised so the knowledge list, chunks "Original document" panel,+ * and file-browser dialog format the same values identically.+ */++/**+ * Format a byte count as a human-readable size (B/KB/MB/GB/TB).+ *+ * Uses bytes for sub-kilobyte files so small text/markdown documents render as+ * e.g. "412 B" instead of rounding down to a misleading "0 KB".+ */+export function formatFileSize(bytes?: number | null): string {+ if (!bytes || bytes < 0) return "0 B";+ const units = ["B", "KB", "MB", "GB", "TB"];+ const i = Math.min(+ Math.floor(Math.log(bytes) / Math.log(1024)),+ units.length - 1,+ );+ return `${(bytes / 1024 ** i).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;+}++const FILE_TYPE_LABELS: Record<string, string> = {+ "application/pdf": "PDF",+ // .txt and .md are both ingested as text/markdown (Langflow .txt->.md+ // workaround), so map both to a sensible label here.+ "text/markdown": "Markdown",+ "text/plain": "Text",+ "text/csv": "CSV",+ "text/html": "HTML",+ "application/json": "JSON",+ "application/msword": "Word Document",+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document":+ "Word Document",+ "application/vnd.ms-excel": "Excel",+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "Excel",+ "application/vnd.ms-powerpoint": "PowerPoint",+ "application/vnd.openxmlformats-officedocument.presentationml.presentation":+ "PowerPoint",+};++/**+ * Map a mimetype to a short human-readable type label. Falls back to a generic+ * group label (Image/Audio/Video/Text) before "Unknown".+ */+export function getFileTypeLabel(mimetype?: string | null): string {+ if (!mimetype) return "Unknown";+ const normalized = mimetype.toLowerCase();+ if (normalized in FILE_TYPE_LABELS) return FILE_TYPE_LABELS[normalized];+ if (normalized.startsWith("image/")) return "Image";+ if (normalized.startsWith("audio/")) return "Audio";+ if (normalized.startsWith("video/")) return "Video";+ if (normalized.startsWith("text/")) return "Text";+ return "Unknown";+}diff --git a/frontend/lib/knowledge-table-state.ts b/frontend/lib/knowledge-table-state.tsindex ba262ec1b..24e425696 100644--- a/frontend/lib/knowledge-table-state.ts+++ b/frontend/lib/knowledge-table-state.ts@@ -100,10 +100,12 @@ export function inferTaskFileConnectorType(}}- if (filePath.includes("/") && !filePath.startsWith("/")) {- return "aws_s3";- }-+ // Bucket connectors (aws_s3, ibm_cos, azure_blob) key task files by a bare+ // object key or composite "<container>::<blob>" id — none of which are+ // distinguishable here — so we rely on the connector_type captured at ingest+ // time (taskConnectorTypesRef) and fall through to "local" otherwise. The+ // processing-row icon for these is tracked as a follow-up+ // (see [[task-connector-type-processing-icon]]).return "local";}diff --git a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yamlindex 0566f48ae..d261c4da4 100644--- a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml+++ b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml@@ -102,6 +102,20 @@ stringData:MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET={{ .Values.global.oauth.microsoft.clientSecret | quote }}{{- end }}+ # Azure Blob Storage connector (optional env defaults; gated by IBM_AUTH_ENABLED or OPENRAG_DEV_AZURE_BLOB)+ {{- if .Values.global.azureBlob.connectionString }}+ AZURE_STORAGE_CONNECTION_STRING={{ .Values.global.azureBlob.connectionString | quote }}+ {{- end }}+ {{- if .Values.global.azureBlob.accountName }}+ AZURE_STORAGE_ACCOUNT_NAME={{ .Values.global.azureBlob.accountName | quote }}+ {{- end }}+ {{- if .Values.global.azureBlob.accountKey }}+ AZURE_STORAGE_ACCOUNT_KEY={{ .Values.global.azureBlob.accountKey | quote }}+ {{- end }}+ {{- if .Values.global.azureBlob.endpoint }}+ AZURE_STORAGE_ENDPOINT={{ .Values.global.azureBlob.endpoint | quote }}+ {{- end }}+# Webhook (continuous ingest){{- if .Values.backend.webhook.baseUrl }}WEBHOOK_BASE_URL={{ .Values.backend.webhook.baseUrl | quote }}diff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yamlindex f7302bd72..3b5e35343 100644--- a/kubernetes/helm/openrag/values.yaml+++ b/kubernetes/helm/openrag/values.yaml@@ -55,6 +55,16 @@ global:accessKeyId: ""secretAccessKey: ""region: ""+ # Azure Blob Storage connector (Enterprise/SaaS; gated by backend.auth.ibmCloud.enabled+ # like the other bucket connectors). Optional env defaults — credentials are+ # normally entered per-connection in the UI. Provide EITHER a connection string+ # OR account name + key (+ optional custom blob endpoint).+ azureBlob:+ enabled: false+ connectionString: "" # AZURE_STORAGE_CONNECTION_STRING+ accountName: "" # AZURE_STORAGE_ACCOUNT_NAME+ accountKey: "" # AZURE_STORAGE_ACCOUNT_KEY+ endpoint: "" # AZURE_STORAGE_ENDPOINT (optional)# ============================================================================# Langflow Configurationdiff --git a/kubernetes/operator/internal/controller/env.go b/kubernetes/operator/internal/controller/env.goindex 86184d462..276677195 100644--- a/kubernetes/operator/internal/controller/env.go+++ b/kubernetes/operator/internal/controller/env.go@@ -147,6 +147,14 @@ func NewEnvVarManager() *EnvVarManager {"WATSONX_ENDPOINT": "","WATSONX_PROJECT_ID": "","DOCLING_SERVE_VERIFY_SSL": "false",++ // Azure Blob Storage connector (Enterprise/SaaS; gated by IBM_AUTH_ENABLED or OPENRAG_DEV_AZURE_BLOB for local dev).+ // Optional env defaults — credentials are normally entered per-connection+ // in the UI. Override via the CR spec.env when env-based defaults are wanted.+ "AZURE_STORAGE_CONNECTION_STRING": "",+ "AZURE_STORAGE_ACCOUNT_NAME": "",+ "AZURE_STORAGE_ACCOUNT_KEY": "",+ "AZURE_STORAGE_ENDPOINT": "",},DefaultOpenRagFEEnvVars: map[string]string{// Frontend environment variables will be added herediff --git a/pyproject.toml b/pyproject.tomlindex 1508ab48f..4f15302e4 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -41,6 +41,7 @@ dependencies = ["uvicorn>=0.35.0","boto3>=1.35.0","ibm-cos-sdk>=2.13.0",+ "azure-storage-blob>=12.20.0","psutil>=7.0.0","rich>=13.0.0","textual>=0.45.0",diff --git a/scripts/connectors/azure/seed_azurite.py b/scripts/connectors/azure/seed_azurite.pynew file mode 100644index 000000000..edc21d642--- /dev/null+++ b/scripts/connectors/azure/seed_azurite.py@@ -0,0 +1,602 @@+"""Seed the local Azurite emulator with test containers + blobs.++Run from the repo root with the project venv:+ uv run python scripts/connectors/azure/seed_azurite.py++Pass --reset to delete and recreate the containers before uploading:+ uv run python scripts/connectors/azure/seed_azurite.py --reset++Two containers are seeded:+ * CONTAINER_1 ("openrag-test-1") — plain text, markdown, and a PDF.+ * CONTAINER_2 ("openrag-test-2") — Office/web document formats+ (docx, xlsx, pptx, csv, html) for exercising the parser end to end.++The Office formats (docx/xlsx/pptx) are built as minimal-but-valid OOXML+packages in pure Python with `zipfile`, matching the no-external-deps style+of the PDF builder below.++Assumes `make azurite-up` is running (Azurite on localhost:10000).+"""++import argparse+import io+import zipfile++from azure.storage.blob import BlobServiceClient++# Well-known Azurite dev connection string. From the host this resolves to+# http://127.0.0.1:10000/devstoreaccount1.+CONN = "UseDevelopmentStorage=true"++CONTAINER_1 = "openrag-test-1"+CONTAINER_2 = "openrag-test-2"+++def _make_sample_pdf() -> bytes:+ """Build a minimal valid single-page PDF in pure Python (no external deps)."""++ def escape(text: str) -> str:+ return text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")++ title = "Fascinating Facts about Azure"+ body_lines = [+ "- The Naming Process: Microsoft originally wanted a name containing",+ ' "cloud". After being advised against it, they chose "Azure"',+ " (a shade of blue) to represent the blue sky behind clouds.",+ ' Clients initially called the name a "dumb idea".',+ "",+ "- Space Travel & Datacenters: Microsoft Azure runs an extension of its",+ " cloud called Azure Orbital, which allows satellite operators to",+ " communicate with their spacecraft directly from Azure datacenters.",+ "",+ "- Massive Infrastructure: The network features over 175,000 miles of",+ " terrestrial and subsea fiber-optic cables and operates in more regions",+ " worldwide than any other cloud provider.",+ "",+ "- Linux Friendly: Despite being built by Microsoft, over half of the",+ " Virtual Machine workloads running on Azure are based on Linux,",+ " reflecting a deep embrace of open-source technology.",+ "",+ "- Extreme Physical Secrecy: Azure's datacenters are so state-of-the-art",+ " that their physical addresses are never publicly listed to ensure",+ " maximum security.",+ ]++ stream_parts = [+ b"BT\n",+ b"14 TL\n",+ b"/F1 14 Tf\n",+ b"72 720 Td\n",+ f"({escape(title)}) Tj T*\n".encode(),+ b"() Tj T*\n",+ b"/F1 11 Tf\n",+ ]+ for line in body_lines:+ stream_parts.append(f"({escape(line)}) Tj T*\n".encode())+ stream_parts.append(b"ET\n")+ stream = b"".join(stream_parts)++ raw_objects: list[bytes] = [+ b"<</Type /Catalog /Pages 2 0 R>>",+ b"<</Type /Pages /Kids [3 0 R] /Count 1>>",+ (+ b"<</Type /Page /Parent 2 0 R"+ b" /MediaBox [0 0 612 792]"+ b" /Contents 4 0 R"+ b" /Resources <</Font <</F1 5 0 R>>>>>>"+ b">>"+ ),+ b"<</Length " + str(len(stream)).encode() + b">>\nstream\n" + stream + b"endstream",+ b"<</Type /Font /Subtype /Type1 /BaseFont /Helvetica>>",+ ]++ body = b"%PDF-1.4\n"+ offsets: list[int] = []+ for idx, obj in enumerate(raw_objects, start=1):+ offsets.append(len(body))+ body += f"{idx} 0 obj\n".encode() + obj + b"\nendobj\n"++ xref_pos = len(body)+ n = len(raw_objects) + 1+ # Each xref entry must be exactly 20 bytes: 10-digit offset, space, 5-digit gen,+ # space, status flag, space, newline.+ xref = f"xref\n0 {n}\n0000000000 65535 f \n"+ for off in offsets:+ xref += f"{off:010d} 00000 n \n"+ trailer = f"trailer\n<</Size {n} /Root 1 0 R>>\nstartxref\n{xref_pos}\n%%EOF\n"++ return body + xref.encode() + trailer.encode()+++def _xml_escape(text: str) -> str:+ """Escape text for safe inclusion in XML element content/attributes."""+ return (+ text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """)+ )+++def _zip_package(parts: dict[str, str]) -> bytes:+ """Pack a mapping of archive-path -> XML/text into an in-memory zip (OOXML)."""+ buf = io.BytesIO()+ with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:+ for path, data in parts.items():+ zf.writestr(path, data)+ return buf.getvalue()+++def _col_letter(idx: int) -> str:+ """Convert a 0-based column index to a spreadsheet column letter (A, B, ...)."""+ letters = ""+ idx += 1+ while idx > 0:+ idx, rem = divmod(idx - 1, 26)+ letters = chr(65 + rem) + letters+ return letters+++def _make_docx(title: str, paragraphs: list[str]) -> bytes:+ """Build a minimal-but-valid .docx (WordprocessingML) in pure Python."""+ body_parts = [+ '<w:p><w:r><w:rPr><w:b/><w:sz w:val="32"/></w:rPr>'+ f'<w:t xml:space="preserve">{_xml_escape(title)}</w:t></w:r></w:p>'+ ]+ for para in paragraphs:+ body_parts.append(+ f'<w:p><w:r><w:t xml:space="preserve">{_xml_escape(para)}</w:t></w:r></w:p>'+ )++ document = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">'+ "<w:body>" + "".join(body_parts) + "<w:sectPr/></w:body></w:document>"+ )+ content_types = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'+ '<Default Extension="rels" '+ 'ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'+ '<Default Extension="xml" ContentType="application/xml"/>'+ '<Override PartName="/word/document.xml" '+ 'ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>'+ "</Types>"+ )+ rels = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'+ '<Relationship Id="rId1" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '+ 'Target="word/document.xml"/>'+ "</Relationships>"+ )+ return _zip_package(+ {+ "[Content_Types].xml": content_types,+ "_rels/.rels": rels,+ "word/document.xml": document,+ }+ )+++def _make_xlsx(sheet_name: str, rows: list[list[str]]) -> bytes:+ """Build a minimal-but-valid .xlsx (SpreadsheetML) with inline-string cells."""+ row_xml = []+ for r, row in enumerate(rows, start=1):+ cells = []+ for c, value in enumerate(row):+ ref = f"{_col_letter(c)}{r}"+ cells.append(+ f'<c r="{ref}" t="inlineStr"><is><t xml:space="preserve">'+ f"{_xml_escape(value)}</t></is></c>"+ )+ row_xml.append(f'<row r="{r}">' + "".join(cells) + "</row>")++ sheet = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">'+ "<sheetData>" + "".join(row_xml) + "</sheetData></worksheet>"+ )+ workbook = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" '+ 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">'+ f'<sheets><sheet name="{_xml_escape(sheet_name)}" sheetId="1" r:id="rId1"/></sheets>'+ "</workbook>"+ )+ workbook_rels = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'+ '<Relationship Id="rId1" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" '+ 'Target="worksheets/sheet1.xml"/>'+ "</Relationships>"+ )+ content_types = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'+ '<Default Extension="rels" '+ 'ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'+ '<Default Extension="xml" ContentType="application/xml"/>'+ '<Override PartName="/xl/workbook.xml" '+ 'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>'+ '<Override PartName="/xl/worksheets/sheet1.xml" '+ 'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>'+ "</Types>"+ )+ rels = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'+ '<Relationship Id="rId1" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '+ 'Target="xl/workbook.xml"/>'+ "</Relationships>"+ )+ return _zip_package(+ {+ "[Content_Types].xml": content_types,+ "_rels/.rels": rels,+ "xl/workbook.xml": workbook,+ "xl/_rels/workbook.xml.rels": workbook_rels,+ "xl/worksheets/sheet1.xml": sheet,+ }+ )+++def _pptx_theme() -> str:+ """Return a minimal theme1.xml required by a valid .pptx package."""+ ph = '<a:solidFill><a:schemeClr val="phClr"/></a:solidFill>'+ line = '<a:ln w="{w}"><a:solidFill><a:schemeClr val="phClr"/></a:solidFill></a:ln>'+ return (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<a:theme xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" '+ 'name="Office Theme"><a:themeElements>'+ '<a:clrScheme name="Office">'+ '<a:dk1><a:sysClr val="windowText" lastClr="000000"/></a:dk1>'+ '<a:lt1><a:sysClr val="window" lastClr="FFFFFF"/></a:lt1>'+ '<a:dk2><a:srgbClr val="44546A"/></a:dk2>'+ '<a:lt2><a:srgbClr val="E7E6E6"/></a:lt2>'+ '<a:accent1><a:srgbClr val="4472C4"/></a:accent1>'+ '<a:accent2><a:srgbClr val="ED7D31"/></a:accent2>'+ '<a:accent3><a:srgbClr val="A5A5A5"/></a:accent3>'+ '<a:accent4><a:srgbClr val="FFC000"/></a:accent4>'+ '<a:accent5><a:srgbClr val="5B9BD5"/></a:accent5>'+ '<a:accent6><a:srgbClr val="70AD47"/></a:accent6>'+ '<a:hlink><a:srgbClr val="0563C1"/></a:hlink>'+ '<a:folHlink><a:srgbClr val="954F72"/></a:folHlink>'+ "</a:clrScheme>"+ '<a:fontScheme name="Office">'+ '<a:majorFont><a:latin typeface="Calibri Light"/><a:ea typeface=""/>'+ '<a:cs typeface=""/></a:majorFont>'+ '<a:minorFont><a:latin typeface="Calibri"/><a:ea typeface=""/>'+ '<a:cs typeface=""/></a:minorFont>'+ "</a:fontScheme>"+ '<a:fmtScheme name="Office">'+ f"<a:fillStyleLst>{ph}{ph}{ph}</a:fillStyleLst>"+ "<a:lnStyleLst>"+ + line.format(w="6350")+ + line.format(w="12700")+ + line.format(w="19050")+ + "</a:lnStyleLst>"+ "<a:effectStyleLst>"+ "<a:effectStyle><a:effectLst/></a:effectStyle>"+ "<a:effectStyle><a:effectLst/></a:effectStyle>"+ "<a:effectStyle><a:effectLst/></a:effectStyle>"+ "</a:effectStyleLst>"+ f"<a:bgFillStyleLst>{ph}{ph}{ph}</a:bgFillStyleLst>"+ "</a:fmtScheme>"+ "</a:themeElements></a:theme>"+ )+++def _make_pptx(title: str, bullets: list[str]) -> bytes:+ """Build a minimal-but-valid single-slide .pptx (PresentationML) in pure Python."""+ p_ns = (+ 'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" '+ 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" '+ 'xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"'+ )++ bullet_paras = "".join(f"<a:p><a:r><a:t>{_xml_escape(b)}</a:t></a:r></a:p>" for b in bullets)+ slide = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ f"<p:sld {p_ns}><p:cSld><p:spTree>"+ '<p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr>'+ "<p:grpSpPr/>"+ "<p:sp><p:nvSpPr>"+ '<p:cNvPr id="2" name="Title 1"/>'+ '<p:cNvSpPr><a:spLocks noGrp="1"/></p:cNvSpPr>'+ '<p:nvPr><p:ph type="title"/></p:nvPr></p:nvSpPr>'+ "<p:spPr/><p:txBody><a:bodyPr/><a:lstStyle/>"+ f"<a:p><a:r><a:t>{_xml_escape(title)}</a:t></a:r></a:p></p:txBody></p:sp>"+ "<p:sp><p:nvSpPr>"+ '<p:cNvPr id="3" name="Content 2"/>'+ '<p:cNvSpPr><a:spLocks noGrp="1"/></p:cNvSpPr>'+ '<p:nvPr><p:ph type="body" idx="1"/></p:nvPr></p:nvSpPr>'+ "<p:spPr/><p:txBody><a:bodyPr/><a:lstStyle/>"+ f"{bullet_paras}</p:txBody></p:sp>"+ "</p:spTree></p:cSld></p:sld>"+ )+ slide_rels = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'+ '<Relationship Id="rId1" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout" '+ 'Target="../slideLayouts/slideLayout1.xml"/>'+ "</Relationships>"+ )+ slide_layout = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ f'<p:sldLayout {p_ns} type="blank" preserve="1"><p:cSld name="Blank"><p:spTree>'+ '<p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr>'+ "<p:grpSpPr/></p:spTree></p:cSld>"+ "<p:clrMapOvr><a:masterClrMapping/></p:clrMapOvr></p:sldLayout>"+ )+ slide_layout_rels = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'+ '<Relationship Id="rId1" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideMaster" '+ 'Target="../slideMasters/slideMaster1.xml"/>'+ "</Relationships>"+ )+ slide_master = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ f"<p:sldMaster {p_ns}><p:cSld><p:spTree>"+ '<p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr>'+ "<p:grpSpPr/></p:spTree></p:cSld>"+ '<p:clrMap bg1="lt1" tx1="dk1" bg2="lt2" tx2="dk2" '+ 'accent1="accent1" accent2="accent2" accent3="accent3" accent4="accent4" '+ 'accent5="accent5" accent6="accent6" hlink="hlink" folHlink="folHlink"/>'+ '<p:sldLayoutIdLst><p:sldLayoutId id="2147483649" r:id="rId1"/></p:sldLayoutIdLst>'+ "</p:sldMaster>"+ )+ slide_master_rels = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'+ '<Relationship Id="rId1" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout" '+ 'Target="../slideLayouts/slideLayout1.xml"/>'+ '<Relationship Id="rId2" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme" '+ 'Target="../theme/theme1.xml"/>'+ "</Relationships>"+ )+ presentation = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ f"<p:presentation {p_ns}>"+ '<p:sldMasterIdLst><p:sldMasterId id="2147483648" r:id="rId1"/></p:sldMasterIdLst>'+ '<p:sldIdLst><p:sldId id="256" r:id="rId2"/></p:sldIdLst>'+ '<p:sldSz cx="9144000" cy="6858000"/><p:notesSz cx="6858000" cy="9144000"/>'+ "</p:presentation>"+ )+ presentation_rels = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'+ '<Relationship Id="rId1" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideMaster" '+ 'Target="slideMasters/slideMaster1.xml"/>'+ '<Relationship Id="rId2" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide" '+ 'Target="slides/slide1.xml"/>'+ '<Relationship Id="rId3" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme" '+ 'Target="theme/theme1.xml"/>'+ "</Relationships>"+ )+ content_types = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'+ '<Default Extension="rels" '+ 'ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'+ '<Default Extension="xml" ContentType="application/xml"/>'+ '<Override PartName="/ppt/presentation.xml" '+ 'ContentType="application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml"/>'+ '<Override PartName="/ppt/slides/slide1.xml" '+ 'ContentType="application/vnd.openxmlformats-officedocument.presentationml.slide+xml"/>'+ '<Override PartName="/ppt/slideLayouts/slideLayout1.xml" '+ 'ContentType="application/vnd.openxmlformats-officedocument.presentationml.slideLayout+xml"/>'+ '<Override PartName="/ppt/slideMasters/slideMaster1.xml" '+ 'ContentType="application/vnd.openxmlformats-officedocument.presentationml.slideMaster+xml"/>'+ '<Override PartName="/ppt/theme/theme1.xml" '+ 'ContentType="application/vnd.openxmlformats-officedocument.theme+xml"/>'+ "</Types>"+ )+ rels = (+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'+ '<Relationship Id="rId1" '+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '+ 'Target="ppt/presentation.xml"/>'+ "</Relationships>"+ )+ return _zip_package(+ {+ "[Content_Types].xml": content_types,+ "_rels/.rels": rels,+ "ppt/presentation.xml": presentation,+ "ppt/_rels/presentation.xml.rels": presentation_rels,+ "ppt/slides/slide1.xml": slide,+ "ppt/slides/_rels/slide1.xml.rels": slide_rels,+ "ppt/slideLayouts/slideLayout1.xml": slide_layout,+ "ppt/slideLayouts/_rels/slideLayout1.xml.rels": slide_layout_rels,+ "ppt/slideMasters/slideMaster1.xml": slide_master,+ "ppt/slideMasters/_rels/slideMaster1.xml.rels": slide_master_rels,+ "ppt/theme/theme1.xml": _pptx_theme(),+ }+ )+++_AZURE_FACTS = [+ "Microsoft originally wanted a cloud name containing the word 'cloud' before "+ "settling on 'Azure', the shade of blue of the sky behind the clouds.",+ "Azure Orbital lets satellite operators communicate with their spacecraft "+ "directly from Azure datacenters.",+ "The Azure network spans over 175,000 miles of terrestrial and subsea "+ "fiber-optic cable and operates in more regions than any other cloud provider.",+ "Over half of all virtual machine workloads running on Azure are Linux-based.",+ "Azure datacenter physical addresses are never publicly listed for security.",+]+++_WORD_DOC = _make_docx(+ "Fascinating Facts about Azure",+ [+ "This Word document was ingested from the local Azurite emulator to "+ "verify the OpenRAG Azure Blob connector handles .docx files end to end.",+ *_AZURE_FACTS,+ ],+)++_EXCEL_DOC = _make_xlsx(+ "Azure Facts",+ [+ ["Service", "Category", "Fact"],+ ["Azure Orbital", "Space", "Talk to spacecraft from Azure datacenters."],+ ["Global Network", "Infrastructure", "175,000+ miles of fiber-optic cable."],+ ["Virtual Machines", "Compute", "Over half of VM workloads run Linux."],+ ["Datacenters", "Security", "Physical addresses are never publicly listed."],+ ],+)++_POWERPOINT_DOC = _make_pptx(+ "Fascinating Facts about Azure",+ _AZURE_FACTS,+)++_CSV_DOC = (+ b"Service,Category,Fact\r\n"+ b"Azure Orbital,Space,Talk to spacecraft from Azure datacenters.\r\n"+ b"Global Network,Infrastructure,175000+ miles of fiber-optic cable.\r\n"+ b"Virtual Machines,Compute,Over half of VM workloads run Linux.\r\n"+ b"Datacenters,Security,Physical addresses are never publicly listed.\r\n"+)++_HTML_DOC = (+ b"<!DOCTYPE html>\n"+ b'<html lang="en">\n<head>\n<meta charset="utf-8"/>\n'+ b"<title>Fascinating Facts about Azure</title>\n</head>\n<body>\n"+ b"<h1>Fascinating Facts about Azure</h1>\n"+ b"<p>This HTML file was ingested from the local Azurite emulator to verify "+ b"the OpenRAG Azure Blob connector handles .html files end to end.</p>\n"+ b"<ul>\n"+ b"<li>Microsoft chose the name 'Azure' for the blue sky behind the clouds.</li>\n"+ b"<li>Azure Orbital connects satellite operators to spacecraft from datacenters.</li>\n"+ b"<li>The network spans over 175,000 miles of fiber-optic cable.</li>\n"+ b"<li>Over half of Azure VM workloads run Linux.</li>\n"+ b"<li>Datacenter physical addresses are never publicly listed.</li>\n"+ b"</ul>\n</body>\n</html>\n"+)+++# Container 1: plain text, markdown, PDF.+BLOBS_1 = {+ "azure-blob-text.txt": b"Hello from Azurite! OpenRAG Azure Blob connector test document.\n",+ "notes/azure-blob-markdown.md": (+ b"# Azure Blob Connector\n\n"+ b"This markdown blob was ingested from the local Azurite emulator "+ b"to verify the OpenRAG Azure Blob connector end to end.\n\n"+ b"Microsoft Azure is a massive global cloud platform offering over 200 services. "+ b"It powers 95% of Fortune 500 companies and is connected by enough fiber-optic cable "+ b"to stretch to the Moon and back three times.\n"+ ),+ "docs/azure-blob-portal-document.pdf": _make_sample_pdf(),+}++# Container 1, "VERSION_2" updates: the same two text/markdown blobs as BLOBS_1+# above, re-tagged with a "VERSION_2" identifier so re-ingestion / change+# detection can be exercised via `--update`.+BLOBS_1_V2 = {+ "azure-blob-text.txt": (+ b"Hello from Azurite! OpenRAG Azure Blob connector test document. VERSION_2\n"+ ),+ "notes/azure-blob-markdown.md": (+ b"# Azure Blob Connector (VERSION_2)\n\n"+ b"This markdown blob was ingested from the local Azurite emulator "+ b"to verify the OpenRAG Azure Blob connector end to end.\n\n"+ b"Microsoft Azure is a massive global cloud platform offering over 200 services. "+ b"It powers 95% of Fortune 500 companies and is connected by enough fiber-optic cable "+ b"to stretch to the Moon and back three times.\n"+ ),+}++# Container 2: Office / web document formats.+BLOBS_2 = {+ "docs/azure-blob-word.docx": _WORD_DOC,+ "spreadsheets/azure-blob-excel.xlsx": _EXCEL_DOC,+ "presentations/azure-blob-powerpoint.pptx": _POWERPOINT_DOC,+ "spreadsheets/azure-blob-comma-separated.csv": _CSV_DOC,+ "azure-blob-hypertext-markup.html": _HTML_DOC,+}+++def _seed_container(+ svc: BlobServiceClient, name: str, blobs: dict[str, bytes], reset: bool+) -> None:+ """Create (optionally resetting) a container and upload the given blobs."""+ container = svc.get_container_client(name)++ if reset:+ try:+ container.delete_container()+ print(f"Deleted existing container {name!r}.")+ except Exception as exc: # ResourceNotFoundError if it didn't exist+ print(f"Container {name!r} did not exist ({type(exc).__name__}), skipping delete.")++ print(f"Ensuring container {name!r} exists...")+ try:+ container.create_container()+ print(" created.")+ except Exception as exc: # ResourceExistsError on re-run+ print(f" already exists ({type(exc).__name__}).")++ for blob_name, data in blobs.items():+ print(f"Uploading blob {blob_name!r} ({len(data)} bytes)...")+ container.get_blob_client(blob_name).upload_blob(data, overwrite=True)++ print(f"\nDone. Blobs in container {name!r}:")+ for b in container.list_blobs():+ print(f" - {b.name} ({b.size} bytes)")+++def _update_blobs(svc: BlobServiceClient, name: str, blobs: dict[str, bytes]) -> None:+ """Overwrite specific blobs in an existing container (no create/delete)."""+ container = svc.get_container_client(name)+ for blob_name, data in blobs.items():+ print(f"Updating blob {blob_name!r} ({len(data)} bytes)...")+ container.get_blob_client(blob_name).upload_blob(data, overwrite=True)++ print(f"\nDone. Updated {len(blobs)} blob(s) in container {name!r}:")+ for blob_name in blobs:+ print(f" - {blob_name}")+++def main() -> None:+ parser = argparse.ArgumentParser(+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter+ )+ parser.add_argument(+ "--reset", action="store_true", help="Delete the containers before seeding."+ )+ parser.add_argument(+ "--update",+ action="store_true",+ help=(+ 'Update the "azure-blob-text.txt" and "notes/azure-blob-markdown.md" '+ f'blobs in {CONTAINER_1!r} in place with a "VERSION_2" identifier, then exit.'+ ),+ )+ args = parser.parse_args()++ print("Connecting to Azurite...")+ svc = BlobServiceClient.from_connection_string(CONN)++ if args.update:+ _update_blobs(svc, CONTAINER_1, BLOBS_1_V2)+ return++ _seed_container(svc, CONTAINER_1, BLOBS_1, args.reset)+ print()+ _seed_container(svc, CONTAINER_2, BLOBS_2, args.reset)+++if __name__ == "__main__":+ main()diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 901148957..b524225f1 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -1,3 +1,4 @@+from datetime import UTC, datetimefrom typing import Anyfrom fastapi import Depends, Request@@ -20,6 +21,7 @@ from services.connector_access_service import (CONNECTOR_TYPES,filter_connectors_for_user,get_access_map,+ is_bucket_connector_type,is_connector_access_policy_enforced,is_connector_allowed_for_request,list_access_for_admin,@@ -194,6 +196,176 @@ async def get_synced_id_to_filename_map(return {}+async def get_synced_id_to_modified_time_map(+ connector_type: str,+ user_id: str,+ session_manager,+ jwt_token: str | None = None,+) -> dict[str, float | None]:+ """Map each ingested connector source id → its stored ``modified_time`` (epoch ms).++ Powers change detection for bucket connectors: callers compare a blob's remote+ ``modified_time`` against the stored value to decide whether a re-ingest is needed.++ A key being **present** means the source id is already ingested under this+ ``connector_type``. A value of **None** means it was ingested but no+ ``modified_time`` was persisted (pre-change-detection docs, or an ingest path that+ didn't enrich it) — callers treat that as *unchanged* to avoid a mass re-ingest.++ Keys cover both ingest layouts (mirrors ``get_synced_file_ids_for_connector``):+ the non-Langflow path stores the connector id in ``connector_file_id`` (where+ ``document_id`` is a content hash), while the Langflow path stores it in+ ``document_id``. ``connector_file_id`` wins when both are present for the same id.+ """+ try:+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)++ query_body = {+ "size": 0,+ "query": {"term": {"connector_type": connector_type}},+ "aggs": {+ "by_connector_file_id": {+ "terms": {"field": "connector_file_id", "size": 10000},+ "aggs": {"latest_modified": {"max": {"field": "modified_time"}}},+ },+ "by_document_id": {+ "terms": {"field": "document_id", "size": 10000},+ "aggs": {"latest_modified": {"max": {"field": "modified_time"}}},+ },+ },+ }++ result = await opensearch_client.search(index=get_index_name(), body=query_body)+ aggs = result.get("aggregations", {})++ mapping: dict[str, float | None] = {}+ # document_id first; connector_file_id overlays it (connector_file_id wins).+ # The content-hash document_ids from the non-Langflow path are harmless noise —+ # they never match an enumerated connector source id.+ for agg_name in ("by_document_id", "by_connector_file_id"):+ for bucket in aggs.get(agg_name, {}).get("buckets", []):+ key = bucket.get("key")+ if not key:+ continue+ mapping[key] = bucket.get("latest_modified", {}).get("value")+ return mapping+ except Exception as e:+ logger.error(+ "Failed to build id→modified_time map",+ connector_type=connector_type,+ error=str(e),+ )+ return {}+++# Tolerance (ms) to avoid a false-positive "changed" when a remote ISO timestamp+# round-trips through the OpenSearch ``date`` field with sub-second rounding.+_CHANGE_DETECTION_TOLERANCE_MS = 1000.0+++def _parse_iso_to_epoch_ms(value: str | None) -> float | None:+ """Parse an ISO-8601 timestamp to epoch milliseconds, or None if unparseable."""+ if not value:+ return None+ try:+ s = value.strip()+ if s.endswith("Z"):+ s = s[:-1] + "+00:00"+ dt = datetime.fromisoformat(s)+ if dt.tzinfo is None:+ dt = dt.replace(tzinfo=UTC)+ return dt.timestamp() * 1000.0+ except (ValueError, TypeError):+ return None+++def remote_is_newer_than_synced(+ file_id: str,+ remote_modified_time: str | None,+ synced_modified_map: dict[str, float | None],+) -> bool:+ """True only when a stored ``modified_time`` exists and the remote is strictly newer.++ Missing/unparseable timestamps (remote or stored) → False, so we never re-ingest+ on ambiguity (backfill-safe).+ """+ stored_ms = synced_modified_map.get(file_id)+ if stored_ms is None:+ return False+ remote_ms = _parse_iso_to_epoch_ms(remote_modified_time)+ if remote_ms is None:+ return False+ return remote_ms - stored_ms > _CHANGE_DETECTION_TOLERANCE_MS+++def classify_remote_file_change(+ file_id: str,+ remote_modified_time: str | None,+ is_ingested: bool,+ synced_modified_map: dict[str, float | None],+) -> str:+ """Classify a remote blob as ``"new"`` / ``"changed"`` / ``"unchanged"``.++ - ``new`` — not yet ingested.+ - ``changed`` — ingested and the remote version is strictly newer than stored.+ - ``unchanged`` — ingested and same-age/older, or the change can't be proven+ (no stored token, e.g. backfill).+ """+ if not is_ingested:+ return "new"+ if remote_is_newer_than_synced(file_id, remote_modified_time, synced_modified_map):+ return "changed"+ return "unchanged"+++async def bucket_changed_file_ids(+ connector,+ connector_type: str,+ user_id: str,+ session_manager,+ jwt_token: str | None,+ existing_file_ids: list[str],+) -> list[str]:+ """Return the already-ingested bucket file ids whose remote copy is newer.++ Updates-only change detection for the Sync path: list the connector's blobs+ once and keep the ids that are (a) already ingested and (b) strictly newer at+ source than the stored ``modified_time``. New blobs that aren't ingested yet+ are intentionally ignored — Sync reconciles existing files; new files are+ added via the connector's "Add from" panel. Listing exceptions propagate to+ the caller (same as the bucket_filter sync path).+ """+ existing_set = set(existing_file_ids)+ if not existing_set:+ return []++ modified_map = await get_synced_id_to_modified_time_map(+ connector_type=connector_type,+ user_id=user_id,+ session_manager=session_manager,+ jwt_token=jwt_token,+ )++ changed_ids: list[str] = []+ page_token = None+ while True:+ result = await connector.list_files(page_token=page_token)+ for f in result.get("files", []):+ fid = f.get("id")+ if not fid or fid not in existing_set:+ continue+ if (+ classify_remote_file_change(fid, f.get("modified_time"), True, modified_map)+ == "changed"+ ):+ changed_ids.append(fid)+ page_token = result.get("next_page_token")+ if not page_token:+ break++ return changed_ids++async def compute_orphans_for_connector_type(connector_type: str,user_id: str,@@ -739,6 +911,10 @@ async def connector_sync(connection_id=working_connection.connection_id,)+ # Branches set either ``task_id`` (single batch) or ``task_ids`` (the+ # bucket_filter path may emit two batches: new files + changed files).+ task_ids: list[str] = []+if selected_files:# Explicit files selected (e.g., from file picker) - sync those specific filesfrom .documents import _ensure_index_exists@@ -775,23 +951,27 @@ async def connector_sync()connector = await connector_service.get_connector(working_connection.connection_id)if body.bucket_filter:- # List only files from the requested buckets, then sync_specific_files+ # List only files from the requested buckets, then reconcile against+ # what's already ingested so we re-fetch only NEW and CHANGED blobs+ # (skipping unchanged ones), rather than re-downloading the whole+ # container every time. Per-file dedup in ConnectorFileProcessor is a+ # backstop, but it runs after download — this pre-filter avoids the+ # redundant fetch/reprocess and the misleading "all files" task view.original_buckets = connector.bucket_namesconnector.bucket_names = body.bucket_filtertry:- all_file_ids = []+ all_files: list[dict[str, Any]] = []page_token = Nonewhile True:result = await connector.list_files(page_token=page_token)- for f in result.get("files", []):- all_file_ids.append(f["id"])+ all_files.extend(result.get("files", []))page_token = result.get("next_page_token")if not page_token:breakfinally:connector.bucket_names = original_buckets- if not all_file_ids:+ if not all_files:return JSONResponse({"status": "no_files",@@ -799,14 +979,83 @@ async def connector_sync(},status_code=200,)- task_id = await connector_service.sync_specific_files(- working_connection.connection_id,- user.user_id,- all_file_ids,++ # Classify each remote blob as new / changed / unchanged.+ existing_ids, _, _ = await get_synced_file_ids_for_connector(+ connector_type=connector_type,+ user_id=user.user_id,+ session_manager=session_manager,+ jwt_token=jwt_token,+ )+ existing_set = set(existing_ids)+ modified_map = await get_synced_id_to_modified_time_map(+ connector_type=connector_type,+ user_id=user.user_id,+ session_manager=session_manager,jwt_token=jwt_token,- ingest_settings=body.settings,- shared=body.shared,)++ new_ids: list[str] = []+ changed_ids: list[str] = []+ for f in all_files:+ fid = f.get("id")+ if not fid:+ continue+ status = classify_remote_file_change(+ fid,+ f.get("modified_time"),+ fid in existing_set,+ modified_map,+ )+ if status == "new":+ new_ids.append(fid)+ elif status == "changed":+ changed_ids.append(fid)+ # "unchanged" → skip; already ingested and not newer at source.++ logger.info(+ "Reconciled bucket selection",+ connector_type=connector_type,+ total=len(all_files),+ new=len(new_ids),+ changed=len(changed_ids),+ skipped=len(all_files) - len(new_ids) - len(changed_ids),+ )++ if not new_ids and not changed_ids:+ return JSONResponse(+ {+ "status": "no_files",+ "message": "All files in the selected buckets are already up to date.",+ },+ status_code=200,+ )++ # Two batches: new files are created; changed files replace the+ # indexed copy (replace_duplicates=True bypasses the filename-skip+ # and deletes stale chunks before re-ingest). replace is batch-level,+ # hence the split.+ if new_ids:+ task_ids.append(+ await connector_service.sync_specific_files(+ working_connection.connection_id,+ user.user_id,+ new_ids,+ jwt_token=jwt_token,+ ingest_settings=body.settings,+ )+ )+ if changed_ids:+ task_ids.append(+ await connector_service.sync_specific_files(+ working_connection.connection_id,+ user.user_id,+ changed_ids,+ jwt_token=jwt_token,+ ingest_settings=body.settings,+ replace_duplicates=True,+ )+ )else:# sync_all: ingest everything the connector can seetask_id = await connector_service.sync_connector_files(@@ -874,15 +1123,50 @@ async def connector_sync(},status_code=200,)- task_id = await connector_service.sync_specific_files(- working_connection.connection_id,- user.user_id,- ids_to_sync,- jwt_token=jwt_token,- ingest_settings=body.settings,- replace_duplicates=_connector_sync_should_replace(connector_type),- shared=body.shared,- )+ if is_bucket_connector_type(connector_type):+ # Bucket Sync is updates-only: re-ingest just the blobs whose+ # remote copy is newer than what's indexed (deleting the stale+ # chunks via replace_duplicates). Unlike replace_duplicates=False+ # this actually propagates content changes; unlike replacing every+ # id it skips unchanged blobs instead of re-fetching the container.+ connector = await connector_service.get_connector(+ working_connection.connection_id+ )+ changed_ids = await bucket_changed_file_ids(+ connector,+ connector_type,+ user.user_id,+ session_manager,+ jwt_token,+ ids_to_sync,+ )+ if not changed_ids:+ return JSONResponse(+ {+ "status": "no_files",+ "message": f"All {connector_type} files are already up to date.",+ },+ status_code=200,+ )+ task_id = await connector_service.sync_specific_files(+ working_connection.connection_id,+ user.user_id,+ changed_ids,+ jwt_token=jwt_token,+ ingest_settings=body.settings,+ replace_duplicates=True,+ shared=body.shared,+ )+ else:+ task_id = await connector_service.sync_specific_files(+ working_connection.connection_id,+ user.user_id,+ ids_to_sync,+ jwt_token=jwt_token,+ ingest_settings=body.settings,+ replace_duplicates=_connector_sync_should_replace(connector_type),+ shared=body.shared,+ )else:# Fallback: use filename filtering (for Langflow-ingested files without document_id)logger.info(@@ -900,7 +1184,10 @@ async def connector_sync(replace_duplicates=_connector_sync_should_replace(connector_type),shared=body.shared,)- task_ids = [task_id]+ # The bucket_filter path may have already populated task_ids (new + changed+ # batches); every other branch sets a single task_id.+ if not task_ids:+ task_ids = [task_id]await TelemetryClient.send_event(Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE)@@ -1427,13 +1714,39 @@ async def sync_all_connectors(if not existing_file_ids:deleted_only_connectors.append(connector_type)continue- task_id = await connector_service.sync_specific_files(- working_connection.connection_id,- user.user_id,- existing_file_ids,- jwt_token=jwt_token,- replace_duplicates=_connector_sync_should_replace(connector_type),- )+ if is_bucket_connector_type(connector_type):+ # Updates-only change detection (see connector_sync): re-ingest+ # only blobs that changed at source, replacing stale chunks.+ connector = await connector_service.get_connector(+ working_connection.connection_id+ )+ changed_ids = await bucket_changed_file_ids(+ connector,+ connector_type,+ user.user_id,+ session_manager,+ jwt_token,+ existing_file_ids,+ )+ if not changed_ids:+ # Nothing changed at source — already up to date.+ skipped_connectors.append(connector_type)+ continue+ task_id = await connector_service.sync_specific_files(+ working_connection.connection_id,+ user.user_id,+ changed_ids,+ jwt_token=jwt_token,+ replace_duplicates=True,+ )+ else:+ task_id = await connector_service.sync_specific_files(+ working_connection.connection_id,+ user.user_id,+ existing_file_ids,+ jwt_token=jwt_token,+ replace_duplicates=_connector_sync_should_replace(connector_type),+ )else:# Fallback: use filename filteringlogger.info(@@ -1851,19 +2164,35 @@ async def browse_connection_files()ingested_set = set(ingested_ids) | set(ingested_filenames)+ # Stored modified_time per ingested source id, for "update available" detection.+ modified_map = await get_synced_id_to_modified_time_map(+ connector_type=connector_type,+ user_id=user.user_id,+ session_manager=session_manager,+ jwt_token=user.jwt_token,+ )+# Merge ingestion status into remote file listenriched_files = []for f in remote_files:- is_ingested = f.get("id", "") in ingested_set or f.get("name", "") in ingested_set+ file_id = f.get("id", "")+ is_ingested = file_id in ingested_set or f.get("name", "") in ingested_set+ # "Update available": ingested, but the source version is newer than what+ # we indexed. The frontend keeps unchanged files disabled but lets the user+ # re-ingest stale ones (with replace_duplicates).+ is_stale = is_ingested and remote_is_newer_than_synced(+ file_id, f.get("modified_time"), modified_map+ )enriched_files.append({- "id": f.get("id", ""),+ "id": file_id,"name": f.get("name", ""),"bucket": f.get("bucket", ""),"key": f.get("key", ""),"size": f.get("size", 0),"modified_time": f.get("modified_time", ""),"is_ingested": is_ingested,+ "is_stale": is_stale,})diff --git a/src/app/container.py b/src/app/container.pyindex 2b32ef0cb..e7a3609cb 100644--- a/src/app/container.py+++ b/src/app/container.py@@ -12,6 +12,7 @@ from config.settings import (config_manager,get_embedding_model,get_index_name,+ is_dev_azure_blob_enabled,is_no_auth_mode,)from connectors.service import ConnectorService@@ -40,6 +41,19 @@ from utils.telemetry import Category, MessageId, TelemetryClientlogger = get_logger(__name__)+def _should_load_persisted_connections() -> bool:+ """Whether to load persisted connector connections at startup.++ Normally skipped in no-auth mode because OAuth connectors are unusable+ there — but dev bucket connectors (e.g. Azure Blob via+ OPENRAG_DEV_AZURE_BLOB) DO work in no-auth mode and persist real+ connections. Without loading them, a saved connection silently disappears+ on restart (the in-memory dict empties), leaving the UI stuck on "Connect".+ OR in those dev flags so their connections survive a restart.+ """+ return not is_no_auth_mode() or is_dev_azure_blob_enabled()++async def initialize_services():"""Initialize all services and their dependencies"""await TelemetryClient.send_event(Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_INIT_START)@@ -131,9 +145,9 @@ async def initialize_services():)# Load persisted connector connections at startup so webhooks and syncs- # can resolve existing subscriptions immediately after server boot- # Skip in no-auth mode since connectors require OAuth- if not is_no_auth_mode():+ # can resolve existing subscriptions immediately after server boot.+ # (See _should_load_persisted_connections for the no-auth/dev-bucket nuance.)+ if _should_load_persisted_connections():try:await connector_service.initialize()loaded_count = len(connector_service.connection_manager.connections)diff --git a/src/config/settings.py b/src/config/settings.pyindex b00b7c15d..32ccd3a9b 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -247,6 +247,17 @@ def is_dev_connector_policy_enabled() -> bool:return raw in ("true", "1", "yes", "on")+def is_dev_azure_blob_enabled() -> bool:+ """Local dev: enable Azure Blob connector without IBM_AUTH_ENABLED.++ Allows testing the Azure Blob connector (e.g. against Azurite) in a local+ environment where IBM auth is not configured. Never enable in production.+ Requires ``OPENRAG_DEV_AZURE_BLOB=true``.+ """+ raw = os.getenv("OPENRAG_DEV_AZURE_BLOB", "false").strip().lower()+ return raw in ("true", "1", "yes", "on")++def is_cloud_context() -> bool:"""True when connector policy and SaaS settings guards should apply."""from utils.run_mode_utils import is_run_mode_saasdiff --git a/src/models/processors.py b/src/models/processors.pyindex af46f5e4a..c767a2b61 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -1066,6 +1066,20 @@ class ConnectorFileProcessor(TaskProcessor):"status": "error","error": "No text content could be extracted from document",}++ # Persist connector metadata (incl. modified_time) onto the+ # Langflow-indexed chunks (keyed by document_id) so bucket-connector+ # change detection has a stored timestamp to compare against on the+ # next sync. Mirrors the standard path's enrichment below.+ if result.get("status") != "error":+ await self.connector_service._update_connector_metadata(+ document,+ self.user_id,+ connector_type,+ self.jwt_token,+ id_field="document_id",+ indexed_filename=file_task.filename,+ )else:# Standard OpenRAG processing pipeline (process_document_standard)standard_kwargs: dict[str, Any] = {}diff --git a/src/services/connector_access_service.py b/src/services/connector_access_service.pyindex 7a64edb17..9948da251 100644--- a/src/services/connector_access_service.py+++ b/src/services/connector_access_service.py@@ -13,7 +13,12 @@ CONNECTOR_ACCESS_SECTION = "connector_access"# without touching this module.CONNECTOR_TYPES: tuple[str, ...] = tuple(cls.CONNECTOR_TYPE for cls in get_connector_classes())-_BUCKET_CONNECTOR_TYPES = frozenset({"aws_s3", "ibm_cos"})+_BUCKET_CONNECTOR_TYPES = frozenset({"aws_s3", "ibm_cos", "azure_blob"})+++def is_bucket_connector_type(connector_type: str) -> bool:+ """True for object-storage "bucket" connectors (S3, IBM COS, Azure Blob)."""+ return connector_type in _BUCKET_CONNECTOR_TYPESdef governable_connector_types() -> tuple[str, ...]:diff --git a/tests/unit/api/test_connector_change_detection.py b/tests/unit/api/test_connector_change_detection.pynew file mode 100644index 000000000..02c46e811--- /dev/null+++ b/tests/unit/api/test_connector_change_detection.py@@ -0,0 +1,522 @@+"""Unit tests for bucket-connector change detection in `src/api/connectors.py`.++Covers:+- the pure timestamp/classification helpers (`_parse_iso_to_epoch_ms`,+ `remote_is_newer_than_synced`, `classify_remote_file_change`),+- the `get_synced_id_to_modified_time_map` aggregation helper,+- the whole-container (`bucket_filter`) reconciliation in `connector_sync`: only+ new + changed blobs are ingested (new as a plain batch, changed with+ replace_duplicates=True), unchanged blobs are skipped.+"""++import json+import sys+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++def _json(response):+ return json.loads(response.body.decode())+++# ---------------------------------------------------------------------------+# Pure helpers+# ---------------------------------------------------------------------------+++def test_parse_iso_to_epoch_ms_handles_z_and_offset_and_naive():+ from api.connectors import _parse_iso_to_epoch_ms++ z = _parse_iso_to_epoch_ms("2024-01-01T00:00:00Z")+ offset = _parse_iso_to_epoch_ms("2024-01-01T00:00:00+00:00")+ naive = _parse_iso_to_epoch_ms("2024-01-01T00:00:00")+ assert z == offset == naive == 1704067200000.0+++def test_parse_iso_to_epoch_ms_returns_none_for_bad_input():+ from api.connectors import _parse_iso_to_epoch_ms++ assert _parse_iso_to_epoch_ms(None) is None+ assert _parse_iso_to_epoch_ms("") is None+ assert _parse_iso_to_epoch_ms("not-a-date") is None+++def test_remote_is_newer_than_synced_true_when_strictly_newer():+ from api.connectors import remote_is_newer_than_synced++ stored = {"c::a": 1704067200000.0} # 2024-01-01+ assert remote_is_newer_than_synced("c::a", "2024-06-01T00:00:00Z", stored) is True+++def test_remote_is_newer_than_synced_false_when_same_or_older():+ from api.connectors import remote_is_newer_than_synced++ stored = {"c::a": 1704067200000.0}+ assert remote_is_newer_than_synced("c::a", "2024-01-01T00:00:00Z", stored) is False+ assert remote_is_newer_than_synced("c::a", "2023-01-01T00:00:00Z", stored) is False+++def test_remote_is_newer_than_synced_tolerance_absorbs_subsecond_jitter():+ from api.connectors import remote_is_newer_than_synced++ stored = {"c::a": 1704067200000.0}+ # 500ms newer is within tolerance → not "changed".+ assert remote_is_newer_than_synced("c::a", "2024-01-01T00:00:00.500Z", stored) is False+ # 2s newer exceeds tolerance → changed.+ assert remote_is_newer_than_synced("c::a", "2024-01-01T00:00:02Z", stored) is True+++def test_remote_is_newer_than_synced_false_when_no_stored_token():+ from api.connectors import remote_is_newer_than_synced++ # Missing id, or ingested-but-no-token (None) → backfill-safe False.+ assert remote_is_newer_than_synced("c::missing", "2024-06-01T00:00:00Z", {}) is False+ assert remote_is_newer_than_synced("c::a", "2024-06-01T00:00:00Z", {"c::a": None}) is False+++def test_classify_remote_file_change():+ from api.connectors import classify_remote_file_change++ stored = {"c::a": 1704067200000.0}+ # Not ingested → new (regardless of timestamps).+ assert classify_remote_file_change("c::new", "2024-06-01T00:00:00Z", False, stored) == "new"+ # Ingested + newer → changed.+ assert classify_remote_file_change("c::a", "2024-06-01T00:00:00Z", True, stored) == "changed"+ # Ingested + same → unchanged.+ assert classify_remote_file_change("c::a", "2024-01-01T00:00:00Z", True, stored) == "unchanged"+ # Ingested but no stored token (backfill) → unchanged.+ assert classify_remote_file_change("c::b", "2024-06-01T00:00:00Z", True, {}) == "unchanged"+++# ---------------------------------------------------------------------------+# get_synced_id_to_modified_time_map+# ---------------------------------------------------------------------------+++async def test_modified_time_map_prefers_connector_file_id_over_document_id(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")++ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(+ return_value={+ "aggregations": {+ "by_connector_file_id": {+ "buckets": [+ {"key": "c::a", "latest_modified": {"value": 1704067200000.0}},+ # connector_file_id present but no modified_time → None+ {"key": "c::b", "latest_modified": {"value": None}},+ ]+ },+ "by_document_id": {+ "buckets": [+ # Langflow-path id (document_id holds the connector source id).+ {"key": "c::lf", "latest_modified": {"value": 1704153600000.0}},+ # A content-hash document_id from the standard path — harmless,+ # never matches an enumerated source id. Overlaid/ignored.+ {"key": "c::a", "latest_modified": {"value": 999.0}},+ ]+ },+ }+ }+ )+ sm = MagicMock()+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ result = await connectors_api.get_synced_id_to_modified_time_map(+ connector_type="azure_blob",+ user_id="alice",+ session_manager=sm,+ jwt_token=None,+ )++ # connector_file_id wins for c::a (1704067200000, not the 999 from document_id).+ assert result["c::a"] == 1704067200000.0+ assert result["c::b"] is None+ assert result["c::lf"] == 1704153600000.0+++async def test_modified_time_map_returns_empty_on_error(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")+ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(side_effect=RuntimeError("boom"))+ sm = MagicMock()+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ result = await connectors_api.get_synced_id_to_modified_time_map(+ connector_type="azure_blob",+ user_id="alice",+ session_manager=sm,+ jwt_token=None,+ )+ assert result == {}+++# ---------------------------------------------------------------------------+# connector_sync — bucket_filter reconciliation+# ---------------------------------------------------------------------------+++def _bucket_sync_service(remote_files, task_id="task-x"):+ connection = SimpleNamespace(connection_id="conn-1", is_active=True)+ connector = MagicMock()+ connector.authenticate = AsyncMock(return_value=True)+ connector.bucket_names = None+ connector.list_files = AsyncMock(return_value={"files": remote_files, "next_page_token": None})++ service = MagicMock()+ service.connection_manager = MagicMock()+ service.connection_manager.list_connections = AsyncMock(return_value=[connection])+ service.get_connector = AsyncMock(return_value=connector)+ service.sync_specific_files = AsyncMock(return_value=task_id)+ return service+++async def test_bucket_filter_ingests_only_new_and_changed(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ # "c::ingested_unchanged" and "c::ingested_changed" are already ingested.+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(+ return_value=(["c::ingested_unchanged", "c::ingested_changed"], [], "connector_file_id")+ ),+ )+ monkeypatch.setattr(+ connectors_api,+ "get_synced_id_to_modified_time_map",+ AsyncMock(+ return_value={+ "c::ingested_unchanged": 1704067200000.0, # 2024-01-01+ "c::ingested_changed": 1704067200000.0, # 2024-01-01+ }+ ),+ )++ remote_files = [+ {"id": "c::new", "modified_time": "2024-01-01T00:00:00Z"},+ {"id": "c::ingested_unchanged", "modified_time": "2024-01-01T00:00:00Z"},+ {"id": "c::ingested_changed", "modified_time": "2024-06-01T00:00:00Z"},+ ]+ service = _bucket_sync_service(remote_files)++ response = await connectors_api.connector_sync(+ "azure_blob",+ connectors_api.ConnectorSyncBody(connection_id="conn-1", bucket_filter=["c"]),+ request=MagicMock(),+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token"),+ session=MagicMock(),+ )++ assert response.status_code == 201+ assert _json(response)["task_ids"] == ["task-x", "task-x"]++ # Two batches: new (replace defaulted False) + changed (replace=True).+ assert service.sync_specific_files.await_count == 2+ new_call, changed_call = service.sync_specific_files.await_args_list+ assert new_call.args[2] == ["c::new"]+ assert new_call.kwargs.get("replace_duplicates", False) is False+ assert changed_call.args[2] == ["c::ingested_changed"]+ assert changed_call.kwargs["replace_duplicates"] is True+++async def test_bucket_filter_all_unchanged_returns_no_files(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["c::a", "c::b"], [], "connector_file_id")),+ )+ monkeypatch.setattr(+ connectors_api,+ "get_synced_id_to_modified_time_map",+ AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),+ )++ remote_files = [+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"},+ {"id": "c::b", "modified_time": "2024-01-01T00:00:00Z"},+ ]+ service = _bucket_sync_service(remote_files)++ response = await connectors_api.connector_sync(+ "azure_blob",+ connectors_api.ConnectorSyncBody(connection_id="conn-1", bucket_filter=["c"]),+ request=MagicMock(),+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token"),+ session=MagicMock(),+ )++ assert response.status_code == 200+ body = _json(response)+ assert body["status"] == "no_files"+ assert "up to date" in body["message"]+ service.sync_specific_files.assert_not_awaited()+++async def test_bucket_filter_only_new_files_single_batch(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=([], [], "connector_file_id")),+ )+ monkeypatch.setattr(+ connectors_api,+ "get_synced_id_to_modified_time_map",+ AsyncMock(return_value={}),+ )++ remote_files = [+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"},+ {"id": "c::b", "modified_time": "2024-01-01T00:00:00Z"},+ ]+ service = _bucket_sync_service(remote_files)++ response = await connectors_api.connector_sync(+ "azure_blob",+ connectors_api.ConnectorSyncBody(connection_id="conn-1", bucket_filter=["c"]),+ request=MagicMock(),+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token"),+ session=MagicMock(),+ )++ assert response.status_code == 201+ service.sync_specific_files.assert_awaited_once()+ call = service.sync_specific_files.await_args+ assert call.args[2] == ["c::a", "c::b"]+ assert call.kwargs.get("replace_duplicates", False) is False+++# ---------------------------------------------------------------------------+# bucket_changed_file_ids — updates-only change detection helper+# ---------------------------------------------------------------------------+++async def test_bucket_changed_file_ids_filters_to_changed_ingested(monkeypatch):+ """Only already-ingested blobs that are newer at source are returned; new+ (un-ingested) blobs are ignored (updates-only)."""+ from api import connectors as connectors_api++ monkeypatch.setattr(+ connectors_api,+ "get_synced_id_to_modified_time_map",+ AsyncMock(+ return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0} # 2024-01-01+ ),+ )++ connector = MagicMock()+ connector.list_files = AsyncMock(+ return_value={+ "files": [+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"}, # unchanged+ {"id": "c::b", "modified_time": "2024-06-01T00:00:00Z"}, # changed+ {"id": "c::new", "modified_time": "2024-06-01T00:00:00Z"}, # new → ignored+ ],+ "next_page_token": None,+ }+ )++ changed = await connectors_api.bucket_changed_file_ids(+ connector,+ "azure_blob",+ "alice",+ MagicMock(),+ "token",+ ["c::a", "c::b"],+ )+ assert changed == ["c::b"]+++async def test_bucket_changed_file_ids_empty_when_no_existing():+ from api import connectors as connectors_api++ connector = MagicMock()+ connector.list_files = AsyncMock()+ changed = await connectors_api.bucket_changed_file_ids(+ connector, "azure_blob", "alice", MagicMock(), "token", []+ )+ assert changed == []+ connector.list_files.assert_not_awaited()+++# ---------------------------------------------------------------------------+# connector_sync (Sync button, no selected_files/sync_all/bucket_filter) —+# updates-only re-ingest for bucket connectors+# ---------------------------------------------------------------------------+++async def test_sync_button_bucket_reingests_only_changed(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["c::a", "c::b"], [], "connector_file_id")),+ )+ monkeypatch.setattr(+ connectors_api,+ "reconcile_orphans_for_connector_type",+ AsyncMock(return_value=[]),+ )+ monkeypatch.setattr(+ connectors_api,+ "get_synced_id_to_modified_time_map",+ AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),+ )++ remote_files = [+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"}, # unchanged+ {"id": "c::b", "modified_time": "2024-06-01T00:00:00Z"}, # changed+ ]+ service = _bucket_sync_service(remote_files)++ response = await connectors_api.connector_sync(+ "azure_blob",+ connectors_api.ConnectorSyncBody(), # plain Sync: no files, no sync_all/bucket_filter+ request=MagicMock(),+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token"),+ session=MagicMock(),+ )++ assert response.status_code == 201+ service.sync_specific_files.assert_awaited_once()+ call = service.sync_specific_files.await_args+ assert call.args[2] == ["c::b"]+ assert call.kwargs["replace_duplicates"] is True+++async def test_sync_button_bucket_all_unchanged_returns_no_files(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["c::a", "c::b"], [], "connector_file_id")),+ )+ monkeypatch.setattr(+ connectors_api,+ "reconcile_orphans_for_connector_type",+ AsyncMock(return_value=[]),+ )+ monkeypatch.setattr(+ connectors_api,+ "get_synced_id_to_modified_time_map",+ AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),+ )++ remote_files = [+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"},+ {"id": "c::b", "modified_time": "2024-01-01T00:00:00Z"},+ ]+ service = _bucket_sync_service(remote_files)++ response = await connectors_api.connector_sync(+ "azure_blob",+ connectors_api.ConnectorSyncBody(),+ request=MagicMock(),+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token"),+ session=MagicMock(),+ )++ assert response.status_code == 200+ body = _json(response)+ assert body["status"] == "no_files"+ assert "up to date" in body["message"]+ service.sync_specific_files.assert_not_awaited()+++# ---------------------------------------------------------------------------+# sync_all_connectors — updates-only re-ingest for bucket connectors+# ---------------------------------------------------------------------------+++async def test_sync_all_bucket_reingests_only_changed(monkeypatch):+ from api import connectors as connectors_api++ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(+ connectors_api,+ "_allowed_connector_types_for_request",+ AsyncMock(return_value=["azure_blob"]),+ )+ monkeypatch.setattr(+ connectors_api,+ "get_synced_file_ids_for_connector",+ AsyncMock(return_value=(["c::a", "c::b"], [], "connector_file_id")),+ )+ monkeypatch.setattr(+ connectors_api,+ "reconcile_orphans_for_connector_type",+ AsyncMock(return_value=[]),+ )+ monkeypatch.setattr(+ connectors_api,+ "get_synced_id_to_modified_time_map",+ AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),+ )++ remote_files = [+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"}, # unchanged+ {"id": "c::b", "modified_time": "2024-06-01T00:00:00Z"}, # changed+ ]+ service = _bucket_sync_service(remote_files)++ response = await connectors_api.sync_all_connectors(+ request=MagicMock(),+ connector_service=service,+ session_manager=MagicMock(),+ user=SimpleNamespace(user_id="alice", jwt_token="token"),+ session=MagicMock(),+ )++ assert response.status_code == 201+ service.sync_specific_files.assert_awaited_once()+ call = service.sync_specific_files.await_args+ assert call.args[2] == ["c::b"]+ assert call.kwargs["replace_duplicates"] is Truediff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.pyindex 9e339ca37..e0e982af1 100644--- a/tests/unit/api/test_reconcile_orphans_for_connector_type.py+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py@@ -435,6 +435,7 @@ async def test_connector_sync_filters_orphan_ids_before_resync(monkeypatch):from api import connectors as connectors_apimonkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))monkeypatch.setattr(connectors_api,"get_synced_file_ids_for_connector",@@ -459,9 +460,11 @@ async def test_connector_sync_filters_orphan_ids_before_resync(monkeypatch):response = await connectors_api.connector_sync("google_drive",connectors_api.ConnectorSyncBody(),+ request=MagicMock(),connector_service=service,session_manager=MagicMock(),user=SimpleNamespace(user_id="alice", jwt_token="token"),+ session=MagicMock(),)assert response.status_code == 201@@ -475,6 +478,7 @@ async def test_connector_sync_returns_no_files_when_all_ids_are_orphans(monkeypafrom api import connectors as connectors_apimonkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))monkeypatch.setattr(connectors_api,"get_synced_file_ids_for_connector",@@ -499,9 +503,11 @@ async def test_connector_sync_returns_no_files_when_all_ids_are_orphans(monkeyparesponse = await connectors_api.connector_sync("google_drive",connectors_api.ConnectorSyncBody(),+ request=MagicMock(),connector_service=service,session_manager=MagicMock(),user=SimpleNamespace(user_id="alice", jwt_token="token"),+ session=MagicMock(),)assert response.status_code == 200@@ -514,6 +520,7 @@ async def test_sync_all_returns_deleted_only_without_error(monkeypatch):from api import connectors as connectors_apimonkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))async def fake_synced_ids(connector_type, *args, **kwargs):if connector_type == "google_drive":@@ -538,9 +545,11 @@ async def test_sync_all_returns_deleted_only_without_error(monkeypatch):service.sync_specific_files = AsyncMock()response = await connectors_api.sync_all_connectors(+ request=MagicMock(),connector_service=service,session_manager=MagicMock(),user=SimpleNamespace(user_id="alice", jwt_token="token"),+ session=MagicMock(),)body = _json(response)diff --git a/tests/unit/connectors/test_azure_blob_connector.py b/tests/unit/connectors/test_azure_blob_connector.pynew file mode 100644index 000000000..05b9dcbb9--- /dev/null+++ b/tests/unit/connectors/test_azure_blob_connector.py@@ -0,0 +1,751 @@+"""Unit tests for the Azure Blob Storage connector.++Covers credential resolution (both auth modes + env fallback), client-factory+validation, file listing (prefix filter, dir-marker skip, max_files), composite+file-id round-trip, blob download → ConnectorDocument mapping (owner-based DLS),+authenticate success/failure, IBM_AUTH_ENABLED gating, config-builder validation,+and the webhook stubs.++The sync azure-storage-blob client is replaced with a MagicMock — the connector+offloads it via asyncio.to_thread, so plain mocks work without async machinery.+"""++import json+import sys+from datetime import UTC, datetime+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++from enhancements.connectors.azure_blob import api as az_api # noqa: E402+from enhancements.connectors.azure_blob import auth as az_auth # noqa: E402+from enhancements.connectors.azure_blob import connector as az_connector # noqa: E402+from enhancements.connectors.azure_blob.connector import ( # noqa: E402+ AzureBlobConnector,+ _make_file_id,+ _split_file_id,+)+from enhancements.connectors.azure_blob.models import AzureBlobConfigureBody # noqa: E402+from enhancements.connectors.azure_blob.support import build_azure_blob_config # noqa: E402++CONN_STR = "AZURE_STORAGE_CONNECTION_STRING"+ACCT_NAME = "AZURE_STORAGE_ACCOUNT_NAME"+ACCT_KEY = "AZURE_STORAGE_ACCOUNT_KEY"+ENDPOINT = "AZURE_STORAGE_ENDPOINT"+++def _clear_azure_env(monkeypatch):+ for var in (CONN_STR, ACCT_NAME, ACCT_KEY, ENDPOINT):+ monkeypatch.delenv(var, raising=False)+++# ---------------------------------------------------------------------------+# Composite file-id helpers+# ---------------------------------------------------------------------------+++def test_make_and_split_file_id_roundtrip():+ file_id = _make_file_id("mycontainer", "path/to/blob.pdf")+ assert file_id == "mycontainer::path/to/blob.pdf"+ container, blob = _split_file_id(file_id)+ assert container == "mycontainer"+ assert blob == "path/to/blob.pdf"+++def test_split_file_id_invalid_raises():+ with pytest.raises(ValueError):+ _split_file_id("no-separator-here")+++def test_split_file_id_blob_with_separator_only_splits_once():+ container, blob = _split_file_id("c::a::b")+ assert container == "c"+ assert blob == "a::b"+++# ---------------------------------------------------------------------------+# Credential resolution + client factory+# ---------------------------------------------------------------------------+++def test_resolve_credentials_prefers_config_over_env(monkeypatch):+ _clear_azure_env(monkeypatch)+ monkeypatch.setenv(CONN_STR, "env-conn-str")+ creds = az_auth._resolve_credentials({"connection_string": "cfg-conn-str"})+ assert creds["connection_string"] == "cfg-conn-str"+++def test_resolve_credentials_env_fallback(monkeypatch):+ _clear_azure_env(monkeypatch)+ monkeypatch.setenv(ACCT_NAME, "acct")+ monkeypatch.setenv(ACCT_KEY, "key")+ monkeypatch.setenv(ENDPOINT, "http://127.0.0.1:10000/devstoreaccount1")+ creds = az_auth._resolve_credentials({"auth_mode": "account_key"})+ assert creds["account_name"] == "acct"+ assert creds["account_key"] == "key"+ assert creds["endpoint_url"] == "http://127.0.0.1:10000/devstoreaccount1"+++def test_create_client_connection_string_mode(monkeypatch):+ _clear_azure_env(monkeypatch)+ # Azurite dev shortcut constructs fully offline.+ client = az_auth.create_blob_service_client(+ {"auth_mode": "connection_string", "connection_string": "UseDevelopmentStorage=true"}+ )+ assert client is not None+++def test_create_client_account_key_mode_with_endpoint(monkeypatch):+ _clear_azure_env(monkeypatch)+ client = az_auth.create_blob_service_client(+ {+ "auth_mode": "account_key",+ "account_name": "devstoreaccount1",+ "account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT==",+ "endpoint_url": "http://127.0.0.1:10000/devstoreaccount1",+ }+ )+ assert client is not None+++def test_create_client_connection_string_missing_raises(monkeypatch):+ _clear_azure_env(monkeypatch)+ with pytest.raises(ValueError, match="Connection string mode requires"):+ az_auth.create_blob_service_client({"auth_mode": "connection_string"})+++def test_create_client_account_key_missing_raises(monkeypatch):+ _clear_azure_env(monkeypatch)+ with pytest.raises(ValueError, match="Account key mode requires"):+ az_auth.create_blob_service_client({"auth_mode": "account_key", "account_name": "acct"})+++# ---------------------------------------------------------------------------+# Account-name resolution (used by get_client_id / status checks)+# ---------------------------------------------------------------------------+++def test_account_name_from_connection_string_dev_storage():+ assert az_auth._account_name_from_connection_string("UseDevelopmentStorage=true") == (+ "devstoreaccount1"+ )+++def test_account_name_from_connection_string_account_name():+ conn = "DefaultEndpointsProtocol=https;AccountName=myacct;AccountKey=ab==;EndpointSuffix=x"+ assert az_auth._account_name_from_connection_string(conn) == "myacct"+++def test_account_name_from_connection_string_sas_host_style():+ conn = "BlobEndpoint=https://myacct.blob.core.windows.net;SharedAccessSignature=sv=2021"+ assert az_auth._account_name_from_connection_string(conn) == "myacct"+++def test_account_name_from_connection_string_sas_path_style():+ conn = "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;SharedAccessSignature=sv=x"+ assert az_auth._account_name_from_connection_string(conn) == "devstoreaccount1"+++def test_account_name_from_connection_string_unparseable_returns_none():+ assert az_auth._account_name_from_connection_string("SharedAccessSignature=sv=2021") is None+++def test_account_name_from_config_prefers_account_name(monkeypatch):+ _clear_azure_env(monkeypatch)+ assert az_auth.account_name_from_config({"account_name": "explicit"}) == "explicit"+++def test_account_name_from_config_parses_connection_string(monkeypatch):+ _clear_azure_env(monkeypatch)+ cfg = {"auth_mode": "connection_string", "connection_string": "UseDevelopmentStorage=true"}+ assert az_auth.account_name_from_config(cfg) == "devstoreaccount1"+++def test_get_client_id_connection_string_mode_does_not_raise(monkeypatch):+ """Regression: connection-string mode (Azurite) must yield a stable id, not raise.++ Otherwise the status endpoint marks an authenticated connection as+ unauthenticated and the UI keeps showing "Connect".+ """+ _clear_azure_env(monkeypatch)+ connector = AzureBlobConnector(+ {"auth_mode": "connection_string", "connection_string": "UseDevelopmentStorage=true"}+ )+ assert connector.get_client_id() == "devstoreaccount1"+++def test_get_client_id_account_key_mode(monkeypatch):+ _clear_azure_env(monkeypatch)+ connector = AzureBlobConnector(+ {"auth_mode": "account_key", "account_name": "acct", "account_key": "k"}+ )+ assert connector.get_client_id() == "acct"+++def test_get_client_id_no_credentials_raises(monkeypatch):+ _clear_azure_env(monkeypatch)+ connector = AzureBlobConnector({"auth_mode": "account_key"})+ with pytest.raises(ValueError, match="Azure Blob credentials not set"):+ connector.get_client_id()+++# ---------------------------------------------------------------------------+# is_available gating (IBM_AUTH_ENABLED / OPENRAG_DEV_AZURE_BLOB)+# ---------------------------------------------------------------------------+++def test_is_available_gated_on_ibm_auth_enabled(monkeypatch):+ monkeypatch.setattr(az_connector, "IBM_AUTH_ENABLED", True)+ monkeypatch.setattr(az_connector, "is_dev_azure_blob_enabled", lambda: False)+ assert AzureBlobConnector.is_available(MagicMock()) is True+ monkeypatch.setattr(az_connector, "IBM_AUTH_ENABLED", False)+ assert AzureBlobConnector.is_available(MagicMock()) is False+++def test_is_available_dev_flag_bypasses_ibm_auth(monkeypatch):+ monkeypatch.setattr(az_connector, "IBM_AUTH_ENABLED", False)+ monkeypatch.setattr(az_connector, "is_dev_azure_blob_enabled", lambda: True)+ assert AzureBlobConnector.is_available(MagicMock()) is True+++# ---------------------------------------------------------------------------+# Fake Azure client helpers+# ---------------------------------------------------------------------------+++def _blob(name, size=10, modified=None):+ b = MagicMock()+ b.name = name+ b.size = size+ b.last_modified = modified+ return b+++def _make_fake_client(containers):+ """containers: dict[name -> list[blob]]."""+ client = MagicMock()+ # NB: MagicMock(name=...) sets the mock's repr name, not a .name attribute,+ # so build the container mocks and assign .name explicitly.+ container_mocks = []+ for cname in containers:+ cm = MagicMock()+ cm.name = cname+ container_mocks.append(cm)+ client.list_containers.return_value = container_mocks++ def _get_container_client(name):+ cc = MagicMock()++ def _list_blobs(name_starts_with=None):+ blobs = containers[name]+ if name_starts_with:+ blobs = [b for b in blobs if b.name.startswith(name_starts_with)]+ return iter(blobs)++ cc.list_blobs.side_effect = _list_blobs+ return cc++ client.get_container_client.side_effect = _get_container_client+ return client+++def patched_factory():+ """Patch create_blob_service_client in the connector module; yield a setter."""+ with patch.object(az_connector, "create_blob_service_client") as factory:+ yield factory+++# ---------------------------------------------------------------------------+# list_files+# ---------------------------------------------------------------------------+++async def test_list_files_single_container(patched_factory):+ ts = datetime(2026, 1, 1, tzinfo=UTC)+ patched_factory.return_value = _make_fake_client(+ {"docs": [_blob("a.pdf", 5, ts), _blob("b.txt", 7, ts)]}+ )+ conn = AzureBlobConnector({"container_names": ["docs"]})+ result = await conn.list_files()+ assert result["next_page_token"] is None+ names = {f["name"] for f in result["files"]}+ assert names == {"a.pdf", "b.txt"}+ first = next(f for f in result["files"] if f["name"] == "a.pdf")+ assert first["id"] == "docs::a.pdf"+ # "bucket" is the shared bucket-connector contract key (matches aws_s3/ibm_cos)+ # that the file browser reads; carries the Azure container name.+ assert first["bucket"] == "docs"+ assert first["modified_time"] == ts.isoformat()+++async def test_list_files_skips_directory_markers(patched_factory):+ patched_factory.return_value = _make_fake_client(+ {"docs": [_blob("folder/"), _blob("folder/real.pdf")]}+ )+ conn = AzureBlobConnector({"container_names": ["docs"]})+ result = await conn.list_files()+ assert [f["key"] for f in result["files"]] == ["folder/real.pdf"]+++async def test_list_files_prefix_filter(patched_factory):+ patched_factory.return_value = _make_fake_client(+ {"docs": [_blob("keep/a.pdf"), _blob("skip/b.pdf")]}+ )+ conn = AzureBlobConnector({"container_names": ["docs"], "prefix": "keep/"})+ result = await conn.list_files()+ assert [f["key"] for f in result["files"]] == ["keep/a.pdf"]+++async def test_list_files_respects_max_files(patched_factory):+ patched_factory.return_value = _make_fake_client(+ {"docs": [_blob(f"f{i}.pdf") for i in range(10)]}+ )+ conn = AzureBlobConnector({"container_names": ["docs"]})+ result = await conn.list_files(max_files=3)+ assert len(result["files"]) == 3+++async def test_list_files_auto_discovers_containers(patched_factory):+ patched_factory.return_value = _make_fake_client(+ {"c1": [_blob("a.pdf")], "c2": [_blob("b.pdf")]}+ )+ conn = AzureBlobConnector({}) # no container_names → auto-discover+ result = await conn.list_files()+ assert {f["bucket"] for f in result["files"]} == {"c1", "c2"}+++# ---------------------------------------------------------------------------+# get_file_content+# ---------------------------------------------------------------------------+++def _patch_download(client, content=b"hello", content_type="", last_modified=None, size=None):+ downloader = MagicMock()+ downloader.readall.return_value = content+ props = MagicMock()+ props.content_settings.content_type = content_type+ props.last_modified = last_modified+ props.size = size if size is not None else len(content)+ downloader.properties = props+ blob_client = MagicMock()+ blob_client.download_blob.return_value = downloader+ client.get_blob_client.return_value = blob_client+++async def test_get_file_content_maps_document(patched_factory):+ ts = datetime(2026, 2, 2, tzinfo=UTC)+ client = _make_fake_client({"docs": []})+ _patch_download(client, content=b"pdfbytes", content_type="application/pdf", last_modified=ts)+ patched_factory.return_value = client++ conn = AzureBlobConnector({"container_names": ["docs"]})+ doc = await conn.get_file_content("docs::report.pdf")++ assert doc.id == "docs::report.pdf"+ assert doc.filename == "report.pdf"+ assert doc.content == b"pdfbytes"+ assert doc.mimetype == "application/pdf"+ assert doc.source_url == "azure://docs/report.pdf"+ assert doc.modified_time == ts+ assert doc.metadata["azure_container"] == "docs"+ assert doc.metadata["azure_blob"] == "report.pdf"+++async def test_get_file_content_owner_based_acl_has_no_principals(patched_factory):+ client = _make_fake_client({"docs": []})+ _patch_download(client, content=b"x", content_type="text/plain")+ patched_factory.return_value = client++ conn = AzureBlobConnector({"container_names": ["docs"]})+ doc = await conn.get_file_content("docs::a.txt")+ assert doc.acl.allowed_principals == []+ assert doc.acl.allowed_users == []+ assert doc.acl.allowed_groups == []+ assert doc.acl.owner is None+++async def test_get_file_content_mime_fallback_to_extension(patched_factory):+ client = _make_fake_client({"docs": []})+ # Generic octet-stream should be ignored in favor of the .pdf extension guess.+ _patch_download(client, content=b"x", content_type="application/octet-stream")+ patched_factory.return_value = client++ conn = AzureBlobConnector({"container_names": ["docs"]})+ doc = await conn.get_file_content("docs::file.pdf")+ assert doc.mimetype == "application/pdf"+++# ---------------------------------------------------------------------------+# authenticate+# ---------------------------------------------------------------------------+++async def test_authenticate_success(patched_factory):+ patched_factory.return_value = _make_fake_client({"docs": []})+ conn = AzureBlobConnector({})+ assert await conn.authenticate() is True+ assert conn.is_authenticated is True+++async def test_authenticate_failure(patched_factory):+ client = MagicMock()+ client.list_containers.side_effect = RuntimeError("bad creds")+ patched_factory.return_value = client+ conn = AzureBlobConnector({})+ assert await conn.authenticate() is False+ assert conn.is_authenticated is False+++# ---------------------------------------------------------------------------+# build_azure_blob_config (support)+# ---------------------------------------------------------------------------+++def test_build_config_connection_string_ok(monkeypatch):+ _clear_azure_env(monkeypatch)+ body = AzureBlobConfigureBody(auth_mode="connection_string", connection_string="cs")+ cfg, err = build_azure_blob_config(body, {})+ assert err is None+ assert cfg == {"auth_mode": "connection_string", "connection_string": "cs"}+++def test_build_config_connection_string_missing(monkeypatch):+ _clear_azure_env(monkeypatch)+ body = AzureBlobConfigureBody(auth_mode="connection_string")+ cfg, err = build_azure_blob_config(body, {})+ assert cfg == {}+ assert "connection_string" in err+++def test_build_config_account_key_ok(monkeypatch):+ _clear_azure_env(monkeypatch)+ body = AzureBlobConfigureBody(+ auth_mode="account_key", account_name="a", account_key="k", endpoint="http://e"+ )+ cfg, err = build_azure_blob_config(body, {})+ assert err is None+ assert cfg["account_name"] == "a"+ assert cfg["account_key"] == "k"+ assert cfg["endpoint_url"] == "http://e"+++def test_build_config_account_key_missing(monkeypatch):+ _clear_azure_env(monkeypatch)+ body = AzureBlobConfigureBody(auth_mode="account_key", account_name="a")+ cfg, err = build_azure_blob_config(body, {})+ assert cfg == {}+ assert "account_name and account_key" in err+++def test_build_config_env_fallback(monkeypatch):+ _clear_azure_env(monkeypatch)+ monkeypatch.setenv(ACCT_NAME, "envname")+ monkeypatch.setenv(ACCT_KEY, "envkey")+ body = AzureBlobConfigureBody(auth_mode="account_key")+ cfg, err = build_azure_blob_config(body, {})+ assert err is None+ assert cfg["account_name"] == "envname"+ assert cfg["account_key"] == "envkey"+++def test_build_config_unknown_mode(monkeypatch):+ _clear_azure_env(monkeypatch)+ body = AzureBlobConfigureBody(auth_mode="sas")+ cfg, err = build_azure_blob_config(body, {})+ assert cfg == {}+ assert "Unknown auth_mode" in err+++def test_build_config_includes_container_names(monkeypatch):+ _clear_azure_env(monkeypatch)+ body = AzureBlobConfigureBody(+ auth_mode="connection_string", connection_string="cs", container_names=["x", "y"]+ )+ cfg, _ = build_azure_blob_config(body, {})+ assert cfg["container_names"] == ["x", "y"]+++# ---------------------------------------------------------------------------+# Webhook / subscription stubs+# ---------------------------------------------------------------------------+++async def test_webhook_stubs_are_noops():+ conn = AzureBlobConnector({})+ assert await conn.setup_subscription() == ""+ assert await conn.handle_webhook({}) == []+ assert conn.extract_webhook_channel_id({}, {}) is None+ assert await conn.cleanup_subscription("sub") is True+++# ---------------------------------------------------------------------------+# azure_blob_test endpoint (non-persisting credential validation + listing)+# ---------------------------------------------------------------------------+++def _test_endpoint_service(existing_connections=None):+ """connector_service mock for azure_blob_test with persistence spies."""+ service = MagicMock()+ cm = service.connection_manager+ cm.list_connections = AsyncMock(return_value=existing_connections or [])+ cm.create_connection = AsyncMock()+ cm.update_connection = AsyncMock()+ cm.get_connection = AsyncMock()+ return service+++def _assert_never_persisted(service):+ service.connection_manager.create_connection.assert_not_called()+ service.connection_manager.update_connection.assert_not_called()+++async def test_azure_blob_test_lists_containers_without_persisting(monkeypatch):+ _clear_azure_env(monkeypatch)+ service = _test_endpoint_service()+ user = MagicMock(user_id="u1")+ body = AzureBlobConfigureBody(auth_mode="connection_string", connection_string="cs")++ with patch.object(az_api, "_list_container_names", return_value=["c1", "c2"]) as lister:+ resp = await az_api.azure_blob_test(body, connector_service=service, user=user)++ assert resp.status_code == 200+ assert json.loads(resp.body) == {"containers": ["c1", "c2"]}+ # Containers listed via the resolved config, and nothing was persisted.+ lister.assert_called_once()+ _assert_never_persisted(service)+++async def test_azure_blob_test_does_not_mutate_existing_connection(monkeypatch):+ """Editing an existing connection's credentials via Test must not save them."""+ _clear_azure_env(monkeypatch)+ existing = MagicMock(connection_id="conn-1", config={"connection_string": "old"})+ service = _test_endpoint_service([existing])+ user = MagicMock(user_id="u1")+ body = AzureBlobConfigureBody(+ auth_mode="connection_string", connection_string="new", connection_id="conn-1"+ )++ with patch.object(az_api, "_list_container_names", return_value=["c1"]):+ resp = await az_api.azure_blob_test(body, connector_service=service, user=user)++ assert resp.status_code == 200+ assert json.loads(resp.body)["containers"] == ["c1"]+ _assert_never_persisted(service)+++async def test_azure_blob_test_invalid_config_returns_400_without_persisting(monkeypatch):+ _clear_azure_env(monkeypatch)+ service = _test_endpoint_service()+ user = MagicMock(user_id="u1")+ # connection_string mode with no string → build_azure_blob_config returns an error.+ body = AzureBlobConfigureBody(auth_mode="connection_string")++ with patch.object(az_api, "_list_container_names") as lister:+ resp = await az_api.azure_blob_test(body, connector_service=service, user=user)++ assert resp.status_code == 400+ assert "connection_string" in json.loads(resp.body)["error"]+ lister.assert_not_called() # never reached the credential test+ _assert_never_persisted(service)+++async def test_azure_blob_test_connection_failure_returns_400_without_persisting(monkeypatch):+ _clear_azure_env(monkeypatch)+ service = _test_endpoint_service()+ user = MagicMock(user_id="u1")+ body = AzureBlobConfigureBody(auth_mode="connection_string", connection_string="cs")++ with patch.object(az_api, "_list_container_names", side_effect=RuntimeError("bad creds")):+ resp = await az_api.azure_blob_test(body, connector_service=service, user=user)++ assert resp.status_code == 400+ assert "Could not connect" in json.loads(resp.body)["error"]+ _assert_never_persisted(service)+++# ---------------------------------------------------------------------------+# azure_blob_container_status endpoint (browse list honors ingestion restriction)+# ---------------------------------------------------------------------------+++def _status_endpoint_deps(config):+ """connector_service + session_manager mocks for azure_blob_container_status."""+ connection = MagicMock(user_id="u1", connector_type="azure_blob", config=config)+ service = MagicMock()+ service.connection_manager.get_connection = AsyncMock(return_value=connection)+ # OpenSearch client raises so doc-count aggregation is skipped (counts → 0).+ session_manager = MagicMock()+ session_manager.get_user_opensearch_client.side_effect = RuntimeError("no opensearch")+ return service, session_manager+++async def test_container_status_restricts_to_allowed_containers():+ """A non-empty container_names allowlist filters out unselected containers."""+ service, session_manager = _status_endpoint_deps({"container_names": ["docs"]})+ user = MagicMock(user_id="u1", jwt_token="t")++ with patch.object(az_api, "_list_container_names", return_value=["docs", "images", "logs"]):+ resp = await az_api.azure_blob_container_status(+ "conn-1",+ connector_service=service,+ session_manager=session_manager,+ user=user,+ )++ names = [c["name"] for c in json.loads(resp.body)["containers"]]+ assert names == ["docs"]+++async def test_container_status_no_restriction_shows_all_containers():+ """An empty/absent allowlist leaves every accessible container browsable."""+ service, session_manager = _status_endpoint_deps({"container_names": []})+ user = MagicMock(user_id="u1", jwt_token="t")++ with patch.object(az_api, "_list_container_names", return_value=["docs", "images"]):+ resp = await az_api.azure_blob_container_status(+ "conn-1",+ connector_service=service,+ session_manager=session_manager,+ user=user,+ )++ names = [c["name"] for c in json.loads(resp.body)["containers"]]+ assert names == ["docs", "images"]+++def _status_deps_with_counts(config, agg_buckets):+ """Deps for azure_blob_container_status with a *working* OpenSearch aggregation.++ Unlike `_status_endpoint_deps` (which raises so counts collapse to 0), this+ returns an AsyncOpenSearch-shaped client whose ``search`` is awaitable and+ yields ``agg_buckets``. Regression guard for the missing-``await`` bug: the+ handler must ``await opensearch_client.search(...)`` — if it doesn't, the+ coroutine's ``.get(...)`` raises, the bare ``except`` swallows it, and every+ count silently reads 0 (which these assertions would then catch).+ """+ connection = MagicMock(user_id="u1", connector_type="azure_blob", config=config)+ service = MagicMock()+ service.connection_manager.get_connection = AsyncMock(return_value=connection)+ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(+ return_value={"aggregations": {"doc_ids": {"buckets": agg_buckets}}}+ )+ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)+ return service, session_manager+++async def test_container_status_reports_ingested_counts():+ """Per-container ingested_count / is_synced reflect the OpenSearch aggregation.++ Guards against the missing-``await`` regression where every container falsely+ reported ``ingested_count: 0, is_synced: false``.+ """+ buckets = [+ {"key": "docs::a.pdf"},+ {"key": "docs::b.pdf"},+ {"key": "images::c.png"},+ ]+ service, session_manager = _status_deps_with_counts({"container_names": []}, buckets)+ user = MagicMock(user_id="u1", jwt_token="t")++ with patch.object(az_api, "_list_container_names", return_value=["docs", "images", "logs"]):+ resp = await az_api.azure_blob_container_status(+ "conn-1",+ connector_service=service,+ session_manager=session_manager,+ user=user,+ )++ by_name = {c["name"]: c for c in json.loads(resp.body)["containers"]}+ assert by_name["docs"]["ingested_count"] == 2+ assert by_name["docs"]["is_synced"] is True+ assert by_name["images"]["ingested_count"] == 1+ assert by_name["images"]["is_synced"] is True+ # No indexed docs → not synced.+ assert by_name["logs"]["ingested_count"] == 0+ assert by_name["logs"]["is_synced"] is False+++async def test_container_status_counts_zero_when_opensearch_unavailable():+ """A failed aggregation degrades gracefully to zero counts (not a 500)."""+ service, session_manager = _status_endpoint_deps({"container_names": []})+ user = MagicMock(user_id="u1", jwt_token="t")++ with patch.object(az_api, "_list_container_names", return_value=["docs"]):+ resp = await az_api.azure_blob_container_status(+ "conn-1",+ connector_service=service,+ session_manager=session_manager,+ user=user,+ )++ entry = json.loads(resp.body)["containers"][0]+ assert entry == {"name": "docs", "ingested_count": 0, "is_synced": False}+++# ---------------------------------------------------------------------------+# azure_blob_list_containers endpoint (also honors the ingestion restriction)+# ---------------------------------------------------------------------------+++def _list_endpoint_service(config):+ """connector_service mock for azure_blob_list_containers."""+ connection = MagicMock(user_id="u1", connector_type="azure_blob", config=config)+ service = MagicMock()+ service.connection_manager.get_connection = AsyncMock(return_value=connection)+ return service+++async def test_list_containers_restricts_to_allowed_containers():+ """A non-empty container_names allowlist filters out unselected containers."""+ service = _list_endpoint_service({"container_names": ["docs"]})+ user = MagicMock(user_id="u1")++ with patch.object(az_api, "_list_container_names", return_value=["docs", "images", "logs"]):+ resp = await az_api.azure_blob_list_containers(+ "conn-1", connector_service=service, user=user+ )++ assert json.loads(resp.body)["containers"] == ["docs"]+++async def test_list_containers_no_restriction_shows_all_containers():+ """An empty/absent allowlist leaves every accessible container listed."""+ service = _list_endpoint_service({"container_names": []})+ user = MagicMock(user_id="u1")++ with patch.object(az_api, "_list_container_names", return_value=["docs", "images"]):+ resp = await az_api.azure_blob_list_containers(+ "conn-1", connector_service=service, user=user+ )++ assert json.loads(resp.body)["containers"] == ["docs", "images"]diff --git a/tests/unit/services/test_connector_access_service.py b/tests/unit/services/test_connector_access_service.pyindex 89d6e438a..ac60d5f12 100644--- a/tests/unit/services/test_connector_access_service.py+++ b/tests/unit/services/test_connector_access_service.py@@ -147,6 +147,19 @@ def test_governable_connector_types_excludes_buckets_in_saas(monkeypatch):assert "sharepoint" in governableassert "aws_s3" not in governableassert "ibm_cos" not in governable+ assert "azure_blob" not in governable+++def test_governable_connector_types_includes_buckets_with_ibm_auth(monkeypatch):+ monkeypatch.setenv("OPENRAG_RUN_MODE", "saas")+ monkeypatch.setattr("config.settings.IBM_AUTH_ENABLED", True)++ governable = governable_connector_types()++ # With IBM auth on, bucket connectors (incl. Azure Blob) are governable.+ assert "azure_blob" in governable+ assert "aws_s3" in governable+ assert "ibm_cos" in governable@pytest.mark.asynciodiff --git a/tests/unit/test_startup_load_connections.py b/tests/unit/test_startup_load_connections.pynew file mode 100644index 000000000..a99654579--- /dev/null+++ b/tests/unit/test_startup_load_connections.py@@ -0,0 +1,44 @@+"""Regression: persisted connector connections must reload at startup for+dev bucket connectors even in no-auth mode.++Bug: `initialize_services()` skipped `connector_service.initialize()` (which+calls `load_connections()`) whenever no-auth mode was active, on the assumption+that connectors require OAuth. But the Azure Blob bucket connector works in+no-auth dev mode via OPENRAG_DEV_AZURE_BLOB=true. Skipping the load left the+in-memory connections dict empty after a restart, so a saved connection+silently reverted to the "Connect" state until re-saved.++`_should_load_persisted_connections()` encodes the fixed decision; these tests+pin its truth table.+"""++import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++from app import container # noqa: E402+++ ("no_auth", "dev_azure", "expected"),+ [+ # Authed deployments always load (today's behaviour, unchanged).+ (False, False, True),+ (False, True, True),+ # No-auth without any dev bucket connector: still skipped.+ (True, False, False),+ # The regression: no-auth + dev Azure Blob must load so the saved+ # connection survives a restart.+ (True, True, True),+ ],+)+def test_should_load_persisted_connections(monkeypatch, no_auth, dev_azure, expected):+ monkeypatch.setattr(container, "is_no_auth_mode", lambda: no_auth)+ monkeypatch.setattr(container, "is_dev_azure_blob_enabled", lambda: dev_azure)+ assert container._should_load_persisted_connections() is expecteddiff --git a/uv.lock b/uv.lockindex f5eff5389..fab2a305a 100644--- a/uv.lock+++ b/uv.lock@@ -295,6 +295,34 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/fb/95/adcb68e20c34162e9135f370d6e31737719c2b6f94bc953fe7ed1f10fe21/authlib-1.7.2-py2.py3-none-any.whl", hash = "sha256:3e1faedc9d87e7d56a164eca3ccb6ace0d61b94abe83e92242f8dc8bba9b4a9f", size = 259548, upload-time = "2026-05-06T08:10:21.436Z" },]+[[package]]+name = "azure-core"+version = "1.41.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "requests" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" },+]++[[package]]+name = "azure-storage-blob"+version = "12.30.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "azure-core" },+ { name = "cryptography" },+ { name = "isodate" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/3b/48/84a820d898267f662b5c06f7cd76fdb8a9e272b44aa9376cef3ec0f6a294/azure_storage_blob-12.30.0.tar.gz", hash = "sha256:2cd74d4d5731e5eb6b8d5c5056ee115a5e88f8fdf22517b739836fda685018be", size = 618229, upload-time = "2026-06-08T11:45:35.575Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/5e/0b/e106f0fd7fa785867d9ffcc47dc9e6237c0e58f51058473b777487a98edc/azure_storage_blob-12.30.0-py3-none-any.whl", hash = "sha256:d415ac50b67a8da6b3ae7e9f1014b1b55cd7aafa0b8d4ca9b380568dc7360423", size = 435610, upload-time = "2026-06-08T11:45:37.213Z" },+]+[[package]]name = "beartype"version = "0.22.9"@@ -1207,6 +1235,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },]+[[package]]+name = "isodate"+version = "0.7.2"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" },+]+[[package]]name = "jaraco-classes"version = "3.4.0"@@ -1841,6 +1878,7 @@ dependencies = [{ name = "alembic" },{ name = "asyncpg" },{ name = "authlib" },+ { name = "azure-storage-blob" },{ name = "boto3" },{ name = "cachetools" },{ name = "cryptography" },@@ -1900,6 +1938,7 @@ requires-dist = [{ name = "asyncpg", specifier = ">=0.30.0" },{ name = "authlib", specifier = ">=1.7.1" },{ name = "authlib", specifier = ">=1.7.2,<2.0.0" },+ { name = "azure-storage-blob", specifier = ">=12.20.0" },{ name = "boto3", specifier = ">=1.35.0" },{ name = "cachetools", specifier = ">=5.5.0" },{ name = "cryptography", specifier = ">=46.0.6" },