Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: Implement support for a new Azure Blob connector (#1974)

Mike Pawlowski77bd3d692239
diff --git a/.env.example b/.env.example
index 97a3f47a4..6917262d0 100644
--- a/.env.example
+++ b/.env.example
@@ -163,6 +163,35 @@ MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET=
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
+# --- Azure Blob Storage connector (Enterprise/SaaS) ------------------------
+# Enterprise/SaaS-only bucket connector, gated by IBM_AUTH_ENABLED (like the
+# AWS S3 / IBM COS connectors). Credentials are normally entered per-connection
+# in the UI; these env vars are OPTIONAL fallbacks / defaults for the config
+# dialog. Two auth modes are supported: a connection string, OR account name +
+# key (+ optional custom blob endpoint).
+#
+# Local dev / Azurite testing: set OPENRAG_DEV_AZURE_BLOB=true to enable the
+# connector without IBM_AUTH_ENABLED. NEVER use in production.
+# OPENRAG_DEV_AZURE_BLOB=false
+#
+# Connection-string mode (covers real Azure accounts and the Azurite emulator):
+# - Real account: copy from Azure Portal -> Storage account -> Access keys
+# - Azurite (local dev): AZURE_STORAGE_CONNECTION_STRING="UseDevelopmentStorage=true"
+# AZURE_STORAGE_CONNECTION_STRING=
+#
+# Account-key mode (set account name + key; endpoint optional):
+# - Azurite (host-side backend):
+# - AZURE_STORAGE_ACCOUNT_NAME=devstoreaccount1
+# - AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+# - AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1
+# - Azurite (in-compose backend):
+# - AZURE_STORAGE_ACCOUNT_NAME=devstoreaccount1
+# - AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+# - AZURE_STORAGE_ENDPOINT=http://azurite:10000/devstoreaccount1
+# AZURE_STORAGE_ACCOUNT_NAME=
+# AZURE_STORAGE_ACCOUNT_KEY=
+# AZURE_STORAGE_ENDPOINT=
+
# OPTIONAL: dns routable from google (etc.) to handle continous ingest (something like ngrok works). This enables continous ingestion
WEBHOOK_BASE_URL=
diff --git a/Makefile b/Makefile
index 3d702e4c5..2eca35150 100644
--- a/Makefile
+++ b/Makefile
@@ -96,6 +96,7 @@ endef
######################
.PHONY: help check_tools help_docker help_dev help_test help_local help_utils help_operator \
dev dev-cpu dev-local dev-local-cpu dev-local-build-lf dev-local-build-lf-cpu stop clean build logs \
+ azurite-up azurite-down \
shell-backend shell-frontend install \
test test-unit test-integration test-ci test-ci-local test-ci-suite test-sdk test-os-jwt lint \
ci-build-images ci-save-images \
@@ -345,6 +346,8 @@ help_local: ## Show local development commands
@echo " $(PURPLE)make frontend$(NC) - Run frontend locally"
@echo " $(PURPLE)make docling$(NC) - Start docling-serve for document processing"
@echo " $(PURPLE)make docling-stop$(NC) - Stop docling-serve"
+ @echo " $(PURPLE)make azurite-up$(NC) - Start Azurite (local Azure Blob emulator) for connector testing"
+ @echo " $(PURPLE)make azurite-down$(NC) - Stop Azurite emulator"
@echo ''
@echo "$(PURPLE)Installation:$(NC)"
@echo " $(PURPLE)make install$(NC) - Install all dependencies"
@@ -673,6 +676,16 @@ docling-stop: ## Stop docling-serve
@uv run python scripts/docling_ctl.py stop
@echo "$(PURPLE)Docling-serve stopped.$(NC)"
+azurite-up: ## Start Azurite (local Azure Blob emulator) for connector testing
+ @echo "$(YELLOW)Starting Azurite (Azure Blob emulator)...$(NC)"
+ $(COMPOSE_CMD) --profile azurite up -d azurite
+ @echo "$(PURPLE)Azurite started on http://localhost:10000 (account: devstoreaccount1).$(NC)"
+
+azurite-down: ## Stop Azurite emulator
+ @echo "$(YELLOW)Stopping Azurite...$(NC)"
+ $(COMPOSE_CMD) --profile azurite stop azurite
+ @echo "$(PURPLE)Azurite stopped.$(NC)"
+
######################
# INSTALLATION
######################
diff --git a/docker-compose.yml b/docker-compose.yml
index 1cc1efc74..3a22a38dc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -219,5 +219,28 @@ services:
# - DEFAULT_FOLDER_NAME=OpenRAG
- HIDE_GETTING_STARTED_PROGRESS=true
+ # Azurite: local Azure Storage emulator for manually testing the Azure Blob
+ # connector without a paid Azure account. Opt-in only — gated behind the
+ # "azurite" compose profile so it never starts with the normal stack.
+ # Start it with `make azurite-up` (or `docker compose --profile azurite up -d azurite`).
+ # Connect the connector with AZURE_STORAGE_CONNECTION_STRING="UseDevelopmentStorage=true"
+ # (from inside the compose network use host "azurite": the well-known dev
+ # endpoint http://azurite:10000/devstoreaccount1).
+ azurite:
+ image: mcr.microsoft.com/azure-storage/azurite:latest
+ container_name: azurite
+ profiles: ["azurite"]
+ # --skipApiVersionCheck: the azure-storage-blob SDK may negotiate a newer
+ # Storage API version than the pinned Azurite image supports; skip the check
+ # so the emulator accepts requests (safe for local dev only).
+ command: "azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0 --location /data --skipApiVersionCheck"
+ ports:
+ - "10000:10000" # Blob
+ - "10001:10001" # Queue
+ - "10002:10002" # Table
+ volumes:
+ - azurite-data:/data
+
volumes:
opensearch-data:
+ azurite-data:
diff --git a/enhancements/__init__.py b/enhancements/__init__.py
index b590503c4..5c86d2f15 100644
--- a/enhancements/__init__.py
+++ b/enhancements/__init__.py
@@ -8,6 +8,7 @@ Strip the imports below to ship a bare OSS build.
from connectors.base import BaseConnector
+from .connectors.azure_blob import AzureBlobConnector
from .connectors.ibm_cos import IBMCOSConnector
-ADDITIONAL_CONNECTORS: list[type[BaseConnector]] = [IBMCOSConnector]
+ADDITIONAL_CONNECTORS: list[type[BaseConnector]] = [IBMCOSConnector, AzureBlobConnector]
diff --git a/enhancements/connectors/azure_blob/__init__.py b/enhancements/connectors/azure_blob/__init__.py
new file mode 100644
index 000000000..b2002733b
--- /dev/null
+++ b/enhancements/connectors/azure_blob/__init__.py
@@ -0,0 +1,19 @@
+from .api import (
+ azure_blob_configure,
+ azure_blob_container_status,
+ azure_blob_defaults,
+ azure_blob_list_containers,
+ azure_blob_test,
+)
+from .connector import AzureBlobConnector
+from .models import AzureBlobConfigureBody
+
+__all__ = [
+ "AzureBlobConnector",
+ "AzureBlobConfigureBody",
+ "azure_blob_defaults",
+ "azure_blob_test",
+ "azure_blob_configure",
+ "azure_blob_list_containers",
+ "azure_blob_container_status",
+]
diff --git a/enhancements/connectors/azure_blob/api.py b/enhancements/connectors/azure_blob/api.py
new file mode 100644
index 000000000..37d44fd98
--- /dev/null
+++ b/enhancements/connectors/azure_blob/api.py
@@ -0,0 +1,246 @@
+"""FastAPI route handlers for Azure Blob-specific endpoints."""
+
+import asyncio
+import os
+
+from fastapi import Depends
+from fastapi.responses import JSONResponse
+
+from config.settings import get_index_name
+from dependencies import get_connector_service, get_current_user, get_session_manager
+from session_manager import User
+from utils.logging_config import get_logger
+
+from .auth import create_blob_service_client
+from .models import AzureBlobConfigureBody
+from .support import build_azure_blob_config
+
+logger = get_logger(__name__)
+
+
+def _list_container_names(config: dict) -> list[str]:
+ """List container names for a resolved Azure Blob config (sync helper)."""
+ client = create_blob_service_client(config)
+ return [c.name for c in client.list_containers()]
+
+
+async def azure_blob_defaults(
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """Return current Azure Blob env-var defaults for pre-filling the config dialog.
+
+ Sensitive values (connection string, account key) are masked — only whether
+ they are set is returned, not the actual values.
+ """
+ connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING", "")
+ account_name = os.getenv("AZURE_STORAGE_ACCOUNT_NAME", "")
+ account_key = os.getenv("AZURE_STORAGE_ACCOUNT_KEY", "")
+ endpoint = os.getenv("AZURE_STORAGE_ENDPOINT", "")
+
+ connections = await connector_service.connection_manager.list_connections(
+ user_id=user.user_id, connector_type="azure_blob"
+ )
+ conn_config = connections[0].config or {} if connections else {}
+
+ def _pick(conn_key, env_val):
+ """Prefer connection config value over env var."""
+ return conn_config.get(conn_key) or env_val
+
+ has_conn_str = bool(connection_string or conn_config.get("connection_string"))
+ has_account_key = bool(account_key or conn_config.get("account_key"))
+
+ return JSONResponse(
+ {
+ "connection_string_set": has_conn_str,
+ "account_name": _pick("account_name", account_name),
+ "account_key_set": has_account_key,
+ "endpoint": _pick("endpoint_url", endpoint),
+ "auth_mode": conn_config.get(
+ "auth_mode",
+ "account_key" if (has_account_key and not has_conn_str) else "connection_string",
+ ),
+ "container_names": conn_config.get("container_names", []),
+ "connection_id": connections[0].connection_id if connections else None,
+ }
+ )
+
+
+async def azure_blob_test(
+ body: AzureBlobConfigureBody,
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """Validate Azure Blob credentials and list containers WITHOUT persisting.
+
+ Backs the settings dialog's "Test Connection" action so testing or refreshing
+ credentials never creates or mutates a saved connection (and so the stored
+ container selection is never clobbered). Persistence happens only in
+ ``azure_blob_configure`` on the final save. Credential resolution mirrors
+ ``azure_blob_configure`` (request body → env → existing connection config).
+ """
+ existing_connections = await connector_service.connection_manager.list_connections(
+ user_id=user.user_id, connector_type="azure_blob"
+ )
+ existing_config = existing_connections[0].config if existing_connections else {}
+
+ conn_config, error = build_azure_blob_config(body, existing_config)
+ if error:
+ return JSONResponse({"error": error}, status_code=400)
+
+ # The azure-storage-blob SDK is sync; offload to keep the event loop free.
+ try:
+ containers = await asyncio.to_thread(_list_container_names, conn_config)
+ except Exception:
+ logger.exception("Failed to connect to Azure Blob during credential test.")
+ return JSONResponse(
+ {"error": "Could not connect to Azure Blob with the provided configuration."},
+ status_code=400,
+ )
+
+ return JSONResponse({"containers": containers})
+
+
+async def azure_blob_configure(
+ body: AzureBlobConfigureBody,
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """Create or update an Azure Blob connection with explicit credentials.
+
+ Tests the credentials by listing containers, then persists the connection.
+ Credentials are stored in the connection config dict (encrypted at rest) so
+ the connector works even without system-level env vars.
+ """
+ existing_connections = await connector_service.connection_manager.list_connections(
+ user_id=user.user_id, connector_type="azure_blob"
+ )
+ existing_config = existing_connections[0].config if existing_connections else {}
+
+ conn_config, error = build_azure_blob_config(body, existing_config)
+ if error:
+ return JSONResponse({"error": error}, status_code=400)
+
+ # Test credentials by listing containers. The azure-storage-blob SDK is sync,
+ # so offload to a worker thread to keep the event loop responsive.
+ try:
+ await asyncio.to_thread(_list_container_names, conn_config)
+ except Exception:
+ logger.exception("Failed to connect to Azure Blob during credential test.")
+ return JSONResponse(
+ {"error": "Could not connect to Azure Blob with the provided configuration."},
+ status_code=400,
+ )
+
+ # Persist: update existing connection or create a new one.
+ if body.connection_id:
+ existing = await connector_service.connection_manager.get_connection(body.connection_id)
+ if existing and existing.user_id == user.user_id:
+ await connector_service.connection_manager.update_connection(
+ connection_id=body.connection_id,
+ config=conn_config,
+ )
+ connector_service.connection_manager.active_connectors.pop(body.connection_id, None)
+ return JSONResponse({"connection_id": body.connection_id, "status": "connected"})
+
+ connection_id = await connector_service.connection_manager.create_connection(
+ connector_type="azure_blob",
+ name="Azure Blob Storage",
+ config=conn_config,
+ user_id=user.user_id,
+ )
+ return JSONResponse({"connection_id": connection_id, "status": "connected"})
+
+
+async def azure_blob_list_containers(
+ connection_id: str,
+ connector_service=Depends(get_connector_service),
+ user: User = Depends(get_current_user),
+):
+ """List containers for an Azure Blob connection, honoring the ingestion restriction.
+
+ When the connection has a non-empty container_names allowlist, only those
+ containers are returned; otherwise all accessible containers are listed.
+ """
+ connection = await connector_service.connection_manager.get_connection(connection_id)
+ if not connection or connection.user_id != user.user_id:
+ return JSONResponse({"error": "Connection not found"}, status_code=404)
+ if connection.connector_type != "azure_blob":
+ return JSONResponse({"error": "Not an Azure Blob connection"}, status_code=400)
+
+ try:
+ containers = await asyncio.to_thread(_list_container_names, connection.config)
+ allowed_containers = connection.config.get("container_names") or []
+ if allowed_containers:
+ allowed_set = set(allowed_containers)
+ containers = [c for c in containers if c in allowed_set]
+ return JSONResponse({"containers": containers})
+ except Exception:
+ logger.exception("Failed to list Azure Blob containers for connection %s", connection_id)
+ return JSONResponse({"error": "Failed to list containers"}, status_code=500)
+
+
+async def azure_blob_container_status(
+ connection_id: str,
+ connector_service=Depends(get_connector_service),
+ session_manager=Depends(get_session_manager),
+ user: User = Depends(get_current_user),
+):
+ """Return all containers for an Azure Blob connection with ingestion status.
+
+ Each entry includes the container name, whether it has been ingested
+ (is_synced), and the count of indexed documents from that container.
+ """
+ connection = await connector_service.connection_manager.get_connection(connection_id)
+ if not connection or connection.user_id != user.user_id:
+ return JSONResponse({"error": "Connection not found"}, status_code=404)
+ if connection.connector_type != "azure_blob":
+ return JSONResponse({"error": "Not an Azure Blob connection"}, status_code=400)
+
+ # 1. List containers, honoring the saved ingestion restriction. When the
+ # connection has a non-empty container_names allowlist, only those
+ # containers are browsable; otherwise all accessible containers are shown.
+ try:
+ all_containers = await asyncio.to_thread(_list_container_names, connection.config)
+ except Exception:
+ logger.exception("Failed to list Azure Blob containers for connection %s", connection_id)
+ return JSONResponse({"error": "Failed to list containers"}, status_code=500)
+
+ allowed_containers = connection.config.get("container_names") or []
+ if allowed_containers:
+ allowed_set = set(allowed_containers)
+ all_containers = [c for c in all_containers if c in allowed_set]
+
+ # 2. Count indexed documents per container from OpenSearch.
+ ingested_counts: dict = {}
+ try:
+ opensearch_client = session_manager.get_user_opensearch_client(user.user_id, user.jwt_token)
+ query_body = {
+ "size": 0,
+ "query": {"term": {"connector_type": "azure_blob"}},
+ "aggs": {"doc_ids": {"terms": {"field": "document_id", "size": 50000}}},
+ }
+ index_name = get_index_name()
+ os_resp = await opensearch_client.search(index=index_name, body=query_body)
+ for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):
+ doc_id = bucket_entry["key"]
+ if "::" in doc_id:
+ container_name = doc_id.split("::")[0]
+ ingested_counts[container_name] = ingested_counts.get(container_name, 0) + 1
+ except Exception:
+ logger.warning(
+ "Failed to aggregate Azure Blob ingestion counts for connection %s; "
+ "container status will show zero counts",
+ connection_id,
+ exc_info=True,
+ ) # OpenSearch unavailable / query failed — show zero counts
+
+ result = [
+ {
+ "name": container,
+ "ingested_count": ingested_counts.get(container, 0),
+ "is_synced": ingested_counts.get(container, 0) > 0,
+ }
+ for container in all_containers
+ ]
+ return JSONResponse({"containers": result})
diff --git a/enhancements/connectors/azure_blob/auth.py b/enhancements/connectors/azure_blob/auth.py
new file mode 100644
index 000000000..a30918123
--- /dev/null
+++ b/enhancements/connectors/azure_blob/auth.py
@@ -0,0 +1,144 @@
+"""Azure Blob Storage authentication and client factory.
+
+Supports two credential modes, both of which work against real Azure Storage
+accounts and the local Azurite emulator:
+
+- ``connection_string``: a full Azure Storage connection string
+ (e.g. from the Azure Portal, or ``UseDevelopmentStorage=true`` for Azurite).
+- ``account_key``: an account name + shared key, with an optional custom
+ ``endpoint_url`` (used to point at Azurite or a sovereign cloud).
+
+Resolution order for each credential: connector config dict → environment
+variable. The sync ``azure-storage-blob`` SDK is used; callers offload blocking
+operations onto a worker thread via ``asyncio.to_thread``.
+"""
+
+import os
+from typing import Any
+
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+# Public Azure Blob endpoint template (account_key mode, no custom endpoint).
+_DEFAULT_BLOB_ENDPOINT = "https://{account}.blob.core.windows.net"
+
+
+def _resolve_credentials(config: dict[str, Any]) -> dict[str, Any]:
+ """Resolve Azure Blob credentials from config dict → environment fallback.
+
+ Returns a dict with the resolved values needed to build a BlobServiceClient.
+ Does not raise — validation happens in the builder so callers get a single,
+ mode-aware error message.
+ """
+ auth_mode = config.get("auth_mode", "connection_string")
+
+ connection_string = config.get("connection_string") or os.getenv(
+ "AZURE_STORAGE_CONNECTION_STRING"
+ )
+ account_name = config.get("account_name") or os.getenv("AZURE_STORAGE_ACCOUNT_NAME")
+ account_key = config.get("account_key") or os.getenv("AZURE_STORAGE_ACCOUNT_KEY")
+ # Optional custom blob endpoint (Azurite / sovereign clouds). For Azurite the
+ # account-style endpoint is e.g. http://127.0.0.1:10000/devstoreaccount1.
+ endpoint_url = config.get("endpoint_url") or os.getenv("AZURE_STORAGE_ENDPOINT") or None
+
+ return {
+ "auth_mode": auth_mode,
+ "connection_string": connection_string,
+ "account_name": account_name,
+ "account_key": account_key,
+ "endpoint_url": endpoint_url,
+ }
+
+
+def _account_name_from_connection_string(connection_string: str) -> str | None:
+ """Best-effort parse of the storage account name from a connection string.
+
+ Handles the Azurite shorthand (``UseDevelopmentStorage=true`` →
+ ``devstoreaccount1``), standard ``AccountName=...`` strings, and SAS-style
+ strings that only carry a ``BlobEndpoint``. Returns None if it can't be
+ determined. Never raises — callers fall back to a stable label.
+ """
+ pairs = (p.split("=", 1) for p in connection_string.split(";") if "=" in p)
+ parts = {k.strip().lower(): v.strip() for k, v in pairs}
+
+ if parts.get("usedevelopmentstorage", "").lower() == "true":
+ return "devstoreaccount1"
+ if parts.get("accountname"):
+ return parts["accountname"]
+
+ # SAS-style strings expose the account only via the blob endpoint host/path,
+ # e.g. https://myacct.blob.core.windows.net or http://host:10000/devstoreaccount1.
+ endpoint = parts.get("blobendpoint")
+ if endpoint:
+ from urllib.parse import urlparse
+
+ parsed = urlparse(endpoint)
+ path = parsed.path.strip("/")
+ if path: # Azurite path-style: account is the first path segment
+ return path.split("/")[0]
+ host = parsed.hostname or ""
+ if host: # production host-style: account is the first label
+ return host.split(".")[0]
+ return None
+
+
+def account_name_from_config(config: dict[str, Any]) -> str | None:
+ """Resolve the storage account name from config (both auth modes).
+
+ Used by the connector's ``get_client_id`` so status checks have a stable,
+ non-raising identifier even in connection-string mode (where ``account_name``
+ is implicit in the string rather than stored separately).
+ """
+ creds = _resolve_credentials(config)
+ if creds["account_name"]:
+ return creds["account_name"]
+ if creds["connection_string"]:
+ return _account_name_from_connection_string(creds["connection_string"])
+ return None
+
+
+def create_blob_service_client(config: dict[str, Any]):
+ """Return a sync ``BlobServiceClient`` for the configured auth mode.
+
+ Args:
+ config: Connector config dict (see module docstring for keys).
+
+ Raises:
+ ImportError: If the ``azure-storage-blob`` package is not installed.
+ ValueError: If required credentials for the selected mode are missing.
+ """
+ try:
+ from azure.storage.blob import BlobServiceClient
+ except ImportError as exc:
+ raise ImportError(
+ "azure-storage-blob is required for the Azure Blob connector. "
+ "Install it with: pip install azure-storage-blob"
+ ) from exc
+
+ creds = _resolve_credentials(config)
+ auth_mode = creds["auth_mode"]
+
+ if auth_mode == "connection_string":
+ if not creds["connection_string"]:
+ raise ValueError(
+ "Connection string mode requires a connection string. Provide "
+ "'connection_string' in the connector config or set "
+ "AZURE_STORAGE_CONNECTION_STRING."
+ )
+ logger.debug("Creating Azure Blob client from connection string")
+ return BlobServiceClient.from_connection_string(creds["connection_string"])
+
+ # account_key mode
+ if not (creds["account_name"] and creds["account_key"]):
+ raise ValueError(
+ "Account key mode requires account_name and account_key. Provide them "
+ "in the connector config or set AZURE_STORAGE_ACCOUNT_NAME / "
+ "AZURE_STORAGE_ACCOUNT_KEY."
+ )
+
+ account_url = creds["endpoint_url"] or _DEFAULT_BLOB_ENDPOINT.format(
+ account=creds["account_name"]
+ )
+ logger.debug("Creating Azure Blob client with account key for %s", account_url)
+ return BlobServiceClient(account_url=account_url, credential=creds["account_key"])
diff --git a/enhancements/connectors/azure_blob/connector.py b/enhancements/connectors/azure_blob/connector.py
new file mode 100644
index 000000000..c97eda53e
--- /dev/null
+++ b/enhancements/connectors/azure_blob/connector.py
@@ -0,0 +1,369 @@
+"""Azure Blob Storage connector for OpenRAG.
+
+Enterprise/SaaS-only ``bucket``-kind connector, gated by ``IBM_AUTH_ENABLED``
+(or ``OPENRAG_DEV_AZURE_BLOB=true`` for local dev/Azurite testing). Uses the
+official sync ``azure-storage-blob`` SDK; all
+blocking SDK calls are offloaded onto a worker thread with ``asyncio.to_thread``
+so the event loop stays responsive while staying consistent with the cached
+client lifecycle used by the AWS S3 / IBM COS connectors.
+
+DLS (v1): owner-based. The connector returns an empty-principal ``DocumentACL``;
+``service.process_connector_document`` assigns ``owner`` to the ingesting
+OpenRAG user. Container→group principal mapping is a planned fast-follow.
+"""
+
+import asyncio
+import mimetypes
+import os
+from datetime import UTC, datetime
+from posixpath import basename
+from typing import Any
+
+from config.settings import IBM_AUTH_ENABLED, is_dev_azure_blob_enabled
+from connectors.base import BaseConnector, ConnectorDocument, DocumentACL
+from utils.logging_config import get_logger
+
+from .auth import account_name_from_config, create_blob_service_client
+
+logger = get_logger(__name__)
+
+# Separator used in composite file IDs: "<container>::<blob>"
+_ID_SEPARATOR = "::"
+
+
+def _make_file_id(container: str, blob: str) -> str:
+ return f"{container}{_ID_SEPARATOR}{blob}"
+
+
+def _split_file_id(file_id: str) -> tuple[str, str]:
+ """Split a composite file ID into (container, blob). Raises ValueError if invalid."""
+ if _ID_SEPARATOR not in file_id:
+ raise ValueError(f"Invalid Azure Blob file ID (missing separator): {file_id!r}")
+ container, blob = file_id.split(_ID_SEPARATOR, 1)
+ return container, blob
+
+
+class AzureBlobConnector(BaseConnector):
+ """Connector for Azure Blob Storage.
+
+ Supports connection-string and account-name/key credential modes. Both work
+ against real Azure accounts and the Azurite emulator. Credentials are read
+ from the connector config dict first, then from environment variables.
+
+ Config dict keys:
+ auth_mode (str): "connection_string" (default) or "account_key".
+ connection_string (str): Overrides AZURE_STORAGE_CONNECTION_STRING.
+ account_name (str): Overrides AZURE_STORAGE_ACCOUNT_NAME.
+ account_key (str): Overrides AZURE_STORAGE_ACCOUNT_KEY.
+ endpoint_url (str): Optional custom blob endpoint (Azurite / sovereign).
+ container_names (list[str]): Containers to ingest from. If empty, all
+ accessible containers are used.
+ prefix (str): Optional blob name prefix filter.
+ connection_id (str): Connection identifier used for logging.
+ """
+
+ CONNECTOR_TYPE = "azure_blob"
+ CONNECTOR_KIND = "bucket"
+ CONNECTOR_NAME = "Azure Blob Storage"
+ CONNECTOR_DESCRIPTION = "Add knowledge from Azure Blob Storage"
+ CONNECTOR_ICON = "azure-blob"
+ # connection_string / account_key are Azure-specific and must be encrypted at rest.
+ SECRET_CONFIG_KEYS = ("connection_string", "account_key")
+
+ # BaseConnector uses these for the default env-availability probe; the
+ # bucket-kind override below makes availability hinge on IBM_AUTH_ENABLED
+ # (or OPENRAG_DEV_AZURE_BLOB for local dev).
+ CLIENT_ID_ENV_VAR = "AZURE_STORAGE_ACCOUNT_NAME"
+ CLIENT_SECRET_ENV_VAR = "AZURE_STORAGE_ACCOUNT_KEY"
+
+ @classmethod
+ def is_available(cls, manager, user_id=None) -> bool:
+ # Gated by feature flag like the other bucket connectors (aws_s3, ibm_cos).
+ # OPENRAG_DEV_AZURE_BLOB=true bypasses the IBM_AUTH_ENABLED requirement for
+ # local dev testing (e.g. against Azurite). Never use in production.
+ return IBM_AUTH_ENABLED or is_dev_azure_blob_enabled()
+
+ @classmethod
+ def register_routes(cls, app) -> None:
+ from .api import (
+ azure_blob_configure,
+ azure_blob_container_status,
+ azure_blob_defaults,
+ azure_blob_list_containers,
+ azure_blob_test,
+ )
+
+ # Registered before generic /{connector_type}/... to avoid shadowing.
+ app.add_api_route(
+ "/connectors/azure_blob/defaults",
+ azure_blob_defaults,
+ methods=["GET"],
+ tags=["internal"],
+ )
+ # Non-persisting credential validation + container listing (Test Connection).
+ app.add_api_route(
+ "/connectors/azure_blob/test",
+ azure_blob_test,
+ methods=["POST"],
+ tags=["internal"],
+ )
+ app.add_api_route(
+ "/connectors/azure_blob/configure",
+ azure_blob_configure,
+ methods=["POST"],
+ tags=["internal"],
+ )
+ app.add_api_route(
+ "/connectors/azure_blob/{connection_id}/containers",
+ azure_blob_list_containers,
+ methods=["GET"],
+ tags=["internal"],
+ )
+ app.add_api_route(
+ "/connectors/azure_blob/{connection_id}/container-status",
+ azure_blob_container_status,
+ methods=["GET"],
+ tags=["internal"],
+ )
+
+ def get_client_id(self) -> str:
+ """Return the storage account name; used by status checks / availability probe.
+
+ Must not raise in connection-string mode: the account name is implicit in
+ the string rather than stored separately, so the status endpoint
+ (api/connectors.py) would otherwise mark an authenticated connection as
+ unauthenticated and the UI would keep showing "Connect". Falls back to a
+ stable label when the account can't be parsed (e.g. SAS-only strings).
+ """
+ val = account_name_from_config(self.config)
+ if val:
+ return val
+ if self.config.get("auth_mode", "connection_string") == "connection_string" and (
+ self.config.get("connection_string") or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+ ):
+ return "azure_blob"
+ raise ValueError(
+ "Azure Blob credentials not set. Provide 'account_name' (account_key mode) "
+ "or a 'connection_string'."
+ )
+
+ def get_client_secret(self) -> str:
+ """Return account key / connection string; used by the availability probe."""
+ val = (
+ self.config.get("account_key")
+ or self.config.get("connection_string")
+ or os.getenv("AZURE_STORAGE_ACCOUNT_KEY")
+ or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+ )
+ if val:
+ return val
+ raise ValueError(
+ "Azure Blob credentials not set. Provide 'account_key' (account_key mode) "
+ "or a 'connection_string'."
+ )
+
+ def __init__(self, config: dict[str, Any]):
+ if config is None:
+ config = {}
+ super().__init__(config)
+
+ self.container_names: list[str] = config.get("container_names") or []
+ self.prefix: str = config.get("prefix", "")
+ self.connection_id: str = config.get("connection_id", "default")
+
+ self._client = None # Lazy-initialised, cached BlobServiceClient
+
+ @property
+ def bucket_names(self) -> list[str]:
+ """Alias of ``container_names`` for the generic ``bucket``-kind sync path.
+
+ The connector API (``api/connectors.py``) drives per-bucket sync and the
+ file-picker bucket filter through ``connector.bucket_names`` (the term
+ S3/IBM COS use). Azure's domain name for the same concept is a
+ *container*, so we expose ``bucket_names`` as a read/write alias to
+ satisfy that contract without renaming the Azure-accurate attribute.
+ """
+ return self.container_names
+
+ @bucket_names.setter
+ def bucket_names(self, value: list[str]) -> None:
+ self.container_names = value or []
+
+ def _get_client(self):
+ """Return (and cache) the BlobServiceClient for this connection."""
+ if self._client is None:
+ self._client = create_blob_service_client(self.config)
+ return self._client
+
+ # ------------------------------------------------------------------
+ # Internal sync helpers (run via asyncio.to_thread)
+ # ------------------------------------------------------------------
+
+ def _list_container_names_sync(self) -> list[str]:
+ client = self._get_client()
+ return [c.name for c in client.list_containers()]
+
+ def _resolve_container_names_sync(self) -> list[str]:
+ """Return configured container names, or auto-discover all accessible ones."""
+ if self.container_names:
+ return self.container_names
+ try:
+ containers = self._list_container_names_sync()
+ logger.debug("Azure Blob auto-discovered %d container(s)", len(containers))
+ return containers
+ except Exception as exc:
+ logger.warning("Azure Blob could not auto-discover containers: %s", exc)
+ return []
+
+ def _list_files_sync(self, max_files: int | None) -> list[dict[str, Any]]:
+ client = self._get_client()
+ files: list[dict[str, Any]] = []
+
+ for container_name in self._resolve_container_names_sync():
+ try:
+ container_client = client.get_container_client(container_name)
+ blob_iter = (
+ container_client.list_blobs(name_starts_with=self.prefix)
+ if self.prefix
+ else container_client.list_blobs()
+ )
+ for blob in blob_iter:
+ if blob.name.endswith("/"):
+ continue # skip virtual directory markers
+ last_modified = getattr(blob, "last_modified", None)
+ files.append(
+ {
+ "id": _make_file_id(container_name, blob.name),
+ "name": basename(blob.name) or blob.name,
+ # "bucket" is the shared bucket-connector contract key
+ # (matches aws_s3 / ibm_cos) that browse_connection_files
+ # and the file browser read; Azure's domain term is
+ # "container", preserved in the id and metadata.
+ "bucket": container_name,
+ "key": blob.name,
+ "size": getattr(blob, "size", 0),
+ "modified_time": last_modified.isoformat() if last_modified else None,
+ }
+ )
+ if max_files and len(files) >= max_files:
+ return files
+ except Exception as exc:
+ logger.error("Failed to list blobs in Azure container %s: %s", container_name, exc)
+ continue
+
+ return files
+
+ def _download_blob_sync(self, container: str, blob: str) -> dict[str, Any]:
+ client = self._get_client()
+ blob_client = client.get_blob_client(container=container, blob=blob)
+ downloader = blob_client.download_blob()
+ content: bytes = downloader.readall()
+ props = getattr(downloader, "properties", None)
+ content_type = ""
+ last_modified = None
+ size = len(content)
+ if props is not None:
+ settings = getattr(props, "content_settings", None)
+ content_type = getattr(settings, "content_type", "") or ""
+ last_modified = getattr(props, "last_modified", None)
+ size = getattr(props, "size", None) or len(content)
+ return {
+ "content": content,
+ "content_type": content_type,
+ "last_modified": last_modified,
+ "size": size,
+ }
+
+ # ------------------------------------------------------------------
+ # BaseConnector abstract method implementations
+ # ------------------------------------------------------------------
+
+ async def authenticate(self) -> bool:
+ """Validate credentials by listing containers on the storage account."""
+ try:
+ await asyncio.to_thread(self._list_container_names_sync)
+ self._authenticated = True
+ logger.debug("Azure Blob authenticated for connection %s", self.connection_id)
+ return True
+ except Exception as exc:
+ logger.warning("Azure Blob authentication failed: %s", exc)
+ self._authenticated = False
+ return False
+
+ async def list_files(
+ self,
+ page_token: str | None = None,
+ max_files: int | None = None,
+ **kwargs,
+ ) -> dict[str, Any]:
+ """List blobs across all configured (or auto-discovered) containers.
+
+ The Azure SDK paginates internally, so a single pass returns all blobs;
+ ``next_page_token`` is always None.
+ """
+ files = await asyncio.to_thread(self._list_files_sync, max_files)
+ return {"files": files, "next_page_token": None}
+
+ async def get_file_content(self, file_id: str) -> ConnectorDocument:
+ """Download a blob and return a ConnectorDocument.
+
+ Args:
+ file_id: Composite ID in the form "<container>::<blob>".
+ """
+ container_name, blob_name = _split_file_id(file_id)
+ result = await asyncio.to_thread(self._download_blob_sync, container_name, blob_name)
+
+ content: bytes = result["content"]
+ last_modified: datetime = result["last_modified"] or datetime.now(UTC)
+ size: int = result["size"]
+
+ # Prefer the blob's stored content type, but fall back to an extension
+ # guess when it is missing or the generic octet-stream default.
+ raw_content_type = result["content_type"]
+ if raw_content_type and raw_content_type != "application/octet-stream":
+ mime_type: str = raw_content_type
+ else:
+ mime_type = mimetypes.guess_type(blob_name)[0] or "application/octet-stream"
+
+ filename = basename(blob_name) or blob_name
+
+ # Owner-based DLS (v1): no per-blob principals — ownership is assigned to
+ # the ingesting OpenRAG user by service.process_connector_document.
+ acl = DocumentACL(owner=None, allowed_users=[], allowed_groups=[], allowed_principals=[])
+
+ return ConnectorDocument(
+ id=file_id,
+ filename=filename,
+ mimetype=mime_type,
+ content=content,
+ source_url=f"azure://{container_name}/{blob_name}",
+ acl=acl,
+ modified_time=last_modified,
+ created_time=last_modified, # Azure Blob does not expose a creation time
+ metadata={
+ "azure_container": container_name,
+ "azure_blob": blob_name,
+ "size": size,
+ },
+ )
+
+ # ------------------------------------------------------------------
+ # Webhook / subscription stubs (Azure Event Grid is out of scope here)
+ # ------------------------------------------------------------------
+
+ async def setup_subscription(self) -> str:
+ """No-op: Azure Blob change notifications are out of scope for this connector."""
+ return ""
+
+ async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:
+ """No-op: webhooks are not supported in this connector version."""
+ return []
+
+ def extract_webhook_channel_id(
+ self, payload: dict[str, Any], headers: dict[str, str]
+ ) -> str | None:
+ return None
+
+ async def cleanup_subscription(self, subscription_id: str) -> bool:
+ """No-op: no subscription to clean up."""
+ return True
diff --git a/enhancements/connectors/azure_blob/models.py b/enhancements/connectors/azure_blob/models.py
new file mode 100644
index 000000000..042eb3152
--- /dev/null
+++ b/enhancements/connectors/azure_blob/models.py
@@ -0,0 +1,17 @@
+"""Pydantic request/response models for Azure Blob API endpoints."""
+
+from pydantic import BaseModel
+
+
+class AzureBlobConfigureBody(BaseModel):
+ auth_mode: str # "connection_string" or "account_key"
+ # connection_string mode
+ connection_string: str | None = None
+ # account_key mode
+ account_name: str | None = None
+ account_key: str | None = None
+ endpoint: str | None = None # optional custom blob endpoint (Azurite / sovereign)
+ # Optional container selection
+ container_names: list[str] | None = None
+ # Optional: update an existing connection
+ connection_id: str | None = None
diff --git a/enhancements/connectors/azure_blob/support.py b/enhancements/connectors/azure_blob/support.py
new file mode 100644
index 000000000..2c0a10dcb
--- /dev/null
+++ b/enhancements/connectors/azure_blob/support.py
@@ -0,0 +1,66 @@
+"""Support helpers for Azure Blob API endpoints.
+
+Pure (non-async) business logic for credential resolution and config dict
+construction, keeping the route handlers thin.
+"""
+
+import os
+
+from .models import AzureBlobConfigureBody
+
+
+def build_azure_blob_config(
+ body: AzureBlobConfigureBody,
+ existing_config: dict,
+) -> tuple[dict, str | None]:
+ """Resolve Azure Blob credentials and build the connection config dict.
+
+ Resolution order for each credential: request body → environment variable
+ → existing connection config.
+
+ Returns:
+ (conn_config, None) on success
+ ({}, error_message) on validation failure
+ """
+ conn_config: dict = {"auth_mode": body.auth_mode}
+
+ if body.auth_mode == "connection_string":
+ connection_string = (
+ body.connection_string
+ or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+ or existing_config.get("connection_string")
+ )
+ if not connection_string:
+ return {}, "Connection string mode requires a connection_string"
+ conn_config["connection_string"] = connection_string
+ elif body.auth_mode == "account_key":
+ account_name = (
+ body.account_name
+ or os.getenv("AZURE_STORAGE_ACCOUNT_NAME")
+ or existing_config.get("account_name")
+ )
+ account_key = (
+ body.account_key
+ or os.getenv("AZURE_STORAGE_ACCOUNT_KEY")
+ or existing_config.get("account_key")
+ )
+ if not account_name or not account_key:
+ return {}, "Account key mode requires account_name and account_key"
+ conn_config["account_name"] = account_name
+ conn_config["account_key"] = account_key
+ endpoint = (
+ body.endpoint
+ or os.getenv("AZURE_STORAGE_ENDPOINT")
+ or existing_config.get("endpoint_url")
+ )
+ if endpoint:
+ conn_config["endpoint_url"] = endpoint
+ else:
+ return {}, f"Unknown auth_mode: {body.auth_mode!r}"
+
+ if body.container_names is not None:
+ conn_config["container_names"] = body.container_names
+ elif existing_config.get("container_names"):
+ conn_config["container_names"] = existing_config["container_names"]
+
+ return conn_config, None
diff --git a/enhancements/connectors/ibm_cos/connector.py b/enhancements/connectors/ibm_cos/connector.py
index dd2d1279a..a6de8411a 100644
--- a/enhancements/connectors/ibm_cos/connector.py
+++ b/enhancements/connectors/ibm_cos/connector.py
@@ -247,11 +247,11 @@ class IBMCOSConnector(BaseConnector):
return {"files": files, "next_page_token": None}
else:
# client API: list_objects_v2 with manual pagination
- kwargs: dict[str, Any] = {"Bucket": bucket_name}
+ list_kwargs: dict[str, Any] = {"Bucket": bucket_name}
if self.prefix:
- kwargs["Prefix"] = self.prefix
+ list_kwargs["Prefix"] = self.prefix
while True:
- resp = handle.list_objects_v2(**kwargs)
+ resp = handle.list_objects_v2(**list_kwargs)
for obj in resp.get("Contents", []):
key = obj["Key"]
if key.endswith("/"):
@@ -271,7 +271,7 @@ class IBMCOSConnector(BaseConnector):
if max_files and len(files) >= max_files:
return {"files": files, "next_page_token": None}
if resp.get("IsTruncated"):
- kwargs["ContinuationToken"] = resp["NextContinuationToken"]
+ list_kwargs["ContinuationToken"] = resp["NextContinuationToken"]
else:
break
diff --git a/frontend/app/api/mutations/useDeleteDocument.ts b/frontend/app/api/mutations/useDeleteDocument.ts
index 8f92c579c..adea15b03 100644
--- a/frontend/app/api/mutations/useDeleteDocument.ts
+++ b/frontend/app/api/mutations/useDeleteDocument.ts
@@ -43,6 +43,9 @@ export const useDeleteDocument = () => {
setTimeout(() => {
queryClient.invalidateQueries({ queryKey: ["search"] });
queryClient.invalidateQueries({ queryKey: ["listFiles"] });
+ // Connector "Browse Files" dialogs cache per-file ingestion state; drop
+ // it so a deleted file no longer shows as "Ingested"/disabled there.
+ queryClient.invalidateQueries({ queryKey: ["browseConnectionFiles"] });
}, 1000);
},
});
diff --git a/frontend/app/api/queries/useBrowseConnectionFiles.ts b/frontend/app/api/queries/useBrowseConnectionFiles.ts
index fa0c12d31..4df961cf7 100644
--- a/frontend/app/api/queries/useBrowseConnectionFiles.ts
+++ b/frontend/app/api/queries/useBrowseConnectionFiles.ts
@@ -12,12 +12,27 @@ export interface RemoteFile {
size: number;
modified_time: string;
is_ingested: boolean;
+ /** Ingested, but the source version is newer than what we indexed. Such files
+ * stay selectable (to re-ingest the newer version) while unchanged ingested
+ * files are disabled. */
+ is_stale: boolean;
}
export interface BrowseConnectionFilesParams {
connectorType: string;
connectionId: string;
bucket?: string;
+ /**
+ * Currently unused. The FileBrowserDialog filters the fetched list in-memory
+ * (client-side) instead of sending this to the backend, because the existing
+ * server-side `search` is deficient: it doesn't filter directly at the
+ * connector level — it fetches the capped blob list (maxFiles) and then
+ * post-filters it in memory, so it can't match files beyond the cap and
+ * re-fetches the whole list per query. Passing it would also remount the
+ * query's loading state on every keystroke.
+ * TODO: implement true server-side filtering at the connector level (push the
+ * search term into the connector's list call), then reinstate this param.
+ */
search?: string;
pageToken?: string;
maxFiles?: number;
@@ -68,6 +83,13 @@ export const useBrowseConnectionFiles = (
queryFn: fetchFiles,
retry: false,
enabled: Boolean(params.connectorType && params.connectionId),
+ // Always reflect live ingestion state when the dialog opens. The global
+ // default staleTime (60s) would otherwise serve a cached `is_ingested`
+ // snapshot — e.g. a file deleted on the Knowledge page would still render
+ // as "Ingested"/disabled here until the cache went stale. Matches the
+ // bucket-status queries (useS3BucketStatusQuery / azure container-status).
+ staleTime: 0,
+ refetchOnMount: "always",
...options,
},
queryClient,
diff --git a/frontend/app/knowledge/chunks/page.tsx b/frontend/app/knowledge/chunks/page.tsx
index 80ef7fca2..2b747db44 100644
--- a/frontend/app/knowledge/chunks/page.tsx
+++ b/frontend/app/knowledge/chunks/page.tsx
@@ -18,6 +18,7 @@ import {
} from "@/components/ui/tooltip";
import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";
import { trackButton } from "@/lib/analytics";
+import { formatFileSize, getFileTypeLabel } from "@/lib/file-format";
import {
type ChunkResult,
EMPTY_SEARCH_RESULT,
@@ -26,13 +27,6 @@ import {
useGetSearchQuery,
} from "../../api/queries/useGetSearchQuery";
-const getFileTypeLabel = (mimetype: string) => {
- if (mimetype === "application/pdf") return "PDF";
- if (mimetype === "text/plain") return "Text";
- if (mimetype === "application/msword") return "Word Document";
- return "Unknown";
-};
-
function ChunksPageContent() {
const router = useRouter();
const searchParams = useSearchParams();
@@ -304,9 +298,7 @@ function ChunksPageContent() {
<div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
<dt className="text-sm/6 text-muted-foreground">Size</dt>
<dd className="mt-1 text-sm/6 text-gray-800 dark:text-gray-100 sm:col-span-2 sm:mt-0">
- {fileData?.size
- ? `${Math.round(fileData.size / 1024)} KB`
- : "Unknown"}
+ {fileData?.size ? formatFileSize(fileData.size) : "Unknown"}
</dd>
</div>
{/* <div className="sm:grid sm:grid-cols-3 sm:gap-4 sm:px-0 mb-2.5">
diff --git a/frontend/app/knowledge/page.tsx b/frontend/app/knowledge/page.tsx
index f572bee4f..2f5fcf02b 100644
--- a/frontend/app/knowledge/page.tsx
+++ b/frontend/app/knowledge/page.tsx
@@ -42,6 +42,7 @@ import {
} from "@/components/ui/tooltip";
import { useIsCloudBrand } from "@/contexts/brand-context";
import { getConnectorDescriptor } from "@/lib/connectors/registry";
+import { formatFileSize } from "@/lib/file-format";
import {
buildKnowledgeTableRows,
getKnowledgeFileIdentity,
@@ -582,7 +583,7 @@ function SearchPage() {
comparator: (valueA?: number, valueB?: number) =>
(valueA || 0) - (valueB || 0),
valueFormatter: (params: ValueFormatterParams<File>) =>
- params.value ? `${Math.round(params.value / 1024)} KB` : "-",
+ params.value ? formatFileSize(params.value) : "-",
cellClass: isCloudBrand ? "text-muted-foreground" : undefined,
},
{
diff --git a/frontend/app/settings/_components/s3-settings-dialog.tsx b/frontend/app/settings/_components/s3-settings-dialog.tsx
index 3c6a45c91..36579a307 100644
--- a/frontend/app/settings/_components/s3-settings-dialog.tsx
+++ b/frontend/app/settings/_components/s3-settings-dialog.tsx
@@ -134,8 +134,8 @@ export default function S3SettingsDialog({
toast.success("Amazon S3 configured", {
description:
selectedBuckets.length > 0
- ? `Will ingest from: ${selectedBuckets.join(", ")}`
- : "Will auto-discover and ingest all accessible buckets.",
+ ? `Ingestion restricted to the following buckets: ${selectedBuckets.join(", ")}.`
+ : "All accessible buckets available for ingestion.",
icon: <AwsLogo className="w-4 h-4" />,
});
diff --git a/frontend/app/settings/_components/s3-settings-form.tsx b/frontend/app/settings/_components/s3-settings-form.tsx
index a9fe98ec6..c16bcd1e3 100644
--- a/frontend/app/settings/_components/s3-settings-form.tsx
+++ b/frontend/app/settings/_components/s3-settings-form.tsx
@@ -168,9 +168,9 @@ export function S3SettingsForm({
<div className="space-y-2">
<div className="flex items-center justify-between">
<Label className="text-sm font-medium">
- Select Buckets to Ingest
+ Restrict Ingestion to Buckets
<span className="ml-1 text-muted-foreground font-normal">
- (leave all unchecked to ingest everything)
+ (optional)
</span>
</Label>
{buckets.length > 1 && (
diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsx
index ab67d7f26..54c59c71e 100644
--- a/frontend/components/connectors/shared-bucket-view.tsx
+++ b/frontend/components/connectors/shared-bucket-view.tsx
@@ -107,13 +107,19 @@ export function SharedBucketView({
onSuccess: (result) => {
invalidate();
if (result.task_ids?.length) {
- addTask(result.task_ids[0], {
- connectorType: connector.type,
- source: "connector",
- });
+ // The container path may return two tasks (new files + changed files);
+ // track them all.
+ for (const id of result.task_ids) {
+ addTask(id, {
+ connectorType: connector.type,
+ source: "connector",
+ });
+ }
onDone();
} else {
- toast.info("No files found in the selected buckets.");
+ toast.info(
+ result.message ?? "No files found in the selected buckets.",
+ );
}
},
onError: (err) => {
diff --git a/frontend/components/file-browser-dialog.tsx b/frontend/components/file-browser-dialog.tsx
index 972714f7e..f57c15c16 100644
--- a/frontend/components/file-browser-dialog.tsx
+++ b/frontend/components/file-browser-dialog.tsx
@@ -1,6 +1,6 @@
"use client";
-import { Check, Download, Loader2, Search } from "lucide-react";
+import { Check, Download, Loader2, RefreshCw, Search } from "lucide-react";
import { useCallback, useMemo, useState } from "react";
import { toast } from "sonner";
import { useSyncConnector } from "@/app/api/mutations/useSyncConnector";
@@ -8,6 +8,7 @@ import {
type RemoteFile,
useBrowseConnectionFiles,
} from "@/app/api/queries/useBrowseConnectionFiles";
+import { formatFileSize } from "@/lib/file-format";
import { Badge } from "./ui/badge";
import { Button } from "./ui/button";
import {
@@ -29,13 +30,6 @@ interface FileBrowserDialogProps {
buckets?: string[];
}
-function formatFileSize(bytes: number): string {
- if (bytes === 0) return "0 B";
- const units = ["B", "KB", "MB", "GB"];
- const i = Math.floor(Math.log(bytes) / Math.log(1024));
- return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
-}
-
export function FileBrowserDialog({
open,
onOpenChange,
@@ -53,18 +47,40 @@ export function FileBrowserDialog({
const syncMutation = useSyncConnector();
+ // The bucket's file set is fetched once per open (search is NOT sent to the
+ // backend — its server-side filter just post-filters this same capped list,
+ // so re-fetching per keystroke only adds latency and remounts the loading
+ // state, making the table jump). We filter the cached list in-memory instead.
const { data, isLoading, error } = useBrowseConnectionFiles(
{
connectorType,
connectionId,
bucket: selectedBucket,
- search: search || undefined,
maxFiles: 500,
},
{ enabled: open },
);
- const files = data?.files ?? [];
+ const allFiles = useMemo(() => data?.files ?? [], [data]);
+
+ const files = useMemo(() => {
+ const q = search.trim().toLowerCase();
+ return q
+ ? allFiles.filter((f) => f.name.toLowerCase().includes(q))
+ : allFiles;
+ }, [allFiles, search]);
+
+ // Selectable files within the current (filtered) view — drives "Select all".
+ // A file is selectable when it's not yet ingested, or it's ingested but stale
+ // (a newer version exists at the source, so re-ingest is allowed).
+ const visibleSelectable = useMemo(
+ () => files.filter((f) => !f.is_ingested || f.is_stale),
+ [files],
+ );
+
+ const allVisibleSelected =
+ visibleSelectable.length > 0 &&
+ visibleSelectable.every((f) => selectedFileIds.has(f.id));
const toggleFile = useCallback((fileId: string) => {
setSelectedFileIds((prev) => {
@@ -79,22 +95,37 @@ export function FileBrowserDialog({
}, []);
const toggleAll = useCallback(() => {
- const unIngestedFiles = files.filter((f) => !f.is_ingested);
- if (selectedFileIds.size === unIngestedFiles.length) {
- setSelectedFileIds(new Set());
- } else {
- setSelectedFileIds(new Set(unIngestedFiles.map((f) => f.id)));
- }
- }, [files, selectedFileIds.size]);
+ setSelectedFileIds((prev) => {
+ const next = new Set(prev);
+ const allSelected =
+ visibleSelectable.length > 0 &&
+ visibleSelectable.every((f) => next.has(f.id));
+ for (const f of visibleSelectable) {
+ if (allSelected) {
+ next.delete(f.id);
+ } else {
+ next.add(f.id);
+ }
+ }
+ return next;
+ });
+ }, [visibleSelectable]);
+ // Resolve selections against the full fetched set so a selection survives
+ // filtering (a selected file hidden by the search box is still ingested).
const selectedFiles = useMemo(
- () => files.filter((f) => selectedFileIds.has(f.id)),
- [files, selectedFileIds],
+ () => allFiles.filter((f) => selectedFileIds.has(f.id)),
+ [allFiles, selectedFileIds],
);
const handleIngest = useCallback(async () => {
if (selectedFiles.length === 0) return;
+ // If any selected file is a stale re-ingest, replace the indexed copy so the
+ // newer version overwrites the old chunks (rather than being skipped as a
+ // duplicate filename). New files are unaffected by this flag.
+ const hasStale = selectedFiles.some((f) => f.is_stale);
+
try {
await syncMutation.mutateAsync({
connectorType,
@@ -105,6 +136,7 @@ export function FileBrowserDialog({
mimeType: "",
size: f.size,
})),
+ ...(hasStale ? { replace_duplicates: true } : {}),
},
});
@@ -121,8 +153,6 @@ export function FileBrowserDialog({
}
}, [selectedFiles, connectorType, syncMutation, onOpenChange]);
- const unIngestedCount = files.filter((f) => !f.is_ingested).length;
-
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="max-w-2xl max-h-[80vh] flex flex-col">
@@ -183,25 +213,24 @@ export function FileBrowserDialog({
</div>
) : files.length === 0 ? (
<div className="p-8 text-center text-muted-foreground text-sm">
- No files found.
+ {search.trim() && allFiles.length > 0
+ ? "No files match your search."
+ : "No files found."}
</div>
) : (
<div className="divide-y">
- {unIngestedCount > 0 && (
+ {visibleSelectable.length > 0 && (
<div className="px-3 py-2 bg-muted/50 flex items-center gap-2 sticky top-0">
<input
type="checkbox"
- checked={
- selectedFileIds.size === unIngestedCount &&
- unIngestedCount > 0
- }
+ checked={allVisibleSelected}
onChange={toggleAll}
className="h-4 w-4 rounded border border-input"
/>
<span className="text-xs text-muted-foreground">
{selectedFileIds.size > 0
? `${selectedFileIds.size} selected`
- : `Select all (${unIngestedCount})`}
+ : `Select all (${visibleSelectable.length})`}
</span>
</div>
)}
@@ -255,21 +284,44 @@ function FileRow({
selected: boolean;
onToggle: () => void;
}) {
+ // The blob/object key is the full path within the bucket/container (e.g.
+ // "invoices/2024/report.pdf"); file.name is only the basename. Surfacing the
+ // directory portion disambiguates same-named files living under different
+ // prefixes (e.g. 2024/report.pdf vs 2025/report.pdf), which otherwise render
+ // identically. Empty for top-level/flat blobs, so flat listings are unchanged.
+ const dir = useMemo(() => {
+ const key = file.key ?? "";
+ const idx = key.lastIndexOf("/");
+ return idx >= 0 ? key.slice(0, idx + 1) : "";
+ }, [file.key]);
+
+ // Unchanged ingested files are locked; stale ones (newer version at source) stay
+ // selectable so the user can re-ingest the update.
+ const locked = file.is_ingested && !file.is_stale;
+
return (
<label
className={`flex items-center gap-3 px-3 py-2.5 cursor-pointer hover:bg-muted/30 transition-colors ${
- file.is_ingested ? "opacity-60" : ""
+ locked ? "opacity-60" : ""
}`}
>
<input
type="checkbox"
- checked={selected || file.is_ingested}
- disabled={file.is_ingested}
+ checked={selected || locked}
+ disabled={locked}
onChange={onToggle}
className="h-4 w-4 rounded border border-input"
/>
<div className="flex-1 min-w-0">
<div className="text-sm truncate font-medium">{file.name}</div>
+ {dir && (
+ <div
+ className="text-xs text-muted-foreground truncate"
+ title={file.key}
+ >
+ {dir}
+ </div>
+ )}
<div className="text-xs text-muted-foreground flex gap-2">
{file.bucket && <span>{file.bucket}</span>}
<span>{formatFileSize(file.size)}</span>
@@ -278,11 +330,18 @@ function FileRow({
)}
</div>
</div>
- {file.is_ingested && (
- <Badge variant="secondary" className="text-xs flex-shrink-0">
- <Check className="h-3 w-3 mr-1" />
- Ingested
+ {file.is_stale ? (
+ <Badge variant="outline" className="text-xs flex-shrink-0">
+ <RefreshCw className="h-3 w-3 mr-1" />
+ Update available
</Badge>
+ ) : (
+ file.is_ingested && (
+ <Badge variant="secondary" className="text-xs flex-shrink-0">
+ <Check className="h-3 w-3 mr-1" />
+ Ingested
+ </Badge>
+ )
)}
</label>
);
diff --git a/frontend/components/knowledge-dropdown.tsx b/frontend/components/knowledge-dropdown.tsx
index f6d1c9a9b..7e431554b 100644
--- a/frontend/components/knowledge-dropdown.tsx
+++ b/frontend/components/knowledge-dropdown.tsx
@@ -31,7 +31,6 @@ import {
} from "@/components/ui/dropdown-menu";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
-import { useAuth } from "@/contexts/auth-context";
import { useIsCloudBrand } from "@/contexts/brand-context";
import { useTask } from "@/contexts/task-context";
import { usePermissions } from "@/hooks/use-permissions";
@@ -122,7 +121,6 @@ const FolderIconWithColor = ({ className }: { className?: string }) => (
);
export function KnowledgeDropdown() {
- const { isIbmAuthMode } = useAuth();
const { can } = usePermissions();
const canUpload = can("knowledge:upload");
const isCloudBrand = useIsCloudBrand();
@@ -142,6 +140,9 @@ export function KnowledgeDropdown() {
const [bucketConnectorConfigured, setBucketConnectorConfigured] = useState<
Record<string, boolean>
>({});
+ const [bucketConnectorAvailable, setBucketConnectorAvailable] = useState<
+ Record<string, boolean>
+ >({});
const [pendingFile, setPendingFile] = useState<File | null>(null);
const [duplicateFilename, setDuplicateFilename] = useState<string>("");
const [pendingFolderUpload, setPendingFolderUpload] = useState<{
@@ -215,6 +216,19 @@ export function KnowledgeDropdown() {
const connectorsRes = await fetch("/api/connectors");
if (connectorsRes.ok) {
const connectorsResult = await connectorsRes.json();
+
+ // Bucket connector availability mirrors the backend `is_available()`
+ // gate (IBM auth, or the dev flag for Azure Blob), so the dropdown
+ // surfaces a bucket entry whenever the connector is actually usable —
+ // not only in IBM auth mode.
+ const bucketAvailable: Record<string, boolean> = {};
+ for (const d of bucketDescriptors) {
+ bucketAvailable[d.connectorType] = Boolean(
+ connectorsResult.connectors?.[d.connectorType]?.available,
+ );
+ }
+ setBucketConnectorAvailable(bucketAvailable);
+
const cloudConnectorTypes = [
"google_drive",
"onedrive",
@@ -700,20 +714,23 @@ export function KnowledgeDropdown() {
};
});
- const bucketConnectorItems = isIbmAuthMode
- ? getConnectorDescriptors()
- .filter(
- (d) =>
- d.kind === "bucket" &&
- d.menuItem &&
- bucketConnectorConfigured[d.connectorType],
- )
- .map((d) => ({
- label: d.menuItem!.label,
- icon: d.Icon,
- onClick: () => router.push(d.menuItem!.route),
- }))
- : [];
+ // Gate each bucket connector on its backend availability (IBM auth, or the
+ // OPENRAG_DEV_AZURE_BLOB dev flag for Azure Blob) AND a saved connection,
+ // rather than the global IBM-auth flag — this keeps S3/IBM COS hidden outside
+ // IBM auth while letting Azure Blob appear in local dev once configured.
+ const bucketConnectorItems = getConnectorDescriptors()
+ .filter(
+ (d) =>
+ d.kind === "bucket" &&
+ d.menuItem &&
+ bucketConnectorAvailable[d.connectorType] &&
+ bucketConnectorConfigured[d.connectorType],
+ )
+ .map((d) => ({
+ label: d.menuItem!.label,
+ icon: d.Icon,
+ onClick: () => router.push(d.menuItem!.route),
+ }));
const menuItems = [
{
diff --git a/frontend/enhancements/connectors/azure-blob/components/bucket-view.tsx b/frontend/enhancements/connectors/azure-blob/components/bucket-view.tsx
new file mode 100644
index 000000000..adfb31854
--- /dev/null
+++ b/frontend/enhancements/connectors/azure-blob/components/bucket-view.tsx
@@ -0,0 +1,48 @@
+"use client";
+
+import type { useSyncConnector } from "@/app/api/mutations/useSyncConnector";
+import type { Connector } from "@/app/api/queries/useGetConnectorsQuery";
+import { SharedBucketView } from "@/components/connectors/shared-bucket-view";
+import { useAzureBlobContainerStatusQuery } from "../useAzureBlobContainerStatusQuery";
+
+export interface AzureBlobBucketViewProps {
+ connector: Connector;
+ syncMutation: ReturnType<typeof useSyncConnector>;
+ addTask: (id: string, options?: { connectorType?: string }) => void;
+ onBack: () => void;
+ onDone: () => void;
+}
+
+export function AzureBlobBucketView({
+ connector,
+ syncMutation,
+ addTask,
+ onBack,
+ onDone,
+}: AzureBlobBucketViewProps) {
+ const {
+ data: containers,
+ isLoading,
+ error: containersError,
+ refetch,
+ } = useAzureBlobContainerStatusQuery(connector.connectionId, {
+ enabled: true,
+ });
+ return (
+ <SharedBucketView
+ connector={connector}
+ buckets={containers}
+ isLoading={isLoading}
+ bucketsError={containersError as Error | null}
+ onRefetch={refetch}
+ invalidateQueryKey={[
+ "azure-blob-container-status",
+ connector.connectionId,
+ ]}
+ syncMutation={syncMutation}
+ addTask={addTask}
+ onBack={onBack}
+ onDone={onDone}
+ />
+ );
+}
diff --git a/frontend/enhancements/connectors/azure-blob/icon.tsx b/frontend/enhancements/connectors/azure-blob/icon.tsx
new file mode 100644
index 000000000..0abc55ef7
--- /dev/null
+++ b/frontend/enhancements/connectors/azure-blob/icon.tsx
@@ -0,0 +1,50 @@
+export default function AzureBlobIcon(props: React.SVGProps<SVGSVGElement>) {
+ return (
+ <svg
+ width="16"
+ height="16"
+ viewBox="0 0 32 32"
+ fill="none"
+ xmlns="http://www.w3.org/2000/svg"
+ {...props}
+ >
+ <rect width="32" height="32" fill="white" fillOpacity="0.01" />
+ <path
+ d="M13.6 5.5h6.05l-6.28 18.6a1 1 0 0 1-.95.68H7.2a1 1 0 0 1-.95-1.32L12.65 6.18a1 1 0 0 1 .95-.68Z"
+ fill="url(#azblob_a)"
+ />
+ <path
+ d="M22.4 20.1H12.7a.46.46 0 0 0-.31.8l6.23 5.82a1 1 0 0 0 .68.27h5.58l-2.46-6.89Z"
+ fill="#0078D4"
+ />
+ <path
+ d="M13.6 5.5a1 1 0 0 0-.95.69L6.26 23.45a1 1 0 0 0 .94 1.33h5.3a1.07 1.07 0 0 0 .82-.7l1.28-3.77 4.57 4.26a1 1 0 0 0 .63.21h5.55l-2.43-6.89-7.09.02 4.34-12.62H13.6Z"
+ fill="url(#azblob_b)"
+ />
+ <defs>
+ <linearGradient
+ id="azblob_a"
+ x1="15.5"
+ y1="6.9"
+ x2="9.1"
+ y2="25.8"
+ gradientUnits="userSpaceOnUse"
+ >
+ <stop stopColor="#114A8B" />
+ <stop offset="1" stopColor="#0669BC" />
+ </linearGradient>
+ <linearGradient
+ id="azblob_b"
+ x1="18.5"
+ y1="6"
+ x2="22"
+ y2="25"
+ gradientUnits="userSpaceOnUse"
+ >
+ <stop stopColor="#3CCBF4" />
+ <stop offset="1" stopColor="#2892DF" />
+ </linearGradient>
+ </defs>
+ </svg>
+ );
+}
diff --git a/frontend/enhancements/connectors/azure-blob/settings-dialog.tsx b/frontend/enhancements/connectors/azure-blob/settings-dialog.tsx
new file mode 100644
index 000000000..c9e6ef5e5
--- /dev/null
+++ b/frontend/enhancements/connectors/azure-blob/settings-dialog.tsx
@@ -0,0 +1,195 @@
+"use client";
+
+import { useQueryClient } from "@tanstack/react-query";
+import { useEffect, useState } from "react";
+import { FormProvider, useForm } from "react-hook-form";
+import { toast } from "sonner";
+import { Button } from "@/components/ui/button";
+import {
+ Dialog,
+ DialogContent,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import AzureBlobIcon from "./icon";
+import { type AzureBlobFormData, AzureBlobSettingsForm } from "./settings-form";
+import { useAzureBlobConfigureMutation } from "./useAzureBlobConfigureMutation";
+import { useAzureBlobDefaultsQuery } from "./useAzureBlobDefaultsQuery";
+
+interface AzureBlobSettingsDialogProps {
+ open: boolean;
+ setOpen: (open: boolean) => void;
+}
+
+export default function AzureBlobSettingsDialog({
+ open,
+ setOpen,
+}: AzureBlobSettingsDialogProps) {
+ const queryClient = useQueryClient();
+
+ const { data: defaults } = useAzureBlobDefaultsQuery({ enabled: open });
+
+ const methods = useForm<AzureBlobFormData>({
+ mode: "onSubmit",
+ values: {
+ auth_mode: defaults?.auth_mode ?? "connection_string",
+ connection_string: "",
+ account_name: defaults?.account_name ?? "",
+ account_key: "",
+ endpoint: defaults?.endpoint ?? "",
+ },
+ });
+
+ const { handleSubmit } = methods;
+
+ const [containers, setContainers] = useState<string[] | null>(
+ defaults?.container_names?.length ? defaults.container_names : null,
+ );
+ const [selectedContainers, setSelectedContainers] = useState<string[]>(
+ defaults?.container_names ?? [],
+ );
+ useEffect(() => {
+ if (defaults?.container_names?.length) {
+ setContainers(defaults.container_names);
+ setSelectedContainers(defaults.container_names);
+ }
+ }, [defaults]);
+
+ const [isFetchingContainers, setIsFetchingContainers] = useState(false);
+ const [containersError, setContainersError] = useState<string | null>(null);
+ const [formError, setFormError] = useState<string | null>(null);
+
+ const configureMutation = useAzureBlobConfigureMutation();
+
+ const handleTestConnection = handleSubmit(async (data) => {
+ setIsFetchingContainers(true);
+ setContainersError(null);
+ setFormError(null);
+
+ try {
+ // Validate credentials + list containers WITHOUT persisting. Saving is
+ // reserved for the Save button (onSubmit), so testing never creates or
+ // mutates the connection or clobbers the stored container selection.
+ const res = await fetch("/api/connectors/azure_blob/test", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ auth_mode: data.auth_mode,
+ connection_string: data.connection_string || undefined,
+ account_name: data.account_name || undefined,
+ account_key: data.account_key || undefined,
+ endpoint: data.endpoint || undefined,
+ connection_id: defaults?.connection_id ?? undefined,
+ }),
+ });
+ const json = await res.json();
+ if (!res.ok) throw new Error(json.error || "Failed to list containers");
+
+ const fetched: string[] = json.containers;
+ setContainers(fetched);
+ setSelectedContainers((prev) => prev.filter((c) => fetched.includes(c)));
+ } catch (err: unknown) {
+ setContainersError(
+ err instanceof Error ? err.message : "Connection failed",
+ );
+ } finally {
+ setIsFetchingContainers(false);
+ }
+ });
+
+ const onSubmit = handleSubmit(async (data) => {
+ setFormError(null);
+ if (containers === null) {
+ setFormError("Test the connection first to validate credentials.");
+ return;
+ }
+
+ try {
+ const latestDefaults = await queryClient.fetchQuery({
+ queryKey: ["azure-blob-defaults"],
+ queryFn: async () => {
+ const res = await fetch("/api/connectors/azure_blob/defaults");
+ return res.json();
+ },
+ staleTime: 0,
+ });
+
+ await configureMutation.mutateAsync({
+ auth_mode: data.auth_mode,
+ connection_string: data.connection_string || undefined,
+ account_name: data.account_name || undefined,
+ account_key: data.account_key || undefined,
+ endpoint: data.endpoint || undefined,
+ container_names: selectedContainers,
+ connection_id:
+ latestDefaults?.connection_id ?? defaults?.connection_id ?? undefined,
+ });
+
+ toast.success("Azure Blob Storage configured", {
+ description:
+ selectedContainers.length > 0
+ ? `Ingestion restricted to the following containers: ${selectedContainers.join(", ")}.`
+ : "All accessible containers available for ingestion.",
+ icon: <AzureBlobIcon className="w-4 h-4" />,
+ });
+
+ queryClient.invalidateQueries({ queryKey: ["connectors"] });
+ setOpen(false);
+ } catch (err: unknown) {
+ setFormError(
+ err instanceof Error ? err.message : "Failed to save configuration",
+ );
+ }
+ });
+
+ return (
+ <Dialog open={open} onOpenChange={setOpen}>
+ <DialogContent
+ autoFocus={false}
+ className="max-w-2xl max-h-[90vh] overflow-y-auto"
+ >
+ <FormProvider {...methods}>
+ <form onSubmit={onSubmit} className="grid gap-4">
+ <DialogHeader className="mb-2">
+ <DialogTitle className="flex items-center gap-3">
+ <div className="w-8 h-8 rounded flex items-center justify-center bg-white border">
+ <AzureBlobIcon />
+ </div>
+ Azure Blob Storage Setup
+ </DialogTitle>
+ </DialogHeader>
+
+ <AzureBlobSettingsForm
+ containers={containers}
+ selectedContainers={selectedContainers}
+ onSelectedContainersChange={setSelectedContainers}
+ isFetchingContainers={isFetchingContainers}
+ containersError={containersError}
+ onTestConnection={handleTestConnection as () => void}
+ connectionStringSet={defaults?.connection_string_set}
+ accountKeySet={defaults?.account_key_set}
+ formError={formError}
+ />
+
+ <DialogFooter className="mt-4">
+ <Button
+ variant="outline"
+ type="button"
+ onClick={() => setOpen(false)}
+ >
+ Cancel
+ </Button>
+ <Button
+ type="submit"
+ disabled={configureMutation.isPending || isFetchingContainers}
+ >
+ {configureMutation.isPending ? "Saving…" : "Save"}
+ </Button>
+ </DialogFooter>
+ </form>
+ </FormProvider>
+ </DialogContent>
+ </Dialog>
+ );
+}
diff --git a/frontend/enhancements/connectors/azure-blob/settings-form.tsx b/frontend/enhancements/connectors/azure-blob/settings-form.tsx
new file mode 100644
index 000000000..872d74c65
--- /dev/null
+++ b/frontend/enhancements/connectors/azure-blob/settings-form.tsx
@@ -0,0 +1,277 @@
+"use client";
+
+import { Loader2, RefreshCcw } from "lucide-react";
+import { Controller, useFormContext } from "react-hook-form";
+import { LabelWrapper } from "@/components/label-wrapper";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
+
+export interface AzureBlobFormData {
+ auth_mode: "connection_string" | "account_key";
+ // connection_string mode
+ connection_string: string;
+ // account_key mode
+ account_name: string;
+ account_key: string;
+ endpoint: string;
+}
+
+interface AzureBlobSettingsFormProps {
+ /** Available containers after a successful test — null means not yet tested */
+ containers: string[] | null;
+ selectedContainers: string[];
+ onSelectedContainersChange: (containers: string[]) => void;
+ isFetchingContainers: boolean;
+ containersError: string | null;
+ onTestConnection: () => void;
+ connectionStringSet?: boolean;
+ accountKeySet?: boolean;
+ formError?: string | null;
+}
+
+export function AzureBlobSettingsForm({
+ containers,
+ selectedContainers,
+ onSelectedContainersChange,
+ isFetchingContainers,
+ containersError,
+ onTestConnection,
+ connectionStringSet,
+ accountKeySet,
+ formError,
+}: AzureBlobSettingsFormProps) {
+ const {
+ register,
+ control,
+ formState: { errors },
+ } = useFormContext<AzureBlobFormData>();
+
+ const toggleContainer = (name: string, checked: boolean) => {
+ if (checked) {
+ onSelectedContainersChange([...selectedContainers, name]);
+ } else {
+ onSelectedContainersChange(selectedContainers.filter((c) => c !== name));
+ }
+ };
+
+ const toggleAll = (checked: boolean) => {
+ onSelectedContainersChange(checked ? (containers ?? []) : []);
+ };
+
+ return (
+ <div className="space-y-5">
+ {/* Auth mode selector using Tabs */}
+ <div className="space-y-2">
+ <Label className="text-sm font-medium">Authentication Method</Label>
+ <Controller
+ control={control}
+ name="auth_mode"
+ render={({ field }) => (
+ <Tabs value={field.value} onValueChange={(v) => field.onChange(v)}>
+ <TabsList className="w-full">
+ <TabsTrigger value="connection_string">
+ <span className="font-semibold text-sm">
+ Connection String
+ </span>
+ <span className="text-xs text-muted-foreground font-normal">
+ Single paste-in string
+ </span>
+ </TabsTrigger>
+ <TabsTrigger value="account_key">
+ <span className="font-semibold text-sm">Account Key</span>
+ <span className="text-xs text-muted-foreground font-normal">
+ Account name + key
+ </span>
+ </TabsTrigger>
+ </TabsList>
+
+ {/* Connection string fields — first tab */}
+ <TabsContent value="connection_string">
+ <div className="space-y-1">
+ <LabelWrapper
+ label="Connection String"
+ helperText="Azure Portal → Storage account → Access keys → Connection string. For the local Azurite emulator use: UseDevelopmentStorage=true"
+ id="azure-conn-str"
+ required
+ >
+ <Input
+ {...register("connection_string", {
+ setValueAs: (v) => v?.trim(),
+ })}
+ id="azure-conn-str"
+ type="password"
+ placeholder={
+ connectionStringSet
+ ? "•••••••• (loaded from env)"
+ : "DefaultEndpointsProtocol=https;AccountName=…"
+ }
+ autoComplete="off"
+ />
+ </LabelWrapper>
+ </div>
+ </TabsContent>
+
+ {/* Account key fields — second tab */}
+ <TabsContent value="account_key">
+ <div className="space-y-4">
+ <div className="space-y-1">
+ <LabelWrapper
+ label="Account Name"
+ helperText="Your Azure Storage account name (e.g. mystorageacct). For Azurite use devstoreaccount1."
+ id="azure-account-name"
+ required
+ >
+ <Input
+ {...register("account_name", {
+ setValueAs: (v) => v?.trim(),
+ })}
+ id="azure-account-name"
+ placeholder="mystorageacct"
+ autoComplete="off"
+ />
+ </LabelWrapper>
+ </div>
+ <div className="space-y-1">
+ <LabelWrapper
+ label="Account Key"
+ helperText="Azure Portal → Storage account → Access keys → key1/key2"
+ id="azure-account-key"
+ required
+ >
+ <Input
+ {...register("account_key", {
+ setValueAs: (v) => v?.trim(),
+ })}
+ id="azure-account-key"
+ type="password"
+ placeholder={
+ accountKeySet
+ ? "•••••••• (loaded from env)"
+ : "base64 account key"
+ }
+ autoComplete="off"
+ />
+ </LabelWrapper>
+ </div>
+ <div className="space-y-1">
+ <LabelWrapper
+ label="Blob Endpoint (optional)"
+ helperText="Leave blank for public Azure. Set for Azurite (http://127.0.0.1:10000/devstoreaccount1) or sovereign clouds."
+ id="azure-endpoint"
+ >
+ <Input
+ {...register("endpoint", {
+ setValueAs: (v) => v?.trim(),
+ })}
+ id="azure-endpoint"
+ placeholder="https://mystorageacct.blob.core.windows.net"
+ />
+ </LabelWrapper>
+ </div>
+ </div>
+ </TabsContent>
+ </Tabs>
+ )}
+ />
+ </div>
+
+ {/* Test connection */}
+ <Button
+ type="button"
+ variant="outline"
+ onClick={onTestConnection}
+ disabled={isFetchingContainers}
+ className="w-full"
+ >
+ {isFetchingContainers ? (
+ <>
+ <Loader2 className="h-4 w-4 mr-2 animate-spin" />
+ Connecting…
+ </>
+ ) : containers !== null ? (
+ <>
+ <RefreshCcw className="h-4 w-4 mr-2" />
+ Refresh Containers
+ </>
+ ) : (
+ "Test Connection & List Containers"
+ )}
+ </Button>
+
+ {containersError && (
+ <p className="text-sm text-destructive rounded-lg border border-destructive/50 p-3">
+ {containersError}
+ </p>
+ )}
+
+ {formError && (
+ <p className="text-sm text-destructive rounded-lg border border-destructive/50 p-3">
+ {formError}
+ </p>
+ )}
+
+ {/* Container selector */}
+ {containers !== null && (
+ <div className="space-y-2">
+ <div className="flex items-center justify-between">
+ <Label className="text-sm font-medium">
+ Restrict Ingestion to Containers
+ <span className="ml-1 text-muted-foreground font-normal">
+ (optional)
+ </span>
+ </Label>
+ {containers.length > 1 && (
+ <button
+ type="button"
+ className="text-xs text-primary underline-offset-2 hover:underline"
+ onClick={() =>
+ toggleAll(selectedContainers.length !== containers.length)
+ }
+ >
+ {selectedContainers.length === containers.length
+ ? "Deselect all"
+ : "Select all"}
+ </button>
+ )}
+ </div>
+
+ {containers.length === 0 ? (
+ <p className="text-sm text-muted-foreground">
+ No containers found for this account.
+ </p>
+ ) : (
+ <div className="max-h-48 overflow-y-auto rounded-lg border p-3 space-y-2">
+ {containers.map((container) => (
+ <label
+ key={container}
+ htmlFor={`container-${container}`}
+ className="flex items-center gap-2.5 cursor-pointer select-none"
+ >
+ <input
+ id={`container-${container}`}
+ type="checkbox"
+ className="h-4 w-4 rounded border-border accent-primary"
+ checked={selectedContainers.includes(container)}
+ onChange={(e) =>
+ toggleContainer(container, e.target.checked)
+ }
+ />
+ <span className="text-sm">{container}</span>
+ </label>
+ ))}
+ </div>
+ )}
+ </div>
+ )}
+
+ {/* Keep errors referenced so unused-var lint stays happy when fields hidden */}
+ {errors.connection_string && (
+ <p className="text-sm text-destructive">
+ {errors.connection_string.message}
+ </p>
+ )}
+ </div>
+ );
+}
diff --git a/frontend/enhancements/connectors/azure-blob/useAzureBlobConfigureMutation.ts b/frontend/enhancements/connectors/azure-blob/useAzureBlobConfigureMutation.ts
new file mode 100644
index 000000000..4120ded90
--- /dev/null
+++ b/frontend/enhancements/connectors/azure-blob/useAzureBlobConfigureMutation.ts
@@ -0,0 +1,38 @@
+import { useMutation, useQueryClient } from "@tanstack/react-query";
+
+export interface AzureBlobConfigurePayload {
+ auth_mode: "connection_string" | "account_key";
+ // connection_string mode
+ connection_string?: string;
+ // account_key mode
+ account_name?: string;
+ account_key?: string;
+ endpoint?: string;
+ // Container selection
+ container_names?: string[];
+ // Updating an existing connection
+ connection_id?: string;
+}
+
+async function configureAzureBlob(payload: AzureBlobConfigurePayload) {
+ const res = await fetch("/api/connectors/azure_blob/configure", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(payload),
+ });
+ const data = await res.json();
+ if (!res.ok) throw new Error(data.error || "Failed to configure Azure Blob");
+ return data as { connection_id: string; status: string };
+}
+
+export function useAzureBlobConfigureMutation() {
+ const queryClient = useQueryClient();
+
+ return useMutation({
+ mutationFn: configureAzureBlob,
+ onSuccess: () => {
+ queryClient.invalidateQueries({ queryKey: ["connectors"] });
+ queryClient.invalidateQueries({ queryKey: ["azure-blob-defaults"] });
+ },
+ });
+}
diff --git a/frontend/enhancements/connectors/azure-blob/useAzureBlobContainerStatusQuery.ts b/frontend/enhancements/connectors/azure-blob/useAzureBlobContainerStatusQuery.ts
new file mode 100644
index 000000000..82e36b027
--- /dev/null
+++ b/frontend/enhancements/connectors/azure-blob/useAzureBlobContainerStatusQuery.ts
@@ -0,0 +1,34 @@
+import { useQuery } from "@tanstack/react-query";
+
+export interface AzureBlobContainerStatus {
+ name: string;
+ ingested_count: number;
+ is_synced: boolean;
+}
+
+async function fetchAzureBlobContainerStatus(
+ connectionId: string,
+): Promise<AzureBlobContainerStatus[]> {
+ const res = await fetch(
+ `/api/connectors/azure_blob/${connectionId}/container-status`,
+ );
+ if (!res.ok) {
+ const err = await res.json().catch(() => ({}));
+ throw new Error(err.error || "Failed to fetch container status");
+ }
+ const data = await res.json();
+ return data.containers as AzureBlobContainerStatus[];
+}
+
+export function useAzureBlobContainerStatusQuery(
+ connectionId: string | null | undefined,
+ options?: { enabled?: boolean },
+) {
+ return useQuery<AzureBlobContainerStatus[]>({
+ queryKey: ["azure-blob-container-status", connectionId],
+ queryFn: () => fetchAzureBlobContainerStatus(connectionId!),
+ enabled: (options?.enabled ?? true) && !!connectionId,
+ staleTime: 0,
+ refetchOnMount: "always",
+ });
+}
diff --git a/frontend/enhancements/connectors/azure-blob/useAzureBlobDefaultsQuery.ts b/frontend/enhancements/connectors/azure-blob/useAzureBlobDefaultsQuery.ts
new file mode 100644
index 000000000..eb9387f6c
--- /dev/null
+++ b/frontend/enhancements/connectors/azure-blob/useAzureBlobDefaultsQuery.ts
@@ -0,0 +1,26 @@
+import { useQuery } from "@tanstack/react-query";
+
+export interface AzureBlobDefaults {
+ connection_string_set: boolean;
+ account_name: string;
+ account_key_set: boolean;
+ endpoint: string;
+ auth_mode: "connection_string" | "account_key";
+ container_names: string[];
+ connection_id: string | null;
+}
+
+async function fetchAzureBlobDefaults(): Promise<AzureBlobDefaults> {
+ const res = await fetch("/api/connectors/azure_blob/defaults");
+ if (!res.ok) throw new Error("Failed to fetch Azure Blob defaults");
+ return res.json();
+}
+
+export function useAzureBlobDefaultsQuery(options?: { enabled?: boolean }) {
+ return useQuery<AzureBlobDefaults>({
+ queryKey: ["azure-blob-defaults"],
+ queryFn: fetchAzureBlobDefaults,
+ enabled: options?.enabled ?? true,
+ staleTime: 0,
+ });
+}
diff --git a/frontend/enhancements/connectors/ibm-cos/settings-dialog.tsx b/frontend/enhancements/connectors/ibm-cos/settings-dialog.tsx
index 2dd31dcf9..dc9a90308 100644
--- a/frontend/enhancements/connectors/ibm-cos/settings-dialog.tsx
+++ b/frontend/enhancements/connectors/ibm-cos/settings-dialog.tsx
@@ -136,8 +136,8 @@ export default function IBMCOSSettingsDialog({
toast.success("IBM Cloud Object Storage configured", {
description:
selectedBuckets.length > 0
- ? `Will ingest from: ${selectedBuckets.join(", ")}`
- : "Will auto-discover and ingest all accessible buckets.",
+ ? `Ingestion restricted to the following containers: ${selectedBuckets.join(", ")}.`
+ : "All accessible buckets available for ingestion.",
icon: <IBMCOSIcon className="w-4 h-4" />,
});
diff --git a/frontend/enhancements/connectors/ibm-cos/settings-form.tsx b/frontend/enhancements/connectors/ibm-cos/settings-form.tsx
index 179e4de76..c647be561 100644
--- a/frontend/enhancements/connectors/ibm-cos/settings-form.tsx
+++ b/frontend/enhancements/connectors/ibm-cos/settings-form.tsx
@@ -275,9 +275,9 @@ export function IBMCOSSettingsForm({
<div className="space-y-2">
<div className="flex items-center justify-between">
<Label className="text-sm font-medium">
- Select Buckets to Ingest
+ Restrict Ingestion to Buckets
<span className="ml-1 text-muted-foreground font-normal">
- (leave all unchecked to ingest everything)
+ (optional)
</span>
</Label>
{buckets.length > 1 && (
diff --git a/frontend/enhancements/index.ts b/frontend/enhancements/index.ts
index 80e637626..e8c68c020 100644
--- a/frontend/enhancements/index.ts
+++ b/frontend/enhancements/index.ts
@@ -1,4 +1,8 @@
import type { ConnectorUIDescriptor } from "@/lib/connectors/types";
+import { AzureBlobBucketView } from "./connectors/azure-blob/components/bucket-view";
+import AzureBlobIcon from "./connectors/azure-blob/icon";
+import AzureBlobSettingsDialog from "./connectors/azure-blob/settings-dialog";
+import { useAzureBlobDefaultsQuery } from "./connectors/azure-blob/useAzureBlobDefaultsQuery";
import { IBMCOSBucketView } from "./connectors/ibm-cos/components/bucket-view";
import IBMCOSIcon from "./connectors/ibm-cos/icon";
import IBMCOSSettingsDialog from "./connectors/ibm-cos/settings-dialog";
@@ -23,4 +27,14 @@ export const ADDITIONAL_CONNECTORS: ConnectorUIDescriptor[] = [
useDefaultsQuery: useIBMCOSDefaultsQuery,
menuItem: { label: "IBM Cloud Object Storage", route: "/upload/ibm_cos" },
},
+ {
+ connectorType: "azure_blob",
+ name: "Azure Blob Storage",
+ Icon: AzureBlobIcon,
+ kind: "bucket",
+ SettingsDialog: AzureBlobSettingsDialog,
+ BucketView: AzureBlobBucketView,
+ useDefaultsQuery: useAzureBlobDefaultsQuery,
+ menuItem: { label: "Azure Blob Storage", route: "/upload/azure_blob" },
+ },
];
diff --git a/frontend/lib/file-format.ts b/frontend/lib/file-format.ts
new file mode 100644
index 000000000..7430bae3a
--- /dev/null
+++ b/frontend/lib/file-format.ts
@@ -0,0 +1,55 @@
+/**
+ * Shared formatting helpers for file metadata (size + human-readable type
+ * label). Centralised so the knowledge list, chunks "Original document" panel,
+ * and file-browser dialog format the same values identically.
+ */
+
+/**
+ * Format a byte count as a human-readable size (B/KB/MB/GB/TB).
+ *
+ * Uses bytes for sub-kilobyte files so small text/markdown documents render as
+ * e.g. "412 B" instead of rounding down to a misleading "0 KB".
+ */
+export function formatFileSize(bytes?: number | null): string {
+ if (!bytes || bytes < 0) return "0 B";
+ const units = ["B", "KB", "MB", "GB", "TB"];
+ const i = Math.min(
+ Math.floor(Math.log(bytes) / Math.log(1024)),
+ units.length - 1,
+ );
+ return `${(bytes / 1024 ** i).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
+}
+
+const FILE_TYPE_LABELS: Record<string, string> = {
+ "application/pdf": "PDF",
+ // .txt and .md are both ingested as text/markdown (Langflow .txt->.md
+ // workaround), so map both to a sensible label here.
+ "text/markdown": "Markdown",
+ "text/plain": "Text",
+ "text/csv": "CSV",
+ "text/html": "HTML",
+ "application/json": "JSON",
+ "application/msword": "Word Document",
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+ "Word Document",
+ "application/vnd.ms-excel": "Excel",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "Excel",
+ "application/vnd.ms-powerpoint": "PowerPoint",
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation":
+ "PowerPoint",
+};
+
+/**
+ * Map a mimetype to a short human-readable type label. Falls back to a generic
+ * group label (Image/Audio/Video/Text) before "Unknown".
+ */
+export function getFileTypeLabel(mimetype?: string | null): string {
+ if (!mimetype) return "Unknown";
+ const normalized = mimetype.toLowerCase();
+ if (normalized in FILE_TYPE_LABELS) return FILE_TYPE_LABELS[normalized];
+ if (normalized.startsWith("image/")) return "Image";
+ if (normalized.startsWith("audio/")) return "Audio";
+ if (normalized.startsWith("video/")) return "Video";
+ if (normalized.startsWith("text/")) return "Text";
+ return "Unknown";
+}
diff --git a/frontend/lib/knowledge-table-state.ts b/frontend/lib/knowledge-table-state.ts
index ba262ec1b..24e425696 100644
--- a/frontend/lib/knowledge-table-state.ts
+++ b/frontend/lib/knowledge-table-state.ts
@@ -100,10 +100,12 @@ export function inferTaskFileConnectorType(
}
}
- if (filePath.includes("/") && !filePath.startsWith("/")) {
- return "aws_s3";
- }
-
+ // Bucket connectors (aws_s3, ibm_cos, azure_blob) key task files by a bare
+ // object key or composite "<container>::<blob>" id — none of which are
+ // distinguishable here — so we rely on the connector_type captured at ingest
+ // time (taskConnectorTypesRef) and fall through to "local" otherwise. The
+ // processing-row icon for these is tracked as a follow-up
+ // (see [[task-connector-type-processing-icon]]).
return "local";
}
diff --git a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
index 0566f48ae..d261c4da4 100644
--- a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
+++ b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
@@ -102,6 +102,20 @@ stringData:
MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET={{ .Values.global.oauth.microsoft.clientSecret | quote }}
{{- end }}
+ # Azure Blob Storage connector (optional env defaults; gated by IBM_AUTH_ENABLED or OPENRAG_DEV_AZURE_BLOB)
+ {{- if .Values.global.azureBlob.connectionString }}
+ AZURE_STORAGE_CONNECTION_STRING={{ .Values.global.azureBlob.connectionString | quote }}
+ {{- end }}
+ {{- if .Values.global.azureBlob.accountName }}
+ AZURE_STORAGE_ACCOUNT_NAME={{ .Values.global.azureBlob.accountName | quote }}
+ {{- end }}
+ {{- if .Values.global.azureBlob.accountKey }}
+ AZURE_STORAGE_ACCOUNT_KEY={{ .Values.global.azureBlob.accountKey | quote }}
+ {{- end }}
+ {{- if .Values.global.azureBlob.endpoint }}
+ AZURE_STORAGE_ENDPOINT={{ .Values.global.azureBlob.endpoint | quote }}
+ {{- end }}
+
# Webhook (continuous ingest)
{{- if .Values.backend.webhook.baseUrl }}
WEBHOOK_BASE_URL={{ .Values.backend.webhook.baseUrl | quote }}
diff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yaml
index f7302bd72..3b5e35343 100644
--- a/kubernetes/helm/openrag/values.yaml
+++ b/kubernetes/helm/openrag/values.yaml
@@ -55,6 +55,16 @@ global:
accessKeyId: ""
secretAccessKey: ""
region: ""
+ # Azure Blob Storage connector (Enterprise/SaaS; gated by backend.auth.ibmCloud.enabled
+ # like the other bucket connectors). Optional env defaults — credentials are
+ # normally entered per-connection in the UI. Provide EITHER a connection string
+ # OR account name + key (+ optional custom blob endpoint).
+ azureBlob:
+ enabled: false
+ connectionString: "" # AZURE_STORAGE_CONNECTION_STRING
+ accountName: "" # AZURE_STORAGE_ACCOUNT_NAME
+ accountKey: "" # AZURE_STORAGE_ACCOUNT_KEY
+ endpoint: "" # AZURE_STORAGE_ENDPOINT (optional)
# ============================================================================
# Langflow Configuration
diff --git a/kubernetes/operator/internal/controller/env.go b/kubernetes/operator/internal/controller/env.go
index 86184d462..276677195 100644
--- a/kubernetes/operator/internal/controller/env.go
+++ b/kubernetes/operator/internal/controller/env.go
@@ -147,6 +147,14 @@ func NewEnvVarManager() *EnvVarManager {
"WATSONX_ENDPOINT": "",
"WATSONX_PROJECT_ID": "",
"DOCLING_SERVE_VERIFY_SSL": "false",
+
+ // Azure Blob Storage connector (Enterprise/SaaS; gated by IBM_AUTH_ENABLED or OPENRAG_DEV_AZURE_BLOB for local dev).
+ // Optional env defaults — credentials are normally entered per-connection
+ // in the UI. Override via the CR spec.env when env-based defaults are wanted.
+ "AZURE_STORAGE_CONNECTION_STRING": "",
+ "AZURE_STORAGE_ACCOUNT_NAME": "",
+ "AZURE_STORAGE_ACCOUNT_KEY": "",
+ "AZURE_STORAGE_ENDPOINT": "",
},
DefaultOpenRagFEEnvVars: map[string]string{
// Frontend environment variables will be added here
diff --git a/pyproject.toml b/pyproject.toml
index 1508ab48f..4f15302e4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,6 +41,7 @@ dependencies = [
"uvicorn>=0.35.0",
"boto3>=1.35.0",
"ibm-cos-sdk>=2.13.0",
+ "azure-storage-blob>=12.20.0",
"psutil>=7.0.0",
"rich>=13.0.0",
"textual>=0.45.0",
diff --git a/scripts/connectors/azure/seed_azurite.py b/scripts/connectors/azure/seed_azurite.py
new file mode 100644
index 000000000..edc21d642
--- /dev/null
+++ b/scripts/connectors/azure/seed_azurite.py
@@ -0,0 +1,602 @@
+"""Seed the local Azurite emulator with test containers + blobs.
+
+Run from the repo root with the project venv:
+ uv run python scripts/connectors/azure/seed_azurite.py
+
+Pass --reset to delete and recreate the containers before uploading:
+ uv run python scripts/connectors/azure/seed_azurite.py --reset
+
+Two containers are seeded:
+ * CONTAINER_1 ("openrag-test-1") — plain text, markdown, and a PDF.
+ * CONTAINER_2 ("openrag-test-2") — Office/web document formats
+ (docx, xlsx, pptx, csv, html) for exercising the parser end to end.
+
+The Office formats (docx/xlsx/pptx) are built as minimal-but-valid OOXML
+packages in pure Python with `zipfile`, matching the no-external-deps style
+of the PDF builder below.
+
+Assumes `make azurite-up` is running (Azurite on localhost:10000).
+"""
+
+import argparse
+import io
+import zipfile
+
+from azure.storage.blob import BlobServiceClient
+
+# Well-known Azurite dev connection string. From the host this resolves to
+# http://127.0.0.1:10000/devstoreaccount1.
+CONN = "UseDevelopmentStorage=true"
+
+CONTAINER_1 = "openrag-test-1"
+CONTAINER_2 = "openrag-test-2"
+
+
+def _make_sample_pdf() -> bytes:
+ """Build a minimal valid single-page PDF in pure Python (no external deps)."""
+
+ def escape(text: str) -> str:
+ return text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
+
+ title = "Fascinating Facts about Azure"
+ body_lines = [
+ "- The Naming Process: Microsoft originally wanted a name containing",
+ ' "cloud". After being advised against it, they chose "Azure"',
+ " (a shade of blue) to represent the blue sky behind clouds.",
+ ' Clients initially called the name a "dumb idea".',
+ "",
+ "- Space Travel & Datacenters: Microsoft Azure runs an extension of its",
+ " cloud called Azure Orbital, which allows satellite operators to",
+ " communicate with their spacecraft directly from Azure datacenters.",
+ "",
+ "- Massive Infrastructure: The network features over 175,000 miles of",
+ " terrestrial and subsea fiber-optic cables and operates in more regions",
+ " worldwide than any other cloud provider.",
+ "",
+ "- Linux Friendly: Despite being built by Microsoft, over half of the",
+ " Virtual Machine workloads running on Azure are based on Linux,",
+ " reflecting a deep embrace of open-source technology.",
+ "",
+ "- Extreme Physical Secrecy: Azure's datacenters are so state-of-the-art",
+ " that their physical addresses are never publicly listed to ensure",
+ " maximum security.",
+ ]
+
+ stream_parts = [
+ b"BT\n",
+ b"14 TL\n",
+ b"/F1 14 Tf\n",
+ b"72 720 Td\n",
+ f"({escape(title)}) Tj T*\n".encode(),
+ b"() Tj T*\n",
+ b"/F1 11 Tf\n",
+ ]
+ for line in body_lines:
+ stream_parts.append(f"({escape(line)}) Tj T*\n".encode())
+ stream_parts.append(b"ET\n")
+ stream = b"".join(stream_parts)
+
+ raw_objects: list[bytes] = [
+ b"<</Type /Catalog /Pages 2 0 R>>",
+ b"<</Type /Pages /Kids [3 0 R] /Count 1>>",
+ (
+ b"<</Type /Page /Parent 2 0 R"
+ b" /MediaBox [0 0 612 792]"
+ b" /Contents 4 0 R"
+ b" /Resources <</Font <</F1 5 0 R>>>>>>"
+ b">>"
+ ),
+ b"<</Length " + str(len(stream)).encode() + b">>\nstream\n" + stream + b"endstream",
+ b"<</Type /Font /Subtype /Type1 /BaseFont /Helvetica>>",
+ ]
+
+ body = b"%PDF-1.4\n"
+ offsets: list[int] = []
+ for idx, obj in enumerate(raw_objects, start=1):
+ offsets.append(len(body))
+ body += f"{idx} 0 obj\n".encode() + obj + b"\nendobj\n"
+
+ xref_pos = len(body)
+ n = len(raw_objects) + 1
+ # Each xref entry must be exactly 20 bytes: 10-digit offset, space, 5-digit gen,
+ # space, status flag, space, newline.
+ xref = f"xref\n0 {n}\n0000000000 65535 f \n"
+ for off in offsets:
+ xref += f"{off:010d} 00000 n \n"
+ trailer = f"trailer\n<</Size {n} /Root 1 0 R>>\nstartxref\n{xref_pos}\n%%EOF\n"
+
+ return body + xref.encode() + trailer.encode()
+
+
+def _xml_escape(text: str) -> str:
+ """Escape text for safe inclusion in XML element content/attributes."""
+ return (
+ text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
+ )
+
+
+def _zip_package(parts: dict[str, str]) -> bytes:
+ """Pack a mapping of archive-path -> XML/text into an in-memory zip (OOXML)."""
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+ for path, data in parts.items():
+ zf.writestr(path, data)
+ return buf.getvalue()
+
+
+def _col_letter(idx: int) -> str:
+ """Convert a 0-based column index to a spreadsheet column letter (A, B, ...)."""
+ letters = ""
+ idx += 1
+ while idx > 0:
+ idx, rem = divmod(idx - 1, 26)
+ letters = chr(65 + rem) + letters
+ return letters
+
+
+def _make_docx(title: str, paragraphs: list[str]) -> bytes:
+ """Build a minimal-but-valid .docx (WordprocessingML) in pure Python."""
+ body_parts = [
+ '<w:p><w:r><w:rPr><w:b/><w:sz w:val="32"/></w:rPr>'
+ f'<w:t xml:space="preserve">{_xml_escape(title)}</w:t></w:r></w:p>'
+ ]
+ for para in paragraphs:
+ body_parts.append(
+ f'<w:p><w:r><w:t xml:space="preserve">{_xml_escape(para)}</w:t></w:r></w:p>'
+ )
+
+ document = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">'
+ "<w:body>" + "".join(body_parts) + "<w:sectPr/></w:body></w:document>"
+ )
+ content_types = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
+ '<Default Extension="rels" '
+ 'ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
+ '<Default Extension="xml" ContentType="application/xml"/>'
+ '<Override PartName="/word/document.xml" '
+ 'ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>'
+ "</Types>"
+ )
+ rels = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId1" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '
+ 'Target="word/document.xml"/>'
+ "</Relationships>"
+ )
+ return _zip_package(
+ {
+ "[Content_Types].xml": content_types,
+ "_rels/.rels": rels,
+ "word/document.xml": document,
+ }
+ )
+
+
+def _make_xlsx(sheet_name: str, rows: list[list[str]]) -> bytes:
+ """Build a minimal-but-valid .xlsx (SpreadsheetML) with inline-string cells."""
+ row_xml = []
+ for r, row in enumerate(rows, start=1):
+ cells = []
+ for c, value in enumerate(row):
+ ref = f"{_col_letter(c)}{r}"
+ cells.append(
+ f'<c r="{ref}" t="inlineStr"><is><t xml:space="preserve">'
+ f"{_xml_escape(value)}</t></is></c>"
+ )
+ row_xml.append(f'<row r="{r}">' + "".join(cells) + "</row>")
+
+ sheet = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">'
+ "<sheetData>" + "".join(row_xml) + "</sheetData></worksheet>"
+ )
+ workbook = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" '
+ 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">'
+ f'<sheets><sheet name="{_xml_escape(sheet_name)}" sheetId="1" r:id="rId1"/></sheets>'
+ "</workbook>"
+ )
+ workbook_rels = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId1" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" '
+ 'Target="worksheets/sheet1.xml"/>'
+ "</Relationships>"
+ )
+ content_types = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
+ '<Default Extension="rels" '
+ 'ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
+ '<Default Extension="xml" ContentType="application/xml"/>'
+ '<Override PartName="/xl/workbook.xml" '
+ 'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>'
+ '<Override PartName="/xl/worksheets/sheet1.xml" '
+ 'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>'
+ "</Types>"
+ )
+ rels = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId1" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '
+ 'Target="xl/workbook.xml"/>'
+ "</Relationships>"
+ )
+ return _zip_package(
+ {
+ "[Content_Types].xml": content_types,
+ "_rels/.rels": rels,
+ "xl/workbook.xml": workbook,
+ "xl/_rels/workbook.xml.rels": workbook_rels,
+ "xl/worksheets/sheet1.xml": sheet,
+ }
+ )
+
+
+def _pptx_theme() -> str:
+ """Return a minimal theme1.xml required by a valid .pptx package."""
+ ph = '<a:solidFill><a:schemeClr val="phClr"/></a:solidFill>'
+ line = '<a:ln w="{w}"><a:solidFill><a:schemeClr val="phClr"/></a:solidFill></a:ln>'
+ return (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<a:theme xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" '
+ 'name="Office Theme"><a:themeElements>'
+ '<a:clrScheme name="Office">'
+ '<a:dk1><a:sysClr val="windowText" lastClr="000000"/></a:dk1>'
+ '<a:lt1><a:sysClr val="window" lastClr="FFFFFF"/></a:lt1>'
+ '<a:dk2><a:srgbClr val="44546A"/></a:dk2>'
+ '<a:lt2><a:srgbClr val="E7E6E6"/></a:lt2>'
+ '<a:accent1><a:srgbClr val="4472C4"/></a:accent1>'
+ '<a:accent2><a:srgbClr val="ED7D31"/></a:accent2>'
+ '<a:accent3><a:srgbClr val="A5A5A5"/></a:accent3>'
+ '<a:accent4><a:srgbClr val="FFC000"/></a:accent4>'
+ '<a:accent5><a:srgbClr val="5B9BD5"/></a:accent5>'
+ '<a:accent6><a:srgbClr val="70AD47"/></a:accent6>'
+ '<a:hlink><a:srgbClr val="0563C1"/></a:hlink>'
+ '<a:folHlink><a:srgbClr val="954F72"/></a:folHlink>'
+ "</a:clrScheme>"
+ '<a:fontScheme name="Office">'
+ '<a:majorFont><a:latin typeface="Calibri Light"/><a:ea typeface=""/>'
+ '<a:cs typeface=""/></a:majorFont>'
+ '<a:minorFont><a:latin typeface="Calibri"/><a:ea typeface=""/>'
+ '<a:cs typeface=""/></a:minorFont>'
+ "</a:fontScheme>"
+ '<a:fmtScheme name="Office">'
+ f"<a:fillStyleLst>{ph}{ph}{ph}</a:fillStyleLst>"
+ "<a:lnStyleLst>"
+ + line.format(w="6350")
+ + line.format(w="12700")
+ + line.format(w="19050")
+ + "</a:lnStyleLst>"
+ "<a:effectStyleLst>"
+ "<a:effectStyle><a:effectLst/></a:effectStyle>"
+ "<a:effectStyle><a:effectLst/></a:effectStyle>"
+ "<a:effectStyle><a:effectLst/></a:effectStyle>"
+ "</a:effectStyleLst>"
+ f"<a:bgFillStyleLst>{ph}{ph}{ph}</a:bgFillStyleLst>"
+ "</a:fmtScheme>"
+ "</a:themeElements></a:theme>"
+ )
+
+
+def _make_pptx(title: str, bullets: list[str]) -> bytes:
+ """Build a minimal-but-valid single-slide .pptx (PresentationML) in pure Python."""
+ p_ns = (
+ 'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" '
+ 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" '
+ 'xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"'
+ )
+
+ bullet_paras = "".join(f"<a:p><a:r><a:t>{_xml_escape(b)}</a:t></a:r></a:p>" for b in bullets)
+ slide = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ f"<p:sld {p_ns}><p:cSld><p:spTree>"
+ '<p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr>'
+ "<p:grpSpPr/>"
+ "<p:sp><p:nvSpPr>"
+ '<p:cNvPr id="2" name="Title 1"/>'
+ '<p:cNvSpPr><a:spLocks noGrp="1"/></p:cNvSpPr>'
+ '<p:nvPr><p:ph type="title"/></p:nvPr></p:nvSpPr>'
+ "<p:spPr/><p:txBody><a:bodyPr/><a:lstStyle/>"
+ f"<a:p><a:r><a:t>{_xml_escape(title)}</a:t></a:r></a:p></p:txBody></p:sp>"
+ "<p:sp><p:nvSpPr>"
+ '<p:cNvPr id="3" name="Content 2"/>'
+ '<p:cNvSpPr><a:spLocks noGrp="1"/></p:cNvSpPr>'
+ '<p:nvPr><p:ph type="body" idx="1"/></p:nvPr></p:nvSpPr>'
+ "<p:spPr/><p:txBody><a:bodyPr/><a:lstStyle/>"
+ f"{bullet_paras}</p:txBody></p:sp>"
+ "</p:spTree></p:cSld></p:sld>"
+ )
+ slide_rels = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId1" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout" '
+ 'Target="../slideLayouts/slideLayout1.xml"/>'
+ "</Relationships>"
+ )
+ slide_layout = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ f'<p:sldLayout {p_ns} type="blank" preserve="1"><p:cSld name="Blank"><p:spTree>'
+ '<p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr>'
+ "<p:grpSpPr/></p:spTree></p:cSld>"
+ "<p:clrMapOvr><a:masterClrMapping/></p:clrMapOvr></p:sldLayout>"
+ )
+ slide_layout_rels = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId1" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideMaster" '
+ 'Target="../slideMasters/slideMaster1.xml"/>'
+ "</Relationships>"
+ )
+ slide_master = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ f"<p:sldMaster {p_ns}><p:cSld><p:spTree>"
+ '<p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr>'
+ "<p:grpSpPr/></p:spTree></p:cSld>"
+ '<p:clrMap bg1="lt1" tx1="dk1" bg2="lt2" tx2="dk2" '
+ 'accent1="accent1" accent2="accent2" accent3="accent3" accent4="accent4" '
+ 'accent5="accent5" accent6="accent6" hlink="hlink" folHlink="folHlink"/>'
+ '<p:sldLayoutIdLst><p:sldLayoutId id="2147483649" r:id="rId1"/></p:sldLayoutIdLst>'
+ "</p:sldMaster>"
+ )
+ slide_master_rels = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId1" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout" '
+ 'Target="../slideLayouts/slideLayout1.xml"/>'
+ '<Relationship Id="rId2" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme" '
+ 'Target="../theme/theme1.xml"/>'
+ "</Relationships>"
+ )
+ presentation = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ f"<p:presentation {p_ns}>"
+ '<p:sldMasterIdLst><p:sldMasterId id="2147483648" r:id="rId1"/></p:sldMasterIdLst>'
+ '<p:sldIdLst><p:sldId id="256" r:id="rId2"/></p:sldIdLst>'
+ '<p:sldSz cx="9144000" cy="6858000"/><p:notesSz cx="6858000" cy="9144000"/>'
+ "</p:presentation>"
+ )
+ presentation_rels = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId1" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideMaster" '
+ 'Target="slideMasters/slideMaster1.xml"/>'
+ '<Relationship Id="rId2" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide" '
+ 'Target="slides/slide1.xml"/>'
+ '<Relationship Id="rId3" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme" '
+ 'Target="theme/theme1.xml"/>'
+ "</Relationships>"
+ )
+ content_types = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
+ '<Default Extension="rels" '
+ 'ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
+ '<Default Extension="xml" ContentType="application/xml"/>'
+ '<Override PartName="/ppt/presentation.xml" '
+ 'ContentType="application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml"/>'
+ '<Override PartName="/ppt/slides/slide1.xml" '
+ 'ContentType="application/vnd.openxmlformats-officedocument.presentationml.slide+xml"/>'
+ '<Override PartName="/ppt/slideLayouts/slideLayout1.xml" '
+ 'ContentType="application/vnd.openxmlformats-officedocument.presentationml.slideLayout+xml"/>'
+ '<Override PartName="/ppt/slideMasters/slideMaster1.xml" '
+ 'ContentType="application/vnd.openxmlformats-officedocument.presentationml.slideMaster+xml"/>'
+ '<Override PartName="/ppt/theme/theme1.xml" '
+ 'ContentType="application/vnd.openxmlformats-officedocument.theme+xml"/>'
+ "</Types>"
+ )
+ rels = (
+ '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
+ '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
+ '<Relationship Id="rId1" '
+ 'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '
+ 'Target="ppt/presentation.xml"/>'
+ "</Relationships>"
+ )
+ return _zip_package(
+ {
+ "[Content_Types].xml": content_types,
+ "_rels/.rels": rels,
+ "ppt/presentation.xml": presentation,
+ "ppt/_rels/presentation.xml.rels": presentation_rels,
+ "ppt/slides/slide1.xml": slide,
+ "ppt/slides/_rels/slide1.xml.rels": slide_rels,
+ "ppt/slideLayouts/slideLayout1.xml": slide_layout,
+ "ppt/slideLayouts/_rels/slideLayout1.xml.rels": slide_layout_rels,
+ "ppt/slideMasters/slideMaster1.xml": slide_master,
+ "ppt/slideMasters/_rels/slideMaster1.xml.rels": slide_master_rels,
+ "ppt/theme/theme1.xml": _pptx_theme(),
+ }
+ )
+
+
+_AZURE_FACTS = [
+ "Microsoft originally wanted a cloud name containing the word 'cloud' before "
+ "settling on 'Azure', the shade of blue of the sky behind the clouds.",
+ "Azure Orbital lets satellite operators communicate with their spacecraft "
+ "directly from Azure datacenters.",
+ "The Azure network spans over 175,000 miles of terrestrial and subsea "
+ "fiber-optic cable and operates in more regions than any other cloud provider.",
+ "Over half of all virtual machine workloads running on Azure are Linux-based.",
+ "Azure datacenter physical addresses are never publicly listed for security.",
+]
+
+
+_WORD_DOC = _make_docx(
+ "Fascinating Facts about Azure",
+ [
+ "This Word document was ingested from the local Azurite emulator to "
+ "verify the OpenRAG Azure Blob connector handles .docx files end to end.",
+ *_AZURE_FACTS,
+ ],
+)
+
+_EXCEL_DOC = _make_xlsx(
+ "Azure Facts",
+ [
+ ["Service", "Category", "Fact"],
+ ["Azure Orbital", "Space", "Talk to spacecraft from Azure datacenters."],
+ ["Global Network", "Infrastructure", "175,000+ miles of fiber-optic cable."],
+ ["Virtual Machines", "Compute", "Over half of VM workloads run Linux."],
+ ["Datacenters", "Security", "Physical addresses are never publicly listed."],
+ ],
+)
+
+_POWERPOINT_DOC = _make_pptx(
+ "Fascinating Facts about Azure",
+ _AZURE_FACTS,
+)
+
+_CSV_DOC = (
+ b"Service,Category,Fact\r\n"
+ b"Azure Orbital,Space,Talk to spacecraft from Azure datacenters.\r\n"
+ b"Global Network,Infrastructure,175000+ miles of fiber-optic cable.\r\n"
+ b"Virtual Machines,Compute,Over half of VM workloads run Linux.\r\n"
+ b"Datacenters,Security,Physical addresses are never publicly listed.\r\n"
+)
+
+_HTML_DOC = (
+ b"<!DOCTYPE html>\n"
+ b'<html lang="en">\n<head>\n<meta charset="utf-8"/>\n'
+ b"<title>Fascinating Facts about Azure</title>\n</head>\n<body>\n"
+ b"<h1>Fascinating Facts about Azure</h1>\n"
+ b"<p>This HTML file was ingested from the local Azurite emulator to verify "
+ b"the OpenRAG Azure Blob connector handles .html files end to end.</p>\n"
+ b"<ul>\n"
+ b"<li>Microsoft chose the name 'Azure' for the blue sky behind the clouds.</li>\n"
+ b"<li>Azure Orbital connects satellite operators to spacecraft from datacenters.</li>\n"
+ b"<li>The network spans over 175,000 miles of fiber-optic cable.</li>\n"
+ b"<li>Over half of Azure VM workloads run Linux.</li>\n"
+ b"<li>Datacenter physical addresses are never publicly listed.</li>\n"
+ b"</ul>\n</body>\n</html>\n"
+)
+
+
+# Container 1: plain text, markdown, PDF.
+BLOBS_1 = {
+ "azure-blob-text.txt": b"Hello from Azurite! OpenRAG Azure Blob connector test document.\n",
+ "notes/azure-blob-markdown.md": (
+ b"# Azure Blob Connector\n\n"
+ b"This markdown blob was ingested from the local Azurite emulator "
+ b"to verify the OpenRAG Azure Blob connector end to end.\n\n"
+ b"Microsoft Azure is a massive global cloud platform offering over 200 services. "
+ b"It powers 95% of Fortune 500 companies and is connected by enough fiber-optic cable "
+ b"to stretch to the Moon and back three times.\n"
+ ),
+ "docs/azure-blob-portal-document.pdf": _make_sample_pdf(),
+}
+
+# Container 1, "VERSION_2" updates: the same two text/markdown blobs as BLOBS_1
+# above, re-tagged with a "VERSION_2" identifier so re-ingestion / change
+# detection can be exercised via `--update`.
+BLOBS_1_V2 = {
+ "azure-blob-text.txt": (
+ b"Hello from Azurite! OpenRAG Azure Blob connector test document. VERSION_2\n"
+ ),
+ "notes/azure-blob-markdown.md": (
+ b"# Azure Blob Connector (VERSION_2)\n\n"
+ b"This markdown blob was ingested from the local Azurite emulator "
+ b"to verify the OpenRAG Azure Blob connector end to end.\n\n"
+ b"Microsoft Azure is a massive global cloud platform offering over 200 services. "
+ b"It powers 95% of Fortune 500 companies and is connected by enough fiber-optic cable "
+ b"to stretch to the Moon and back three times.\n"
+ ),
+}
+
+# Container 2: Office / web document formats.
+BLOBS_2 = {
+ "docs/azure-blob-word.docx": _WORD_DOC,
+ "spreadsheets/azure-blob-excel.xlsx": _EXCEL_DOC,
+ "presentations/azure-blob-powerpoint.pptx": _POWERPOINT_DOC,
+ "spreadsheets/azure-blob-comma-separated.csv": _CSV_DOC,
+ "azure-blob-hypertext-markup.html": _HTML_DOC,
+}
+
+
+def _seed_container(
+ svc: BlobServiceClient, name: str, blobs: dict[str, bytes], reset: bool
+) -> None:
+ """Create (optionally resetting) a container and upload the given blobs."""
+ container = svc.get_container_client(name)
+
+ if reset:
+ try:
+ container.delete_container()
+ print(f"Deleted existing container {name!r}.")
+ except Exception as exc: # ResourceNotFoundError if it didn't exist
+ print(f"Container {name!r} did not exist ({type(exc).__name__}), skipping delete.")
+
+ print(f"Ensuring container {name!r} exists...")
+ try:
+ container.create_container()
+ print(" created.")
+ except Exception as exc: # ResourceExistsError on re-run
+ print(f" already exists ({type(exc).__name__}).")
+
+ for blob_name, data in blobs.items():
+ print(f"Uploading blob {blob_name!r} ({len(data)} bytes)...")
+ container.get_blob_client(blob_name).upload_blob(data, overwrite=True)
+
+ print(f"\nDone. Blobs in container {name!r}:")
+ for b in container.list_blobs():
+ print(f" - {b.name} ({b.size} bytes)")
+
+
+def _update_blobs(svc: BlobServiceClient, name: str, blobs: dict[str, bytes]) -> None:
+ """Overwrite specific blobs in an existing container (no create/delete)."""
+ container = svc.get_container_client(name)
+ for blob_name, data in blobs.items():
+ print(f"Updating blob {blob_name!r} ({len(data)} bytes)...")
+ container.get_blob_client(blob_name).upload_blob(data, overwrite=True)
+
+ print(f"\nDone. Updated {len(blobs)} blob(s) in container {name!r}:")
+ for blob_name in blobs:
+ print(f" - {blob_name}")
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+ )
+ parser.add_argument(
+ "--reset", action="store_true", help="Delete the containers before seeding."
+ )
+ parser.add_argument(
+ "--update",
+ action="store_true",
+ help=(
+ 'Update the "azure-blob-text.txt" and "notes/azure-blob-markdown.md" '
+ f'blobs in {CONTAINER_1!r} in place with a "VERSION_2" identifier, then exit.'
+ ),
+ )
+ args = parser.parse_args()
+
+ print("Connecting to Azurite...")
+ svc = BlobServiceClient.from_connection_string(CONN)
+
+ if args.update:
+ _update_blobs(svc, CONTAINER_1, BLOBS_1_V2)
+ return
+
+ _seed_container(svc, CONTAINER_1, BLOBS_1, args.reset)
+ print()
+ _seed_container(svc, CONTAINER_2, BLOBS_2, args.reset)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 901148957..b524225f1 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -1,3 +1,4 @@
+from datetime import UTC, datetime
from typing import Any
from fastapi import Depends, Request
@@ -20,6 +21,7 @@ from services.connector_access_service import (
CONNECTOR_TYPES,
filter_connectors_for_user,
get_access_map,
+ is_bucket_connector_type,
is_connector_access_policy_enforced,
is_connector_allowed_for_request,
list_access_for_admin,
@@ -194,6 +196,176 @@ async def get_synced_id_to_filename_map(
return {}
+async def get_synced_id_to_modified_time_map(
+ connector_type: str,
+ user_id: str,
+ session_manager,
+ jwt_token: str | None = None,
+) -> dict[str, float | None]:
+ """Map each ingested connector source id → its stored ``modified_time`` (epoch ms).
+
+ Powers change detection for bucket connectors: callers compare a blob's remote
+ ``modified_time`` against the stored value to decide whether a re-ingest is needed.
+
+ A key being **present** means the source id is already ingested under this
+ ``connector_type``. A value of **None** means it was ingested but no
+ ``modified_time`` was persisted (pre-change-detection docs, or an ingest path that
+ didn't enrich it) — callers treat that as *unchanged* to avoid a mass re-ingest.
+
+ Keys cover both ingest layouts (mirrors ``get_synced_file_ids_for_connector``):
+ the non-Langflow path stores the connector id in ``connector_file_id`` (where
+ ``document_id`` is a content hash), while the Langflow path stores it in
+ ``document_id``. ``connector_file_id`` wins when both are present for the same id.
+ """
+ try:
+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
+
+ query_body = {
+ "size": 0,
+ "query": {"term": {"connector_type": connector_type}},
+ "aggs": {
+ "by_connector_file_id": {
+ "terms": {"field": "connector_file_id", "size": 10000},
+ "aggs": {"latest_modified": {"max": {"field": "modified_time"}}},
+ },
+ "by_document_id": {
+ "terms": {"field": "document_id", "size": 10000},
+ "aggs": {"latest_modified": {"max": {"field": "modified_time"}}},
+ },
+ },
+ }
+
+ result = await opensearch_client.search(index=get_index_name(), body=query_body)
+ aggs = result.get("aggregations", {})
+
+ mapping: dict[str, float | None] = {}
+ # document_id first; connector_file_id overlays it (connector_file_id wins).
+ # The content-hash document_ids from the non-Langflow path are harmless noise —
+ # they never match an enumerated connector source id.
+ for agg_name in ("by_document_id", "by_connector_file_id"):
+ for bucket in aggs.get(agg_name, {}).get("buckets", []):
+ key = bucket.get("key")
+ if not key:
+ continue
+ mapping[key] = bucket.get("latest_modified", {}).get("value")
+ return mapping
+ except Exception as e:
+ logger.error(
+ "Failed to build id→modified_time map",
+ connector_type=connector_type,
+ error=str(e),
+ )
+ return {}
+
+
+# Tolerance (ms) to avoid a false-positive "changed" when a remote ISO timestamp
+# round-trips through the OpenSearch ``date`` field with sub-second rounding.
+_CHANGE_DETECTION_TOLERANCE_MS = 1000.0
+
+
+def _parse_iso_to_epoch_ms(value: str | None) -> float | None:
+ """Parse an ISO-8601 timestamp to epoch milliseconds, or None if unparseable."""
+ if not value:
+ return None
+ try:
+ s = value.strip()
+ if s.endswith("Z"):
+ s = s[:-1] + "+00:00"
+ dt = datetime.fromisoformat(s)
+ if dt.tzinfo is None:
+ dt = dt.replace(tzinfo=UTC)
+ return dt.timestamp() * 1000.0
+ except (ValueError, TypeError):
+ return None
+
+
+def remote_is_newer_than_synced(
+ file_id: str,
+ remote_modified_time: str | None,
+ synced_modified_map: dict[str, float | None],
+) -> bool:
+ """True only when a stored ``modified_time`` exists and the remote is strictly newer.
+
+ Missing/unparseable timestamps (remote or stored) → False, so we never re-ingest
+ on ambiguity (backfill-safe).
+ """
+ stored_ms = synced_modified_map.get(file_id)
+ if stored_ms is None:
+ return False
+ remote_ms = _parse_iso_to_epoch_ms(remote_modified_time)
+ if remote_ms is None:
+ return False
+ return remote_ms - stored_ms > _CHANGE_DETECTION_TOLERANCE_MS
+
+
+def classify_remote_file_change(
+ file_id: str,
+ remote_modified_time: str | None,
+ is_ingested: bool,
+ synced_modified_map: dict[str, float | None],
+) -> str:
+ """Classify a remote blob as ``"new"`` / ``"changed"`` / ``"unchanged"``.
+
+ - ``new`` — not yet ingested.
+ - ``changed`` — ingested and the remote version is strictly newer than stored.
+ - ``unchanged`` — ingested and same-age/older, or the change can't be proven
+ (no stored token, e.g. backfill).
+ """
+ if not is_ingested:
+ return "new"
+ if remote_is_newer_than_synced(file_id, remote_modified_time, synced_modified_map):
+ return "changed"
+ return "unchanged"
+
+
+async def bucket_changed_file_ids(
+ connector,
+ connector_type: str,
+ user_id: str,
+ session_manager,
+ jwt_token: str | None,
+ existing_file_ids: list[str],
+) -> list[str]:
+ """Return the already-ingested bucket file ids whose remote copy is newer.
+
+ Updates-only change detection for the Sync path: list the connector's blobs
+ once and keep the ids that are (a) already ingested and (b) strictly newer at
+ source than the stored ``modified_time``. New blobs that aren't ingested yet
+ are intentionally ignored — Sync reconciles existing files; new files are
+ added via the connector's "Add from" panel. Listing exceptions propagate to
+ the caller (same as the bucket_filter sync path).
+ """
+ existing_set = set(existing_file_ids)
+ if not existing_set:
+ return []
+
+ modified_map = await get_synced_id_to_modified_time_map(
+ connector_type=connector_type,
+ user_id=user_id,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ )
+
+ changed_ids: list[str] = []
+ page_token = None
+ while True:
+ result = await connector.list_files(page_token=page_token)
+ for f in result.get("files", []):
+ fid = f.get("id")
+ if not fid or fid not in existing_set:
+ continue
+ if (
+ classify_remote_file_change(fid, f.get("modified_time"), True, modified_map)
+ == "changed"
+ ):
+ changed_ids.append(fid)
+ page_token = result.get("next_page_token")
+ if not page_token:
+ break
+
+ return changed_ids
+
+
async def compute_orphans_for_connector_type(
connector_type: str,
user_id: str,
@@ -739,6 +911,10 @@ async def connector_sync(
connection_id=working_connection.connection_id,
)
+ # Branches set either ``task_id`` (single batch) or ``task_ids`` (the
+ # bucket_filter path may emit two batches: new files + changed files).
+ task_ids: list[str] = []
+
if selected_files:
# Explicit files selected (e.g., from file picker) - sync those specific files
from .documents import _ensure_index_exists
@@ -775,23 +951,27 @@ async def connector_sync(
)
connector = await connector_service.get_connector(working_connection.connection_id)
if body.bucket_filter:
- # List only files from the requested buckets, then sync_specific_files
+ # List only files from the requested buckets, then reconcile against
+ # what's already ingested so we re-fetch only NEW and CHANGED blobs
+ # (skipping unchanged ones), rather than re-downloading the whole
+ # container every time. Per-file dedup in ConnectorFileProcessor is a
+ # backstop, but it runs after download — this pre-filter avoids the
+ # redundant fetch/reprocess and the misleading "all files" task view.
original_buckets = connector.bucket_names
connector.bucket_names = body.bucket_filter
try:
- all_file_ids = []
+ all_files: list[dict[str, Any]] = []
page_token = None
while True:
result = await connector.list_files(page_token=page_token)
- for f in result.get("files", []):
- all_file_ids.append(f["id"])
+ all_files.extend(result.get("files", []))
page_token = result.get("next_page_token")
if not page_token:
break
finally:
connector.bucket_names = original_buckets
- if not all_file_ids:
+ if not all_files:
return JSONResponse(
{
"status": "no_files",
@@ -799,14 +979,83 @@ async def connector_sync(
},
status_code=200,
)
- task_id = await connector_service.sync_specific_files(
- working_connection.connection_id,
- user.user_id,
- all_file_ids,
+
+ # Classify each remote blob as new / changed / unchanged.
+ existing_ids, _, _ = await get_synced_file_ids_for_connector(
+ connector_type=connector_type,
+ user_id=user.user_id,
+ session_manager=session_manager,
+ jwt_token=jwt_token,
+ )
+ existing_set = set(existing_ids)
+ modified_map = await get_synced_id_to_modified_time_map(
+ connector_type=connector_type,
+ user_id=user.user_id,
+ session_manager=session_manager,
jwt_token=jwt_token,
- ingest_settings=body.settings,
- shared=body.shared,
)
+
+ new_ids: list[str] = []
+ changed_ids: list[str] = []
+ for f in all_files:
+ fid = f.get("id")
+ if not fid:
+ continue
+ status = classify_remote_file_change(
+ fid,
+ f.get("modified_time"),
+ fid in existing_set,
+ modified_map,
+ )
+ if status == "new":
+ new_ids.append(fid)
+ elif status == "changed":
+ changed_ids.append(fid)
+ # "unchanged" → skip; already ingested and not newer at source.
+
+ logger.info(
+ "Reconciled bucket selection",
+ connector_type=connector_type,
+ total=len(all_files),
+ new=len(new_ids),
+ changed=len(changed_ids),
+ skipped=len(all_files) - len(new_ids) - len(changed_ids),
+ )
+
+ if not new_ids and not changed_ids:
+ return JSONResponse(
+ {
+ "status": "no_files",
+ "message": "All files in the selected buckets are already up to date.",
+ },
+ status_code=200,
+ )
+
+ # Two batches: new files are created; changed files replace the
+ # indexed copy (replace_duplicates=True bypasses the filename-skip
+ # and deletes stale chunks before re-ingest). replace is batch-level,
+ # hence the split.
+ if new_ids:
+ task_ids.append(
+ await connector_service.sync_specific_files(
+ working_connection.connection_id,
+ user.user_id,
+ new_ids,
+ jwt_token=jwt_token,
+ ingest_settings=body.settings,
+ )
+ )
+ if changed_ids:
+ task_ids.append(
+ await connector_service.sync_specific_files(
+ working_connection.connection_id,
+ user.user_id,
+ changed_ids,
+ jwt_token=jwt_token,
+ ingest_settings=body.settings,
+ replace_duplicates=True,
+ )
+ )
else:
# sync_all: ingest everything the connector can see
task_id = await connector_service.sync_connector_files(
@@ -874,15 +1123,50 @@ async def connector_sync(
},
status_code=200,
)
- task_id = await connector_service.sync_specific_files(
- working_connection.connection_id,
- user.user_id,
- ids_to_sync,
- jwt_token=jwt_token,
- ingest_settings=body.settings,
- replace_duplicates=_connector_sync_should_replace(connector_type),
- shared=body.shared,
- )
+ if is_bucket_connector_type(connector_type):
+ # Bucket Sync is updates-only: re-ingest just the blobs whose
+ # remote copy is newer than what's indexed (deleting the stale
+ # chunks via replace_duplicates). Unlike replace_duplicates=False
+ # this actually propagates content changes; unlike replacing every
+ # id it skips unchanged blobs instead of re-fetching the container.
+ connector = await connector_service.get_connector(
+ working_connection.connection_id
+ )
+ changed_ids = await bucket_changed_file_ids(
+ connector,
+ connector_type,
+ user.user_id,
+ session_manager,
+ jwt_token,
+ ids_to_sync,
+ )
+ if not changed_ids:
+ return JSONResponse(
+ {
+ "status": "no_files",
+ "message": f"All {connector_type} files are already up to date.",
+ },
+ status_code=200,
+ )
+ task_id = await connector_service.sync_specific_files(
+ working_connection.connection_id,
+ user.user_id,
+ changed_ids,
+ jwt_token=jwt_token,
+ ingest_settings=body.settings,
+ replace_duplicates=True,
+ shared=body.shared,
+ )
+ else:
+ task_id = await connector_service.sync_specific_files(
+ working_connection.connection_id,
+ user.user_id,
+ ids_to_sync,
+ jwt_token=jwt_token,
+ ingest_settings=body.settings,
+ replace_duplicates=_connector_sync_should_replace(connector_type),
+ shared=body.shared,
+ )
else:
# Fallback: use filename filtering (for Langflow-ingested files without document_id)
logger.info(
@@ -900,7 +1184,10 @@ async def connector_sync(
replace_duplicates=_connector_sync_should_replace(connector_type),
shared=body.shared,
)
- task_ids = [task_id]
+ # The bucket_filter path may have already populated task_ids (new + changed
+ # batches); every other branch sets a single task_id.
+ if not task_ids:
+ task_ids = [task_id]
await TelemetryClient.send_event(
Category.CONNECTOR_OPERATIONS, MessageId.ORB_CONN_SYNC_COMPLETE
)
@@ -1427,13 +1714,39 @@ async def sync_all_connectors(
if not existing_file_ids:
deleted_only_connectors.append(connector_type)
continue
- task_id = await connector_service.sync_specific_files(
- working_connection.connection_id,
- user.user_id,
- existing_file_ids,
- jwt_token=jwt_token,
- replace_duplicates=_connector_sync_should_replace(connector_type),
- )
+ if is_bucket_connector_type(connector_type):
+ # Updates-only change detection (see connector_sync): re-ingest
+ # only blobs that changed at source, replacing stale chunks.
+ connector = await connector_service.get_connector(
+ working_connection.connection_id
+ )
+ changed_ids = await bucket_changed_file_ids(
+ connector,
+ connector_type,
+ user.user_id,
+ session_manager,
+ jwt_token,
+ existing_file_ids,
+ )
+ if not changed_ids:
+ # Nothing changed at source — already up to date.
+ skipped_connectors.append(connector_type)
+ continue
+ task_id = await connector_service.sync_specific_files(
+ working_connection.connection_id,
+ user.user_id,
+ changed_ids,
+ jwt_token=jwt_token,
+ replace_duplicates=True,
+ )
+ else:
+ task_id = await connector_service.sync_specific_files(
+ working_connection.connection_id,
+ user.user_id,
+ existing_file_ids,
+ jwt_token=jwt_token,
+ replace_duplicates=_connector_sync_should_replace(connector_type),
+ )
else:
# Fallback: use filename filtering
logger.info(
@@ -1851,19 +2164,35 @@ async def browse_connection_files(
)
ingested_set = set(ingested_ids) | set(ingested_filenames)
+ # Stored modified_time per ingested source id, for "update available" detection.
+ modified_map = await get_synced_id_to_modified_time_map(
+ connector_type=connector_type,
+ user_id=user.user_id,
+ session_manager=session_manager,
+ jwt_token=user.jwt_token,
+ )
+
# Merge ingestion status into remote file list
enriched_files = []
for f in remote_files:
- is_ingested = f.get("id", "") in ingested_set or f.get("name", "") in ingested_set
+ file_id = f.get("id", "")
+ is_ingested = file_id in ingested_set or f.get("name", "") in ingested_set
+ # "Update available": ingested, but the source version is newer than what
+ # we indexed. The frontend keeps unchanged files disabled but lets the user
+ # re-ingest stale ones (with replace_duplicates).
+ is_stale = is_ingested and remote_is_newer_than_synced(
+ file_id, f.get("modified_time"), modified_map
+ )
enriched_files.append(
{
- "id": f.get("id", ""),
+ "id": file_id,
"name": f.get("name", ""),
"bucket": f.get("bucket", ""),
"key": f.get("key", ""),
"size": f.get("size", 0),
"modified_time": f.get("modified_time", ""),
"is_ingested": is_ingested,
+ "is_stale": is_stale,
}
)
diff --git a/src/app/container.py b/src/app/container.py
index 2b32ef0cb..e7a3609cb 100644
--- a/src/app/container.py
+++ b/src/app/container.py
@@ -12,6 +12,7 @@ from config.settings import (
config_manager,
get_embedding_model,
get_index_name,
+ is_dev_azure_blob_enabled,
is_no_auth_mode,
)
from connectors.service import ConnectorService
@@ -40,6 +41,19 @@ from utils.telemetry import Category, MessageId, TelemetryClient
logger = get_logger(__name__)
+def _should_load_persisted_connections() -> bool:
+ """Whether to load persisted connector connections at startup.
+
+ Normally skipped in no-auth mode because OAuth connectors are unusable
+ there — but dev bucket connectors (e.g. Azure Blob via
+ OPENRAG_DEV_AZURE_BLOB) DO work in no-auth mode and persist real
+ connections. Without loading them, a saved connection silently disappears
+ on restart (the in-memory dict empties), leaving the UI stuck on "Connect".
+ OR in those dev flags so their connections survive a restart.
+ """
+ return not is_no_auth_mode() or is_dev_azure_blob_enabled()
+
+
async def initialize_services():
"""Initialize all services and their dependencies"""
await TelemetryClient.send_event(Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_INIT_START)
@@ -131,9 +145,9 @@ async def initialize_services():
)
# Load persisted connector connections at startup so webhooks and syncs
- # can resolve existing subscriptions immediately after server boot
- # Skip in no-auth mode since connectors require OAuth
- if not is_no_auth_mode():
+ # can resolve existing subscriptions immediately after server boot.
+ # (See _should_load_persisted_connections for the no-auth/dev-bucket nuance.)
+ if _should_load_persisted_connections():
try:
await connector_service.initialize()
loaded_count = len(connector_service.connection_manager.connections)
diff --git a/src/config/settings.py b/src/config/settings.py
index b00b7c15d..32ccd3a9b 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -247,6 +247,17 @@ def is_dev_connector_policy_enabled() -> bool:
return raw in ("true", "1", "yes", "on")
+def is_dev_azure_blob_enabled() -> bool:
+ """Local dev: enable Azure Blob connector without IBM_AUTH_ENABLED.
+
+ Allows testing the Azure Blob connector (e.g. against Azurite) in a local
+ environment where IBM auth is not configured. Never enable in production.
+ Requires ``OPENRAG_DEV_AZURE_BLOB=true``.
+ """
+ raw = os.getenv("OPENRAG_DEV_AZURE_BLOB", "false").strip().lower()
+ return raw in ("true", "1", "yes", "on")
+
+
def is_cloud_context() -> bool:
"""True when connector policy and SaaS settings guards should apply."""
from utils.run_mode_utils import is_run_mode_saas
diff --git a/src/models/processors.py b/src/models/processors.py
index af46f5e4a..c767a2b61 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -1066,6 +1066,20 @@ class ConnectorFileProcessor(TaskProcessor):
"status": "error",
"error": "No text content could be extracted from document",
}
+
+ # Persist connector metadata (incl. modified_time) onto the
+ # Langflow-indexed chunks (keyed by document_id) so bucket-connector
+ # change detection has a stored timestamp to compare against on the
+ # next sync. Mirrors the standard path's enrichment below.
+ if result.get("status") != "error":
+ await self.connector_service._update_connector_metadata(
+ document,
+ self.user_id,
+ connector_type,
+ self.jwt_token,
+ id_field="document_id",
+ indexed_filename=file_task.filename,
+ )
else:
# Standard OpenRAG processing pipeline (process_document_standard)
standard_kwargs: dict[str, Any] = {}
diff --git a/src/services/connector_access_service.py b/src/services/connector_access_service.py
index 7a64edb17..9948da251 100644
--- a/src/services/connector_access_service.py
+++ b/src/services/connector_access_service.py
@@ -13,7 +13,12 @@ CONNECTOR_ACCESS_SECTION = "connector_access"
# without touching this module.
CONNECTOR_TYPES: tuple[str, ...] = tuple(cls.CONNECTOR_TYPE for cls in get_connector_classes())
-_BUCKET_CONNECTOR_TYPES = frozenset({"aws_s3", "ibm_cos"})
+_BUCKET_CONNECTOR_TYPES = frozenset({"aws_s3", "ibm_cos", "azure_blob"})
+
+
+def is_bucket_connector_type(connector_type: str) -> bool:
+ """True for object-storage "bucket" connectors (S3, IBM COS, Azure Blob)."""
+ return connector_type in _BUCKET_CONNECTOR_TYPES
def governable_connector_types() -> tuple[str, ...]:
diff --git a/tests/unit/api/test_connector_change_detection.py b/tests/unit/api/test_connector_change_detection.py
new file mode 100644
index 000000000..02c46e811
--- /dev/null
+++ b/tests/unit/api/test_connector_change_detection.py
@@ -0,0 +1,522 @@
+"""Unit tests for bucket-connector change detection in `src/api/connectors.py`.
+
+Covers:
+- the pure timestamp/classification helpers (`_parse_iso_to_epoch_ms`,
+ `remote_is_newer_than_synced`, `classify_remote_file_change`),
+- the `get_synced_id_to_modified_time_map` aggregation helper,
+- the whole-container (`bucket_filter`) reconciliation in `connector_sync`: only
+ new + changed blobs are ingested (new as a plain batch, changed with
+ replace_duplicates=True), unchanged blobs are skipped.
+"""
+
+import json
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def _json(response):
+ return json.loads(response.body.decode())
+
+
+# ---------------------------------------------------------------------------
+# Pure helpers
+# ---------------------------------------------------------------------------
+
+
+def test_parse_iso_to_epoch_ms_handles_z_and_offset_and_naive():
+ from api.connectors import _parse_iso_to_epoch_ms
+
+ z = _parse_iso_to_epoch_ms("2024-01-01T00:00:00Z")
+ offset = _parse_iso_to_epoch_ms("2024-01-01T00:00:00+00:00")
+ naive = _parse_iso_to_epoch_ms("2024-01-01T00:00:00")
+ assert z == offset == naive == 1704067200000.0
+
+
+def test_parse_iso_to_epoch_ms_returns_none_for_bad_input():
+ from api.connectors import _parse_iso_to_epoch_ms
+
+ assert _parse_iso_to_epoch_ms(None) is None
+ assert _parse_iso_to_epoch_ms("") is None
+ assert _parse_iso_to_epoch_ms("not-a-date") is None
+
+
+def test_remote_is_newer_than_synced_true_when_strictly_newer():
+ from api.connectors import remote_is_newer_than_synced
+
+ stored = {"c::a": 1704067200000.0} # 2024-01-01
+ assert remote_is_newer_than_synced("c::a", "2024-06-01T00:00:00Z", stored) is True
+
+
+def test_remote_is_newer_than_synced_false_when_same_or_older():
+ from api.connectors import remote_is_newer_than_synced
+
+ stored = {"c::a": 1704067200000.0}
+ assert remote_is_newer_than_synced("c::a", "2024-01-01T00:00:00Z", stored) is False
+ assert remote_is_newer_than_synced("c::a", "2023-01-01T00:00:00Z", stored) is False
+
+
+def test_remote_is_newer_than_synced_tolerance_absorbs_subsecond_jitter():
+ from api.connectors import remote_is_newer_than_synced
+
+ stored = {"c::a": 1704067200000.0}
+ # 500ms newer is within tolerance → not "changed".
+ assert remote_is_newer_than_synced("c::a", "2024-01-01T00:00:00.500Z", stored) is False
+ # 2s newer exceeds tolerance → changed.
+ assert remote_is_newer_than_synced("c::a", "2024-01-01T00:00:02Z", stored) is True
+
+
+def test_remote_is_newer_than_synced_false_when_no_stored_token():
+ from api.connectors import remote_is_newer_than_synced
+
+ # Missing id, or ingested-but-no-token (None) → backfill-safe False.
+ assert remote_is_newer_than_synced("c::missing", "2024-06-01T00:00:00Z", {}) is False
+ assert remote_is_newer_than_synced("c::a", "2024-06-01T00:00:00Z", {"c::a": None}) is False
+
+
+def test_classify_remote_file_change():
+ from api.connectors import classify_remote_file_change
+
+ stored = {"c::a": 1704067200000.0}
+ # Not ingested → new (regardless of timestamps).
+ assert classify_remote_file_change("c::new", "2024-06-01T00:00:00Z", False, stored) == "new"
+ # Ingested + newer → changed.
+ assert classify_remote_file_change("c::a", "2024-06-01T00:00:00Z", True, stored) == "changed"
+ # Ingested + same → unchanged.
+ assert classify_remote_file_change("c::a", "2024-01-01T00:00:00Z", True, stored) == "unchanged"
+ # Ingested but no stored token (backfill) → unchanged.
+ assert classify_remote_file_change("c::b", "2024-06-01T00:00:00Z", True, {}) == "unchanged"
+
+
+# ---------------------------------------------------------------------------
+# get_synced_id_to_modified_time_map
+# ---------------------------------------------------------------------------
+
+
+async def test_modified_time_map_prefers_connector_file_id_over_document_id(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(
+ return_value={
+ "aggregations": {
+ "by_connector_file_id": {
+ "buckets": [
+ {"key": "c::a", "latest_modified": {"value": 1704067200000.0}},
+ # connector_file_id present but no modified_time → None
+ {"key": "c::b", "latest_modified": {"value": None}},
+ ]
+ },
+ "by_document_id": {
+ "buckets": [
+ # Langflow-path id (document_id holds the connector source id).
+ {"key": "c::lf", "latest_modified": {"value": 1704153600000.0}},
+ # A content-hash document_id from the standard path — harmless,
+ # never matches an enumerated source id. Overlaid/ignored.
+ {"key": "c::a", "latest_modified": {"value": 999.0}},
+ ]
+ },
+ }
+ }
+ )
+ sm = MagicMock()
+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ result = await connectors_api.get_synced_id_to_modified_time_map(
+ connector_type="azure_blob",
+ user_id="alice",
+ session_manager=sm,
+ jwt_token=None,
+ )
+
+ # connector_file_id wins for c::a (1704067200000, not the 999 from document_id).
+ assert result["c::a"] == 1704067200000.0
+ assert result["c::b"] is None
+ assert result["c::lf"] == 1704153600000.0
+
+
+async def test_modified_time_map_returns_empty_on_error(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api, "get_index_name", lambda: "idx")
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(side_effect=RuntimeError("boom"))
+ sm = MagicMock()
+ sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ result = await connectors_api.get_synced_id_to_modified_time_map(
+ connector_type="azure_blob",
+ user_id="alice",
+ session_manager=sm,
+ jwt_token=None,
+ )
+ assert result == {}
+
+
+# ---------------------------------------------------------------------------
+# connector_sync — bucket_filter reconciliation
+# ---------------------------------------------------------------------------
+
+
+def _bucket_sync_service(remote_files, task_id="task-x"):
+ connection = SimpleNamespace(connection_id="conn-1", is_active=True)
+ connector = MagicMock()
+ connector.authenticate = AsyncMock(return_value=True)
+ connector.bucket_names = None
+ connector.list_files = AsyncMock(return_value={"files": remote_files, "next_page_token": None})
+
+ service = MagicMock()
+ service.connection_manager = MagicMock()
+ service.connection_manager.list_connections = AsyncMock(return_value=[connection])
+ service.get_connector = AsyncMock(return_value=connector)
+ service.sync_specific_files = AsyncMock(return_value=task_id)
+ return service
+
+
+async def test_bucket_filter_ingests_only_new_and_changed(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ # "c::ingested_unchanged" and "c::ingested_changed" are already ingested.
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(
+ return_value=(["c::ingested_unchanged", "c::ingested_changed"], [], "connector_file_id")
+ ),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_id_to_modified_time_map",
+ AsyncMock(
+ return_value={
+ "c::ingested_unchanged": 1704067200000.0, # 2024-01-01
+ "c::ingested_changed": 1704067200000.0, # 2024-01-01
+ }
+ ),
+ )
+
+ remote_files = [
+ {"id": "c::new", "modified_time": "2024-01-01T00:00:00Z"},
+ {"id": "c::ingested_unchanged", "modified_time": "2024-01-01T00:00:00Z"},
+ {"id": "c::ingested_changed", "modified_time": "2024-06-01T00:00:00Z"},
+ ]
+ service = _bucket_sync_service(remote_files)
+
+ response = await connectors_api.connector_sync(
+ "azure_blob",
+ connectors_api.ConnectorSyncBody(connection_id="conn-1", bucket_filter=["c"]),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
+ )
+
+ assert response.status_code == 201
+ assert _json(response)["task_ids"] == ["task-x", "task-x"]
+
+ # Two batches: new (replace defaulted False) + changed (replace=True).
+ assert service.sync_specific_files.await_count == 2
+ new_call, changed_call = service.sync_specific_files.await_args_list
+ assert new_call.args[2] == ["c::new"]
+ assert new_call.kwargs.get("replace_duplicates", False) is False
+ assert changed_call.args[2] == ["c::ingested_changed"]
+ assert changed_call.kwargs["replace_duplicates"] is True
+
+
+async def test_bucket_filter_all_unchanged_returns_no_files(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["c::a", "c::b"], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_id_to_modified_time_map",
+ AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),
+ )
+
+ remote_files = [
+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"},
+ {"id": "c::b", "modified_time": "2024-01-01T00:00:00Z"},
+ ]
+ service = _bucket_sync_service(remote_files)
+
+ response = await connectors_api.connector_sync(
+ "azure_blob",
+ connectors_api.ConnectorSyncBody(connection_id="conn-1", bucket_filter=["c"]),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
+ )
+
+ assert response.status_code == 200
+ body = _json(response)
+ assert body["status"] == "no_files"
+ assert "up to date" in body["message"]
+ service.sync_specific_files.assert_not_awaited()
+
+
+async def test_bucket_filter_only_new_files_single_batch(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=([], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_id_to_modified_time_map",
+ AsyncMock(return_value={}),
+ )
+
+ remote_files = [
+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"},
+ {"id": "c::b", "modified_time": "2024-01-01T00:00:00Z"},
+ ]
+ service = _bucket_sync_service(remote_files)
+
+ response = await connectors_api.connector_sync(
+ "azure_blob",
+ connectors_api.ConnectorSyncBody(connection_id="conn-1", bucket_filter=["c"]),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
+ )
+
+ assert response.status_code == 201
+ service.sync_specific_files.assert_awaited_once()
+ call = service.sync_specific_files.await_args
+ assert call.args[2] == ["c::a", "c::b"]
+ assert call.kwargs.get("replace_duplicates", False) is False
+
+
+# ---------------------------------------------------------------------------
+# bucket_changed_file_ids — updates-only change detection helper
+# ---------------------------------------------------------------------------
+
+
+async def test_bucket_changed_file_ids_filters_to_changed_ingested(monkeypatch):
+ """Only already-ingested blobs that are newer at source are returned; new
+ (un-ingested) blobs are ignored (updates-only)."""
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_id_to_modified_time_map",
+ AsyncMock(
+ return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0} # 2024-01-01
+ ),
+ )
+
+ connector = MagicMock()
+ connector.list_files = AsyncMock(
+ return_value={
+ "files": [
+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"}, # unchanged
+ {"id": "c::b", "modified_time": "2024-06-01T00:00:00Z"}, # changed
+ {"id": "c::new", "modified_time": "2024-06-01T00:00:00Z"}, # new → ignored
+ ],
+ "next_page_token": None,
+ }
+ )
+
+ changed = await connectors_api.bucket_changed_file_ids(
+ connector,
+ "azure_blob",
+ "alice",
+ MagicMock(),
+ "token",
+ ["c::a", "c::b"],
+ )
+ assert changed == ["c::b"]
+
+
+async def test_bucket_changed_file_ids_empty_when_no_existing():
+ from api import connectors as connectors_api
+
+ connector = MagicMock()
+ connector.list_files = AsyncMock()
+ changed = await connectors_api.bucket_changed_file_ids(
+ connector, "azure_blob", "alice", MagicMock(), "token", []
+ )
+ assert changed == []
+ connector.list_files.assert_not_awaited()
+
+
+# ---------------------------------------------------------------------------
+# connector_sync (Sync button, no selected_files/sync_all/bucket_filter) —
+# updates-only re-ingest for bucket connectors
+# ---------------------------------------------------------------------------
+
+
+async def test_sync_button_bucket_reingests_only_changed(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["c::a", "c::b"], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "reconcile_orphans_for_connector_type",
+ AsyncMock(return_value=[]),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_id_to_modified_time_map",
+ AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),
+ )
+
+ remote_files = [
+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"}, # unchanged
+ {"id": "c::b", "modified_time": "2024-06-01T00:00:00Z"}, # changed
+ ]
+ service = _bucket_sync_service(remote_files)
+
+ response = await connectors_api.connector_sync(
+ "azure_blob",
+ connectors_api.ConnectorSyncBody(), # plain Sync: no files, no sync_all/bucket_filter
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
+ )
+
+ assert response.status_code == 201
+ service.sync_specific_files.assert_awaited_once()
+ call = service.sync_specific_files.await_args
+ assert call.args[2] == ["c::b"]
+ assert call.kwargs["replace_duplicates"] is True
+
+
+async def test_sync_button_bucket_all_unchanged_returns_no_files(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["c::a", "c::b"], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "reconcile_orphans_for_connector_type",
+ AsyncMock(return_value=[]),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_id_to_modified_time_map",
+ AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),
+ )
+
+ remote_files = [
+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"},
+ {"id": "c::b", "modified_time": "2024-01-01T00:00:00Z"},
+ ]
+ service = _bucket_sync_service(remote_files)
+
+ response = await connectors_api.connector_sync(
+ "azure_blob",
+ connectors_api.ConnectorSyncBody(),
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
+ )
+
+ assert response.status_code == 200
+ body = _json(response)
+ assert body["status"] == "no_files"
+ assert "up to date" in body["message"]
+ service.sync_specific_files.assert_not_awaited()
+
+
+# ---------------------------------------------------------------------------
+# sync_all_connectors — updates-only re-ingest for bucket connectors
+# ---------------------------------------------------------------------------
+
+
+async def test_sync_all_bucket_reingests_only_changed(monkeypatch):
+ from api import connectors as connectors_api
+
+ monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(
+ connectors_api,
+ "_allowed_connector_types_for_request",
+ AsyncMock(return_value=["azure_blob"]),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_file_ids_for_connector",
+ AsyncMock(return_value=(["c::a", "c::b"], [], "connector_file_id")),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "reconcile_orphans_for_connector_type",
+ AsyncMock(return_value=[]),
+ )
+ monkeypatch.setattr(
+ connectors_api,
+ "get_synced_id_to_modified_time_map",
+ AsyncMock(return_value={"c::a": 1704067200000.0, "c::b": 1704067200000.0}),
+ )
+
+ remote_files = [
+ {"id": "c::a", "modified_time": "2024-01-01T00:00:00Z"}, # unchanged
+ {"id": "c::b", "modified_time": "2024-06-01T00:00:00Z"}, # changed
+ ]
+ service = _bucket_sync_service(remote_files)
+
+ response = await connectors_api.sync_all_connectors(
+ request=MagicMock(),
+ connector_service=service,
+ session_manager=MagicMock(),
+ user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
+ )
+
+ assert response.status_code == 201
+ service.sync_specific_files.assert_awaited_once()
+ call = service.sync_specific_files.await_args
+ assert call.args[2] == ["c::b"]
+ assert call.kwargs["replace_duplicates"] is True
diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
index 9e339ca37..e0e982af1 100644
--- a/tests/unit/api/test_reconcile_orphans_for_connector_type.py
+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
@@ -435,6 +435,7 @@ async def test_connector_sync_filters_orphan_ids_before_resync(monkeypatch):
from api import connectors as connectors_api
monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
monkeypatch.setattr(
connectors_api,
"get_synced_file_ids_for_connector",
@@ -459,9 +460,11 @@ async def test_connector_sync_filters_orphan_ids_before_resync(monkeypatch):
response = await connectors_api.connector_sync(
"google_drive",
connectors_api.ConnectorSyncBody(),
+ request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
)
assert response.status_code == 201
@@ -475,6 +478,7 @@ async def test_connector_sync_returns_no_files_when_all_ids_are_orphans(monkeypa
from api import connectors as connectors_api
monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
monkeypatch.setattr(
connectors_api,
"get_synced_file_ids_for_connector",
@@ -499,9 +503,11 @@ async def test_connector_sync_returns_no_files_when_all_ids_are_orphans(monkeypa
response = await connectors_api.connector_sync(
"google_drive",
connectors_api.ConnectorSyncBody(),
+ request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
)
assert response.status_code == 200
@@ -514,6 +520,7 @@ async def test_sync_all_returns_deleted_only_without_error(monkeypatch):
from api import connectors as connectors_api
monkeypatch.setattr(connectors_api.TelemetryClient, "send_event", AsyncMock())
+ monkeypatch.setattr(connectors_api, "_connector_access_denied", AsyncMock(return_value=None))
async def fake_synced_ids(connector_type, *args, **kwargs):
if connector_type == "google_drive":
@@ -538,9 +545,11 @@ async def test_sync_all_returns_deleted_only_without_error(monkeypatch):
service.sync_specific_files = AsyncMock()
response = await connectors_api.sync_all_connectors(
+ request=MagicMock(),
connector_service=service,
session_manager=MagicMock(),
user=SimpleNamespace(user_id="alice", jwt_token="token"),
+ session=MagicMock(),
)
body = _json(response)
diff --git a/tests/unit/connectors/test_azure_blob_connector.py b/tests/unit/connectors/test_azure_blob_connector.py
new file mode 100644
index 000000000..05b9dcbb9
--- /dev/null
+++ b/tests/unit/connectors/test_azure_blob_connector.py
@@ -0,0 +1,751 @@
+"""Unit tests for the Azure Blob Storage connector.
+
+Covers credential resolution (both auth modes + env fallback), client-factory
+validation, file listing (prefix filter, dir-marker skip, max_files), composite
+file-id round-trip, blob download → ConnectorDocument mapping (owner-based DLS),
+authenticate success/failure, IBM_AUTH_ENABLED gating, config-builder validation,
+and the webhook stubs.
+
+The sync azure-storage-blob client is replaced with a MagicMock — the connector
+offloads it via asyncio.to_thread, so plain mocks work without async machinery.
+"""
+
+import json
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+from enhancements.connectors.azure_blob import api as az_api # noqa: E402
+from enhancements.connectors.azure_blob import auth as az_auth # noqa: E402
+from enhancements.connectors.azure_blob import connector as az_connector # noqa: E402
+from enhancements.connectors.azure_blob.connector import ( # noqa: E402
+ AzureBlobConnector,
+ _make_file_id,
+ _split_file_id,
+)
+from enhancements.connectors.azure_blob.models import AzureBlobConfigureBody # noqa: E402
+from enhancements.connectors.azure_blob.support import build_azure_blob_config # noqa: E402
+
+CONN_STR = "AZURE_STORAGE_CONNECTION_STRING"
+ACCT_NAME = "AZURE_STORAGE_ACCOUNT_NAME"
+ACCT_KEY = "AZURE_STORAGE_ACCOUNT_KEY"
+ENDPOINT = "AZURE_STORAGE_ENDPOINT"
+
+
+def _clear_azure_env(monkeypatch):
+ for var in (CONN_STR, ACCT_NAME, ACCT_KEY, ENDPOINT):
+ monkeypatch.delenv(var, raising=False)
+
+
+# ---------------------------------------------------------------------------
+# Composite file-id helpers
+# ---------------------------------------------------------------------------
+
+
+def test_make_and_split_file_id_roundtrip():
+ file_id = _make_file_id("mycontainer", "path/to/blob.pdf")
+ assert file_id == "mycontainer::path/to/blob.pdf"
+ container, blob = _split_file_id(file_id)
+ assert container == "mycontainer"
+ assert blob == "path/to/blob.pdf"
+
+
+def test_split_file_id_invalid_raises():
+ with pytest.raises(ValueError):
+ _split_file_id("no-separator-here")
+
+
+def test_split_file_id_blob_with_separator_only_splits_once():
+ container, blob = _split_file_id("c::a::b")
+ assert container == "c"
+ assert blob == "a::b"
+
+
+# ---------------------------------------------------------------------------
+# Credential resolution + client factory
+# ---------------------------------------------------------------------------
+
+
+def test_resolve_credentials_prefers_config_over_env(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ monkeypatch.setenv(CONN_STR, "env-conn-str")
+ creds = az_auth._resolve_credentials({"connection_string": "cfg-conn-str"})
+ assert creds["connection_string"] == "cfg-conn-str"
+
+
+def test_resolve_credentials_env_fallback(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ monkeypatch.setenv(ACCT_NAME, "acct")
+ monkeypatch.setenv(ACCT_KEY, "key")
+ monkeypatch.setenv(ENDPOINT, "http://127.0.0.1:10000/devstoreaccount1")
+ creds = az_auth._resolve_credentials({"auth_mode": "account_key"})
+ assert creds["account_name"] == "acct"
+ assert creds["account_key"] == "key"
+ assert creds["endpoint_url"] == "http://127.0.0.1:10000/devstoreaccount1"
+
+
+def test_create_client_connection_string_mode(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ # Azurite dev shortcut constructs fully offline.
+ client = az_auth.create_blob_service_client(
+ {"auth_mode": "connection_string", "connection_string": "UseDevelopmentStorage=true"}
+ )
+ assert client is not None
+
+
+def test_create_client_account_key_mode_with_endpoint(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ client = az_auth.create_blob_service_client(
+ {
+ "auth_mode": "account_key",
+ "account_name": "devstoreaccount1",
+ "account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT==",
+ "endpoint_url": "http://127.0.0.1:10000/devstoreaccount1",
+ }
+ )
+ assert client is not None
+
+
+def test_create_client_connection_string_missing_raises(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ with pytest.raises(ValueError, match="Connection string mode requires"):
+ az_auth.create_blob_service_client({"auth_mode": "connection_string"})
+
+
+def test_create_client_account_key_missing_raises(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ with pytest.raises(ValueError, match="Account key mode requires"):
+ az_auth.create_blob_service_client({"auth_mode": "account_key", "account_name": "acct"})
+
+
+# ---------------------------------------------------------------------------
+# Account-name resolution (used by get_client_id / status checks)
+# ---------------------------------------------------------------------------
+
+
+def test_account_name_from_connection_string_dev_storage():
+ assert az_auth._account_name_from_connection_string("UseDevelopmentStorage=true") == (
+ "devstoreaccount1"
+ )
+
+
+def test_account_name_from_connection_string_account_name():
+ conn = "DefaultEndpointsProtocol=https;AccountName=myacct;AccountKey=ab==;EndpointSuffix=x"
+ assert az_auth._account_name_from_connection_string(conn) == "myacct"
+
+
+def test_account_name_from_connection_string_sas_host_style():
+ conn = "BlobEndpoint=https://myacct.blob.core.windows.net;SharedAccessSignature=sv=2021"
+ assert az_auth._account_name_from_connection_string(conn) == "myacct"
+
+
+def test_account_name_from_connection_string_sas_path_style():
+ conn = "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;SharedAccessSignature=sv=x"
+ assert az_auth._account_name_from_connection_string(conn) == "devstoreaccount1"
+
+
+def test_account_name_from_connection_string_unparseable_returns_none():
+ assert az_auth._account_name_from_connection_string("SharedAccessSignature=sv=2021") is None
+
+
+def test_account_name_from_config_prefers_account_name(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ assert az_auth.account_name_from_config({"account_name": "explicit"}) == "explicit"
+
+
+def test_account_name_from_config_parses_connection_string(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ cfg = {"auth_mode": "connection_string", "connection_string": "UseDevelopmentStorage=true"}
+ assert az_auth.account_name_from_config(cfg) == "devstoreaccount1"
+
+
+def test_get_client_id_connection_string_mode_does_not_raise(monkeypatch):
+ """Regression: connection-string mode (Azurite) must yield a stable id, not raise.
+
+ Otherwise the status endpoint marks an authenticated connection as
+ unauthenticated and the UI keeps showing "Connect".
+ """
+ _clear_azure_env(monkeypatch)
+ connector = AzureBlobConnector(
+ {"auth_mode": "connection_string", "connection_string": "UseDevelopmentStorage=true"}
+ )
+ assert connector.get_client_id() == "devstoreaccount1"
+
+
+def test_get_client_id_account_key_mode(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ connector = AzureBlobConnector(
+ {"auth_mode": "account_key", "account_name": "acct", "account_key": "k"}
+ )
+ assert connector.get_client_id() == "acct"
+
+
+def test_get_client_id_no_credentials_raises(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ connector = AzureBlobConnector({"auth_mode": "account_key"})
+ with pytest.raises(ValueError, match="Azure Blob credentials not set"):
+ connector.get_client_id()
+
+
+# ---------------------------------------------------------------------------
+# is_available gating (IBM_AUTH_ENABLED / OPENRAG_DEV_AZURE_BLOB)
+# ---------------------------------------------------------------------------
+
+
+def test_is_available_gated_on_ibm_auth_enabled(monkeypatch):
+ monkeypatch.setattr(az_connector, "IBM_AUTH_ENABLED", True)
+ monkeypatch.setattr(az_connector, "is_dev_azure_blob_enabled", lambda: False)
+ assert AzureBlobConnector.is_available(MagicMock()) is True
+ monkeypatch.setattr(az_connector, "IBM_AUTH_ENABLED", False)
+ assert AzureBlobConnector.is_available(MagicMock()) is False
+
+
+def test_is_available_dev_flag_bypasses_ibm_auth(monkeypatch):
+ monkeypatch.setattr(az_connector, "IBM_AUTH_ENABLED", False)
+ monkeypatch.setattr(az_connector, "is_dev_azure_blob_enabled", lambda: True)
+ assert AzureBlobConnector.is_available(MagicMock()) is True
+
+
+# ---------------------------------------------------------------------------
+# Fake Azure client helpers
+# ---------------------------------------------------------------------------
+
+
+def _blob(name, size=10, modified=None):
+ b = MagicMock()
+ b.name = name
+ b.size = size
+ b.last_modified = modified
+ return b
+
+
+def _make_fake_client(containers):
+ """containers: dict[name -> list[blob]]."""
+ client = MagicMock()
+ # NB: MagicMock(name=...) sets the mock's repr name, not a .name attribute,
+ # so build the container mocks and assign .name explicitly.
+ container_mocks = []
+ for cname in containers:
+ cm = MagicMock()
+ cm.name = cname
+ container_mocks.append(cm)
+ client.list_containers.return_value = container_mocks
+
+ def _get_container_client(name):
+ cc = MagicMock()
+
+ def _list_blobs(name_starts_with=None):
+ blobs = containers[name]
+ if name_starts_with:
+ blobs = [b for b in blobs if b.name.startswith(name_starts_with)]
+ return iter(blobs)
+
+ cc.list_blobs.side_effect = _list_blobs
+ return cc
+
+ client.get_container_client.side_effect = _get_container_client
+ return client
+
+
+def patched_factory():
+ """Patch create_blob_service_client in the connector module; yield a setter."""
+ with patch.object(az_connector, "create_blob_service_client") as factory:
+ yield factory
+
+
+# ---------------------------------------------------------------------------
+# list_files
+# ---------------------------------------------------------------------------
+
+
+async def test_list_files_single_container(patched_factory):
+ ts = datetime(2026, 1, 1, tzinfo=UTC)
+ patched_factory.return_value = _make_fake_client(
+ {"docs": [_blob("a.pdf", 5, ts), _blob("b.txt", 7, ts)]}
+ )
+ conn = AzureBlobConnector({"container_names": ["docs"]})
+ result = await conn.list_files()
+ assert result["next_page_token"] is None
+ names = {f["name"] for f in result["files"]}
+ assert names == {"a.pdf", "b.txt"}
+ first = next(f for f in result["files"] if f["name"] == "a.pdf")
+ assert first["id"] == "docs::a.pdf"
+ # "bucket" is the shared bucket-connector contract key (matches aws_s3/ibm_cos)
+ # that the file browser reads; carries the Azure container name.
+ assert first["bucket"] == "docs"
+ assert first["modified_time"] == ts.isoformat()
+
+
+async def test_list_files_skips_directory_markers(patched_factory):
+ patched_factory.return_value = _make_fake_client(
+ {"docs": [_blob("folder/"), _blob("folder/real.pdf")]}
+ )
+ conn = AzureBlobConnector({"container_names": ["docs"]})
+ result = await conn.list_files()
+ assert [f["key"] for f in result["files"]] == ["folder/real.pdf"]
+
+
+async def test_list_files_prefix_filter(patched_factory):
+ patched_factory.return_value = _make_fake_client(
+ {"docs": [_blob("keep/a.pdf"), _blob("skip/b.pdf")]}
+ )
+ conn = AzureBlobConnector({"container_names": ["docs"], "prefix": "keep/"})
+ result = await conn.list_files()
+ assert [f["key"] for f in result["files"]] == ["keep/a.pdf"]
+
+
+async def test_list_files_respects_max_files(patched_factory):
+ patched_factory.return_value = _make_fake_client(
+ {"docs": [_blob(f"f{i}.pdf") for i in range(10)]}
+ )
+ conn = AzureBlobConnector({"container_names": ["docs"]})
+ result = await conn.list_files(max_files=3)
+ assert len(result["files"]) == 3
+
+
+async def test_list_files_auto_discovers_containers(patched_factory):
+ patched_factory.return_value = _make_fake_client(
+ {"c1": [_blob("a.pdf")], "c2": [_blob("b.pdf")]}
+ )
+ conn = AzureBlobConnector({}) # no container_names → auto-discover
+ result = await conn.list_files()
+ assert {f["bucket"] for f in result["files"]} == {"c1", "c2"}
+
+
+# ---------------------------------------------------------------------------
+# get_file_content
+# ---------------------------------------------------------------------------
+
+
+def _patch_download(client, content=b"hello", content_type="", last_modified=None, size=None):
+ downloader = MagicMock()
+ downloader.readall.return_value = content
+ props = MagicMock()
+ props.content_settings.content_type = content_type
+ props.last_modified = last_modified
+ props.size = size if size is not None else len(content)
+ downloader.properties = props
+ blob_client = MagicMock()
+ blob_client.download_blob.return_value = downloader
+ client.get_blob_client.return_value = blob_client
+
+
+async def test_get_file_content_maps_document(patched_factory):
+ ts = datetime(2026, 2, 2, tzinfo=UTC)
+ client = _make_fake_client({"docs": []})
+ _patch_download(client, content=b"pdfbytes", content_type="application/pdf", last_modified=ts)
+ patched_factory.return_value = client
+
+ conn = AzureBlobConnector({"container_names": ["docs"]})
+ doc = await conn.get_file_content("docs::report.pdf")
+
+ assert doc.id == "docs::report.pdf"
+ assert doc.filename == "report.pdf"
+ assert doc.content == b"pdfbytes"
+ assert doc.mimetype == "application/pdf"
+ assert doc.source_url == "azure://docs/report.pdf"
+ assert doc.modified_time == ts
+ assert doc.metadata["azure_container"] == "docs"
+ assert doc.metadata["azure_blob"] == "report.pdf"
+
+
+async def test_get_file_content_owner_based_acl_has_no_principals(patched_factory):
+ client = _make_fake_client({"docs": []})
+ _patch_download(client, content=b"x", content_type="text/plain")
+ patched_factory.return_value = client
+
+ conn = AzureBlobConnector({"container_names": ["docs"]})
+ doc = await conn.get_file_content("docs::a.txt")
+ assert doc.acl.allowed_principals == []
+ assert doc.acl.allowed_users == []
+ assert doc.acl.allowed_groups == []
+ assert doc.acl.owner is None
+
+
+async def test_get_file_content_mime_fallback_to_extension(patched_factory):
+ client = _make_fake_client({"docs": []})
+ # Generic octet-stream should be ignored in favor of the .pdf extension guess.
+ _patch_download(client, content=b"x", content_type="application/octet-stream")
+ patched_factory.return_value = client
+
+ conn = AzureBlobConnector({"container_names": ["docs"]})
+ doc = await conn.get_file_content("docs::file.pdf")
+ assert doc.mimetype == "application/pdf"
+
+
+# ---------------------------------------------------------------------------
+# authenticate
+# ---------------------------------------------------------------------------
+
+
+async def test_authenticate_success(patched_factory):
+ patched_factory.return_value = _make_fake_client({"docs": []})
+ conn = AzureBlobConnector({})
+ assert await conn.authenticate() is True
+ assert conn.is_authenticated is True
+
+
+async def test_authenticate_failure(patched_factory):
+ client = MagicMock()
+ client.list_containers.side_effect = RuntimeError("bad creds")
+ patched_factory.return_value = client
+ conn = AzureBlobConnector({})
+ assert await conn.authenticate() is False
+ assert conn.is_authenticated is False
+
+
+# ---------------------------------------------------------------------------
+# build_azure_blob_config (support)
+# ---------------------------------------------------------------------------
+
+
+def test_build_config_connection_string_ok(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ body = AzureBlobConfigureBody(auth_mode="connection_string", connection_string="cs")
+ cfg, err = build_azure_blob_config(body, {})
+ assert err is None
+ assert cfg == {"auth_mode": "connection_string", "connection_string": "cs"}
+
+
+def test_build_config_connection_string_missing(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ body = AzureBlobConfigureBody(auth_mode="connection_string")
+ cfg, err = build_azure_blob_config(body, {})
+ assert cfg == {}
+ assert "connection_string" in err
+
+
+def test_build_config_account_key_ok(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ body = AzureBlobConfigureBody(
+ auth_mode="account_key", account_name="a", account_key="k", endpoint="http://e"
+ )
+ cfg, err = build_azure_blob_config(body, {})
+ assert err is None
+ assert cfg["account_name"] == "a"
+ assert cfg["account_key"] == "k"
+ assert cfg["endpoint_url"] == "http://e"
+
+
+def test_build_config_account_key_missing(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ body = AzureBlobConfigureBody(auth_mode="account_key", account_name="a")
+ cfg, err = build_azure_blob_config(body, {})
+ assert cfg == {}
+ assert "account_name and account_key" in err
+
+
+def test_build_config_env_fallback(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ monkeypatch.setenv(ACCT_NAME, "envname")
+ monkeypatch.setenv(ACCT_KEY, "envkey")
+ body = AzureBlobConfigureBody(auth_mode="account_key")
+ cfg, err = build_azure_blob_config(body, {})
+ assert err is None
+ assert cfg["account_name"] == "envname"
+ assert cfg["account_key"] == "envkey"
+
+
+def test_build_config_unknown_mode(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ body = AzureBlobConfigureBody(auth_mode="sas")
+ cfg, err = build_azure_blob_config(body, {})
+ assert cfg == {}
+ assert "Unknown auth_mode" in err
+
+
+def test_build_config_includes_container_names(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ body = AzureBlobConfigureBody(
+ auth_mode="connection_string", connection_string="cs", container_names=["x", "y"]
+ )
+ cfg, _ = build_azure_blob_config(body, {})
+ assert cfg["container_names"] == ["x", "y"]
+
+
+# ---------------------------------------------------------------------------
+# Webhook / subscription stubs
+# ---------------------------------------------------------------------------
+
+
+async def test_webhook_stubs_are_noops():
+ conn = AzureBlobConnector({})
+ assert await conn.setup_subscription() == ""
+ assert await conn.handle_webhook({}) == []
+ assert conn.extract_webhook_channel_id({}, {}) is None
+ assert await conn.cleanup_subscription("sub") is True
+
+
+# ---------------------------------------------------------------------------
+# azure_blob_test endpoint (non-persisting credential validation + listing)
+# ---------------------------------------------------------------------------
+
+
+def _test_endpoint_service(existing_connections=None):
+ """connector_service mock for azure_blob_test with persistence spies."""
+ service = MagicMock()
+ cm = service.connection_manager
+ cm.list_connections = AsyncMock(return_value=existing_connections or [])
+ cm.create_connection = AsyncMock()
+ cm.update_connection = AsyncMock()
+ cm.get_connection = AsyncMock()
+ return service
+
+
+def _assert_never_persisted(service):
+ service.connection_manager.create_connection.assert_not_called()
+ service.connection_manager.update_connection.assert_not_called()
+
+
+async def test_azure_blob_test_lists_containers_without_persisting(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ service = _test_endpoint_service()
+ user = MagicMock(user_id="u1")
+ body = AzureBlobConfigureBody(auth_mode="connection_string", connection_string="cs")
+
+ with patch.object(az_api, "_list_container_names", return_value=["c1", "c2"]) as lister:
+ resp = await az_api.azure_blob_test(body, connector_service=service, user=user)
+
+ assert resp.status_code == 200
+ assert json.loads(resp.body) == {"containers": ["c1", "c2"]}
+ # Containers listed via the resolved config, and nothing was persisted.
+ lister.assert_called_once()
+ _assert_never_persisted(service)
+
+
+async def test_azure_blob_test_does_not_mutate_existing_connection(monkeypatch):
+ """Editing an existing connection's credentials via Test must not save them."""
+ _clear_azure_env(monkeypatch)
+ existing = MagicMock(connection_id="conn-1", config={"connection_string": "old"})
+ service = _test_endpoint_service([existing])
+ user = MagicMock(user_id="u1")
+ body = AzureBlobConfigureBody(
+ auth_mode="connection_string", connection_string="new", connection_id="conn-1"
+ )
+
+ with patch.object(az_api, "_list_container_names", return_value=["c1"]):
+ resp = await az_api.azure_blob_test(body, connector_service=service, user=user)
+
+ assert resp.status_code == 200
+ assert json.loads(resp.body)["containers"] == ["c1"]
+ _assert_never_persisted(service)
+
+
+async def test_azure_blob_test_invalid_config_returns_400_without_persisting(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ service = _test_endpoint_service()
+ user = MagicMock(user_id="u1")
+ # connection_string mode with no string → build_azure_blob_config returns an error.
+ body = AzureBlobConfigureBody(auth_mode="connection_string")
+
+ with patch.object(az_api, "_list_container_names") as lister:
+ resp = await az_api.azure_blob_test(body, connector_service=service, user=user)
+
+ assert resp.status_code == 400
+ assert "connection_string" in json.loads(resp.body)["error"]
+ lister.assert_not_called() # never reached the credential test
+ _assert_never_persisted(service)
+
+
+async def test_azure_blob_test_connection_failure_returns_400_without_persisting(monkeypatch):
+ _clear_azure_env(monkeypatch)
+ service = _test_endpoint_service()
+ user = MagicMock(user_id="u1")
+ body = AzureBlobConfigureBody(auth_mode="connection_string", connection_string="cs")
+
+ with patch.object(az_api, "_list_container_names", side_effect=RuntimeError("bad creds")):
+ resp = await az_api.azure_blob_test(body, connector_service=service, user=user)
+
+ assert resp.status_code == 400
+ assert "Could not connect" in json.loads(resp.body)["error"]
+ _assert_never_persisted(service)
+
+
+# ---------------------------------------------------------------------------
+# azure_blob_container_status endpoint (browse list honors ingestion restriction)
+# ---------------------------------------------------------------------------
+
+
+def _status_endpoint_deps(config):
+ """connector_service + session_manager mocks for azure_blob_container_status."""
+ connection = MagicMock(user_id="u1", connector_type="azure_blob", config=config)
+ service = MagicMock()
+ service.connection_manager.get_connection = AsyncMock(return_value=connection)
+ # OpenSearch client raises so doc-count aggregation is skipped (counts → 0).
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client.side_effect = RuntimeError("no opensearch")
+ return service, session_manager
+
+
+async def test_container_status_restricts_to_allowed_containers():
+ """A non-empty container_names allowlist filters out unselected containers."""
+ service, session_manager = _status_endpoint_deps({"container_names": ["docs"]})
+ user = MagicMock(user_id="u1", jwt_token="t")
+
+ with patch.object(az_api, "_list_container_names", return_value=["docs", "images", "logs"]):
+ resp = await az_api.azure_blob_container_status(
+ "conn-1",
+ connector_service=service,
+ session_manager=session_manager,
+ user=user,
+ )
+
+ names = [c["name"] for c in json.loads(resp.body)["containers"]]
+ assert names == ["docs"]
+
+
+async def test_container_status_no_restriction_shows_all_containers():
+ """An empty/absent allowlist leaves every accessible container browsable."""
+ service, session_manager = _status_endpoint_deps({"container_names": []})
+ user = MagicMock(user_id="u1", jwt_token="t")
+
+ with patch.object(az_api, "_list_container_names", return_value=["docs", "images"]):
+ resp = await az_api.azure_blob_container_status(
+ "conn-1",
+ connector_service=service,
+ session_manager=session_manager,
+ user=user,
+ )
+
+ names = [c["name"] for c in json.loads(resp.body)["containers"]]
+ assert names == ["docs", "images"]
+
+
+def _status_deps_with_counts(config, agg_buckets):
+ """Deps for azure_blob_container_status with a *working* OpenSearch aggregation.
+
+ Unlike `_status_endpoint_deps` (which raises so counts collapse to 0), this
+ returns an AsyncOpenSearch-shaped client whose ``search`` is awaitable and
+ yields ``agg_buckets``. Regression guard for the missing-``await`` bug: the
+ handler must ``await opensearch_client.search(...)`` — if it doesn't, the
+ coroutine's ``.get(...)`` raises, the bare ``except`` swallows it, and every
+ count silently reads 0 (which these assertions would then catch).
+ """
+ connection = MagicMock(user_id="u1", connector_type="azure_blob", config=config)
+ service = MagicMock()
+ service.connection_manager.get_connection = AsyncMock(return_value=connection)
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(
+ return_value={"aggregations": {"doc_ids": {"buckets": agg_buckets}}}
+ )
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+ return service, session_manager
+
+
+async def test_container_status_reports_ingested_counts():
+ """Per-container ingested_count / is_synced reflect the OpenSearch aggregation.
+
+ Guards against the missing-``await`` regression where every container falsely
+ reported ``ingested_count: 0, is_synced: false``.
+ """
+ buckets = [
+ {"key": "docs::a.pdf"},
+ {"key": "docs::b.pdf"},
+ {"key": "images::c.png"},
+ ]
+ service, session_manager = _status_deps_with_counts({"container_names": []}, buckets)
+ user = MagicMock(user_id="u1", jwt_token="t")
+
+ with patch.object(az_api, "_list_container_names", return_value=["docs", "images", "logs"]):
+ resp = await az_api.azure_blob_container_status(
+ "conn-1",
+ connector_service=service,
+ session_manager=session_manager,
+ user=user,
+ )
+
+ by_name = {c["name"]: c for c in json.loads(resp.body)["containers"]}
+ assert by_name["docs"]["ingested_count"] == 2
+ assert by_name["docs"]["is_synced"] is True
+ assert by_name["images"]["ingested_count"] == 1
+ assert by_name["images"]["is_synced"] is True
+ # No indexed docs → not synced.
+ assert by_name["logs"]["ingested_count"] == 0
+ assert by_name["logs"]["is_synced"] is False
+
+
+async def test_container_status_counts_zero_when_opensearch_unavailable():
+ """A failed aggregation degrades gracefully to zero counts (not a 500)."""
+ service, session_manager = _status_endpoint_deps({"container_names": []})
+ user = MagicMock(user_id="u1", jwt_token="t")
+
+ with patch.object(az_api, "_list_container_names", return_value=["docs"]):
+ resp = await az_api.azure_blob_container_status(
+ "conn-1",
+ connector_service=service,
+ session_manager=session_manager,
+ user=user,
+ )
+
+ entry = json.loads(resp.body)["containers"][0]
+ assert entry == {"name": "docs", "ingested_count": 0, "is_synced": False}
+
+
+# ---------------------------------------------------------------------------
+# azure_blob_list_containers endpoint (also honors the ingestion restriction)
+# ---------------------------------------------------------------------------
+
+
+def _list_endpoint_service(config):
+ """connector_service mock for azure_blob_list_containers."""
+ connection = MagicMock(user_id="u1", connector_type="azure_blob", config=config)
+ service = MagicMock()
+ service.connection_manager.get_connection = AsyncMock(return_value=connection)
+ return service
+
+
+async def test_list_containers_restricts_to_allowed_containers():
+ """A non-empty container_names allowlist filters out unselected containers."""
+ service = _list_endpoint_service({"container_names": ["docs"]})
+ user = MagicMock(user_id="u1")
+
+ with patch.object(az_api, "_list_container_names", return_value=["docs", "images", "logs"]):
+ resp = await az_api.azure_blob_list_containers(
+ "conn-1", connector_service=service, user=user
+ )
+
+ assert json.loads(resp.body)["containers"] == ["docs"]
+
+
+async def test_list_containers_no_restriction_shows_all_containers():
+ """An empty/absent allowlist leaves every accessible container listed."""
+ service = _list_endpoint_service({"container_names": []})
+ user = MagicMock(user_id="u1")
+
+ with patch.object(az_api, "_list_container_names", return_value=["docs", "images"]):
+ resp = await az_api.azure_blob_list_containers(
+ "conn-1", connector_service=service, user=user
+ )
+
+ assert json.loads(resp.body)["containers"] == ["docs", "images"]
diff --git a/tests/unit/services/test_connector_access_service.py b/tests/unit/services/test_connector_access_service.py
index 89d6e438a..ac60d5f12 100644
--- a/tests/unit/services/test_connector_access_service.py
+++ b/tests/unit/services/test_connector_access_service.py
@@ -147,6 +147,19 @@ def test_governable_connector_types_excludes_buckets_in_saas(monkeypatch):
assert "sharepoint" in governable
assert "aws_s3" not in governable
assert "ibm_cos" not in governable
+ assert "azure_blob" not in governable
+
+
+def test_governable_connector_types_includes_buckets_with_ibm_auth(monkeypatch):
+ monkeypatch.setenv("OPENRAG_RUN_MODE", "saas")
+ monkeypatch.setattr("config.settings.IBM_AUTH_ENABLED", True)
+
+ governable = governable_connector_types()
+
+ # With IBM auth on, bucket connectors (incl. Azure Blob) are governable.
+ assert "azure_blob" in governable
+ assert "aws_s3" in governable
+ assert "ibm_cos" in governable
@pytest.mark.asyncio
diff --git a/tests/unit/test_startup_load_connections.py b/tests/unit/test_startup_load_connections.py
new file mode 100644
index 000000000..a99654579
--- /dev/null
+++ b/tests/unit/test_startup_load_connections.py
@@ -0,0 +1,44 @@
+"""Regression: persisted connector connections must reload at startup for
+dev bucket connectors even in no-auth mode.
+
+Bug: `initialize_services()` skipped `connector_service.initialize()` (which
+calls `load_connections()`) whenever no-auth mode was active, on the assumption
+that connectors require OAuth. But the Azure Blob bucket connector works in
+no-auth dev mode via OPENRAG_DEV_AZURE_BLOB=true. Skipping the load left the
+in-memory connections dict empty after a restart, so a saved connection
+silently reverted to the "Connect" state until re-saved.
+
+`_should_load_persisted_connections()` encodes the fixed decision; these tests
+pin its truth table.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+from app import container # noqa: E402
+
+
+ ("no_auth", "dev_azure", "expected"),
+ [
+ # Authed deployments always load (today's behaviour, unchanged).
+ (False, False, True),
+ (False, True, True),
+ # No-auth without any dev bucket connector: still skipped.
+ (True, False, False),
+ # The regression: no-auth + dev Azure Blob must load so the saved
+ # connection survives a restart.
+ (True, True, True),
+ ],
+)
+def test_should_load_persisted_connections(monkeypatch, no_auth, dev_azure, expected):
+ monkeypatch.setattr(container, "is_no_auth_mode", lambda: no_auth)
+ monkeypatch.setattr(container, "is_dev_azure_blob_enabled", lambda: dev_azure)
+ assert container._should_load_persisted_connections() is expected
diff --git a/uv.lock b/uv.lock
index f5eff5389..fab2a305a 100644
--- a/uv.lock
+++ b/uv.lock
@@ -295,6 +295,34 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fb/95/adcb68e20c34162e9135f370d6e31737719c2b6f94bc953fe7ed1f10fe21/authlib-1.7.2-py2.py3-none-any.whl", hash = "sha256:3e1faedc9d87e7d56a164eca3ccb6ace0d61b94abe83e92242f8dc8bba9b4a9f", size = 259548, upload-time = "2026-05-06T08:10:21.436Z" },
]
+[[package]]
+name = "azure-core"
+version = "1.41.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "requests" },
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a6/f3/b416179e408990df5db0d516283022dde0f5d0111d98c1a848e41853e81c/azure_core-1.41.0.tar.gz", hash = "sha256:f46ff5dfcd230f25cf1c19e8a34b8dc08a337b2503e268bb600a16c00db8ad5a", size = 381042, upload-time = "2026-05-07T23:30:54.302Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5b/db/325c6d7312d2200251c52323878281045aaffcb5586612296484e4280eaa/azure_core-1.41.0-py3-none-any.whl", hash = "sha256:522b4011e8180b1a3dcd2024396a4e7fe9ac37fb8597db47163d230b5efe892d", size = 220920, upload-time = "2026-05-07T23:30:56.357Z" },
+]
+
+[[package]]
+name = "azure-storage-blob"
+version = "12.30.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "azure-core" },
+ { name = "cryptography" },
+ { name = "isodate" },
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3b/48/84a820d898267f662b5c06f7cd76fdb8a9e272b44aa9376cef3ec0f6a294/azure_storage_blob-12.30.0.tar.gz", hash = "sha256:2cd74d4d5731e5eb6b8d5c5056ee115a5e88f8fdf22517b739836fda685018be", size = 618229, upload-time = "2026-06-08T11:45:35.575Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/5e/0b/e106f0fd7fa785867d9ffcc47dc9e6237c0e58f51058473b777487a98edc/azure_storage_blob-12.30.0-py3-none-any.whl", hash = "sha256:d415ac50b67a8da6b3ae7e9f1014b1b55cd7aafa0b8d4ca9b380568dc7360423", size = 435610, upload-time = "2026-06-08T11:45:37.213Z" },
+]
+
[[package]]
name = "beartype"
version = "0.22.9"
@@ -1207,6 +1235,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
]
+[[package]]
+name = "isodate"
+version = "0.7.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" },
+]
+
[[package]]
name = "jaraco-classes"
version = "3.4.0"
@@ -1841,6 +1878,7 @@ dependencies = [
{ name = "alembic" },
{ name = "asyncpg" },
{ name = "authlib" },
+ { name = "azure-storage-blob" },
{ name = "boto3" },
{ name = "cachetools" },
{ name = "cryptography" },
@@ -1900,6 +1938,7 @@ requires-dist = [
{ name = "asyncpg", specifier = ">=0.30.0" },
{ name = "authlib", specifier = ">=1.7.1" },
{ name = "authlib", specifier = ">=1.7.2,<2.0.0" },
+ { name = "azure-storage-blob", specifier = ">=12.20.0" },
{ name = "boto3", specifier = ">=1.35.0" },
{ name = "cachetools", specifier = ">=5.5.0" },
{ name = "cryptography", specifier = ">=46.0.6" },