Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(go-syncer): add Azure Blob Storage connector with checkpoint resume (#18701)

mkaaade5b1adc9fab5
diff --git a/go.mod b/go.mod
index 64284133f0..7aa97bb9f0 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,8 @@ go 1.26.4
require (
cloud.google.com/go/storage v1.63.0
github.com/AkmalOt/gomsg v0.0.0-20260407083308-985c3a1a76b7
+ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
+ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/JohannesKaufmann/html-to-markdown v1.6.0
github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0
@@ -32,7 +34,7 @@ require (
github.com/glebarez/sqlite v1.11.0
github.com/go-sql-driver/mysql v1.7.0
github.com/goccy/go-json v0.10.6
- github.com/golang-jwt/jwt/v5 v5.3.0
+ github.com/golang-jwt/jwt/v5 v5.3.1
github.com/gomarkdown/markdown v0.0.0-20260614204949-e08cff860f76
github.com/google/go-cmp v0.7.0
github.com/google/uuid v1.6.0
@@ -97,6 +99,7 @@ require (
cloud.google.com/go/iam v1.11.0 // indirect
cloud.google.com/go/monitoring v1.29.0 // indirect
connectrpc.com/connect v1.20.0 // indirect
+ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect
diff --git a/go.sum b/go.sum
index dfaeba4f1b..0eb0f2815e 100644
--- a/go.sum
+++ b/go.sum
@@ -26,8 +26,20 @@ cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfj
connectrpc.com/connect v1.20.0 h1:6TNDAB+WeNd2uolWNlYczB5E0KNNaVMNUEx8JEUsPmQ=
connectrpc.com/connect v1.20.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4=
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
@@ -266,8 +278,8 @@ github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7Lk
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
-github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
+github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
+github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
@@ -363,6 +375,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/larksuite/oapi-sdk-go/v3 v3.9.9 h1:qzVK5U1AuT/n0Z4LqCN2ATU6MlYp2Iipa7q+fVK5Nh8=
github.com/larksuite/oapi-sdk-go/v3 v3.9.9/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI=
github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8=
@@ -430,6 +444,8 @@ github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJ
github.com/phpdave11/gofpdi v1.0.14-0.20211212211723-1f10f9844311 h1:zyWXQ6vu27ETMpYsEMAsisQ+GqJ4e1TPvSNfdOPF0no=
github.com/phpdave11/gofpdi v1.0.14-0.20211212211723-1f10f9844311/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
diff --git a/internal/syncer/connector/azure_blob.go b/internal/syncer/connector/azure_blob.go
new file mode 100644
index 0000000000..143899306d
--- /dev/null
+++ b/internal/syncer/connector/azure_blob.go
@@ -0,0 +1,608 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore"
+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+
+ "ragflow/internal/utility"
+)
+
+const (
+ azureBlobSource = "azure_blob"
+ azureBlobDefaultBatchSize = 2
+ azureBlobEndpointSuffix = "blob.core.windows.net"
+ azureBlobErrorMessageLimit = 300
+)
+
+// azureAssertURLSafe is the SSRF guard used for caller-supplied endpoint URLs.
+// It is an indirection over utility.AssertURLSafe so unit tests can substitute
+// a stub without touching the shared global guard.
+var azureAssertURLSafe = utility.AssertURLSafe
+
+// azureBlobTextExtensions mirrors the extension set used by the OneDrive
+// family of file sources in the blob connector.
+var azureBlobTextExtensions = map[string]struct{}{
+ ".pdf": {},
+ ".docx": {},
+ ".doc": {},
+ ".xlsx": {},
+ ".xls": {},
+ ".pptx": {},
+ ".ppt": {},
+ ".txt": {},
+ ".md": {},
+ ".csv": {},
+ ".html": {},
+ ".htm": {},
+ ".json": {},
+ ".xml": {},
+}
+
+var azureBlobImageExtensions = map[string]struct{}{
+ ".png": {},
+ ".jpg": {},
+ ".jpeg": {},
+ ".webp": {},
+ ".gif": {},
+ ".bmp": {},
+ ".tiff": {},
+}
+
+// AzureBlobStorageConnector reads blobs from an Azure Blob Storage container.
+//
+// Authentication supports three mutually exclusive modes selected by
+// auth_mode (connection_string, account_key, sas_token). When auth_mode is
+// absent, the mode is derived from whichever credential fields are populated,
+// mirroring the blob connector's fallback precedence.
+type AzureBlobStorageConnector struct {
+ batchSize int
+ prefix string
+ allowImages bool
+
+ authMode string
+ containerName string
+ accountName string
+ accountKey string
+ connectionString string
+ containerURL string // sas_token mode base URL (no trailing slash, no SAS query)
+ sasToken string // sas_token mode token (no leading "?")
+
+ containerClient *container.Client
+
+ listBlobs func(ctx context.Context, prefix, marker string, maxResults int32) ([]azureBlobObject, string, bool, error)
+ downloadBlob func(ctx context.Context, name string) ([]byte, error)
+ validateContainer func(ctx context.Context) error
+}
+
+// NewAzureBlobStorageConnector creates an Azure Blob connector from
+// Python-compatible config.
+func NewAzureBlobStorageConnector(config map[string]any) (*AzureBlobStorageConnector, error) {
+ credentials := configAnyMap(config["credentials"])
+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), azureBlobDefaultBatchSize)
+ if batchSize <= 0 {
+ batchSize = azureBlobDefaultBatchSize
+ }
+ connector := &AzureBlobStorageConnector{
+ batchSize: batchSize,
+ prefix: normalizeAzureBlobPrefix(stringConfig(config["prefix"])),
+ allowImages: configBoolDefault(config["allow_images"], false),
+ authMode: strings.ToLower(strings.TrimSpace(stringConfig(config["auth_mode"]))),
+ containerName: strings.TrimSpace(stringConfig(credentials["container_name"])),
+ accountName: strings.TrimSpace(stringConfig(credentials["account_name"])),
+ accountKey: stringConfig(credentials["account_key"]),
+ connectionString: stringConfig(credentials["connection_string"]),
+ containerURL: strings.TrimRight(strings.TrimSpace(stringConfig(credentials["container_url"])), "/"),
+ sasToken: strings.TrimLeft(stringConfig(credentials["sas_token"]), "?"),
+ }
+ if connector.authMode == "" {
+ switch {
+ case connector.connectionString != "":
+ connector.authMode = "connection_string"
+ case connector.accountName != "" && connector.accountKey != "":
+ connector.authMode = "account_key"
+ case connector.containerURL != "" && connector.sasToken != "":
+ connector.authMode = "sas_token"
+ }
+ }
+ if err := connector.validateCredentials(); err != nil {
+ return nil, err
+ }
+ return connector, nil
+}
+
+// Validate validates Azure Blob connector settings and credentials.
+func (c *AzureBlobStorageConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("Azure Blob connector is nil")
+ }
+ if c.batchSize <= 0 {
+ return &ConnectorValidationError{Message: "Azure Blob: batch_size must be a positive integer"}
+ }
+ if err := c.validateCredentials(); err != nil {
+ return err
+ }
+ if err := c.validateURLSafety(); err != nil {
+ return err
+ }
+ if c.validateContainer != nil {
+ return c.validateContainer(ctx)
+ }
+ return c.defaultValidateContainer(ctx)
+}
+
+// ValidateConnectorSetting validates Azure Blob settings from an unsaved
+// config by constructing a temporary connector from the request.
+func (c *AzureBlobStorageConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ connector, err := NewAzureBlobStorageConnector(request)
+ if err != nil {
+ return err
+ }
+ return connector.Validate(ctx)
+}
+
+// OpenSync opens one Azure Blob sync session.
+func (c *AzureBlobStorageConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ session := &azureBlobSyncSession{
+ connector: c,
+ request: request,
+ batchSize: c.batchSize,
+ }
+ if request.Resume != nil {
+ session.applyResume(request.Resume)
+ }
+ return session, nil
+}
+
+// OpenPrune opens one complete Azure Blob prune snapshot session.
+func (c *AzureBlobStorageConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ names, err := c.collectBlobNames(ctx)
+ if err != nil {
+ return nil, err
+ }
+ documents := make([]SlimDocument, 0, len(names))
+ for _, name := range names {
+ if name == "" || !azureBlobSupportedExtension(name, c.allowImages) {
+ continue
+ }
+ documents = append(documents, SlimDocument{SourceID: name})
+ }
+ return &azureBlobPruneSession{documents: documents, batchSize: c.batchSize}, nil
+}
+
+// Fetch downloads an Azure Blob body referenced by a previous sync batch.
+func (c *AzureBlobStorageConnector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ var fetch azureBlobFetchReference
+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {
+ return nil, err
+ }
+ return c.download(ctx, fetch.Name)
+}
+
+// validateCredentials checks that the selected auth mode has all required
+// credential fields populated.
+func (c *AzureBlobStorageConnector) validateCredentials() error {
+ switch c.authMode {
+ case "connection_string":
+ if c.connectionString == "" {
+ return &ConnectorMissingCredentialError{Message: "Azure Blob: connection_string is required for the connection_string auth mode"}
+ }
+ if c.containerName == "" {
+ return &ConnectorMissingCredentialError{Message: "Azure Blob: container_name is required together with connection_string"}
+ }
+ case "account_key":
+ if c.accountName == "" || c.accountKey == "" {
+ return &ConnectorMissingCredentialError{Message: "Azure Blob: account_name and account_key are required for the account_key auth mode"}
+ }
+ if c.containerName == "" {
+ return &ConnectorMissingCredentialError{Message: "Azure Blob: container_name is required together with account_name + account_key"}
+ }
+ case "sas_token":
+ if c.containerURL == "" || c.sasToken == "" {
+ return &ConnectorMissingCredentialError{Message: "Azure Blob: container_url and sas_token are required for the sas_token auth mode"}
+ }
+ default:
+ return &ConnectorMissingCredentialError{
+ Message: "Azure Blob credentials are incomplete. Provide one of: (a) connection_string + container_name, (b) account_name + account_key + container_name, (c) container_url + sas_token.",
+ }
+ }
+ return nil
+}
+
+// validateURLSafety rejects caller-supplied endpoints that are not HTTPS and
+// do not resolve to a public host. The connection-string mode is a single
+// opaque blob whose endpoint is embedded by Azure, so it is not routed by the
+// caller and is intentionally left alone.
+func (c *AzureBlobStorageConnector) validateURLSafety() error {
+ endpoint := ""
+ switch c.authMode {
+ case "account_key":
+ endpoint = fmt.Sprintf("https://%s.%s", c.accountName, azureBlobEndpointSuffix)
+ case "sas_token":
+ endpoint = c.containerURL
+ }
+ if endpoint == "" {
+ return nil
+ }
+ if _, _, err := azureAssertURLSafe(endpoint); err != nil {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Azure Blob: unsafe endpoint URL: %v", err)}
+ }
+ return nil
+}
+
+func (c *AzureBlobStorageConnector) ensureClient(ctx context.Context) (*container.Client, error) {
+ if c.containerClient != nil {
+ return c.containerClient, nil
+ }
+ var (
+ client *container.Client
+ err error
+ )
+ switch c.authMode {
+ case "connection_string":
+ client, err = container.NewClientFromConnectionString(c.connectionString, c.containerName, nil)
+ case "account_key":
+ var cred *azblob.SharedKeyCredential
+ cred, err = azblob.NewSharedKeyCredential(c.accountName, c.accountKey)
+ if err == nil {
+ accountURL := fmt.Sprintf("https://%s.%s/%s", c.accountName, azureBlobEndpointSuffix, c.containerName)
+ client, err = container.NewClientWithSharedKeyCredential(accountURL, cred, nil)
+ }
+ case "sas_token":
+ fullURL := c.containerURL + "?" + c.sasToken
+ client, err = container.NewClientWithNoCredential(fullURL, nil)
+ default:
+ return nil, &ConnectorMissingCredentialError{Message: "Azure Blob credentials are incomplete."}
+ }
+ if err != nil {
+ return nil, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to initialise Azure Blob client: %v", err)}
+ }
+ c.containerClient = client
+ return client, nil
+}
+
+// defaultValidateContainer issues the container properties call that proves
+// both the credential and the container name are valid, classifying errors the
+// same way the blob connector does.
+func (c *AzureBlobStorageConnector) defaultValidateContainer(ctx context.Context) error {
+ client, err := c.ensureClient(ctx)
+ if err != nil {
+ return err
+ }
+ if _, err := client.GetProperties(ctx, nil); err != nil {
+ return c.classifyValidationError(err)
+ }
+ return nil
+}
+
+func (c *AzureBlobStorageConnector) classifyValidationError(err error) error {
+ msg := truncateAzureBlobMessage(err.Error())
+ var respErr *azcore.ResponseError
+ if errors.As(err, &respErr) {
+ switch {
+ case respErr.StatusCode == http.StatusUnauthorized ||
+ strings.Contains(msg, "AuthenticationFailed") ||
+ strings.Contains(msg, "InvalidAuthenticationInfo"):
+ return &ConnectorMissingCredentialError{Message: fmt.Sprintf("Azure Blob credential rejected: %s", msg)}
+ case respErr.StatusCode == http.StatusForbidden ||
+ strings.Contains(msg, "AuthorizationPermissionMismatch") ||
+ strings.Contains(msg, "403"):
+ return &ConnectorValidationError{Message: fmt.Sprintf("Azure Blob: insufficient permissions on container: %s", msg)}
+ case respErr.StatusCode == http.StatusNotFound ||
+ strings.Contains(msg, "ContainerNotFound") ||
+ strings.Contains(msg, "404"):
+ return &ConnectorValidationError{Message: fmt.Sprintf("Azure Blob: container not found: %s", msg)}
+ default:
+ return &ConnectorValidationError{Message: fmt.Sprintf("Azure Blob validation failed (%s): %s", respErr.ErrorCode, msg)}
+ }
+ }
+ return &ConnectorValidationError{Message: fmt.Sprintf("Azure Blob validation failed: %s", msg)}
+}
+
+func (c *AzureBlobStorageConnector) listBlobPage(ctx context.Context, prefix, marker string, maxResults int32) ([]azureBlobObject, string, bool, error) {
+ if c.listBlobs != nil {
+ return c.listBlobs(ctx, prefix, marker, maxResults)
+ }
+ client, err := c.ensureClient(ctx)
+ if err != nil {
+ return nil, "", false, err
+ }
+ options := &container.ListBlobsFlatOptions{MaxResults: &maxResults}
+ if prefix != "" {
+ options.Prefix = &prefix
+ }
+ if marker != "" {
+ options.Marker = &marker
+ }
+ page, err := client.NewListBlobsFlatPager(options).NextPage(ctx)
+ if err != nil {
+ return nil, "", false, err
+ }
+ objects := make([]azureBlobObject, 0, len(page.Segment.BlobItems))
+ for _, item := range page.Segment.BlobItems {
+ if object, ok := azureBlobObjectFromItem(item); ok {
+ objects = append(objects, object)
+ }
+ }
+ next := ""
+ if page.NextMarker != nil {
+ next = *page.NextMarker
+ }
+ return objects, next, next != "", nil
+}
+
+func (c *AzureBlobStorageConnector) collectBlobNames(ctx context.Context) ([]string, error) {
+ var names []string
+ marker := ""
+ for {
+ objects, next, hasMore, err := c.listBlobPage(ctx, c.prefix, marker, int32(c.batchSize))
+ if err != nil {
+ return nil, err
+ }
+ for _, object := range objects {
+ names = append(names, object.Name)
+ }
+ if !hasMore {
+ break
+ }
+ marker = next
+ }
+ sort.Strings(names)
+ return names, nil
+}
+
+func (c *AzureBlobStorageConnector) download(ctx context.Context, name string) ([]byte, error) {
+ if c.downloadBlob != nil {
+ return c.downloadBlob(ctx, name)
+ }
+ client, err := c.ensureClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ response, err := client.NewBlockBlobClient(name).DownloadStream(ctx, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+ return io.ReadAll(response.Body)
+}
+
+func (c *AzureBlobStorageConnector) sourceDocument(name string, object azureBlobObject) (SourceDocument, bool) {
+ if name == "" || !azureBlobSupportedExtension(name, c.allowImages) {
+ return SourceDocument{}, false
+ }
+ fetch := azureBlobFetchReference{Name: name}
+ fetchKey, _ := json.Marshal(fetch)
+ return SourceDocument{
+ SourceID: name,
+ SemanticIdentifier: name,
+ Extension: azureBlobExtension(name),
+ FetchRef: &FetchReference{Key: string(fetchKey), SizeHint: object.Size},
+ UpdatedAt: object.LastModified,
+ SizeBytes: object.Size,
+ Metadata: map[string]any{
+ "container": c.containerName,
+ "etag": azureBlobETag(object.ETag),
+ "prefix": c.prefix,
+ },
+ Fingerprint: normalizedAzureBlobETag(object.ETag),
+ }, true
+}
+
+// azureBlobObject is the normalized minimal blob snapshot used by the
+// connector and its tests.
+type azureBlobObject struct {
+ Name string
+ LastModified time.Time
+ Size int64
+ ETag string
+}
+
+func azureBlobObjectFromItem(item *container.BlobItem) (azureBlobObject, bool) {
+ if item == nil || item.Name == nil || item.Properties == nil {
+ return azureBlobObject{}, false
+ }
+ object := azureBlobObject{Name: *item.Name}
+ if item.Properties.LastModified != nil {
+ object.LastModified = item.Properties.LastModified.UTC()
+ }
+ if item.Properties.ContentLength != nil {
+ object.Size = *item.Properties.ContentLength
+ }
+ if item.Properties.ETag != nil {
+ object.ETag = string(*item.Properties.ETag)
+ }
+ return object, true
+}
+
+type azureBlobSyncSession struct {
+ connector *AzureBlobStorageConnector
+ request SyncRequest
+ batchSize int
+ marker string
+ skipUntil string
+ done bool
+}
+
+// NextBatch returns the next Azure Blob document batch.
+func (s *azureBlobSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ for {
+ if s.done {
+ return SyncBatch{}, io.EOF
+ }
+ objects, next, hasMore, err := s.connector.listBlobPage(ctx, s.connector.prefix, s.marker, int32(s.batchSize))
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if !hasMore {
+ s.done = true
+ }
+ s.marker = next
+
+ documents := make([]SourceDocument, 0, len(objects))
+ for _, object := range objects {
+ if s.skipUntil != "" && object.Name <= s.skipUntil {
+ continue
+ }
+ if !includeAzureBlobObject(s.request, object) {
+ continue
+ }
+ if document, ok := s.connector.sourceDocument(object.Name, object); ok {
+ documents = append(documents, document)
+ }
+ }
+ if len(documents) == 0 {
+ continue
+ }
+ last := documents[len(documents)-1]
+ updatedAt := last.UpdatedAt
+ return SyncBatch{
+ Documents: documents,
+ Checkpoint: &SyncCheckpoint{
+ Cursor: last.SourceID,
+ SourceID: last.SourceID,
+ UpdatedAt: &updatedAt,
+ },
+ }, nil
+ }
+}
+
+// Close closes the Azure Blob sync session.
+func (s *azureBlobSyncSession) Close() error {
+ return nil
+}
+
+// Fetch downloads a delayed Azure Blob document body.
+func (s *azureBlobSyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ return s.connector.Fetch(ctx, ref)
+}
+
+// applyResume configures the session to skip every blob at or before the
+// checkpoint cursor. Azure lists blobs in lexicographic name order, so skipping
+// <= cursor and re-listing guarantees no blob is dropped across a resume.
+func (s *azureBlobSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ s.skipUntil = firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+}
+
+type azureBlobPruneSession struct {
+ documents []SlimDocument
+ batchSize int
+ batchIndex int
+}
+
+// NextBatch returns the next Azure Blob prune snapshot batch.
+func (s *azureBlobPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ if s.batchIndex >= len(s.documents) {
+ return PruneBatch{}, io.EOF
+ }
+ end := s.batchIndex + s.batchSize
+ if end > len(s.documents) {
+ end = len(s.documents)
+ }
+ documents := s.documents[s.batchIndex:end]
+ s.batchIndex = end
+ return PruneBatch{Documents: documents}, nil
+}
+
+// Close closes the Azure Blob prune session.
+func (s *azureBlobPruneSession) Close() error {
+ return nil
+}
+
+type azureBlobFetchReference struct {
+ Name string `json:"name"`
+}
+
+func includeAzureBlobObject(request SyncRequest, object azureBlobObject) bool {
+ if request.FromBeginning {
+ return true
+ }
+ if object.LastModified.IsZero() {
+ return true
+ }
+ if len(request.Fingerprints) > 0 {
+ fingerprint := normalizedAzureBlobETag(object.ETag)
+ stored, ok := request.Fingerprints[object.Name]
+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint
+ }
+ return !beforeOrAtWindowStart(object.LastModified, request.WindowStart) && !afterWindowEnd(object.LastModified, request.WindowEnd)
+}
+
+func azureBlobExtension(name string) string {
+ index := strings.LastIndex(name, ".")
+ if index < 0 {
+ return ""
+ }
+ return "." + strings.ToLower(name[index+1:])
+}
+
+func azureBlobSupportedExtension(name string, allowImages bool) bool {
+ extension := azureBlobExtension(name)
+ if _, ok := azureBlobTextExtensions[extension]; ok {
+ return true
+ }
+ if allowImages {
+ _, ok := azureBlobImageExtensions[extension]
+ return ok
+ }
+ return false
+}
+
+func normalizeAzureBlobPrefix(prefix string) string {
+ return strings.TrimLeft(strings.TrimSpace(prefix), "/")
+}
+
+func normalizedAzureBlobETag(rawETag string) string {
+ rawETag = strings.Trim(strings.TrimSpace(rawETag), `"`)
+ if rawETag == "" {
+ return ""
+ }
+ return contentFingerprint([]byte(rawETag))
+}
+
+// azureBlobETag returns the raw ETag with surrounding quotes stripped, matching
+// the metadata emitted by the blob connector.
+func azureBlobETag(rawETag string) string {
+ return strings.Trim(strings.TrimSpace(rawETag), `"`)
+}
+
+func truncateAzureBlobMessage(msg string) string {
+ if len(msg) > azureBlobErrorMessageLimit {
+ return msg[:azureBlobErrorMessageLimit]
+ }
+ return msg
+}
diff --git a/internal/syncer/connector/azure_blob_test.go b/internal/syncer/connector/azure_blob_test.go
new file mode 100644
index 0000000000..f2689fb17c
--- /dev/null
+++ b/internal/syncer/connector/azure_blob_test.go
@@ -0,0 +1,508 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "reflect"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore"
+)
+
+func TestAzureBlobStorageConnectorDerivesAuthMode(t *testing.T) {
+ tests := []struct {
+ name string
+ config map[string]any
+ want string
+ wantErr bool
+ }{
+ {
+ name: "explicit connection_string",
+ config: map[string]any{
+ "auth_mode": "connection_string",
+ "credentials": map[string]any{
+ "connection_string": "DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=key;",
+ "container_name": "container",
+ },
+ },
+ want: "connection_string",
+ },
+ {
+ name: "explicit account_key",
+ config: map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{
+ "account_name": "acct",
+ "account_key": "key",
+ "container_name": "container",
+ },
+ },
+ want: "account_key",
+ },
+ {
+ name: "explicit sas_token",
+ config: map[string]any{
+ "auth_mode": "sas_token",
+ "credentials": map[string]any{
+ "container_url": "https://acct.blob.core.windows.net/container",
+ "sas_token": "?sv=1&sig=abc",
+ },
+ },
+ want: "sas_token",
+ },
+ {
+ name: "fallback connection_string",
+ config: map[string]any{
+ "credentials": map[string]any{
+ "connection_string": "DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=key;",
+ "container_name": "container",
+ },
+ },
+ want: "connection_string",
+ },
+ {
+ name: "fallback account_key",
+ config: map[string]any{
+ "credentials": map[string]any{
+ "account_name": "acct",
+ "account_key": "key",
+ "container_name": "container",
+ },
+ },
+ want: "account_key",
+ },
+ {
+ name: "fallback sas_token",
+ config: map[string]any{
+ "credentials": map[string]any{
+ "container_url": "https://acct.blob.core.windows.net/container",
+ "sas_token": "sv=1&sig=abc",
+ },
+ },
+ want: "sas_token",
+ },
+ {
+ name: "missing credentials",
+ config: map[string]any{"credentials": map[string]any{}},
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ connector, err := NewAzureBlobStorageConnector(tt.config)
+ if tt.wantErr {
+ if err == nil {
+ t.Fatalf("expected error, got nil")
+ }
+ var missing *ConnectorMissingCredentialError
+ if !errors.As(err, &missing) {
+ t.Fatalf("error type = %T, want ConnectorMissingCredentialError", err)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatalf("NewAzureBlobStorageConnector: %v", err)
+ }
+ if connector.authMode != tt.want {
+ t.Fatalf("auth mode = %q, want %q", connector.authMode, tt.want)
+ }
+ })
+ }
+}
+
+func TestAzureBlobStorageConnectorSasNormalization(t *testing.T) {
+ connector, err := NewAzureBlobStorageConnector(map[string]any{
+ "auth_mode": "sas_token",
+ "credentials": map[string]any{
+ "container_url": "https://acct.blob.core.windows.net/container/",
+ "sas_token": "?sv=1&sig=abc",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewAzureBlobStorageConnector: %v", err)
+ }
+ if connector.containerURL != "https://acct.blob.core.windows.net/container" {
+ t.Fatalf("container URL = %q", connector.containerURL)
+ }
+ if connector.sasToken != "sv=1&sig=abc" {
+ t.Fatalf("sas token = %q", connector.sasToken)
+ }
+}
+
+func TestAzureBlobStorageConnectorOpenSyncUsesFetchAndFingerprint(t *testing.T) {
+ connector := newTestAzureBlobStorageConnector(t, map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{
+ "account_name": "acct",
+ "account_key": "key",
+ "container_name": "container",
+ },
+ }, []azureBlobObject{
+ {Name: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 5, ETag: `"etag-a"`},
+ {Name: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 5, ETag: `"etag-b"`},
+ })
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != "docs/a.txt" || doc.SemanticIdentifier != "docs/a.txt" {
+ t.Fatalf("source id = %q, semantic = %q", doc.SourceID, doc.SemanticIdentifier)
+ }
+ if doc.Extension != ".txt" {
+ t.Fatalf("extension = %q", doc.Extension)
+ }
+ if doc.Fingerprint != normalizedAzureBlobETag(`"etag-a"`) {
+ t.Fatalf("fingerprint = %q", doc.Fingerprint)
+ }
+ if doc.Metadata["container"] != "container" || doc.Metadata["etag"] != "etag-a" {
+ t.Fatalf("metadata = %+v", doc.Metadata)
+ }
+ if doc.FetchRef == nil {
+ t.Fatalf("fetch ref is nil")
+ }
+ fetcher, ok := session.(Fetcher)
+ if !ok {
+ t.Fatalf("session does not implement Fetcher")
+ }
+ blob, err := fetcher.Fetch(context.Background(), *doc.FetchRef)
+ if err != nil {
+ t.Fatalf("Fetch: %v", err)
+ }
+ if string(blob) != "body:docs/a.txt" {
+ t.Fatalf("blob = %q", blob)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestAzureBlobStorageConnectorOpenSyncDefersListing(t *testing.T) {
+ connector := newTestAzureBlobStorageConnector(t, map[string]any{
+ "auth_mode": "sas_token",
+ "credentials": map[string]any{
+ "container_url": "https://acct.blob.core.windows.net/container",
+ "sas_token": "sv=1",
+ },
+ }, []azureBlobObject{{Name: "a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"}})
+
+ var listCalls int
+ baseList := connector.listBlobs
+ connector.listBlobs = func(ctx context.Context, prefix, marker string, max int32) ([]azureBlobObject, string, bool, error) {
+ listCalls++
+ return baseList(ctx, prefix, marker, max)
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ if listCalls != 0 {
+ t.Fatalf("OpenSync list calls = %d, want 0", listCalls)
+ }
+ if _, err := session.NextBatch(context.Background()); err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if listCalls != 1 {
+ t.Fatalf("NextBatch list calls = %d, want 1", listCalls)
+ }
+}
+
+func TestAzureBlobStorageConnectorFiltersImagesUnlessAllowed(t *testing.T) {
+ config := map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},
+ }
+ objects := []azureBlobObject{
+ {Name: "docs/a.png", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Name: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ }
+ connector := newTestAzureBlobStorageConnector(t, config, objects)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "docs/b.txt" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+
+ allowed := newTestAzureBlobStorageConnector(t, config, objects)
+ allowed.allowImages = true
+ session, err = allowed.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("allow images OpenSync: %v", err)
+ }
+ batch, err = session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("allow images NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("allow images documents len = %d, want 2", len(batch.Documents))
+ }
+}
+
+func TestAzureBlobStorageConnectorTimeWindowBoundaries(t *testing.T) {
+ since := mustTime(t, "2026-01-01T00:00:00Z")
+ until := mustTime(t, "2026-01-03T00:00:00Z")
+ connector := newTestAzureBlobStorageConnector(t, map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},
+ }, []azureBlobObject{
+ {Name: "at_since.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Name: "mid.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Name: "until.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ {Name: "above.txt", LastModified: mustTime(t, "2026-01-04T00:00:00Z"), Size: 1, ETag: "d"},
+ })
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &since,
+ WindowEnd: until,
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ got := make([]string, 0, len(batch.Documents))
+ for _, doc := range batch.Documents {
+ got = append(got, doc.SourceID)
+ }
+ want := []string{"mid.txt", "until.txt"}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("window documents = %v, want %v", got, want)
+ }
+}
+
+func TestAzureBlobStorageConnectorFingerprintSkipsUnchanged(t *testing.T) {
+ connector := newTestAzureBlobStorageConnector(t, map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},
+ }, []azureBlobObject{
+ {Name: "a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: `"etag-a"`},
+ {Name: "b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: `"etag-b"`},
+ })
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ Fingerprints: map[string]string{
+ "a.txt": normalizedAzureBlobETag(`"etag-a"`),
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "b.txt" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestAzureBlobStorageConnectorResumeSkipsBlobs(t *testing.T) {
+ connector := newTestAzureBlobStorageConnector(t, map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},
+ }, []azureBlobObject{
+ {Name: "a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Name: "b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Name: "c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ FromBeginning: true,
+ Resume: &SyncCheckpoint{Cursor: "a.txt", SourceID: "b.txt"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "c.txt" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+ if batch.Checkpoint == nil || batch.Checkpoint.Cursor != "c.txt" {
+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)
+ }
+}
+
+func TestAzureBlobStorageConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {
+ connector := newTestAzureBlobStorageConnector(t, map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},
+ }, []azureBlobObject{
+ {Name: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},
+ {Name: "docs/b.md", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},
+ {Name: "docs/c.png", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "c"},
+ })
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))
+ }
+ if batch.Documents[0].SourceID != "docs/a.txt" || batch.Documents[1].SourceID != "docs/b.md" {
+ t.Fatalf("documents = %+v", batch.Documents)
+ }
+}
+
+func TestAzureBlobStorageConnectorValidateClassification(t *testing.T) {
+ c := &AzureBlobStorageConnector{}
+ var missing *ConnectorMissingCredentialError
+ var validation *ConnectorValidationError
+
+ err := c.classifyValidationError(&azcore.ResponseError{StatusCode: http.StatusUnauthorized})
+ if !errors.As(err, &missing) {
+ t.Fatalf("401 error type = %T, want ConnectorMissingCredentialError", err)
+ }
+
+ err = c.classifyValidationError(&azcore.ResponseError{StatusCode: http.StatusForbidden})
+ if !errors.As(err, &validation) {
+ t.Fatalf("403 error type = %T, want ConnectorValidationError", err)
+ }
+
+ err = c.classifyValidationError(&azcore.ResponseError{StatusCode: http.StatusNotFound})
+ if !errors.As(err, &validation) {
+ t.Fatalf("404 error type = %T, want ConnectorValidationError", err)
+ }
+
+ err = c.classifyValidationError(errors.New("boom"))
+ if !errors.As(err, &validation) {
+ t.Fatalf("generic error type = %T, want ConnectorValidationError", err)
+ }
+}
+
+func TestAzureBlobStorageConnectorValidateRejectsUnsafeURL(t *testing.T) {
+ connector, err := NewAzureBlobStorageConnector(map[string]any{
+ "auth_mode": "sas_token",
+ "credentials": map[string]any{
+ "container_url": "http://192.168.1.10/container",
+ "sas_token": "sv=1",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewAzureBlobStorageConnector: %v", err)
+ }
+ restore := azureAssertURLSafe
+ azureAssertURLSafe = func(rawURL string) (string, string, error) {
+ return "", "", errors.New("URL resolves to a non-public address (192.168.1.10), which is not allowed")
+ }
+ defer func() { azureAssertURLSafe = restore }()
+
+ err = connector.Validate(context.Background())
+ var validation *ConnectorValidationError
+ if !errors.As(err, &validation) {
+ t.Fatalf("Validate error type = %T, want ConnectorValidationError", err)
+ }
+}
+
+func TestAzureBlobStorageConnectorValidateConnectorSettingUsesRequest(t *testing.T) {
+ // The receiver holds valid credentials, but the request is missing them.
+ // ValidateConnectorSetting must reject based on the unsaved request.
+ receiver, err := NewAzureBlobStorageConnector(map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{
+ "account_name": "acct",
+ "account_key": "key",
+ "container_name": "container",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewAzureBlobStorageConnector: %v", err)
+ }
+ err = receiver.ValidateConnectorSetting(context.Background(), map[string]any{
+ "auth_mode": "account_key",
+ "credentials": map[string]any{"account_name": "acct"}, // missing account_key
+ })
+ var missing *ConnectorMissingCredentialError
+ if !errors.As(err, &missing) {
+ t.Fatalf("ValidateConnectorSetting error type = %T, want ConnectorMissingCredentialError", err)
+ }
+}
+
+// newTestAzureBlobStorageConnector builds a connector with injected listing,
+// download and validation hooks so unit tests need no external Blob service.
+func newTestAzureBlobStorageConnector(t *testing.T, config map[string]any, objects []azureBlobObject) *AzureBlobStorageConnector {
+ t.Helper()
+ connector, err := NewAzureBlobStorageConnector(config)
+ if err != nil {
+ t.Fatalf("NewAzureBlobStorageConnector: %v", err)
+ }
+ restore := azureAssertURLSafe
+ azureAssertURLSafe = func(rawURL string) (string, string, error) { return "host", "1.2.3.4", nil }
+ t.Cleanup(func() { azureAssertURLSafe = restore })
+
+ connector.validateContainer = func(ctx context.Context) error { return nil }
+ connector.listBlobs = func(ctx context.Context, prefix, marker string, max int32) ([]azureBlobObject, string, bool, error) {
+ var filtered []azureBlobObject
+ for _, object := range objects {
+ if prefix != "" && !strings.HasPrefix(object.Name, prefix) {
+ continue
+ }
+ if marker != "" && object.Name <= marker {
+ continue
+ }
+ filtered = append(filtered, object)
+ }
+ sort.SliceStable(filtered, func(i, j int) bool { return filtered[i].Name < filtered[j].Name })
+ if len(filtered) == 0 {
+ return nil, "", false, nil
+ }
+ end := int(max)
+ if end > len(filtered) {
+ end = len(filtered)
+ }
+ page := filtered[:end]
+ hasMore := end < len(filtered)
+ next := ""
+ if hasMore {
+ next = page[len(page)-1].Name
+ }
+ return page, next, hasMore, nil
+ }
+ connector.downloadBlob = func(ctx context.Context, name string) ([]byte, error) {
+ return []byte("body:" + name), nil
+ }
+ return connector
+}
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 1a1a3ee395..5f44befdbb 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -37,6 +37,7 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "google-drive", NewGoogleDriveConnector)
registerBuiltIn(registry, "google_drive", NewGoogleDriveConnector)
registerBuiltIn(registry, "google_cloud_storage", NewGoogleCloudStorageConnector)
+ registerBuiltIn(registry, "azure_blob", NewAzureBlobStorageConnector)
registerBuiltIn(registry, "r2", NewR2Connector)
registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)
registerBuiltIn(registry, "imap", NewIMAPConnector)