RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): add Azure Blob Storage connector with checkpoint resume (#18701)
mkaaade5b1adc9fab5
diff --git a/go.mod b/go.modindex 64284133f0..7aa97bb9f0 100644--- a/go.mod+++ b/go.mod@@ -5,6 +5,8 @@ go 1.26.4require (cloud.google.com/go/storage v1.63.0github.com/AkmalOt/gomsg v0.0.0-20260407083308-985c3a1a76b7+ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0+ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0github.com/DATA-DOG/go-sqlmock v1.5.2github.com/JohannesKaufmann/html-to-markdown v1.6.0github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0@@ -32,7 +34,7 @@ require (github.com/glebarez/sqlite v1.11.0github.com/go-sql-driver/mysql v1.7.0github.com/goccy/go-json v0.10.6- github.com/golang-jwt/jwt/v5 v5.3.0+ github.com/golang-jwt/jwt/v5 v5.3.1github.com/gomarkdown/markdown v0.0.0-20260614204949-e08cff860f76github.com/google/go-cmp v0.7.0github.com/google/uuid v1.6.0@@ -97,6 +99,7 @@ require (cloud.google.com/go/iam v1.11.0 // indirectcloud.google.com/go/monitoring v1.29.0 // indirectconnectrpc.com/connect v1.20.0 // indirect+ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirectgithub.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirectgithub.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirectgithub.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirectdiff --git a/go.sum b/go.sumindex dfaeba4f1b..0eb0f2815e 100644--- a/go.sum+++ b/go.sum@@ -26,8 +26,20 @@ cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjconnectrpc.com/connect v1.20.0 h1:6TNDAB+WeNd2uolWNlYczB5E0KNNaVMNUEx8JEUsPmQ=connectrpc.com/connect v1.20.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4=github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE=+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM=github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=+github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=+github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw=+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y=+github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=+github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=@@ -266,8 +278,8 @@ github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7Lkgithub.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=-github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=-github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=+github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=+github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=@@ -363,6 +375,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=github.com/larksuite/oapi-sdk-go/v3 v3.9.9 h1:qzVK5U1AuT/n0Z4LqCN2ATU6MlYp2Iipa7q+fVK5Nh8=github.com/larksuite/oapi-sdk-go/v3 v3.9.9/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI=github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8=@@ -430,6 +444,8 @@ github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJgithub.com/phpdave11/gofpdi v1.0.14-0.20211212211723-1f10f9844311 h1:zyWXQ6vu27ETMpYsEMAsisQ+GqJ4e1TPvSNfdOPF0no=github.com/phpdave11/gofpdi v1.0.14-0.20211212211723-1f10f9844311/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=diff --git a/internal/syncer/connector/azure_blob.go b/internal/syncer/connector/azure_blob.gonew file mode 100644index 0000000000..143899306d--- /dev/null+++ b/internal/syncer/connector/azure_blob.go@@ -0,0 +1,608 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "sort"+ "strings"+ "time"++ "github.com/Azure/azure-sdk-for-go/sdk/azcore"+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"++ "ragflow/internal/utility"+)++const (+ azureBlobSource = "azure_blob"+ azureBlobDefaultBatchSize = 2+ azureBlobEndpointSuffix = "blob.core.windows.net"+ azureBlobErrorMessageLimit = 300+)++// azureAssertURLSafe is the SSRF guard used for caller-supplied endpoint URLs.+// It is an indirection over utility.AssertURLSafe so unit tests can substitute+// a stub without touching the shared global guard.+var azureAssertURLSafe = utility.AssertURLSafe++// azureBlobTextExtensions mirrors the extension set used by the OneDrive+// family of file sources in the blob connector.+var azureBlobTextExtensions = map[string]struct{}{+ ".pdf": {},+ ".docx": {},+ ".doc": {},+ ".xlsx": {},+ ".xls": {},+ ".pptx": {},+ ".ppt": {},+ ".txt": {},+ ".md": {},+ ".csv": {},+ ".html": {},+ ".htm": {},+ ".json": {},+ ".xml": {},+}++var azureBlobImageExtensions = map[string]struct{}{+ ".png": {},+ ".jpg": {},+ ".jpeg": {},+ ".webp": {},+ ".gif": {},+ ".bmp": {},+ ".tiff": {},+}++// AzureBlobStorageConnector reads blobs from an Azure Blob Storage container.+//+// Authentication supports three mutually exclusive modes selected by+// auth_mode (connection_string, account_key, sas_token). When auth_mode is+// absent, the mode is derived from whichever credential fields are populated,+// mirroring the blob connector's fallback precedence.+type AzureBlobStorageConnector struct {+ batchSize int+ prefix string+ allowImages bool++ authMode string+ containerName string+ accountName string+ accountKey string+ connectionString string+ containerURL string // sas_token mode base URL (no trailing slash, no SAS query)+ sasToken string // sas_token mode token (no leading "?")++ containerClient *container.Client++ listBlobs func(ctx context.Context, prefix, marker string, maxResults int32) ([]azureBlobObject, string, bool, error)+ downloadBlob func(ctx context.Context, name string) ([]byte, error)+ validateContainer func(ctx context.Context) error+}++// NewAzureBlobStorageConnector creates an Azure Blob connector from+// Python-compatible config.+func NewAzureBlobStorageConnector(config map[string]any) (*AzureBlobStorageConnector, error) {+ credentials := configAnyMap(config["credentials"])+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), azureBlobDefaultBatchSize)+ if batchSize <= 0 {+ batchSize = azureBlobDefaultBatchSize+ }+ connector := &AzureBlobStorageConnector{+ batchSize: batchSize,+ prefix: normalizeAzureBlobPrefix(stringConfig(config["prefix"])),+ allowImages: configBoolDefault(config["allow_images"], false),+ authMode: strings.ToLower(strings.TrimSpace(stringConfig(config["auth_mode"]))),+ containerName: strings.TrimSpace(stringConfig(credentials["container_name"])),+ accountName: strings.TrimSpace(stringConfig(credentials["account_name"])),+ accountKey: stringConfig(credentials["account_key"]),+ connectionString: stringConfig(credentials["connection_string"]),+ containerURL: strings.TrimRight(strings.TrimSpace(stringConfig(credentials["container_url"])), "/"),+ sasToken: strings.TrimLeft(stringConfig(credentials["sas_token"]), "?"),+ }+ if connector.authMode == "" {+ switch {+ case connector.connectionString != "":+ connector.authMode = "connection_string"+ case connector.accountName != "" && connector.accountKey != "":+ connector.authMode = "account_key"+ case connector.containerURL != "" && connector.sasToken != "":+ connector.authMode = "sas_token"+ }+ }+ if err := connector.validateCredentials(); err != nil {+ return nil, err+ }+ return connector, nil+}++// Validate validates Azure Blob connector settings and credentials.+func (c *AzureBlobStorageConnector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("Azure Blob connector is nil")+ }+ if c.batchSize <= 0 {+ return &ConnectorValidationError{Message: "Azure Blob: batch_size must be a positive integer"}+ }+ if err := c.validateCredentials(); err != nil {+ return err+ }+ if err := c.validateURLSafety(); err != nil {+ return err+ }+ if c.validateContainer != nil {+ return c.validateContainer(ctx)+ }+ return c.defaultValidateContainer(ctx)+}++// ValidateConnectorSetting validates Azure Blob settings from an unsaved+// config by constructing a temporary connector from the request.+func (c *AzureBlobStorageConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ connector, err := NewAzureBlobStorageConnector(request)+ if err != nil {+ return err+ }+ return connector.Validate(ctx)+}++// OpenSync opens one Azure Blob sync session.+func (c *AzureBlobStorageConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ session := &azureBlobSyncSession{+ connector: c,+ request: request,+ batchSize: c.batchSize,+ }+ if request.Resume != nil {+ session.applyResume(request.Resume)+ }+ return session, nil+}++// OpenPrune opens one complete Azure Blob prune snapshot session.+func (c *AzureBlobStorageConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ names, err := c.collectBlobNames(ctx)+ if err != nil {+ return nil, err+ }+ documents := make([]SlimDocument, 0, len(names))+ for _, name := range names {+ if name == "" || !azureBlobSupportedExtension(name, c.allowImages) {+ continue+ }+ documents = append(documents, SlimDocument{SourceID: name})+ }+ return &azureBlobPruneSession{documents: documents, batchSize: c.batchSize}, nil+}++// Fetch downloads an Azure Blob body referenced by a previous sync batch.+func (c *AzureBlobStorageConnector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ var fetch azureBlobFetchReference+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {+ return nil, err+ }+ return c.download(ctx, fetch.Name)+}++// validateCredentials checks that the selected auth mode has all required+// credential fields populated.+func (c *AzureBlobStorageConnector) validateCredentials() error {+ switch c.authMode {+ case "connection_string":+ if c.connectionString == "" {+ return &ConnectorMissingCredentialError{Message: "Azure Blob: connection_string is required for the connection_string auth mode"}+ }+ if c.containerName == "" {+ return &ConnectorMissingCredentialError{Message: "Azure Blob: container_name is required together with connection_string"}+ }+ case "account_key":+ if c.accountName == "" || c.accountKey == "" {+ return &ConnectorMissingCredentialError{Message: "Azure Blob: account_name and account_key are required for the account_key auth mode"}+ }+ if c.containerName == "" {+ return &ConnectorMissingCredentialError{Message: "Azure Blob: container_name is required together with account_name + account_key"}+ }+ case "sas_token":+ if c.containerURL == "" || c.sasToken == "" {+ return &ConnectorMissingCredentialError{Message: "Azure Blob: container_url and sas_token are required for the sas_token auth mode"}+ }+ default:+ return &ConnectorMissingCredentialError{+ Message: "Azure Blob credentials are incomplete. Provide one of: (a) connection_string + container_name, (b) account_name + account_key + container_name, (c) container_url + sas_token.",+ }+ }+ return nil+}++// validateURLSafety rejects caller-supplied endpoints that are not HTTPS and+// do not resolve to a public host. The connection-string mode is a single+// opaque blob whose endpoint is embedded by Azure, so it is not routed by the+// caller and is intentionally left alone.+func (c *AzureBlobStorageConnector) validateURLSafety() error {+ endpoint := ""+ switch c.authMode {+ case "account_key":+ endpoint = fmt.Sprintf("https://%s.%s", c.accountName, azureBlobEndpointSuffix)+ case "sas_token":+ endpoint = c.containerURL+ }+ if endpoint == "" {+ return nil+ }+ if _, _, err := azureAssertURLSafe(endpoint); err != nil {+ return &ConnectorValidationError{Message: fmt.Sprintf("Azure Blob: unsafe endpoint URL: %v", err)}+ }+ return nil+}++func (c *AzureBlobStorageConnector) ensureClient(ctx context.Context) (*container.Client, error) {+ if c.containerClient != nil {+ return c.containerClient, nil+ }+ var (+ client *container.Client+ err error+ )+ switch c.authMode {+ case "connection_string":+ client, err = container.NewClientFromConnectionString(c.connectionString, c.containerName, nil)+ case "account_key":+ var cred *azblob.SharedKeyCredential+ cred, err = azblob.NewSharedKeyCredential(c.accountName, c.accountKey)+ if err == nil {+ accountURL := fmt.Sprintf("https://%s.%s/%s", c.accountName, azureBlobEndpointSuffix, c.containerName)+ client, err = container.NewClientWithSharedKeyCredential(accountURL, cred, nil)+ }+ case "sas_token":+ fullURL := c.containerURL + "?" + c.sasToken+ client, err = container.NewClientWithNoCredential(fullURL, nil)+ default:+ return nil, &ConnectorMissingCredentialError{Message: "Azure Blob credentials are incomplete."}+ }+ if err != nil {+ return nil, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to initialise Azure Blob client: %v", err)}+ }+ c.containerClient = client+ return client, nil+}++// defaultValidateContainer issues the container properties call that proves+// both the credential and the container name are valid, classifying errors the+// same way the blob connector does.+func (c *AzureBlobStorageConnector) defaultValidateContainer(ctx context.Context) error {+ client, err := c.ensureClient(ctx)+ if err != nil {+ return err+ }+ if _, err := client.GetProperties(ctx, nil); err != nil {+ return c.classifyValidationError(err)+ }+ return nil+}++func (c *AzureBlobStorageConnector) classifyValidationError(err error) error {+ msg := truncateAzureBlobMessage(err.Error())+ var respErr *azcore.ResponseError+ if errors.As(err, &respErr) {+ switch {+ case respErr.StatusCode == http.StatusUnauthorized ||+ strings.Contains(msg, "AuthenticationFailed") ||+ strings.Contains(msg, "InvalidAuthenticationInfo"):+ return &ConnectorMissingCredentialError{Message: fmt.Sprintf("Azure Blob credential rejected: %s", msg)}+ case respErr.StatusCode == http.StatusForbidden ||+ strings.Contains(msg, "AuthorizationPermissionMismatch") ||+ strings.Contains(msg, "403"):+ return &ConnectorValidationError{Message: fmt.Sprintf("Azure Blob: insufficient permissions on container: %s", msg)}+ case respErr.StatusCode == http.StatusNotFound ||+ strings.Contains(msg, "ContainerNotFound") ||+ strings.Contains(msg, "404"):+ return &ConnectorValidationError{Message: fmt.Sprintf("Azure Blob: container not found: %s", msg)}+ default:+ return &ConnectorValidationError{Message: fmt.Sprintf("Azure Blob validation failed (%s): %s", respErr.ErrorCode, msg)}+ }+ }+ return &ConnectorValidationError{Message: fmt.Sprintf("Azure Blob validation failed: %s", msg)}+}++func (c *AzureBlobStorageConnector) listBlobPage(ctx context.Context, prefix, marker string, maxResults int32) ([]azureBlobObject, string, bool, error) {+ if c.listBlobs != nil {+ return c.listBlobs(ctx, prefix, marker, maxResults)+ }+ client, err := c.ensureClient(ctx)+ if err != nil {+ return nil, "", false, err+ }+ options := &container.ListBlobsFlatOptions{MaxResults: &maxResults}+ if prefix != "" {+ options.Prefix = &prefix+ }+ if marker != "" {+ options.Marker = &marker+ }+ page, err := client.NewListBlobsFlatPager(options).NextPage(ctx)+ if err != nil {+ return nil, "", false, err+ }+ objects := make([]azureBlobObject, 0, len(page.Segment.BlobItems))+ for _, item := range page.Segment.BlobItems {+ if object, ok := azureBlobObjectFromItem(item); ok {+ objects = append(objects, object)+ }+ }+ next := ""+ if page.NextMarker != nil {+ next = *page.NextMarker+ }+ return objects, next, next != "", nil+}++func (c *AzureBlobStorageConnector) collectBlobNames(ctx context.Context) ([]string, error) {+ var names []string+ marker := ""+ for {+ objects, next, hasMore, err := c.listBlobPage(ctx, c.prefix, marker, int32(c.batchSize))+ if err != nil {+ return nil, err+ }+ for _, object := range objects {+ names = append(names, object.Name)+ }+ if !hasMore {+ break+ }+ marker = next+ }+ sort.Strings(names)+ return names, nil+}++func (c *AzureBlobStorageConnector) download(ctx context.Context, name string) ([]byte, error) {+ if c.downloadBlob != nil {+ return c.downloadBlob(ctx, name)+ }+ client, err := c.ensureClient(ctx)+ if err != nil {+ return nil, err+ }+ response, err := client.NewBlockBlobClient(name).DownloadStream(ctx, nil)+ if err != nil {+ return nil, err+ }+ defer response.Body.Close()+ return io.ReadAll(response.Body)+}++func (c *AzureBlobStorageConnector) sourceDocument(name string, object azureBlobObject) (SourceDocument, bool) {+ if name == "" || !azureBlobSupportedExtension(name, c.allowImages) {+ return SourceDocument{}, false+ }+ fetch := azureBlobFetchReference{Name: name}+ fetchKey, _ := json.Marshal(fetch)+ return SourceDocument{+ SourceID: name,+ SemanticIdentifier: name,+ Extension: azureBlobExtension(name),+ FetchRef: &FetchReference{Key: string(fetchKey), SizeHint: object.Size},+ UpdatedAt: object.LastModified,+ SizeBytes: object.Size,+ Metadata: map[string]any{+ "container": c.containerName,+ "etag": azureBlobETag(object.ETag),+ "prefix": c.prefix,+ },+ Fingerprint: normalizedAzureBlobETag(object.ETag),+ }, true+}++// azureBlobObject is the normalized minimal blob snapshot used by the+// connector and its tests.+type azureBlobObject struct {+ Name string+ LastModified time.Time+ Size int64+ ETag string+}++func azureBlobObjectFromItem(item *container.BlobItem) (azureBlobObject, bool) {+ if item == nil || item.Name == nil || item.Properties == nil {+ return azureBlobObject{}, false+ }+ object := azureBlobObject{Name: *item.Name}+ if item.Properties.LastModified != nil {+ object.LastModified = item.Properties.LastModified.UTC()+ }+ if item.Properties.ContentLength != nil {+ object.Size = *item.Properties.ContentLength+ }+ if item.Properties.ETag != nil {+ object.ETag = string(*item.Properties.ETag)+ }+ return object, true+}++type azureBlobSyncSession struct {+ connector *AzureBlobStorageConnector+ request SyncRequest+ batchSize int+ marker string+ skipUntil string+ done bool+}++// NextBatch returns the next Azure Blob document batch.+func (s *azureBlobSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ for {+ if s.done {+ return SyncBatch{}, io.EOF+ }+ objects, next, hasMore, err := s.connector.listBlobPage(ctx, s.connector.prefix, s.marker, int32(s.batchSize))+ if err != nil {+ return SyncBatch{}, err+ }+ if !hasMore {+ s.done = true+ }+ s.marker = next++ documents := make([]SourceDocument, 0, len(objects))+ for _, object := range objects {+ if s.skipUntil != "" && object.Name <= s.skipUntil {+ continue+ }+ if !includeAzureBlobObject(s.request, object) {+ continue+ }+ if document, ok := s.connector.sourceDocument(object.Name, object); ok {+ documents = append(documents, document)+ }+ }+ if len(documents) == 0 {+ continue+ }+ last := documents[len(documents)-1]+ updatedAt := last.UpdatedAt+ return SyncBatch{+ Documents: documents,+ Checkpoint: &SyncCheckpoint{+ Cursor: last.SourceID,+ SourceID: last.SourceID,+ UpdatedAt: &updatedAt,+ },+ }, nil+ }+}++// Close closes the Azure Blob sync session.+func (s *azureBlobSyncSession) Close() error {+ return nil+}++// Fetch downloads a delayed Azure Blob document body.+func (s *azureBlobSyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ return s.connector.Fetch(ctx, ref)+}++// applyResume configures the session to skip every blob at or before the+// checkpoint cursor. Azure lists blobs in lexicographic name order, so skipping+// <= cursor and re-listing guarantees no blob is dropped across a resume.+func (s *azureBlobSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ s.skipUntil = firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+}++type azureBlobPruneSession struct {+ documents []SlimDocument+ batchSize int+ batchIndex int+}++// NextBatch returns the next Azure Blob prune snapshot batch.+func (s *azureBlobPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ if s.batchIndex >= len(s.documents) {+ return PruneBatch{}, io.EOF+ }+ end := s.batchIndex + s.batchSize+ if end > len(s.documents) {+ end = len(s.documents)+ }+ documents := s.documents[s.batchIndex:end]+ s.batchIndex = end+ return PruneBatch{Documents: documents}, nil+}++// Close closes the Azure Blob prune session.+func (s *azureBlobPruneSession) Close() error {+ return nil+}++type azureBlobFetchReference struct {+ Name string `json:"name"`+}++func includeAzureBlobObject(request SyncRequest, object azureBlobObject) bool {+ if request.FromBeginning {+ return true+ }+ if object.LastModified.IsZero() {+ return true+ }+ if len(request.Fingerprints) > 0 {+ fingerprint := normalizedAzureBlobETag(object.ETag)+ stored, ok := request.Fingerprints[object.Name]+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint+ }+ return !beforeOrAtWindowStart(object.LastModified, request.WindowStart) && !afterWindowEnd(object.LastModified, request.WindowEnd)+}++func azureBlobExtension(name string) string {+ index := strings.LastIndex(name, ".")+ if index < 0 {+ return ""+ }+ return "." + strings.ToLower(name[index+1:])+}++func azureBlobSupportedExtension(name string, allowImages bool) bool {+ extension := azureBlobExtension(name)+ if _, ok := azureBlobTextExtensions[extension]; ok {+ return true+ }+ if allowImages {+ _, ok := azureBlobImageExtensions[extension]+ return ok+ }+ return false+}++func normalizeAzureBlobPrefix(prefix string) string {+ return strings.TrimLeft(strings.TrimSpace(prefix), "/")+}++func normalizedAzureBlobETag(rawETag string) string {+ rawETag = strings.Trim(strings.TrimSpace(rawETag), `"`)+ if rawETag == "" {+ return ""+ }+ return contentFingerprint([]byte(rawETag))+}++// azureBlobETag returns the raw ETag with surrounding quotes stripped, matching+// the metadata emitted by the blob connector.+func azureBlobETag(rawETag string) string {+ return strings.Trim(strings.TrimSpace(rawETag), `"`)+}++func truncateAzureBlobMessage(msg string) string {+ if len(msg) > azureBlobErrorMessageLimit {+ return msg[:azureBlobErrorMessageLimit]+ }+ return msg+}diff --git a/internal/syncer/connector/azure_blob_test.go b/internal/syncer/connector/azure_blob_test.gonew file mode 100644index 0000000000..f2689fb17c--- /dev/null+++ b/internal/syncer/connector/azure_blob_test.go@@ -0,0 +1,508 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "errors"+ "io"+ "net/http"+ "reflect"+ "sort"+ "strings"+ "testing"++ "github.com/Azure/azure-sdk-for-go/sdk/azcore"+)++func TestAzureBlobStorageConnectorDerivesAuthMode(t *testing.T) {+ tests := []struct {+ name string+ config map[string]any+ want string+ wantErr bool+ }{+ {+ name: "explicit connection_string",+ config: map[string]any{+ "auth_mode": "connection_string",+ "credentials": map[string]any{+ "connection_string": "DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=key;",+ "container_name": "container",+ },+ },+ want: "connection_string",+ },+ {+ name: "explicit account_key",+ config: map[string]any{+ "auth_mode": "account_key",+ "credentials": map[string]any{+ "account_name": "acct",+ "account_key": "key",+ "container_name": "container",+ },+ },+ want: "account_key",+ },+ {+ name: "explicit sas_token",+ config: map[string]any{+ "auth_mode": "sas_token",+ "credentials": map[string]any{+ "container_url": "https://acct.blob.core.windows.net/container",+ "sas_token": "?sv=1&sig=abc",+ },+ },+ want: "sas_token",+ },+ {+ name: "fallback connection_string",+ config: map[string]any{+ "credentials": map[string]any{+ "connection_string": "DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=key;",+ "container_name": "container",+ },+ },+ want: "connection_string",+ },+ {+ name: "fallback account_key",+ config: map[string]any{+ "credentials": map[string]any{+ "account_name": "acct",+ "account_key": "key",+ "container_name": "container",+ },+ },+ want: "account_key",+ },+ {+ name: "fallback sas_token",+ config: map[string]any{+ "credentials": map[string]any{+ "container_url": "https://acct.blob.core.windows.net/container",+ "sas_token": "sv=1&sig=abc",+ },+ },+ want: "sas_token",+ },+ {+ name: "missing credentials",+ config: map[string]any{"credentials": map[string]any{}},+ wantErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ connector, err := NewAzureBlobStorageConnector(tt.config)+ if tt.wantErr {+ if err == nil {+ t.Fatalf("expected error, got nil")+ }+ var missing *ConnectorMissingCredentialError+ if !errors.As(err, &missing) {+ t.Fatalf("error type = %T, want ConnectorMissingCredentialError", err)+ }+ return+ }+ if err != nil {+ t.Fatalf("NewAzureBlobStorageConnector: %v", err)+ }+ if connector.authMode != tt.want {+ t.Fatalf("auth mode = %q, want %q", connector.authMode, tt.want)+ }+ })+ }+}++func TestAzureBlobStorageConnectorSasNormalization(t *testing.T) {+ connector, err := NewAzureBlobStorageConnector(map[string]any{+ "auth_mode": "sas_token",+ "credentials": map[string]any{+ "container_url": "https://acct.blob.core.windows.net/container/",+ "sas_token": "?sv=1&sig=abc",+ },+ })+ if err != nil {+ t.Fatalf("NewAzureBlobStorageConnector: %v", err)+ }+ if connector.containerURL != "https://acct.blob.core.windows.net/container" {+ t.Fatalf("container URL = %q", connector.containerURL)+ }+ if connector.sasToken != "sv=1&sig=abc" {+ t.Fatalf("sas token = %q", connector.sasToken)+ }+}++func TestAzureBlobStorageConnectorOpenSyncUsesFetchAndFingerprint(t *testing.T) {+ connector := newTestAzureBlobStorageConnector(t, map[string]any{+ "auth_mode": "account_key",+ "credentials": map[string]any{+ "account_name": "acct",+ "account_key": "key",+ "container_name": "container",+ },+ }, []azureBlobObject{+ {Name: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 5, ETag: `"etag-a"`},+ {Name: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 5, ETag: `"etag-b"`},+ })++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != "docs/a.txt" || doc.SemanticIdentifier != "docs/a.txt" {+ t.Fatalf("source id = %q, semantic = %q", doc.SourceID, doc.SemanticIdentifier)+ }+ if doc.Extension != ".txt" {+ t.Fatalf("extension = %q", doc.Extension)+ }+ if doc.Fingerprint != normalizedAzureBlobETag(`"etag-a"`) {+ t.Fatalf("fingerprint = %q", doc.Fingerprint)+ }+ if doc.Metadata["container"] != "container" || doc.Metadata["etag"] != "etag-a" {+ t.Fatalf("metadata = %+v", doc.Metadata)+ }+ if doc.FetchRef == nil {+ t.Fatalf("fetch ref is nil")+ }+ fetcher, ok := session.(Fetcher)+ if !ok {+ t.Fatalf("session does not implement Fetcher")+ }+ blob, err := fetcher.Fetch(context.Background(), *doc.FetchRef)+ if err != nil {+ t.Fatalf("Fetch: %v", err)+ }+ if string(blob) != "body:docs/a.txt" {+ t.Fatalf("blob = %q", blob)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestAzureBlobStorageConnectorOpenSyncDefersListing(t *testing.T) {+ connector := newTestAzureBlobStorageConnector(t, map[string]any{+ "auth_mode": "sas_token",+ "credentials": map[string]any{+ "container_url": "https://acct.blob.core.windows.net/container",+ "sas_token": "sv=1",+ },+ }, []azureBlobObject{{Name: "a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"}})++ var listCalls int+ baseList := connector.listBlobs+ connector.listBlobs = func(ctx context.Context, prefix, marker string, max int32) ([]azureBlobObject, string, bool, error) {+ listCalls+++ return baseList(ctx, prefix, marker, max)+ }++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ if listCalls != 0 {+ t.Fatalf("OpenSync list calls = %d, want 0", listCalls)+ }+ if _, err := session.NextBatch(context.Background()); err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if listCalls != 1 {+ t.Fatalf("NextBatch list calls = %d, want 1", listCalls)+ }+}++func TestAzureBlobStorageConnectorFiltersImagesUnlessAllowed(t *testing.T) {+ config := map[string]any{+ "auth_mode": "account_key",+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},+ }+ objects := []azureBlobObject{+ {Name: "docs/a.png", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Name: "docs/b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ }+ connector := newTestAzureBlobStorageConnector(t, config, objects)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "docs/b.txt" {+ t.Fatalf("documents = %+v", batch.Documents)+ }++ allowed := newTestAzureBlobStorageConnector(t, config, objects)+ allowed.allowImages = true+ session, err = allowed.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("allow images OpenSync: %v", err)+ }+ batch, err = session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("allow images NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("allow images documents len = %d, want 2", len(batch.Documents))+ }+}++func TestAzureBlobStorageConnectorTimeWindowBoundaries(t *testing.T) {+ since := mustTime(t, "2026-01-01T00:00:00Z")+ until := mustTime(t, "2026-01-03T00:00:00Z")+ connector := newTestAzureBlobStorageConnector(t, map[string]any{+ "auth_mode": "account_key",+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},+ }, []azureBlobObject{+ {Name: "at_since.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Name: "mid.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Name: "until.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},+ {Name: "above.txt", LastModified: mustTime(t, "2026-01-04T00:00:00Z"), Size: 1, ETag: "d"},+ })++ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &since,+ WindowEnd: until,+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ got := make([]string, 0, len(batch.Documents))+ for _, doc := range batch.Documents {+ got = append(got, doc.SourceID)+ }+ want := []string{"mid.txt", "until.txt"}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("window documents = %v, want %v", got, want)+ }+}++func TestAzureBlobStorageConnectorFingerprintSkipsUnchanged(t *testing.T) {+ connector := newTestAzureBlobStorageConnector(t, map[string]any{+ "auth_mode": "account_key",+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},+ }, []azureBlobObject{+ {Name: "a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: `"etag-a"`},+ {Name: "b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: `"etag-b"`},+ })++ session, err := connector.OpenSync(context.Background(), SyncRequest{+ Fingerprints: map[string]string{+ "a.txt": normalizedAzureBlobETag(`"etag-a"`),+ },+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "b.txt" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestAzureBlobStorageConnectorResumeSkipsBlobs(t *testing.T) {+ connector := newTestAzureBlobStorageConnector(t, map[string]any{+ "auth_mode": "account_key",+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},+ }, []azureBlobObject{+ {Name: "a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Name: "b.txt", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Name: "c.txt", LastModified: mustTime(t, "2026-01-03T00:00:00Z"), Size: 1, ETag: "c"},+ })++ session, err := connector.OpenSync(context.Background(), SyncRequest{+ FromBeginning: true,+ Resume: &SyncCheckpoint{Cursor: "a.txt", SourceID: "b.txt"},+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "c.txt" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.Cursor != "c.txt" {+ t.Fatalf("checkpoint = %+v", batch.Checkpoint)+ }+}++func TestAzureBlobStorageConnectorOpenPruneReturnsSlimSnapshot(t *testing.T) {+ connector := newTestAzureBlobStorageConnector(t, map[string]any{+ "auth_mode": "account_key",+ "credentials": map[string]any{"account_name": "acct", "account_key": "key", "container_name": "container"},+ }, []azureBlobObject{+ {Name: "docs/a.txt", LastModified: mustTime(t, "2026-01-01T00:00:00Z"), Size: 1, ETag: "a"},+ {Name: "docs/b.md", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "b"},+ {Name: "docs/c.png", LastModified: mustTime(t, "2026-01-02T00:00:00Z"), Size: 1, ETag: "c"},+ })+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != "docs/a.txt" || batch.Documents[1].SourceID != "docs/b.md" {+ t.Fatalf("documents = %+v", batch.Documents)+ }+}++func TestAzureBlobStorageConnectorValidateClassification(t *testing.T) {+ c := &AzureBlobStorageConnector{}+ var missing *ConnectorMissingCredentialError+ var validation *ConnectorValidationError++ err := c.classifyValidationError(&azcore.ResponseError{StatusCode: http.StatusUnauthorized})+ if !errors.As(err, &missing) {+ t.Fatalf("401 error type = %T, want ConnectorMissingCredentialError", err)+ }++ err = c.classifyValidationError(&azcore.ResponseError{StatusCode: http.StatusForbidden})+ if !errors.As(err, &validation) {+ t.Fatalf("403 error type = %T, want ConnectorValidationError", err)+ }++ err = c.classifyValidationError(&azcore.ResponseError{StatusCode: http.StatusNotFound})+ if !errors.As(err, &validation) {+ t.Fatalf("404 error type = %T, want ConnectorValidationError", err)+ }++ err = c.classifyValidationError(errors.New("boom"))+ if !errors.As(err, &validation) {+ t.Fatalf("generic error type = %T, want ConnectorValidationError", err)+ }+}++func TestAzureBlobStorageConnectorValidateRejectsUnsafeURL(t *testing.T) {+ connector, err := NewAzureBlobStorageConnector(map[string]any{+ "auth_mode": "sas_token",+ "credentials": map[string]any{+ "container_url": "http://192.168.1.10/container",+ "sas_token": "sv=1",+ },+ })+ if err != nil {+ t.Fatalf("NewAzureBlobStorageConnector: %v", err)+ }+ restore := azureAssertURLSafe+ azureAssertURLSafe = func(rawURL string) (string, string, error) {+ return "", "", errors.New("URL resolves to a non-public address (192.168.1.10), which is not allowed")+ }+ defer func() { azureAssertURLSafe = restore }()++ err = connector.Validate(context.Background())+ var validation *ConnectorValidationError+ if !errors.As(err, &validation) {+ t.Fatalf("Validate error type = %T, want ConnectorValidationError", err)+ }+}++func TestAzureBlobStorageConnectorValidateConnectorSettingUsesRequest(t *testing.T) {+ // The receiver holds valid credentials, but the request is missing them.+ // ValidateConnectorSetting must reject based on the unsaved request.+ receiver, err := NewAzureBlobStorageConnector(map[string]any{+ "auth_mode": "account_key",+ "credentials": map[string]any{+ "account_name": "acct",+ "account_key": "key",+ "container_name": "container",+ },+ })+ if err != nil {+ t.Fatalf("NewAzureBlobStorageConnector: %v", err)+ }+ err = receiver.ValidateConnectorSetting(context.Background(), map[string]any{+ "auth_mode": "account_key",+ "credentials": map[string]any{"account_name": "acct"}, // missing account_key+ })+ var missing *ConnectorMissingCredentialError+ if !errors.As(err, &missing) {+ t.Fatalf("ValidateConnectorSetting error type = %T, want ConnectorMissingCredentialError", err)+ }+}++// newTestAzureBlobStorageConnector builds a connector with injected listing,+// download and validation hooks so unit tests need no external Blob service.+func newTestAzureBlobStorageConnector(t *testing.T, config map[string]any, objects []azureBlobObject) *AzureBlobStorageConnector {+ t.Helper()+ connector, err := NewAzureBlobStorageConnector(config)+ if err != nil {+ t.Fatalf("NewAzureBlobStorageConnector: %v", err)+ }+ restore := azureAssertURLSafe+ azureAssertURLSafe = func(rawURL string) (string, string, error) { return "host", "1.2.3.4", nil }+ t.Cleanup(func() { azureAssertURLSafe = restore })++ connector.validateContainer = func(ctx context.Context) error { return nil }+ connector.listBlobs = func(ctx context.Context, prefix, marker string, max int32) ([]azureBlobObject, string, bool, error) {+ var filtered []azureBlobObject+ for _, object := range objects {+ if prefix != "" && !strings.HasPrefix(object.Name, prefix) {+ continue+ }+ if marker != "" && object.Name <= marker {+ continue+ }+ filtered = append(filtered, object)+ }+ sort.SliceStable(filtered, func(i, j int) bool { return filtered[i].Name < filtered[j].Name })+ if len(filtered) == 0 {+ return nil, "", false, nil+ }+ end := int(max)+ if end > len(filtered) {+ end = len(filtered)+ }+ page := filtered[:end]+ hasMore := end < len(filtered)+ next := ""+ if hasMore {+ next = page[len(page)-1].Name+ }+ return page, next, hasMore, nil+ }+ connector.downloadBlob = func(ctx context.Context, name string) ([]byte, error) {+ return []byte("body:" + name), nil+ }+ return connector+}diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 1a1a3ee395..5f44befdbb 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -37,6 +37,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "google-drive", NewGoogleDriveConnector)registerBuiltIn(registry, "google_drive", NewGoogleDriveConnector)registerBuiltIn(registry, "google_cloud_storage", NewGoogleCloudStorageConnector)+ registerBuiltIn(registry, "azure_blob", NewAzureBlobStorageConnector)registerBuiltIn(registry, "r2", NewR2Connector)registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)registerBuiltIn(registry, "imap", NewIMAPConnector)