Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

azureblob: use the released Azure SDK for Apache Arrow listing

Nick Craig-Wood3b9f0a05b55f
diff --git a/backend/azureblob/arrow.go b/backend/azureblob/arrow.go
index baaf51eb4..210400f82 100644
--- a/backend/azureblob/arrow.go
+++ b/backend/azureblob/arrow.go
@@ -18,24 +18,20 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
- "github.com/rclone/rclone/backend/azureblob/arrowlist"
"github.com/rclone/rclone/fs"
"golang.org/x/sync/errgroup"
)
+// arrowContentType is the Content-Type of an Apache Arrow IPC stream listing
+// response, used to tell whether the service honoured an Arrow request.
+const arrowContentType = "application/vnd.apache.arrow.stream"
+
// listArrowParallel lists (containerName, directory) by splitting the blob name
// keyspace into shards and listing them concurrently. opts must already request
// the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
// returns the number of raw items seen. directory/prefix are already normalised
// by the caller.
-func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
- // Parallel listing needs the arrowlist client (server-side endBefore is
- // only honoured on the Arrow path) - without it fall back to sequential.
- if _, err := f.arrowCntSVC(containerName); err != nil {
- fs.Debugf(f, "Not using parallel Arrow listing: %v", err)
- return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
- }
-
+func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
// Split the keyspace by a single-case character ladder (see
// arrowLadderBoundaries for why single-case matters).
points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
@@ -115,16 +111,13 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
// isEndBeforeUnsupported reports whether err means the service can't honour
// the endBefore listing bound, so sharded parallel listing must not be used.
//
-// This happens two ways: the arrowlist pager's ErrEndBeforeXMLFallback (the
-// server answered XML with endBefore set), or the service rejecting the
-// endBefore parameter with 400 OperationNotSupportedWithFeatureMissing "The
-// requested operation is not allowed as EndBefore parameter support is
-// missing on this account" - endBefore is only supported on the Arrow
-// listing path of accounts with Blob Listing with Apache Arrow enabled.
+// The service rejects the endBefore parameter with 400
+// OperationNotSupportedWithFeatureMissing "The requested operation is not
+// allowed as EndBefore parameter support is missing on this account" -
+// endBefore is only supported on the Arrow listing path of accounts with
+// Blob Listing with Apache Arrow enabled. (The SDK pager would silently
+// accept an XML page with endBefore set, but the service never sends one.)
func isEndBeforeUnsupported(err error) bool {
- if errors.Is(err, arrowlist.ErrEndBeforeXMLFallback) {
- return true
- }
var respErr *azcore.ResponseError
return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
}
diff --git a/backend/azureblob/arrow_internal_test.go b/backend/azureblob/arrow_internal_test.go
index b1bd96868..a77ab3607 100644
--- a/backend/azureblob/arrow_internal_test.go
+++ b/backend/azureblob/arrow_internal_test.go
@@ -10,7 +10,6 @@ import (
"testing"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
- "github.com/rclone/rclone/backend/azureblob/arrowlist"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
@@ -212,9 +211,8 @@ func TestArrowLadderBoundaries(t *testing.T) {
}
func TestIsEndBeforeUnsupported(t *testing.T) {
- assert.True(t, isEndBeforeUnsupported(arrowlist.ErrEndBeforeXMLFallback))
- assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", arrowlist.ErrEndBeforeXMLFallback)))
assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
+ assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", &azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"})))
assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
assert.False(t, isEndBeforeUnsupported(nil))
diff --git a/backend/azureblob/arrowlist/arrow.go b/backend/azureblob/arrowlist/arrow.go
deleted file mode 100644
index e4e788553..000000000
--- a/backend/azureblob/arrowlist/arrow.go
+++ /dev/null
@@ -1,421 +0,0 @@
-//go:build !plan9 && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/arrow/arrow.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
-// c6fa341ca22b (branch feature/storage/bifrost), retyped onto the SDK's
-// public type aliases so it compiles against the released azblob module.
-
-package arrowlist
-
-import (
- "encoding/base64"
- "fmt"
- "io"
- "net/http"
- "strconv"
- "time"
-
- "github.com/Azure/azure-sdk-for-go/sdk/azcore"
- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
- "github.com/apache/arrow-go/v18/arrow"
- "github.com/apache/arrow-go/v18/arrow/array"
- "github.com/apache/arrow-go/v18/arrow/ipc"
- "github.com/apache/arrow-go/v18/arrow/memory"
-)
-
-const (
- // ArrowContentType is the Content-Type for Apache Arrow IPC stream responses.
- ArrowContentType = "application/vnd.apache.arrow.stream"
-
- // ArrowAcceptHeader is the Accept header value to request Arrow format.
- ArrowAcceptHeader = ArrowContentType
-
- // resourceTypeBlobPrefix is the ResourceType value that identifies a virtual directory prefix
- // in hierarchy listing responses. In XML, prefixes are separate <BlobPrefix> elements;
- // in Arrow, all rows share the same schema and are distinguished by this field value.
- resourceTypeBlobPrefix = "blobprefix"
-)
-
-// HandleFlatListResponse parses an Arrow IPC stream response into a ContainerClientListBlobFlatSegmentResponse.
-func HandleFlatListResponse(resp *http.Response) (container.ListBlobsFlatResponse, error) {
- result := container.ListBlobsFlatResponse{}
-
- // Extract response headers
- extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-
- // Parse Arrow IPC stream
- items, nextMarker, err := parseArrowStream(resp.Body)
- if err != nil {
- return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
- }
-
- result.ListBlobsFlatSegmentResponse = container.ListBlobsFlatSegmentResponse{
- Segment: &container.BlobFlatListSegment{
- BlobItems: items,
- },
- NextMarker: nextMarker,
- }
-
- return result, nil
-}
-
-// HandleHierarchyListResponse parses an Arrow IPC stream response into a ContainerClientListBlobHierarchySegmentResponse.
-func HandleHierarchyListResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
- result := container.ListBlobsHierarchyResponse{}
-
- extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-
- items, nextMarker, err := parseArrowStream(resp.Body)
- if err != nil {
- return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
- }
-
- // Separate blob items from blob prefixes based on ResourceType field.
- var blobItems []*container.BlobItem
- var blobPrefixes []*container.BlobPrefix
- for _, item := range items {
- if item.Properties != nil && item.Properties.ResourceType != nil && *item.Properties.ResourceType == resourceTypeBlobPrefix {
- blobPrefixes = append(blobPrefixes, &container.BlobPrefix{
- Name: item.Name,
- Properties: item.Properties,
- })
- } else {
- blobItems = append(blobItems, item)
- }
- }
-
- result.ListBlobsHierarchySegmentResponse = container.ListBlobsHierarchySegmentResponse{
- Segment: &container.BlobHierarchyListSegment{
- BlobItems: blobItems,
- BlobPrefixes: blobPrefixes,
- },
- NextMarker: nextMarker,
- }
-
- return result, nil
-}
-
-func extractResponseHeaders(resp *http.Response, clientRequestID, contentType **string, date **time.Time, requestID, version **string) {
- if val := resp.Header.Get("x-ms-client-request-id"); val != "" {
- *clientRequestID = &val
- }
- if val := resp.Header.Get("Content-Type"); val != "" {
- *contentType = &val
- }
- if val := resp.Header.Get("Date"); val != "" {
- if t, err := time.Parse(time.RFC1123, val); err == nil {
- *date = &t
- }
- }
- if val := resp.Header.Get("x-ms-request-id"); val != "" {
- *requestID = &val
- }
- if val := resp.Header.Get("x-ms-version"); val != "" {
- *version = &val
- }
-}
-
-// parseArrowStream reads an Arrow IPC stream and converts it to BlobItems.
-func parseArrowStream(body io.Reader) ([]*container.BlobItem, *string, error) {
- reader, err := ipc.NewReader(body, ipc.WithAllocator(memory.DefaultAllocator))
- if err != nil {
- return nil, nil, fmt.Errorf("failed to create Arrow IPC reader: %w", err)
- }
- defer reader.Release()
-
- // Extract NextMarker from schema metadata
- var nextMarker *string
- var numRecords int
- if md := reader.Schema().Metadata(); md.Len() > 0 {
- if idx := md.FindKey("NextMarker"); idx >= 0 {
- val := md.Values()[idx]
- if val != "" {
- nextMarker = &val
- }
- }
- if idx := md.FindKey("NumberOfRecords"); idx >= 0 {
- if n, err := strconv.Atoi(md.Values()[idx]); err == nil {
- numRecords = n
- }
- }
- }
-
- // Build column index for the schema
- schema := reader.Schema()
- colIndex := buildColumnIndex(schema)
-
- // Pre-allocate if we know the count
- items := make([]*container.BlobItem, 0, numRecords)
-
- // Iterate over record batches
- for reader.Next() {
- rec := reader.RecordBatch()
- rows := int(rec.NumRows())
-
- for row := range rows {
- item := &container.BlobItem{
- Properties: &container.BlobProperties{},
- }
- populateBlobItem(item, rec, row, colIndex)
- items = append(items, item)
- }
- }
-
- if err := reader.Err(); err != nil {
- return nil, nil, fmt.Errorf("error reading Arrow record batches: %w", err)
- }
-
- return items, nextMarker, nil
-}
-
-// columnIndex maps column names to their indices in the record batch for O(1) lookup.
-type columnIndex map[string]int
-
-func buildColumnIndex(schema *arrow.Schema) columnIndex {
- idx := make(columnIndex, len(schema.Fields()))
- for i, f := range schema.Fields() {
- idx[f.Name] = i
- }
- return idx
-}
-
-// populateBlobItem fills a BlobItem from a single row across all columns.
-func populateBlobItem(item *container.BlobItem, rec arrow.RecordBatch, row int, colIdx columnIndex) {
- // Top-level BlobItem fields
- item.Name = getStringField(rec, row, colIdx, "Name")
- item.Deleted = getBoolField(rec, row, colIdx, "Deleted")
- item.Snapshot = getStringField(rec, row, colIdx, "Snapshot")
- item.VersionID = getStringField(rec, row, colIdx, "VersionId")
- item.IsCurrentVersion = getBoolField(rec, row, colIdx, "IsCurrentVersion")
- item.HasVersionsOnly = getBoolField(rec, row, colIdx, "HasVersionsOnly")
-
- // Metadata maps
- item.Metadata = getMapField(rec, row, colIdx, "Metadata")
- item.OrMetadata = getMapField(rec, row, colIdx, "OrMetadata")
-
- // Tags
- if tags := getMapField(rec, row, colIdx, "Tags"); tags != nil {
- var blobTags []*container.BlobTag
- for k, v := range tags {
- key := k
- blobTags = append(blobTags, &container.BlobTag{Key: &key, Value: v})
- }
- item.BlobTags = &container.BlobTags{BlobTagSet: blobTags}
- }
-
- // BlobProperties fields
- p := item.Properties
- p.CreationTime = getTimestampField(rec, row, colIdx, "Creation-Time")
- p.LastModified = getTimestampField(rec, row, colIdx, "Last-Modified")
- p.ETag = getETagField(rec, row, colIdx, "Etag")
- p.ContentLength = getInt64Field(rec, row, colIdx, "Content-Length")
- p.ContentType = getStringField(rec, row, colIdx, "Content-Type")
- p.ContentEncoding = getStringField(rec, row, colIdx, "Content-Encoding")
- p.ContentLanguage = getStringField(rec, row, colIdx, "Content-Language")
- p.ContentDisposition = getStringField(rec, row, colIdx, "Content-Disposition")
- p.CacheControl = getStringField(rec, row, colIdx, "Cache-Control")
- p.ContentMD5 = getBase64BytesField(rec, row, colIdx, "Content-MD5")
- p.BlobType = getEnumField[container.BlobType](rec, row, colIdx, "BlobType")
- p.AccessTier = getEnumField[container.AccessTier](rec, row, colIdx, "AccessTier")
- p.AccessTierInferred = getBoolField(rec, row, colIdx, "AccessTierInferred")
- p.AccessTierChangeTime = getTimestampField(rec, row, colIdx, "AccessTierChangeTime")
- p.LeaseState = getEnumField[lease.StateType](rec, row, colIdx, "LeaseState")
- p.LeaseStatus = getEnumField[lease.StatusType](rec, row, colIdx, "LeaseStatus")
- p.LeaseDuration = getEnumField[lease.DurationType](rec, row, colIdx, "LeaseDuration")
- p.ServerEncrypted = getBoolField(rec, row, colIdx, "ServerEncrypted")
- p.CustomerProvidedKeySHA256 = getStringField(rec, row, colIdx, "CustomerProvidedKeySha256")
- p.EncryptionScope = getStringField(rec, row, colIdx, "EncryptionScope")
- p.IncrementalCopy = getBoolField(rec, row, colIdx, "IncrementalCopy")
- p.IsSealed = getBoolField(rec, row, colIdx, "Sealed")
- p.ArchiveStatus = getEnumField[container.ArchiveStatus](rec, row, colIdx, "ArchiveStatus")
- p.RehydratePriority = getEnumField[blob.RehydratePriority](rec, row, colIdx, "RehydratePriority")
- p.CopyID = getStringField(rec, row, colIdx, "CopyId")
- p.CopyStatus = getEnumField[blob.CopyStatusType](rec, row, colIdx, "CopyStatus")
- p.CopySource = getStringField(rec, row, colIdx, "CopySource")
- p.CopyProgress = getStringField(rec, row, colIdx, "CopyProgress")
- p.CopyCompletionTime = getTimestampField(rec, row, colIdx, "CopyCompletionTime")
- p.CopyStatusDescription = getStringField(rec, row, colIdx, "CopyStatusDescription")
- p.DestinationSnapshot = getStringField(rec, row, colIdx, "CopyDestinationSnapshot")
- p.ImmutabilityPolicyExpiresOn = getTimestampField(rec, row, colIdx, "ImmutabilityPolicyUntilDate")
- p.ImmutabilityPolicyMode = getEnumField[container.ImmutabilityPolicyMode](rec, row, colIdx, "ImmutabilityPolicyMode")
- p.LegalHold = getBoolField(rec, row, colIdx, "LegalHold")
- p.DeletedTime = getTimestampField(rec, row, colIdx, "DeletedTime")
- p.RemainingRetentionDays = getInt32Field(rec, row, colIdx, "RemainingRetentionDays")
- p.LastAccessedOn = getTimestampField(rec, row, colIdx, "LastAccessTime")
- p.TagCount = getInt32Field(rec, row, colIdx, "TagCount")
- p.BlobSequenceNumber = getInt64Field(rec, row, colIdx, "x-ms-blob-sequence-number")
- p.ResourceType = getStringField(rec, row, colIdx, "ResourceType")
-
- // Content-CRC64 is a string field in Arrow but not directly mapped to BlobProperties.
- // The field exists in the Arrow schema but has no corresponding field in the Go SDK BlobProperties.
- // It is silently ignored for forward compatibility.
-}
-
-// Field accessor helpers that safely handle missing columns and null values.
-
-func getStringField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *string {
- idx, ok := colIdx[name]
- if !ok {
- return nil
- }
- col := rec.Column(idx)
- if col.IsNull(row) {
- return nil
- }
- if arr, ok := col.(*array.String); ok {
- val := arr.Value(row)
- return &val
- }
- return nil
-}
-
-func getBoolField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *bool {
- idx, ok := colIdx[name]
- if !ok {
- return nil
- }
- col := rec.Column(idx)
- if col.IsNull(row) {
- return nil
- }
- if arr, ok := col.(*array.Boolean); ok {
- val := arr.Value(row)
- return &val
- }
- return nil
-}
-
-func getTimestampField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *time.Time {
- idx, ok := colIdx[name]
- if !ok {
- return nil
- }
- col := rec.Column(idx)
- if col.IsNull(row) {
- return nil
- }
- if arr, ok := col.(*array.Timestamp); ok {
- val := arr.Value(row)
- // Arrow timestamps are stored as int64 with a unit; convert to time.Time
- dt := val.ToTime(arr.DataType().(*arrow.TimestampType).Unit)
- return &dt
- }
- return nil
-}
-
-func getInt64Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int64 {
- idx, ok := colIdx[name]
- if !ok {
- return nil
- }
- col := rec.Column(idx)
- if col.IsNull(row) {
- return nil
- }
- if arr, ok := col.(*array.Uint64); ok {
- val := int64(arr.Value(row))
- return &val
- }
- if arr, ok := col.(*array.Int64); ok {
- val := arr.Value(row)
- return &val
- }
- return nil
-}
-
-func getInt32Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int32 {
- idx, ok := colIdx[name]
- if !ok {
- return nil
- }
- col := rec.Column(idx)
- if col.IsNull(row) {
- return nil
- }
- if arr, ok := col.(*array.Uint64); ok {
- val := int32(arr.Value(row))
- return &val
- }
- if arr, ok := col.(*array.Int32); ok {
- val := arr.Value(row)
- return &val
- }
- return nil
-}
-
-func getETagField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *azcore.ETag {
- s := getStringField(rec, row, colIdx, name)
- if s == nil {
- return nil
- }
- etag := azcore.ETag(*s)
- return &etag
-}
-
-func getBase64BytesField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) []byte {
- s := getStringField(rec, row, colIdx, name)
- if s == nil {
- return nil
- }
- decoded, err := base64.StdEncoding.DecodeString(*s)
- if err != nil {
- return nil
- }
- return decoded
-}
-
-// getEnumField returns a pointer to a string-typed enum value.
-type stringEnum interface {
- ~string
-}
-
-func getEnumField[T stringEnum](rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *T {
- s := getStringField(rec, row, colIdx, name)
- if s == nil {
- return nil
- }
- val := T(*s)
- return &val
-}
-
-func getMapField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) map[string]*string {
- idx, ok := colIdx[name]
- if !ok {
- return nil
- }
- col := rec.Column(idx)
- if col.IsNull(row) {
- return nil
- }
- mapArr, ok := col.(*array.Map)
- if !ok {
- return nil
- }
-
- // Get the offsets for this row's map entries
- start := int(mapArr.Offsets()[row])
- end := int(mapArr.Offsets()[row+1])
- if start == end {
- return nil
- }
-
- keys := mapArr.Keys().(*array.String)
- values := mapArr.Items().(*array.String)
-
- result := make(map[string]*string, end-start)
- for i := start; i < end; i++ {
- k := keys.Value(i)
- if values.IsNull(i) {
- result[k] = nil
- } else {
- v := values.Value(i)
- result[k] = &v
- }
- }
- return result
-}
diff --git a/backend/azureblob/arrowlist/arrow_test.go b/backend/azureblob/arrowlist/arrow_test.go
deleted file mode 100644
index 9c0c24f5d..000000000
--- a/backend/azureblob/arrowlist/arrow_test.go
+++ /dev/null
@@ -1,374 +0,0 @@
-//go:build !plan9 && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/arrow/arrow_test.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
-// c6fa341ca22b (branch feature/storage/bifrost).
-
-package arrowlist
-
-import (
- "bytes"
- "io"
- "net/http"
- "testing"
-
- "github.com/apache/arrow-go/v18/arrow"
- arrowArray "github.com/apache/arrow-go/v18/arrow/array"
- "github.com/apache/arrow-go/v18/arrow/ipc"
- "github.com/apache/arrow-go/v18/arrow/memory"
- "github.com/stretchr/testify/require"
-)
-
-// coreSchema returns a basic Arrow schema for testing with common blob fields.
-func coreSchema(md *arrow.Metadata) *arrow.Schema {
- fields := []arrow.Field{
- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
- {Name: "Creation-Time", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
- {Name: "Last-Modified", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
- {Name: "BlobType", Type: arrow.BinaryTypes.String, Nullable: true},
- {Name: "Etag", Type: arrow.BinaryTypes.String, Nullable: true},
- {Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
- {Name: "Content-Type", Type: arrow.BinaryTypes.String, Nullable: true},
- {Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
- {Name: "AccessTier", Type: arrow.BinaryTypes.String, Nullable: true},
- {Name: "LeaseState", Type: arrow.BinaryTypes.String, Nullable: true},
- {Name: "LeaseStatus", Type: arrow.BinaryTypes.String, Nullable: true},
- }
- return arrow.NewSchema(fields, md)
-}
-
-// buildArrowStream constructs a synthetic Arrow IPC stream from a schema and a populate function.
-func buildArrowStream(t *testing.T, schema *arrow.Schema, populate func(builder *arrowArray.RecordBuilder)) []byte {
- t.Helper()
- alloc := memory.DefaultAllocator
-
- var buf bytes.Buffer
- w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
-
- builder := arrowArray.NewRecordBuilder(alloc, schema)
- defer builder.Release()
-
- populate(builder)
-
- rec := builder.NewRecordBatch()
- defer rec.Release()
-
- require.NoError(t, w.Write(rec))
- require.NoError(t, w.Close())
-
- return buf.Bytes()
-}
-
-func makeHTTPResponse(data []byte, headers map[string]string) *http.Response {
- h := http.Header{}
- h.Set("Content-Type", ArrowContentType)
- for k, v := range headers {
- h.Set(k, v)
- }
- return &http.Response{
- StatusCode: http.StatusOK,
- Header: h,
- Body: io.NopCloser(bytes.NewReader(data)),
- }
-}
-
-func TestHandleFlatListResponse_BasicParsing(t *testing.T) {
- md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"marker123", "2"})
- schema := coreSchema(&md)
-
- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
- b.Field(0).(*arrowArray.StringBuilder).Append("blob1.txt")
- b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
- b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
- b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
- b.Field(4).(*arrowArray.StringBuilder).Append("0x1234")
- b.Field(5).(*arrowArray.Uint64Builder).Append(1024)
- b.Field(6).(*arrowArray.StringBuilder).Append("application/octet-stream")
- b.Field(7).(*arrowArray.BooleanBuilder).Append(true)
- b.Field(8).(*arrowArray.StringBuilder).Append("Hot")
- b.Field(9).(*arrowArray.StringBuilder).Append("available")
- b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
-
- b.Field(0).(*arrowArray.StringBuilder).Append("blob2.txt")
- b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
- b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
- b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
- b.Field(4).(*arrowArray.StringBuilder).Append("0x5678")
- b.Field(5).(*arrowArray.Uint64Builder).Append(2048)
- b.Field(6).(*arrowArray.StringBuilder).Append("text/plain")
- b.Field(7).(*arrowArray.BooleanBuilder).Append(false)
- b.Field(8).(*arrowArray.StringBuilder).Append("Cool")
- b.Field(9).(*arrowArray.StringBuilder).Append("available")
- b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
- })
-
- resp := makeHTTPResponse(data, nil)
- result, err := HandleFlatListResponse(resp)
- require.NoError(t, err)
-
- require.NotNil(t, result.NextMarker)
- require.Equal(t, "marker123", *result.NextMarker)
- require.Len(t, result.Segment.BlobItems, 2)
-
- item := result.Segment.BlobItems[0]
- require.Equal(t, "blob1.txt", *item.Name)
- require.Equal(t, "BlockBlob", string(*item.Properties.BlobType))
- require.Equal(t, int64(1024), *item.Properties.ContentLength)
- require.Equal(t, "Hot", string(*item.Properties.AccessTier))
-}
-
-func TestHandleFlatListResponse_EmptyRecordBatch(t *testing.T) {
- md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "0"})
- schema := coreSchema(&md)
-
- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
- // No rows added
- })
-
- resp := makeHTTPResponse(data, nil)
- result, err := HandleFlatListResponse(resp)
- require.NoError(t, err)
- require.Nil(t, result.NextMarker)
- require.Empty(t, result.Segment.BlobItems)
-}
-
-func TestHandleFlatListResponse_NullFields(t *testing.T) {
- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
- schema := coreSchema(&md)
-
- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
- b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
- b.Field(1).(*arrowArray.TimestampBuilder).AppendNull()
- b.Field(2).(*arrowArray.TimestampBuilder).AppendNull()
- b.Field(3).(*arrowArray.StringBuilder).AppendNull()
- b.Field(4).(*arrowArray.StringBuilder).AppendNull()
- b.Field(5).(*arrowArray.Uint64Builder).AppendNull()
- b.Field(6).(*arrowArray.StringBuilder).AppendNull()
- b.Field(7).(*arrowArray.BooleanBuilder).AppendNull()
- b.Field(8).(*arrowArray.StringBuilder).AppendNull()
- b.Field(9).(*arrowArray.StringBuilder).AppendNull()
- b.Field(10).(*arrowArray.StringBuilder).AppendNull()
- })
-
- resp := makeHTTPResponse(data, nil)
- result, err := HandleFlatListResponse(resp)
- require.NoError(t, err)
- require.Len(t, result.Segment.BlobItems, 1)
-
- item := result.Segment.BlobItems[0]
- require.Equal(t, "blob.txt", *item.Name)
- require.Nil(t, item.Properties.CreationTime)
- require.Nil(t, item.Properties.BlobType)
- require.Nil(t, item.Properties.ETag)
- require.Nil(t, item.Properties.ContentLength)
- require.Nil(t, item.Properties.ContentType)
- require.Nil(t, item.Properties.ServerEncrypted)
-}
-
-func TestHandleFlatListResponse_UnknownColumnsIgnored(t *testing.T) {
- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
- fields := []arrow.Field{
- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
- {Name: "FutureField", Type: arrow.BinaryTypes.String, Nullable: true},
- }
- schema := arrow.NewSchema(fields, &md)
-
- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
- b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
- b.Field(1).(*arrowArray.StringBuilder).Append("some-future-value")
- })
-
- resp := makeHTTPResponse(data, nil)
- result, err := HandleFlatListResponse(resp)
- require.NoError(t, err)
- require.Len(t, result.Segment.BlobItems, 1)
- require.Equal(t, "blob.txt", *result.Segment.BlobItems[0].Name)
-}
-
-func TestHandleFlatListResponse_MultiRecordBatch(t *testing.T) {
- alloc := memory.DefaultAllocator
- md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "3"})
- schema := coreSchema(&md)
-
- var buf bytes.Buffer
- w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
-
- // First batch: 2 rows
- builder := arrowArray.NewRecordBuilder(alloc, schema)
- for range 2 {
- builder.Field(0).(*arrowArray.StringBuilder).Append("batch1_blob")
- for j := 1; j < 11; j++ {
- builder.Field(j).AppendNull()
- }
- }
- rec := builder.NewRecordBatch()
- require.NoError(t, w.Write(rec))
- rec.Release()
- builder.Release()
-
- // Second batch: 1 row
- builder = arrowArray.NewRecordBuilder(alloc, schema)
- builder.Field(0).(*arrowArray.StringBuilder).Append("batch2_blob")
- for j := 1; j < 11; j++ {
- builder.Field(j).AppendNull()
- }
- rec = builder.NewRecordBatch()
- require.NoError(t, w.Write(rec))
- rec.Release()
- builder.Release()
-
- require.NoError(t, w.Close())
-
- resp := makeHTTPResponse(buf.Bytes(), nil)
- result, err := HandleFlatListResponse(resp)
- require.NoError(t, err)
- require.Len(t, result.Segment.BlobItems, 3)
-}
-
-func TestHandleHierarchyListResponse_SeparatesBlobsAndPrefixes(t *testing.T) {
- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
- fields := []arrow.Field{
- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
- {Name: "ResourceType", Type: arrow.BinaryTypes.String, Nullable: true},
- }
- schema := arrow.NewSchema(fields, &md)
-
- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
- // A regular blob
- b.Field(0).(*arrowArray.StringBuilder).Append("folder/file.txt")
- b.Field(1).(*arrowArray.StringBuilder).AppendNull()
-
- // A blob prefix (virtual directory)
- b.Field(0).(*arrowArray.StringBuilder).Append("folder/subfolder/")
- b.Field(1).(*arrowArray.StringBuilder).Append("blobprefix")
- })
-
- resp := makeHTTPResponse(data, nil)
- result, err := HandleHierarchyListResponse(resp)
- require.NoError(t, err)
-
- require.Len(t, result.Segment.BlobItems, 1)
- require.Equal(t, "folder/file.txt", *result.Segment.BlobItems[0].Name)
-
- require.Len(t, result.Segment.BlobPrefixes, 1)
- require.Equal(t, "folder/subfolder/", *result.Segment.BlobPrefixes[0].Name)
-}
-
-func TestHandleFlatListResponse_ResponseHeaders(t *testing.T) {
- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
- fields := []arrow.Field{
- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
- }
- schema := arrow.NewSchema(fields, &md)
-
- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
- b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
- })
-
- resp := makeHTTPResponse(data, map[string]string{
- "x-ms-request-id": "req-123",
- "x-ms-version": "2026-10-06",
- "x-ms-client-request-id": "client-req-456",
- "Date": "Mon, 01 Jan 2024 00:00:00 GMT",
- })
-
- result, err := HandleFlatListResponse(resp)
- require.NoError(t, err)
- require.Equal(t, "req-123", *result.RequestID)
- require.Equal(t, "2026-10-06", *result.Version)
- require.Equal(t, "client-req-456", *result.ClientRequestID)
- require.NotNil(t, result.Date)
-}
-
-func TestHandleFlatListResponse_MapFields(t *testing.T) {
- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
- fields := []arrow.Field{
- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
- {Name: "Metadata", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
- {Name: "Tags", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
- }
- schema := arrow.NewSchema(fields, &md)
-
- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
- b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-
- metaBuilder := b.Field(1).(*arrowArray.MapBuilder)
- metaBuilder.Append(true)
- metaBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("key1")
- metaBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("value1")
-
- tagBuilder := b.Field(2).(*arrowArray.MapBuilder)
- tagBuilder.Append(true)
- tagBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("env")
- tagBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("prod")
- })
-
- resp := makeHTTPResponse(data, nil)
- result, err := HandleFlatListResponse(resp)
- require.NoError(t, err)
- require.Len(t, result.Segment.BlobItems, 1)
-
- item := result.Segment.BlobItems[0]
- require.NotNil(t, item.Metadata)
- require.Equal(t, "value1", *item.Metadata["key1"])
-
- require.NotNil(t, item.BlobTags)
- require.Len(t, item.BlobTags.BlobTagSet, 1)
-}
-
-func TestHandleFlatListResponse_BooleanFields(t *testing.T) {
- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
- fields := []arrow.Field{
- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
- {Name: "Deleted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
- {Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
- {Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
- }
- schema := arrow.NewSchema(fields, &md)
-
- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
- b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
- b.Field(1).(*arrowArray.BooleanBuilder).Append(true)
- b.Field(2).(*arrowArray.BooleanBuilder).Append(false)
- b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
- })
-
- resp := makeHTTPResponse(data, nil)
- result, err := HandleFlatListResponse(resp)
- require.NoError(t, err)
-
- item := result.Segment.BlobItems[0]
- require.True(t, *item.Deleted)
- require.False(t, *item.IsCurrentVersion)
- require.True(t, *item.Properties.ServerEncrypted)
-}
-
-func TestHandleFlatListResponse_VersionAndSnapshot(t *testing.T) {
- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
- fields := []arrow.Field{
- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
- {Name: "VersionId", Type: arrow.BinaryTypes.String, Nullable: true},
- {Name: "Snapshot", Type: arrow.BinaryTypes.String, Nullable: true},
- {Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
- }
- schema := arrow.NewSchema(fields, &md)
-
- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
- b.Field(0).(*arrowArray.StringBuilder).Append("versioned-blob.txt")
- b.Field(1).(*arrowArray.StringBuilder).Append("2024-01-01T00:00:00.0000000Z")
- b.Field(2).(*arrowArray.StringBuilder).Append("snap-123")
- b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
- })
-
- resp := makeHTTPResponse(data, nil)
- result, err := HandleFlatListResponse(resp)
- require.NoError(t, err)
-
- item := result.Segment.BlobItems[0]
- require.Equal(t, "2024-01-01T00:00:00.0000000Z", *item.VersionID)
- require.Equal(t, "snap-123", *item.Snapshot)
- require.True(t, *item.IsCurrentVersion)
-}
diff --git a/backend/azureblob/arrowlist/arrowlist.go b/backend/azureblob/arrowlist/arrowlist.go
deleted file mode 100644
index a9246adef..000000000
--- a/backend/azureblob/arrowlist/arrowlist.go
+++ /dev/null
@@ -1,259 +0,0 @@
-//go:build !plan9 && !js
-
-// Package arrowlist implements the experimental "Blob Listing with Apache
-// Arrow" feature on top of the released Azure azblob SDK.
-//
-// The Azure SDK for Go has support for Arrow listing on the unreleased
-// feature/storage/bifrost branch (commit c6fa341ca22b) where the listing
-// pagers decode Arrow IPC stream responses into the ordinary ListBlobs
-// response types. Until that ships in a tagged azblob release this package
-// fills the gap: it exposes the same options and pager interface as the
-// experimental SDK, built only on the public API of the released SDK plus
-// copies of two small internal auth policies (see sharedkey.go and
-// challenge_policy.go).
-//
-// This package is temporary. When Arrow listing ships in a released azblob
-// the whole package should be deleted and its callers pointed back at
-// container.Client.NewListBlobsHierarchyPager - the option and response
-// types here are deliberately source compatible with the SDK's to make that
-// a mechanical change.
-package arrowlist
-
-import (
- "context"
- "errors"
- "net/http"
- "strconv"
- "strings"
-
- "github.com/Azure/azure-sdk-for-go/sdk/azcore"
- "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
- "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-)
-
-const (
- // moduleName and moduleVersion identify this package in the azcore
- // telemetry policy's User-Agent fragment.
- moduleName = "github.com/rclone/rclone/backend/azureblob/arrowlist"
- moduleVersion = "v0.0.1"
-
- // serviceVersion is the x-ms-version sent with listing requests. Arrow
- // responses and the startFrom/endBefore parameters need at least
- // 2026-06-06.
- serviceVersion = "2026-06-06"
-
- // tokenScope is the OAuth scope for Azure Storage, as used by the SDK's
- // own clients.
- tokenScope = "https://storage.azure.com/.default"
-)
-
-// ErrEndBeforeXMLFallback is returned by the pager when the server answered
-// with XML instead of Arrow while EndBefore was set. The service only honours
-// endBefore on the Arrow listing path, so an XML page could silently contain
-// blobs outside the requested range and cannot be used.
-var ErrEndBeforeXMLFallback = errors.New("arrow listing fell back to XML with endBefore set")
-
-// ListBlobsHierarchyOptions provides the configuration for
-// Client.NewListBlobsHierarchyPager.
-//
-// It mirrors the experimental SDK's container.ListBlobsHierarchyOptions:
-// the released SDK's options are embedded (providing Include, Marker,
-// MaxResults, Prefix and StartFrom by promotion) with the two fields the
-// released SDK lacks added alongside.
-type ListBlobsHierarchyOptions struct {
- container.ListBlobsHierarchyOptions
- // EndBefore limits listing to blobs whose full container path is
- // lexically before it (exclusive). Only honoured on Arrow responses.
- EndBefore *string
- // UseArrowFormat requests the response as an Apache Arrow IPC stream.
- UseArrowFormat *bool
-}
-
-// ClientOptions contains the optional parameters for client creation.
-type ClientOptions struct {
- azcore.ClientOptions
-}
-
-// Client issues Arrow listing requests to a single container.
-type Client struct {
- endpoint string
- azClient *azcore.Client
-}
-
-// newClient creates a Client for the container at containerURL sending
-// requests through a pipeline with the given auth policy (which may be nil
-// for anonymous or SAS access).
-func newClient(containerURL string, authPolicy policy.Policy, options *ClientOptions) (*Client, error) {
- if options == nil {
- options = &ClientOptions{}
- }
- plOpts := runtime.PipelineOptions{}
- if authPolicy != nil {
- plOpts.PerRetry = []policy.Policy{authPolicy}
- }
- azClient, err := azcore.NewClient(moduleName, moduleVersion, plOpts, &options.ClientOptions)
- if err != nil {
- return nil, err
- }
- return &Client{
- endpoint: containerURL,
- azClient: azClient,
- }, nil
-}
-
-// NewClient creates a Client authenticating with a token credential.
-func NewClient(containerURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
- return newClient(containerURL, NewStorageChallengePolicy(cred, tokenScope, false), options)
-}
-
-// NewClientWithNoCredential creates a Client with no authentication, for
-// anonymous access or a containerURL carrying a SAS token in its query.
-func NewClientWithNoCredential(containerURL string, options *ClientOptions) (*Client, error) {
- return newClient(containerURL, nil, options)
-}
-
-// NewClientWithSharedKeyCredential creates a Client signing requests with the
-// account's shared key.
-func NewClientWithSharedKeyCredential(containerURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
- return newClient(containerURL, NewSharedKeyCredPolicy(cred), options)
-}
-
-// NewListBlobsHierarchyPager returns a pager over the container's blobs,
-// requesting Arrow responses when o.UseArrowFormat is set and returning the
-// same response type as container.Client.NewListBlobsHierarchyPager.
-//
-// If the service does not support Arrow listing (the feature not enabled on
-// the account, or an HNS account) it answers with XML, which is decoded
-// transparently - unless EndBefore is set, in which case the fetcher returns
-// ErrEndBeforeXMLFallback (see there).
-func (c *Client) NewListBlobsHierarchyPager(delimiter string, o *ListBlobsHierarchyOptions) *runtime.Pager[container.ListBlobsHierarchyResponse] {
- // Copy the options so advancing the Marker doesn't mutate the caller's struct.
- opts := ListBlobsHierarchyOptions{}
- if o != nil {
- opts = *o
- }
- useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
- return runtime.NewPager(runtime.PagingHandler[container.ListBlobsHierarchyResponse]{
- More: func(page container.ListBlobsHierarchyResponse) bool {
- return page.NextMarker != nil && len(*page.NextMarker) > 0
- },
- Fetcher: func(ctx context.Context, page *container.ListBlobsHierarchyResponse) (container.ListBlobsHierarchyResponse, error) {
- if page != nil {
- opts.Marker = page.NextMarker
- }
- req, err := c.listCreateRequest(ctx, delimiter, &opts, useArrow)
- if err != nil {
- return container.ListBlobsHierarchyResponse{}, err
- }
- resp, err := c.azClient.Pipeline().Do(req)
- if err != nil {
- return container.ListBlobsHierarchyResponse{}, err
- }
- if !runtime.HasStatusCode(resp, http.StatusOK) {
- return container.ListBlobsHierarchyResponse{}, runtime.NewResponseError(resp)
- }
- if useArrow && resp.Header.Get("Content-Type") == ArrowContentType {
- return HandleHierarchyListResponse(resp)
- }
- if opts.EndBefore != nil {
- return container.ListBlobsHierarchyResponse{}, ErrEndBeforeXMLFallback
- }
- return handleXMLResponse(resp)
- },
- })
-}
-
-// listCreateRequest builds the ListBlobs request, mirroring the generated
-// request builders in the SDK.
-func (c *Client) listCreateRequest(ctx context.Context, delimiter string, opts *ListBlobsHierarchyOptions, useArrow bool) (*policy.Request, error) {
- req, err := runtime.NewRequest(ctx, http.MethodGet, c.endpoint)
- if err != nil {
- return nil, err
- }
- // Reading the existing query first preserves any SAS token in the endpoint.
- reqQP := req.Raw().URL.Query()
- reqQP.Set("comp", "list")
- reqQP.Set("restype", "container")
- reqQP.Set("delimiter", delimiter)
- if include := formatInclude(opts.Include); include != "" {
- reqQP.Set("include", include)
- }
- if opts.Marker != nil {
- reqQP.Set("marker", *opts.Marker)
- }
- if opts.MaxResults != nil {
- reqQP.Set("maxresults", strconv.FormatInt(int64(*opts.MaxResults), 10))
- }
- if opts.Prefix != nil {
- reqQP.Set("prefix", *opts.Prefix)
- }
- if opts.StartFrom != nil {
- reqQP.Set("startFrom", *opts.StartFrom)
- }
- if useArrow && opts.EndBefore != nil {
- reqQP.Set("endBefore", *opts.EndBefore)
- }
- req.Raw().URL.RawQuery = strings.ReplaceAll(reqQP.Encode(), "+", "%20")
- if useArrow {
- // Stream the Arrow body rather than buffering it in the pipeline.
- runtime.SkipBodyDownload(req)
- req.Raw().Header["Accept"] = []string{ArrowAcceptHeader}
- } else {
- req.Raw().Header["Accept"] = []string{"application/xml"}
- }
- req.Raw().Header["x-ms-version"] = []string{serviceVersion}
- return req, nil
-}
-
-// handleXMLResponse decodes an XML ListBlobs response, mirroring the SDK's
-// generated ListBlobHierarchySegmentHandleResponse.
-func handleXMLResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
- result := container.ListBlobsHierarchyResponse{}
- extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
- if err := runtime.UnmarshalAsXML(resp, &result.ListBlobsHierarchySegmentResponse); err != nil {
- return container.ListBlobsHierarchyResponse{}, err
- }
- return result, nil
-}
-
-// formatInclude renders the include datasets as the comma separated query
-// parameter value, in the same order as the SDK's unexported
-// ListBlobsInclude.format.
-func formatInclude(l container.ListBlobsInclude) string {
- var include []string
- if l.Copy {
- include = append(include, "copy")
- }
- if l.Deleted {
- include = append(include, "deleted")
- }
- if l.DeletedWithVersions {
- include = append(include, "deletedwithversions")
- }
- if l.ImmutabilityPolicy {
- include = append(include, "immutabilitypolicy")
- }
- if l.LegalHold {
- include = append(include, "legalhold")
- }
- if l.Metadata {
- include = append(include, "metadata")
- }
- if l.Snapshots {
- include = append(include, "snapshots")
- }
- if l.Tags {
- include = append(include, "tags")
- }
- if l.UncommittedBlobs {
- include = append(include, "uncommittedblobs")
- }
- if l.Versions {
- include = append(include, "versions")
- }
- if l.Permissions {
- include = append(include, "permissions")
- }
- return strings.Join(include, ",")
-}
diff --git a/backend/azureblob/arrowlist/arrowlist_test.go b/backend/azureblob/arrowlist/arrowlist_test.go
deleted file mode 100644
index f121aac6b..000000000
--- a/backend/azureblob/arrowlist/arrowlist_test.go
+++ /dev/null
@@ -1,198 +0,0 @@
-//go:build !plan9 && !js
-
-package arrowlist
-
-import (
- "context"
- "encoding/base64"
- "net/http"
- "net/http/httptest"
- "strings"
- "testing"
-
- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
- "github.com/apache/arrow-go/v18/arrow"
- arrowArray "github.com/apache/arrow-go/v18/arrow/array"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-// arrowPage builds a one row Arrow IPC listing page with the given blob name
-// and NextMarker.
-func arrowPage(t *testing.T, name, nextMarker string) []byte {
- md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{nextMarker, "1"})
- fields := []arrow.Field{
- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
- {Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
- }
- schema := arrow.NewSchema(fields, &md)
- return buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
- b.Field(0).(*arrowArray.StringBuilder).Append(name)
- b.Field(1).(*arrowArray.Uint64Builder).Append(42)
- })
-}
-
-func TestPagerArrow(t *testing.T) {
- var requests []*http.Request
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- requests = append(requests, r.Clone(context.Background()))
- w.Header().Set("Content-Type", ArrowContentType)
- if r.URL.Query().Get("marker") == "" {
- _, _ = w.Write(arrowPage(t, "blob1.txt", "marker1"))
- } else {
- _, _ = w.Write(arrowPage(t, "blob2.txt", ""))
- }
- }))
- defer srv.Close()
-
- // SAS style query on the endpoint must be preserved in requests.
- client, err := NewClientWithNoCredential(srv.URL+"/testcontainer?sv=fakesas", nil)
- require.NoError(t, err)
-
- opts := &ListBlobsHierarchyOptions{
- ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
- Include: container.ListBlobsInclude{Metadata: true, Tags: true},
- Prefix: new("dir/"),
- MaxResults: new(int32(1000)),
- StartFrom: new("testcontainer/dir/a"),
- },
- EndBefore: new("testcontainer/dir/n"),
- UseArrowFormat: new(true),
- }
- pager := client.NewListBlobsHierarchyPager("/", opts)
-
- var names []string
- for pager.More() {
- page, err := pager.NextPage(context.Background())
- require.NoError(t, err)
- assert.Equal(t, ArrowContentType, *page.ContentType)
- for _, item := range page.Segment.BlobItems {
- names = append(names, *item.Name)
- assert.Equal(t, int64(42), *item.Properties.ContentLength)
- }
- }
- assert.Equal(t, []string{"blob1.txt", "blob2.txt"}, names)
-
- require.Len(t, requests, 2)
- q := requests[0].URL.Query()
- assert.Equal(t, "list", q.Get("comp"))
- assert.Equal(t, "container", q.Get("restype"))
- assert.Equal(t, "/", q.Get("delimiter"))
- assert.Equal(t, "metadata,tags", q.Get("include"))
- assert.Equal(t, "dir/", q.Get("prefix"))
- assert.Equal(t, "1000", q.Get("maxresults"))
- assert.Equal(t, "testcontainer/dir/a", q.Get("startFrom"))
- assert.Equal(t, "testcontainer/dir/n", q.Get("endBefore"))
- assert.Equal(t, "fakesas", q.Get("sv"))
- assert.Empty(t, q.Get("marker"))
- assert.Equal(t, ArrowAcceptHeader, requests[0].Header.Get("Accept"))
- assert.Equal(t, serviceVersion, requests[0].Header.Get("x-ms-version"))
- assert.Equal(t, "marker1", requests[1].URL.Query().Get("marker"))
-
- // The caller's options must not have been mutated by paging.
- assert.Nil(t, opts.Marker)
-}
-
-const xmlListResponse = `<?xml version="1.0" encoding="utf-8"?>
-<EnumerationResults ServiceEndpoint="https://example.com/" ContainerName="testcontainer">
- <Blobs>
- <Blob>
- <Name>file.txt</Name>
- <Properties>
- <Last-Modified>Mon, 01 Jan 2024 00:00:00 GMT</Last-Modified>
- <Etag>0x8D1</Etag>
- <Content-Length>7</Content-Length>
- <BlobType>BlockBlob</BlobType>
- </Properties>
- </Blob>
- <BlobPrefix>
- <Name>subdir/</Name>
- </BlobPrefix>
- </Blobs>
- <NextMarker />
-</EnumerationResults>`
-
-func TestPagerXMLFallback(t *testing.T) {
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "application/xml")
- _, _ = w.Write([]byte(xmlListResponse))
- }))
- defer srv.Close()
-
- client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
- require.NoError(t, err)
-
- pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
- UseArrowFormat: new(true),
- })
- page, err := pager.NextPage(context.Background())
- require.NoError(t, err)
- assert.False(t, pager.More())
- assert.Equal(t, "application/xml", *page.ContentType)
- require.Len(t, page.Segment.BlobItems, 1)
- assert.Equal(t, "file.txt", *page.Segment.BlobItems[0].Name)
- assert.Equal(t, int64(7), *page.Segment.BlobItems[0].Properties.ContentLength)
- require.Len(t, page.Segment.BlobPrefixes, 1)
- assert.Equal(t, "subdir/", *page.Segment.BlobPrefixes[0].Name)
-}
-
-func TestPagerXMLFallbackWithEndBefore(t *testing.T) {
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "application/xml")
- _, _ = w.Write([]byte(xmlListResponse))
- }))
- defer srv.Close()
-
- client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
- require.NoError(t, err)
-
- pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
- EndBefore: new("testcontainer/n"),
- UseArrowFormat: new(true),
- })
- _, err = pager.NextPage(context.Background())
- require.ErrorIs(t, err, ErrEndBeforeXMLFallback)
-}
-
-func TestPagerSharedKeyAuth(t *testing.T) {
- var gotAuth, gotDate string
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- gotAuth = r.Header.Get("Authorization")
- gotDate = r.Header.Get("x-ms-date")
- w.Header().Set("Content-Type", ArrowContentType)
- _, _ = w.Write(arrowPage(t, "blob.txt", ""))
- }))
- defer srv.Close()
-
- cred, err := NewSharedKeyCredential("testaccount", base64.StdEncoding.EncodeToString([]byte("testkey")))
- require.NoError(t, err)
- client, err := NewClientWithSharedKeyCredential(srv.URL+"/testcontainer", cred, nil)
- require.NoError(t, err)
-
- pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
- UseArrowFormat: new(true),
- })
- _, err = pager.NextPage(context.Background())
- require.NoError(t, err)
- assert.True(t, strings.HasPrefix(gotAuth, "SharedKey testaccount:"), "Authorization = %q", gotAuth)
- assert.NotEmpty(t, gotDate)
-}
-
-func TestPagerResponseError(t *testing.T) {
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("x-ms-error-code", string(bloberror.ContainerNotFound))
- w.WriteHeader(http.StatusNotFound)
- }))
- defer srv.Close()
-
- client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
- require.NoError(t, err)
-
- pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
- UseArrowFormat: new(true),
- })
- _, err = pager.NextPage(context.Background())
- require.Error(t, err)
- assert.True(t, bloberror.HasCode(err, bloberror.ContainerNotFound), "expected ContainerNotFound, got %v", err)
-}
diff --git a/backend/azureblob/arrowlist/challenge_policy.go b/backend/azureblob/arrowlist/challenge_policy.go
deleted file mode 100644
index 90699c350..000000000
--- a/backend/azureblob/arrowlist/challenge_policy.go
+++ /dev/null
@@ -1,121 +0,0 @@
-//go:build !plan9 && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/shared/challenge_policy.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0. The
-// released SDK does not export its storage bearer-challenge policy, which is
-// needed for token authentication with the same semantics as the SDK's own
-// clients.
-
-package arrowlist
-
-import (
- "errors"
- "net/http"
- "strings"
-
- "github.com/Azure/azure-sdk-for-go/sdk/azcore"
- "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
- "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-)
-
-type storageAuthorizer struct {
- scopes []string
- tenantID string
-}
-
-// NewStorageChallengePolicy returns a bearer token policy for cred which
-// handles the storage service's authentication challenges.
-func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy {
- s := storageAuthorizer{scopes: []string{audience}}
- return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{
- AuthorizationHandler: policy.AuthorizationHandler{
- OnRequest: s.onRequest,
- OnChallenge: s.onChallenge,
- },
- InsecureAllowCredentialWithHTTP: allowHTTP,
- })
-}
-
-func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error {
- return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
-}
-
-func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error {
- // parse the challenge
- err := s.parseChallenge(resp)
- if err != nil {
- return err
- }
- // TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841
- return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
-}
-
-type challengePolicyError struct {
- err error
-}
-
-func (c *challengePolicyError) Error() string {
- return c.err.Error()
-}
-
-func (*challengePolicyError) NonRetriable() {
- // marker method
-}
-
-func (c *challengePolicyError) Unwrap() error {
- return c.err
-}
-
-// parses Tenant ID from auth challenge
-// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize
-func parseTenant(url string) string {
- if url == "" {
- return ""
- }
- parts := strings.Split(url, "/")
- if len(parts) >= 3 {
- tenant := parts[3]
- tenant = strings.ReplaceAll(tenant, ",", "")
- return tenant
- }
- return ""
-}
-
-func (s *storageAuthorizer) parseChallenge(resp *http.Response) error {
- authHeader := resp.Header.Get("WWW-Authenticate")
- if authHeader == "" {
- return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")}
- }
-
- // Strip down to auth and resource
- // Format is "Bearer authorization_uri=\"<site>\" resource_id=\"<site>\""
- authHeader = strings.ReplaceAll(authHeader, "Bearer ", "")
-
- parts := strings.Split(authHeader, " ")
-
- vals := map[string]string{}
- for _, part := range parts {
- subParts := strings.Split(part, "=")
- if len(subParts) == 2 {
- stripped := strings.ReplaceAll(subParts[1], "\"", "")
- stripped = strings.TrimSuffix(stripped, ",")
- vals[subParts[0]] = stripped
- }
- }
-
- s.tenantID = parseTenant(vals["authorization_uri"])
-
- scope := vals["resource_id"]
- if scope == "" {
- return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")}
- }
-
- if !strings.HasSuffix(scope, "/.default") {
- scope += "/.default"
- }
- s.scopes = []string{scope}
- return nil
-}
diff --git a/backend/azureblob/arrowlist/sharedkey.go b/backend/azureblob/arrowlist/sharedkey.go
deleted file mode 100644
index 4044d0265..000000000
--- a/backend/azureblob/arrowlist/sharedkey.go
+++ /dev/null
@@ -1,318 +0,0 @@
-//go:build !plan9 && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/exported/shared_key_credential.go in the Azure SDK
-// for Go github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0,
-// with the internal header constants inlined. The released SDK does not
-// export its shared key signing policy, so a copy is needed to sign the
-// Arrow listing requests this package sends through its own pipeline.
-
-package arrowlist
-
-import (
- "bytes"
- "crypto/hmac"
- "crypto/sha256"
- "encoding/base64"
- "fmt"
- "net/http"
- "net/url"
- "sort"
- "strings"
- "sync/atomic"
- "time"
-
- "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
- "github.com/rclone/rclone/fs"
-)
-
-// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the
-// storage account's name and either its primary or secondary key.
-func NewSharedKeyCredential(accountName string, accountKey string) (*SharedKeyCredential, error) {
- c := SharedKeyCredential{accountName: accountName}
- if err := c.SetAccountKey(accountKey); err != nil {
- return nil, err
- }
- return &c, nil
-}
-
-// SharedKeyCredential contains an account's name and its primary or secondary key.
-type SharedKeyCredential struct {
- // Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only
- accountName string
- accountKey atomic.Value // []byte
-}
-
-// AccountName returns the Storage account's name.
-func (c *SharedKeyCredential) AccountName() string {
- return c.accountName
-}
-
-// SetAccountKey replaces the existing account key with the specified account key.
-func (c *SharedKeyCredential) SetAccountKey(accountKey string) error {
- _bytes, err := base64.StdEncoding.DecodeString(accountKey)
- if err != nil {
- return fmt.Errorf("decode account key: %w", err)
- }
- c.accountKey.Store(_bytes)
- return nil
-}
-
-// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.
-func (c *SharedKeyCredential) computeHMACSHA256(message string) (string, error) {
- h := hmac.New(sha256.New, c.accountKey.Load().([]byte))
- _, err := h.Write([]byte(message))
- return base64.StdEncoding.EncodeToString(h.Sum(nil)), err
-}
-
-func (c *SharedKeyCredential) buildStringToSign(req *http.Request) (string, error) {
- // https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
- headers := req.Header
- contentLength := getHeader("Content-Length", headers)
- if contentLength == "0" {
- contentLength = ""
- }
-
- canonicalizedResource, err := c.buildCanonicalizedResource(req.URL)
- if err != nil {
- return "", err
- }
-
- stringToSign := strings.Join([]string{
- req.Method,
- getHeader("Content-Encoding", headers),
- getHeader("Content-Language", headers),
- contentLength,
- getHeader("Content-MD5", headers),
- getHeader("Content-Type", headers),
- "", // Empty date because x-ms-date is expected (as per web page above)
- getHeader("If-Modified-Since", headers),
- getHeader("If-Match", headers),
- getHeader("If-None-Match", headers),
- getHeader("If-Unmodified-Since", headers),
- getHeader("Range", headers),
- c.buildCanonicalizedHeader(headers),
- canonicalizedResource,
- }, "\n")
- return stringToSign, nil
-}
-
-func getHeader(key string, headers map[string][]string) string {
- if headers == nil {
- return ""
- }
- if v, ok := headers[key]; ok {
- if len(v) > 0 {
- return v[0]
- }
- }
-
- return ""
-}
-
-func getWeightTables() [][]int {
- tableLv0 := [...]int{
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,
- 0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0xd03, 0xd1a, 0xd1c, 0xd1e,
- 0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
- 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,
- 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,
- 0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,
- 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,
- 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
- }
- tableLv2 := [...]int{
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
- }
- tables := [][]int{tableLv0[:], tableLv2[:]}
- return tables
-}
-
-// NewHeaderStringComparer performs a multi-level, weight-based comparison of two strings
-func compareHeaders(lhs, rhs string, tables [][]int) int {
- currLevel, i, j := 0, 0, 0
- n := len(tables)
- lhsLen := len(lhs)
- rhsLen := len(rhs)
-
- for currLevel < n {
- if currLevel == (n-1) && i != j {
- if i > j {
- return -1
- }
- if i < j {
- return 1
- }
- return 0
- }
-
- var w1, w2 int
-
- // Check bounds before accessing lhs[i]
- if i < lhsLen {
- w1 = tables[currLevel][lhs[i]]
- } else {
- w1 = 0x1
- }
-
- // Check bounds before accessing rhs[j]
- if j < rhsLen {
- w2 = tables[currLevel][rhs[j]]
- } else {
- w2 = 0x1
- }
-
- if w1 == 0x1 && w2 == 0x1 {
- i = 0
- j = 0
- currLevel++
- } else if w1 == w2 {
- i++
- j++
- } else if w1 == 0 {
- i++
- } else if w2 == 0 {
- j++
- } else {
- if w1 < w2 {
- return -1
- }
- if w1 > w2 {
- return 1
- }
- return 0
- }
- }
- return 0
-}
-
-func (c *SharedKeyCredential) buildCanonicalizedHeader(headers http.Header) string {
- cm := map[string][]string{}
- for k, v := range headers {
- headerName := strings.TrimSpace(strings.ToLower(k))
- if strings.HasPrefix(headerName, "x-ms-") {
- cm[headerName] = v // NOTE: the value must not have any whitespace around it.
- }
- }
- if len(cm) == 0 {
- return ""
- }
-
- keys := make([]string, 0, len(cm))
- for key := range cm {
- keys = append(keys, key)
- }
- tables := getWeightTables()
- // Sort the keys using the custom comparator
- sort.Slice(keys, func(i, j int) bool {
- return compareHeaders(keys[i], keys[j], tables) < 0
- })
- ch := bytes.NewBufferString("")
- for i, key := range keys {
- if i > 0 {
- ch.WriteRune('\n')
- }
- ch.WriteString(key)
- ch.WriteRune(':')
- ch.WriteString(strings.Join(cm[key], ","))
- }
- return ch.String()
-}
-
-func (c *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {
- // https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
- cr := bytes.NewBufferString("/")
- cr.WriteString(c.accountName)
-
- if len(u.Path) > 0 {
- // Any portion of the CanonicalizedResource string that is derived from
- // the resource's URI should be encoded exactly as it is in the URI.
- // -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
- cr.WriteString(u.EscapedPath())
- } else {
- // a slash is required to indicate the root path
- cr.WriteString("/")
- }
-
- // params is a map[string][]string; param name is key; params values is []string
- params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values
- if err != nil {
- return "", fmt.Errorf("failed to parse query params: %w", err)
- }
-
- if len(params) > 0 { // There is at least 1 query parameter
- var paramNames []string // We use this to sort the parameter key names
- for paramName := range params {
- paramNames = append(paramNames, paramName) // paramNames must be lowercase
- }
- sort.Strings(paramNames)
-
- for _, paramName := range paramNames {
- paramValues := params[paramName]
- sort.Strings(paramValues)
-
- // Join the sorted key values separated by ','
- // Then prepend "keyName:"; then add this string to the buffer
- cr.WriteString("\n" + strings.ToLower(paramName) + ":" + strings.Join(paramValues, ","))
- }
- }
- return cr.String(), nil
-}
-
-// ComputeHMACSHA256 is a helper for computing the signed string outside of this package.
-func ComputeHMACSHA256(cred *SharedKeyCredential, message string) (string, error) {
- return cred.computeHMACSHA256(message)
-}
-
-// SharedKeyCredPolicy is a pipeline policy that signs each request with the
-// account's shared key.
-type SharedKeyCredPolicy struct {
- cred *SharedKeyCredential
-}
-
-// NewSharedKeyCredPolicy creates a SharedKeyCredPolicy signing with cred.
-func NewSharedKeyCredPolicy(cred *SharedKeyCredential) *SharedKeyCredPolicy {
- return &SharedKeyCredPolicy{cred: cred}
-}
-
-// Do signs the request and sends it via the next policy in the pipeline.
-func (s *SharedKeyCredPolicy) Do(req *policy.Request) (*http.Response, error) {
- // skip adding the authorization header if no SharedKeyCredential was provided.
- // this prevents a panic that might be hard to diagnose and allows testing
- // against http endpoints that don't require authentication.
- if s.cred == nil {
- return req.Next()
- }
-
- if d := getHeader("x-ms-date", req.Raw().Header); d == "" {
- req.Raw().Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))
- }
- stringToSign, err := s.cred.buildStringToSign(req.Raw())
- if err != nil {
- return nil, err
- }
- signature, err := s.cred.computeHMACSHA256(stringToSign)
- if err != nil {
- return nil, err
- }
- authHeader := strings.Join([]string{"SharedKey ", s.cred.AccountName(), ":", signature}, "")
- req.Raw().Header.Set("Authorization", authHeader)
-
- response, err := req.Next()
- if err != nil && response != nil && response.StatusCode == http.StatusForbidden {
- // Service failed to authenticate request, log it
- fs.Debugf(nil, "azureblob arrowlist: HTTP Forbidden status, String-to-Sign: %q", stringToSign)
- }
- return response, err
-}
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 642d34d2e..77ecec922 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -26,14 +26,12 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
- "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
- "github.com/rclone/rclone/backend/azureblob/arrowlist"
"github.com/rclone/rclone/backend/azureblob/auth"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/chunksize"
@@ -241,40 +239,38 @@ avoid the time out.`,
Advanced: true,
}, {
Name: "use_arrow_list",
- Help: `Use the experimental Apache Arrow listing format.
+ Help: `Use the Apache Arrow listing format.
If set, directory listings are fetched using the ListBlobs Apache
-Arrow response format instead of XML. This can be faster for very
-large containers.
-
-This is EXPERIMENTAL and requires the "Blob Listing with Apache Arrow"
-preview feature to be enabled on the storage account. It is NOT
-supported on accounts with a hierarchical namespace (ADLS Gen2) -
-those return a 409 error. If the feature is not enabled the server
-returns XML and the listing transparently falls back to the normal XML
-path (logged at debug level).
-
-Not supported with connection_string auth - falls back to normal
-listing.`,
+Arrow response format instead of XML. Arrow responses are smaller and
+much cheaper to parse, making listings of large containers several
+times faster. Combine with "list_parallelism" for the biggest gains.
+
+"Blob Listing with Apache Arrow" is in public preview at Microsoft and
+is only supported on flat namespace accounts. On accounts with a
+hierarchical namespace (ADLS Gen2), or where the feature is otherwise
+unavailable, the server returns XML and the listing transparently
+falls back to the normal XML path (logged at debug level).`,
Default: false,
Advanced: true,
- Hide: fs.OptionHideBoth,
}, {
Name: "list_parallelism",
Help: `Number of parallel shards to list a directory with.
-EXPERIMENTAL. If set greater than 1, the blob name keyspace of each
-directory is split into this many ranges which are listed concurrently
-using the Arrow startFrom/endBefore range parameters. This can
-dramatically speed up listing containers with millions of objects, for
-both recursive (ListR) and single directory listings.
+If set greater than 1, the blob name keyspace of each directory is
+split into ranges which are listed concurrently using the Arrow
+startFrom/endBefore range parameters. This can dramatically speed up
+listing containers with millions of objects, for both recursive
+(ListR) and single directory listings. Speed keeps improving up to a
+parallelism of around 30.
This has no effect unless "use_arrow_list" is also set, as Arrow is the
-only listing path that supports server-side name ranges. The default of
-0 (or 1) lists sequentially.`,
+only listing path that supports server-side name ranges. If the
+account does not support range listing (e.g. it has a hierarchical
+namespace) the listing falls back to sequential. The default of 0 (or
+1) lists sequentially.`,
Default: 0,
Advanced: true,
- Hide: fs.OptionHideBoth,
}, {
Name: "access_tier",
Help: `Access tier of blob: hot, cool, cold or archive.
@@ -449,10 +445,8 @@ type Fs struct {
opt Options // parsed config options
ci *fs.ConfigInfo // global config
features *fs.Features // optional features
- cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache and arrowCntSVCcache
+ cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache
cntSVCcache map[string]*container.Client // reference to containerClient per container
- arrowCntSVCcache map[string]*arrowlist.Client // reference to arrowlist client per container
- arrowClientOpts *arrowlist.ClientOptions // client options for the arrowlist clients
arrowXMLFallback atomic.Bool // set once the server answers XML so parallel listing is not retried
svc *service.Client // client to access azblob
cred azcore.TokenCredential // how to generate tokens (may be nil)
@@ -670,15 +664,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci := fs.GetConfig(ctx)
f := &Fs{
- name: name,
- opt: *opt,
- ci: ci,
- pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
- uploadToken: pacer.NewTokenDispenser(ci.Transfers),
- copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
- cache: bucket.NewCache(),
- cntSVCcache: make(map[string]*container.Client, 1),
- arrowCntSVCcache: make(map[string]*arrowlist.Client, 1),
+ name: name,
+ opt: *opt,
+ ci: ci,
+ pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ uploadToken: pacer.NewTokenDispenser(ci.Transfers),
+ copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
+ cache: bucket.NewCache(),
+ cntSVCcache: make(map[string]*container.Client, 1),
}
f.publicAccess = container.PublicAccessType(opt.PublicAccess)
f.setRoot(root)
@@ -725,15 +718,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
f.usingSharedKeyCred = res.UsingSharedKeyCred
f.anonymous = res.Anonymous
- // Client options for the arrowlist clients, using the same transport and
- // gzip policy as the SDK clients built above.
- f.arrowClientOpts = &arrowlist.ClientOptions{
- ClientOptions: azcore.ClientOptions{
- Transport: auth.Transporter(ctx),
- PerCallPolicies: []policy.Policy{setAcceptEncodingGzip{}},
- },
- }
-
// if using Container level SAS put the container client into the cache
if opt.SASURL != "" && res.Container != "" {
_ = f.cntSVC(res.Container)
@@ -772,44 +756,6 @@ func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
return containerClient
}
-// errArrowAuthUnsupported is returned by arrowCntSVC when the configured
-// credentials can't be used for Arrow listing.
-var errArrowAuthUnsupported = errors.New("credentials not supported for Arrow listing (connection_string auth is not supported)")
-
-// return the arrowlist client for the container passed in
-//
-// Returns errArrowAuthUnsupported if the configured credentials can't be
-// reused for the arrowlist client's pipeline.
-func (f *Fs) arrowCntSVC(containerName string) (client *arrowlist.Client, err error) {
- // The container URL includes any SAS token in its query
- url := f.cntSVC(containerName).URL()
- f.cntSVCcacheMu.Lock()
- defer f.cntSVCcacheMu.Unlock()
- if client, ok := f.arrowCntSVCcache[containerName]; ok {
- return client, nil
- }
- switch {
- case f.usingSharedKeyCred:
- // Covers account+key and the emulator (auth fills in Account/Key)
- var cred *arrowlist.SharedKeyCredential
- cred, err = arrowlist.NewSharedKeyCredential(f.opt.Account, f.opt.Key)
- if err == nil {
- client, err = arrowlist.NewClientWithSharedKeyCredential(url, cred, f.arrowClientOpts)
- }
- case f.cred != nil:
- client, err = arrowlist.NewClient(url, f.cred, f.arrowClientOpts)
- case f.anonymous || f.opt.SASURL != "":
- client, err = arrowlist.NewClientWithNoCredential(url, f.arrowClientOpts)
- default:
- return nil, errArrowAuthUnsupported
- }
- if err != nil {
- return nil, err
- }
- f.arrowCntSVCcache[containerName] = client
- return client, nil
-}
-
// Return an Object from a path
//
// If it can't be found it returns the error fs.ErrorObjectNotFound.
@@ -1186,27 +1132,25 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
delimiter = "/"
}
- opts := &arrowlist.ListBlobsHierarchyOptions{
- ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
- // Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
- Include: container.ListBlobsInclude{
- Copy: false,
- Metadata: true,
- Snapshots: false,
- UncommittedBlobs: false,
- Deleted: false,
- },
- Prefix: &directory,
- MaxResults: &maxResults,
+ opts := &container.ListBlobsHierarchyOptions{
+ // Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
+ Include: container.ListBlobsInclude{
+ Copy: false,
+ Metadata: true,
+ Snapshots: false,
+ UncommittedBlobs: false,
+ Deleted: false,
},
+ Prefix: &directory,
+ MaxResults: &maxResults,
}
- // Experimental: request the Apache Arrow listing format. The arrowlist
- // pager requests an Arrow IPC stream and decodes it, falling back to XML
- // if the account doesn't have Arrow listing enabled. Skip the
- // maxResults==1 probe (isEmpty) which doesn't benefit.
+ // Request the Apache Arrow listing format. The SDK pager requests an
+ // Arrow IPC stream and decodes it, falling back to XML if the account
+ // doesn't have Arrow listing enabled. Skip the maxResults==1 probe
+ // (isEmpty) which doesn't benefit.
useArrow := f.opt.UseArrowList && maxResults != 1
if useArrow {
- opts.UseArrowFormat = new(true)
+ opts.ResponseFormat = container.StorageResponseFormatArrow
}
var foundItems int
@@ -1237,25 +1181,11 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
// listBlobsPager runs the hierarchy listing described by opts, calling fn for
// each blob and subdirectory, and returns the number of raw items seen.
// delimiter selects flat (recurse) vs hierarchical listing. If opts requests
-// the Apache Arrow format the listing goes through the arrowlist pager
-// (falling back to the SDK pager if the credentials don't support it); if the
-// service then answers with XML (Arrow listing not enabled) a debug message
-// is logged.
-func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
- useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
- var pager *runtime.Pager[container.ListBlobsHierarchyResponse]
- if useArrow {
- arrowSVC, err := f.arrowCntSVC(containerName)
- if err != nil {
- fs.Debugf(f, "Not using Arrow listing: %v", err)
- useArrow = false
- } else {
- pager = arrowSVC.NewListBlobsHierarchyPager(delimiter, opts)
- }
- }
- if pager == nil {
- pager = f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &opts.ListBlobsHierarchyOptions)
- }
+// the Apache Arrow format and the service answers with XML (Arrow listing
+// not enabled) a debug message is logged.
+func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
+ useArrow := opts.ResponseFormat == container.StorageResponseFormatArrow
+ pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, opts)
checkedArrow := false
for pager.More() {
var response container.ListBlobsHierarchyResponse
@@ -1271,7 +1201,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
// but not accelerated.
if useArrow && !checkedArrow && err == nil {
checkedArrow = true
- if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowlist.ArrowContentType) {
+ if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowContentType) {
fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")
}
}
diff --git a/go.mod b/go.mod
index d344a18d3..cbdca57db 100644
--- a/go.mod
+++ b/go.mod
@@ -6,9 +6,9 @@ godebug x509negativeserial=1
require (
bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5
- github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
+ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
- github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
+ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0
github.com/Azure/go-ntlmssp v0.1.1
github.com/FilenCloudDienste/filen-sdk-go v0.0.39
@@ -21,7 +21,7 @@ require (
github.com/adrg/xdg v0.5.3
github.com/anacrolix/dms v1.7.2
github.com/anacrolix/log v0.17.0
- github.com/apache/arrow-go/v18 v18.7.0
+ github.com/apache/arrow-go/v18 v18.7.0 // indirect
github.com/atotto/clipboard v0.1.4
github.com/aws/aws-sdk-go-v2 v1.43.7
github.com/aws/aws-sdk-go-v2/config v1.32.30
@@ -52,7 +52,7 @@ require (
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb
github.com/josephspurrier/goversioninfo v1.7.0
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004
- github.com/klauspost/compress v1.19.0
+ github.com/klauspost/compress v1.19.2
github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6
github.com/lanrat/extsort v1.4.2
@@ -82,7 +82,7 @@ require (
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10
- github.com/stretchr/testify v1.11.1
+ github.com/stretchr/testify v1.12.1
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04
github.com/unknwon/goconfig v1.0.0
github.com/willscott/go-nfs v0.0.4
@@ -115,7 +115,7 @@ require (
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
- github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
+ github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
github.com/ProtonMail/go-srp v0.0.7 // indirect
@@ -157,7 +157,6 @@ require (
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
github.com/creasty/defaults v1.8.0 // indirect
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
- github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dromara/dongle v1.0.1 // indirect
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
@@ -222,7 +221,7 @@ require (
github.com/panjf2000/ants/v2 v2.12.1 // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
- github.com/pierrec/lz4/v4 v4.1.27 // indirect
+ github.com/pierrec/lz4/v4 v4.1.28 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
@@ -258,11 +257,11 @@ require (
go.opentelemetry.io/otel/metric v1.45.0 // indirect
go.opentelemetry.io/otel/trace v1.45.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
- go.yaml.in/yaml/v3 v3.0.4 // indirect
+ go.yaml.in/yaml/v3 v3.0.5 // indirect
go4.org v0.0.0-20260112195520-a5071408f32f // indirect
- golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect
+ golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirect
golang.org/x/image v0.45.0 // indirect
- golang.org/x/tools v0.48.0 // indirect
+ golang.org/x/tools v0.49.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect
google.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e // indirect
google.golang.org/protobuf v1.36.12 // indirect
diff --git a/go.sum b/go.sum
index f27afaefd..9fc4ff90e 100644
--- a/go.sum
+++ b/go.sum
@@ -6,8 +6,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA=
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs=
@@ -16,16 +16,16 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6Xu
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1 h1:gkBLVmB3Z/HnGP/Jo4o12/RDpi0agnKav6sCKsX5Vu0=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1/go.mod h1:e3/1P5K+jIUi9JevDRklq/tFeTvbBb75bNAjU4xd31w=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 h1:cuiKf1UVyWHu+XSQghPZR/qEF43JIcuk2CDqMlPiT6M=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0/go.mod h1:9JSyvgXLPAOC4jfhgZg58XeU2FJHsGmxuYSclCFQ4ZY=
github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjws122HrwskaEE/igY=
github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=
github.com/Files-com/files-sdk-go/v3 v3.3.194 h1:dtOFxSTWWRpkmvXa6ycNiw8dVDu1wkgzcXyVV1VafNc=
@@ -184,8 +184,6 @@ github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 h1:tLWBZgPg
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/diskfs/go-diskfs v1.9.4 h1:0j2d7eG4IjyxL6+ChWbDPocdBCF6HQ4HBWU2WDYWVnc=
github.com/diskfs/go-diskfs v1.9.4/go.mod h1:TePJORO83Adh5pb2SqsxAwaP0fofFxKLkxctiS/9OQc=
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
@@ -357,8 +355,8 @@ github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXw
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
-github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
+github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
@@ -458,8 +456,8 @@ github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw=
github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
-github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk=
-github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
+github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio=
+github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
@@ -552,8 +550,9 @@ github.com/stangelandcl/ppmd v0.1.1/go.mod h1:Rrv7M+/2P5jYr/GMLhBl7Ug3uJ1bUiVzr5
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
+github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -564,8 +563,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
+github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 h1:s30A8dMuZ55lUOi5xUTh1hlfLqHpVBsRMfpk2iKCayk=
github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo=
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
@@ -641,8 +641,9 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
+go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw=
go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0=
goftp.io/server/v2 v2.0.3 h1:iz6Gxj7f2SFQVxrj0s1is+gueE6O9yTc+Ab0vtQ6Zn4=
@@ -660,8 +661,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
-golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g=
-golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=
+golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY=
+golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk=
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
golang.org/x/mobile v0.0.0-20260709172247-6129f5bee9d5 h1:Mn1OzFmF0ZKX/ZayHz/UdnWHufPp1wlD9lZ5U8LRDFY=
@@ -744,8 +745,8 @@ golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
-golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
+golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
+golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=