"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
azureblob: use the released Azure SDK for Apache Arrow listing
Nick Craig-Wood3b9f0a05b55f
diff --git a/backend/azureblob/arrow.go b/backend/azureblob/arrow.goindex baaf51eb4..210400f82 100644--- a/backend/azureblob/arrow.go+++ b/backend/azureblob/arrow.go@@ -18,24 +18,20 @@ import ("github.com/Azure/azure-sdk-for-go/sdk/azcore""github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"- "github.com/rclone/rclone/backend/azureblob/arrowlist""github.com/rclone/rclone/fs""golang.org/x/sync/errgroup")+// arrowContentType is the Content-Type of an Apache Arrow IPC stream listing+// response, used to tell whether the service honoured an Arrow request.+const arrowContentType = "application/vnd.apache.arrow.stream"+// listArrowParallel lists (containerName, directory) by splitting the blob name// keyspace into shards and listing them concurrently. opts must already request// the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It// returns the number of raw items seen. directory/prefix are already normalised// by the caller.-func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {- // Parallel listing needs the arrowlist client (server-side endBefore is- // only honoured on the Arrow path) - without it fall back to sequential.- if _, err := f.arrowCntSVC(containerName); err != nil {- fs.Debugf(f, "Not using parallel Arrow listing: %v", err)- return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)- }-+func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {// Split the keyspace by a single-case character ladder (see// arrowLadderBoundaries for why single-case matters).points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))@@ -115,16 +111,13 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr// isEndBeforeUnsupported reports whether err means the service can't honour// the endBefore listing bound, so sharded parallel listing must not be used.//-// This happens two ways: the arrowlist pager's ErrEndBeforeXMLFallback (the-// server answered XML with endBefore set), or the service rejecting the-// endBefore parameter with 400 OperationNotSupportedWithFeatureMissing "The-// requested operation is not allowed as EndBefore parameter support is-// missing on this account" - endBefore is only supported on the Arrow-// listing path of accounts with Blob Listing with Apache Arrow enabled.+// The service rejects the endBefore parameter with 400+// OperationNotSupportedWithFeatureMissing "The requested operation is not+// allowed as EndBefore parameter support is missing on this account" -+// endBefore is only supported on the Arrow listing path of accounts with+// Blob Listing with Apache Arrow enabled. (The SDK pager would silently+// accept an XML page with endBefore set, but the service never sends one.)func isEndBeforeUnsupported(err error) bool {- if errors.Is(err, arrowlist.ErrEndBeforeXMLFallback) {- return true- }var respErr *azcore.ResponseErrorreturn errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"}diff --git a/backend/azureblob/arrow_internal_test.go b/backend/azureblob/arrow_internal_test.goindex b1bd96868..a77ab3607 100644--- a/backend/azureblob/arrow_internal_test.go+++ b/backend/azureblob/arrow_internal_test.go@@ -10,7 +10,6 @@ import ("testing""github.com/Azure/azure-sdk-for-go/sdk/azcore"- "github.com/rclone/rclone/backend/azureblob/arrowlist""github.com/rclone/rclone/fs""github.com/rclone/rclone/fstest""github.com/rclone/rclone/fstest/fstests"@@ -212,9 +211,8 @@ func TestArrowLadderBoundaries(t *testing.T) {}func TestIsEndBeforeUnsupported(t *testing.T) {- assert.True(t, isEndBeforeUnsupported(arrowlist.ErrEndBeforeXMLFallback))- assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", arrowlist.ErrEndBeforeXMLFallback)))assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))+ assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", &azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"})))assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))assert.False(t, isEndBeforeUnsupported(nil))diff --git a/backend/azureblob/arrowlist/arrow.go b/backend/azureblob/arrowlist/arrow.godeleted file mode 100644index e4e788553..000000000--- a/backend/azureblob/arrowlist/arrow.go+++ /dev/null@@ -1,421 +0,0 @@-//go:build !plan9 && !js--// Copyright (c) Microsoft Corporation. All rights reserved.-// Licensed under the MIT License. See License.txt in the project root for license information.-//-// Adapted from internal/arrow/arrow.go in the Azure SDK for Go-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit-// c6fa341ca22b (branch feature/storage/bifrost), retyped onto the SDK's-// public type aliases so it compiles against the released azblob module.--package arrowlist--import (- "encoding/base64"- "fmt"- "io"- "net/http"- "strconv"- "time"-- "github.com/Azure/azure-sdk-for-go/sdk/azcore"- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"- "github.com/apache/arrow-go/v18/arrow"- "github.com/apache/arrow-go/v18/arrow/array"- "github.com/apache/arrow-go/v18/arrow/ipc"- "github.com/apache/arrow-go/v18/arrow/memory"-)--const (- // ArrowContentType is the Content-Type for Apache Arrow IPC stream responses.- ArrowContentType = "application/vnd.apache.arrow.stream"-- // ArrowAcceptHeader is the Accept header value to request Arrow format.- ArrowAcceptHeader = ArrowContentType-- // resourceTypeBlobPrefix is the ResourceType value that identifies a virtual directory prefix- // in hierarchy listing responses. In XML, prefixes are separate <BlobPrefix> elements;- // in Arrow, all rows share the same schema and are distinguished by this field value.- resourceTypeBlobPrefix = "blobprefix"-)--// HandleFlatListResponse parses an Arrow IPC stream response into a ContainerClientListBlobFlatSegmentResponse.-func HandleFlatListResponse(resp *http.Response) (container.ListBlobsFlatResponse, error) {- result := container.ListBlobsFlatResponse{}-- // Extract response headers- extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)-- // Parse Arrow IPC stream- items, nextMarker, err := parseArrowStream(resp.Body)- if err != nil {- return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)- }-- result.ListBlobsFlatSegmentResponse = container.ListBlobsFlatSegmentResponse{- Segment: &container.BlobFlatListSegment{- BlobItems: items,- },- NextMarker: nextMarker,- }-- return result, nil-}--// HandleHierarchyListResponse parses an Arrow IPC stream response into a ContainerClientListBlobHierarchySegmentResponse.-func HandleHierarchyListResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {- result := container.ListBlobsHierarchyResponse{}-- extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)-- items, nextMarker, err := parseArrowStream(resp.Body)- if err != nil {- return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)- }-- // Separate blob items from blob prefixes based on ResourceType field.- var blobItems []*container.BlobItem- var blobPrefixes []*container.BlobPrefix- for _, item := range items {- if item.Properties != nil && item.Properties.ResourceType != nil && *item.Properties.ResourceType == resourceTypeBlobPrefix {- blobPrefixes = append(blobPrefixes, &container.BlobPrefix{- Name: item.Name,- Properties: item.Properties,- })- } else {- blobItems = append(blobItems, item)- }- }-- result.ListBlobsHierarchySegmentResponse = container.ListBlobsHierarchySegmentResponse{- Segment: &container.BlobHierarchyListSegment{- BlobItems: blobItems,- BlobPrefixes: blobPrefixes,- },- NextMarker: nextMarker,- }-- return result, nil-}--func extractResponseHeaders(resp *http.Response, clientRequestID, contentType **string, date **time.Time, requestID, version **string) {- if val := resp.Header.Get("x-ms-client-request-id"); val != "" {- *clientRequestID = &val- }- if val := resp.Header.Get("Content-Type"); val != "" {- *contentType = &val- }- if val := resp.Header.Get("Date"); val != "" {- if t, err := time.Parse(time.RFC1123, val); err == nil {- *date = &t- }- }- if val := resp.Header.Get("x-ms-request-id"); val != "" {- *requestID = &val- }- if val := resp.Header.Get("x-ms-version"); val != "" {- *version = &val- }-}--// parseArrowStream reads an Arrow IPC stream and converts it to BlobItems.-func parseArrowStream(body io.Reader) ([]*container.BlobItem, *string, error) {- reader, err := ipc.NewReader(body, ipc.WithAllocator(memory.DefaultAllocator))- if err != nil {- return nil, nil, fmt.Errorf("failed to create Arrow IPC reader: %w", err)- }- defer reader.Release()-- // Extract NextMarker from schema metadata- var nextMarker *string- var numRecords int- if md := reader.Schema().Metadata(); md.Len() > 0 {- if idx := md.FindKey("NextMarker"); idx >= 0 {- val := md.Values()[idx]- if val != "" {- nextMarker = &val- }- }- if idx := md.FindKey("NumberOfRecords"); idx >= 0 {- if n, err := strconv.Atoi(md.Values()[idx]); err == nil {- numRecords = n- }- }- }-- // Build column index for the schema- schema := reader.Schema()- colIndex := buildColumnIndex(schema)-- // Pre-allocate if we know the count- items := make([]*container.BlobItem, 0, numRecords)-- // Iterate over record batches- for reader.Next() {- rec := reader.RecordBatch()- rows := int(rec.NumRows())-- for row := range rows {- item := &container.BlobItem{- Properties: &container.BlobProperties{},- }- populateBlobItem(item, rec, row, colIndex)- items = append(items, item)- }- }-- if err := reader.Err(); err != nil {- return nil, nil, fmt.Errorf("error reading Arrow record batches: %w", err)- }-- return items, nextMarker, nil-}--// columnIndex maps column names to their indices in the record batch for O(1) lookup.-type columnIndex map[string]int--func buildColumnIndex(schema *arrow.Schema) columnIndex {- idx := make(columnIndex, len(schema.Fields()))- for i, f := range schema.Fields() {- idx[f.Name] = i- }- return idx-}--// populateBlobItem fills a BlobItem from a single row across all columns.-func populateBlobItem(item *container.BlobItem, rec arrow.RecordBatch, row int, colIdx columnIndex) {- // Top-level BlobItem fields- item.Name = getStringField(rec, row, colIdx, "Name")- item.Deleted = getBoolField(rec, row, colIdx, "Deleted")- item.Snapshot = getStringField(rec, row, colIdx, "Snapshot")- item.VersionID = getStringField(rec, row, colIdx, "VersionId")- item.IsCurrentVersion = getBoolField(rec, row, colIdx, "IsCurrentVersion")- item.HasVersionsOnly = getBoolField(rec, row, colIdx, "HasVersionsOnly")-- // Metadata maps- item.Metadata = getMapField(rec, row, colIdx, "Metadata")- item.OrMetadata = getMapField(rec, row, colIdx, "OrMetadata")-- // Tags- if tags := getMapField(rec, row, colIdx, "Tags"); tags != nil {- var blobTags []*container.BlobTag- for k, v := range tags {- key := k- blobTags = append(blobTags, &container.BlobTag{Key: &key, Value: v})- }- item.BlobTags = &container.BlobTags{BlobTagSet: blobTags}- }-- // BlobProperties fields- p := item.Properties- p.CreationTime = getTimestampField(rec, row, colIdx, "Creation-Time")- p.LastModified = getTimestampField(rec, row, colIdx, "Last-Modified")- p.ETag = getETagField(rec, row, colIdx, "Etag")- p.ContentLength = getInt64Field(rec, row, colIdx, "Content-Length")- p.ContentType = getStringField(rec, row, colIdx, "Content-Type")- p.ContentEncoding = getStringField(rec, row, colIdx, "Content-Encoding")- p.ContentLanguage = getStringField(rec, row, colIdx, "Content-Language")- p.ContentDisposition = getStringField(rec, row, colIdx, "Content-Disposition")- p.CacheControl = getStringField(rec, row, colIdx, "Cache-Control")- p.ContentMD5 = getBase64BytesField(rec, row, colIdx, "Content-MD5")- p.BlobType = getEnumField[container.BlobType](rec, row, colIdx, "BlobType")- p.AccessTier = getEnumField[container.AccessTier](rec, row, colIdx, "AccessTier")- p.AccessTierInferred = getBoolField(rec, row, colIdx, "AccessTierInferred")- p.AccessTierChangeTime = getTimestampField(rec, row, colIdx, "AccessTierChangeTime")- p.LeaseState = getEnumField[lease.StateType](rec, row, colIdx, "LeaseState")- p.LeaseStatus = getEnumField[lease.StatusType](rec, row, colIdx, "LeaseStatus")- p.LeaseDuration = getEnumField[lease.DurationType](rec, row, colIdx, "LeaseDuration")- p.ServerEncrypted = getBoolField(rec, row, colIdx, "ServerEncrypted")- p.CustomerProvidedKeySHA256 = getStringField(rec, row, colIdx, "CustomerProvidedKeySha256")- p.EncryptionScope = getStringField(rec, row, colIdx, "EncryptionScope")- p.IncrementalCopy = getBoolField(rec, row, colIdx, "IncrementalCopy")- p.IsSealed = getBoolField(rec, row, colIdx, "Sealed")- p.ArchiveStatus = getEnumField[container.ArchiveStatus](rec, row, colIdx, "ArchiveStatus")- p.RehydratePriority = getEnumField[blob.RehydratePriority](rec, row, colIdx, "RehydratePriority")- p.CopyID = getStringField(rec, row, colIdx, "CopyId")- p.CopyStatus = getEnumField[blob.CopyStatusType](rec, row, colIdx, "CopyStatus")- p.CopySource = getStringField(rec, row, colIdx, "CopySource")- p.CopyProgress = getStringField(rec, row, colIdx, "CopyProgress")- p.CopyCompletionTime = getTimestampField(rec, row, colIdx, "CopyCompletionTime")- p.CopyStatusDescription = getStringField(rec, row, colIdx, "CopyStatusDescription")- p.DestinationSnapshot = getStringField(rec, row, colIdx, "CopyDestinationSnapshot")- p.ImmutabilityPolicyExpiresOn = getTimestampField(rec, row, colIdx, "ImmutabilityPolicyUntilDate")- p.ImmutabilityPolicyMode = getEnumField[container.ImmutabilityPolicyMode](rec, row, colIdx, "ImmutabilityPolicyMode")- p.LegalHold = getBoolField(rec, row, colIdx, "LegalHold")- p.DeletedTime = getTimestampField(rec, row, colIdx, "DeletedTime")- p.RemainingRetentionDays = getInt32Field(rec, row, colIdx, "RemainingRetentionDays")- p.LastAccessedOn = getTimestampField(rec, row, colIdx, "LastAccessTime")- p.TagCount = getInt32Field(rec, row, colIdx, "TagCount")- p.BlobSequenceNumber = getInt64Field(rec, row, colIdx, "x-ms-blob-sequence-number")- p.ResourceType = getStringField(rec, row, colIdx, "ResourceType")-- // Content-CRC64 is a string field in Arrow but not directly mapped to BlobProperties.- // The field exists in the Arrow schema but has no corresponding field in the Go SDK BlobProperties.- // It is silently ignored for forward compatibility.-}--// Field accessor helpers that safely handle missing columns and null values.--func getStringField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *string {- idx, ok := colIdx[name]- if !ok {- return nil- }- col := rec.Column(idx)- if col.IsNull(row) {- return nil- }- if arr, ok := col.(*array.String); ok {- val := arr.Value(row)- return &val- }- return nil-}--func getBoolField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *bool {- idx, ok := colIdx[name]- if !ok {- return nil- }- col := rec.Column(idx)- if col.IsNull(row) {- return nil- }- if arr, ok := col.(*array.Boolean); ok {- val := arr.Value(row)- return &val- }- return nil-}--func getTimestampField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *time.Time {- idx, ok := colIdx[name]- if !ok {- return nil- }- col := rec.Column(idx)- if col.IsNull(row) {- return nil- }- if arr, ok := col.(*array.Timestamp); ok {- val := arr.Value(row)- // Arrow timestamps are stored as int64 with a unit; convert to time.Time- dt := val.ToTime(arr.DataType().(*arrow.TimestampType).Unit)- return &dt- }- return nil-}--func getInt64Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int64 {- idx, ok := colIdx[name]- if !ok {- return nil- }- col := rec.Column(idx)- if col.IsNull(row) {- return nil- }- if arr, ok := col.(*array.Uint64); ok {- val := int64(arr.Value(row))- return &val- }- if arr, ok := col.(*array.Int64); ok {- val := arr.Value(row)- return &val- }- return nil-}--func getInt32Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int32 {- idx, ok := colIdx[name]- if !ok {- return nil- }- col := rec.Column(idx)- if col.IsNull(row) {- return nil- }- if arr, ok := col.(*array.Uint64); ok {- val := int32(arr.Value(row))- return &val- }- if arr, ok := col.(*array.Int32); ok {- val := arr.Value(row)- return &val- }- return nil-}--func getETagField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *azcore.ETag {- s := getStringField(rec, row, colIdx, name)- if s == nil {- return nil- }- etag := azcore.ETag(*s)- return &etag-}--func getBase64BytesField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) []byte {- s := getStringField(rec, row, colIdx, name)- if s == nil {- return nil- }- decoded, err := base64.StdEncoding.DecodeString(*s)- if err != nil {- return nil- }- return decoded-}--// getEnumField returns a pointer to a string-typed enum value.-type stringEnum interface {- ~string-}--func getEnumField[T stringEnum](rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *T {- s := getStringField(rec, row, colIdx, name)- if s == nil {- return nil- }- val := T(*s)- return &val-}--func getMapField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) map[string]*string {- idx, ok := colIdx[name]- if !ok {- return nil- }- col := rec.Column(idx)- if col.IsNull(row) {- return nil- }- mapArr, ok := col.(*array.Map)- if !ok {- return nil- }-- // Get the offsets for this row's map entries- start := int(mapArr.Offsets()[row])- end := int(mapArr.Offsets()[row+1])- if start == end {- return nil- }-- keys := mapArr.Keys().(*array.String)- values := mapArr.Items().(*array.String)-- result := make(map[string]*string, end-start)- for i := start; i < end; i++ {- k := keys.Value(i)- if values.IsNull(i) {- result[k] = nil- } else {- v := values.Value(i)- result[k] = &v- }- }- return result-}diff --git a/backend/azureblob/arrowlist/arrow_test.go b/backend/azureblob/arrowlist/arrow_test.godeleted file mode 100644index 9c0c24f5d..000000000--- a/backend/azureblob/arrowlist/arrow_test.go+++ /dev/null@@ -1,374 +0,0 @@-//go:build !plan9 && !js--// Copyright (c) Microsoft Corporation. All rights reserved.-// Licensed under the MIT License. See License.txt in the project root for license information.-//-// Adapted from internal/arrow/arrow_test.go in the Azure SDK for Go-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit-// c6fa341ca22b (branch feature/storage/bifrost).--package arrowlist--import (- "bytes"- "io"- "net/http"- "testing"-- "github.com/apache/arrow-go/v18/arrow"- arrowArray "github.com/apache/arrow-go/v18/arrow/array"- "github.com/apache/arrow-go/v18/arrow/ipc"- "github.com/apache/arrow-go/v18/arrow/memory"- "github.com/stretchr/testify/require"-)--// coreSchema returns a basic Arrow schema for testing with common blob fields.-func coreSchema(md *arrow.Metadata) *arrow.Schema {- fields := []arrow.Field{- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},- {Name: "Creation-Time", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},- {Name: "Last-Modified", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},- {Name: "BlobType", Type: arrow.BinaryTypes.String, Nullable: true},- {Name: "Etag", Type: arrow.BinaryTypes.String, Nullable: true},- {Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},- {Name: "Content-Type", Type: arrow.BinaryTypes.String, Nullable: true},- {Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},- {Name: "AccessTier", Type: arrow.BinaryTypes.String, Nullable: true},- {Name: "LeaseState", Type: arrow.BinaryTypes.String, Nullable: true},- {Name: "LeaseStatus", Type: arrow.BinaryTypes.String, Nullable: true},- }- return arrow.NewSchema(fields, md)-}--// buildArrowStream constructs a synthetic Arrow IPC stream from a schema and a populate function.-func buildArrowStream(t *testing.T, schema *arrow.Schema, populate func(builder *arrowArray.RecordBuilder)) []byte {- t.Helper()- alloc := memory.DefaultAllocator-- var buf bytes.Buffer- w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))-- builder := arrowArray.NewRecordBuilder(alloc, schema)- defer builder.Release()-- populate(builder)-- rec := builder.NewRecordBatch()- defer rec.Release()-- require.NoError(t, w.Write(rec))- require.NoError(t, w.Close())-- return buf.Bytes()-}--func makeHTTPResponse(data []byte, headers map[string]string) *http.Response {- h := http.Header{}- h.Set("Content-Type", ArrowContentType)- for k, v := range headers {- h.Set(k, v)- }- return &http.Response{- StatusCode: http.StatusOK,- Header: h,- Body: io.NopCloser(bytes.NewReader(data)),- }-}--func TestHandleFlatListResponse_BasicParsing(t *testing.T) {- md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"marker123", "2"})- schema := coreSchema(&md)-- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {- b.Field(0).(*arrowArray.StringBuilder).Append("blob1.txt")- b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))- b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))- b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")- b.Field(4).(*arrowArray.StringBuilder).Append("0x1234")- b.Field(5).(*arrowArray.Uint64Builder).Append(1024)- b.Field(6).(*arrowArray.StringBuilder).Append("application/octet-stream")- b.Field(7).(*arrowArray.BooleanBuilder).Append(true)- b.Field(8).(*arrowArray.StringBuilder).Append("Hot")- b.Field(9).(*arrowArray.StringBuilder).Append("available")- b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")-- b.Field(0).(*arrowArray.StringBuilder).Append("blob2.txt")- b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))- b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))- b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")- b.Field(4).(*arrowArray.StringBuilder).Append("0x5678")- b.Field(5).(*arrowArray.Uint64Builder).Append(2048)- b.Field(6).(*arrowArray.StringBuilder).Append("text/plain")- b.Field(7).(*arrowArray.BooleanBuilder).Append(false)- b.Field(8).(*arrowArray.StringBuilder).Append("Cool")- b.Field(9).(*arrowArray.StringBuilder).Append("available")- b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")- })-- resp := makeHTTPResponse(data, nil)- result, err := HandleFlatListResponse(resp)- require.NoError(t, err)-- require.NotNil(t, result.NextMarker)- require.Equal(t, "marker123", *result.NextMarker)- require.Len(t, result.Segment.BlobItems, 2)-- item := result.Segment.BlobItems[0]- require.Equal(t, "blob1.txt", *item.Name)- require.Equal(t, "BlockBlob", string(*item.Properties.BlobType))- require.Equal(t, int64(1024), *item.Properties.ContentLength)- require.Equal(t, "Hot", string(*item.Properties.AccessTier))-}--func TestHandleFlatListResponse_EmptyRecordBatch(t *testing.T) {- md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "0"})- schema := coreSchema(&md)-- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {- // No rows added- })-- resp := makeHTTPResponse(data, nil)- result, err := HandleFlatListResponse(resp)- require.NoError(t, err)- require.Nil(t, result.NextMarker)- require.Empty(t, result.Segment.BlobItems)-}--func TestHandleFlatListResponse_NullFields(t *testing.T) {- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})- schema := coreSchema(&md)-- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {- b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")- b.Field(1).(*arrowArray.TimestampBuilder).AppendNull()- b.Field(2).(*arrowArray.TimestampBuilder).AppendNull()- b.Field(3).(*arrowArray.StringBuilder).AppendNull()- b.Field(4).(*arrowArray.StringBuilder).AppendNull()- b.Field(5).(*arrowArray.Uint64Builder).AppendNull()- b.Field(6).(*arrowArray.StringBuilder).AppendNull()- b.Field(7).(*arrowArray.BooleanBuilder).AppendNull()- b.Field(8).(*arrowArray.StringBuilder).AppendNull()- b.Field(9).(*arrowArray.StringBuilder).AppendNull()- b.Field(10).(*arrowArray.StringBuilder).AppendNull()- })-- resp := makeHTTPResponse(data, nil)- result, err := HandleFlatListResponse(resp)- require.NoError(t, err)- require.Len(t, result.Segment.BlobItems, 1)-- item := result.Segment.BlobItems[0]- require.Equal(t, "blob.txt", *item.Name)- require.Nil(t, item.Properties.CreationTime)- require.Nil(t, item.Properties.BlobType)- require.Nil(t, item.Properties.ETag)- require.Nil(t, item.Properties.ContentLength)- require.Nil(t, item.Properties.ContentType)- require.Nil(t, item.Properties.ServerEncrypted)-}--func TestHandleFlatListResponse_UnknownColumnsIgnored(t *testing.T) {- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})- fields := []arrow.Field{- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},- {Name: "FutureField", Type: arrow.BinaryTypes.String, Nullable: true},- }- schema := arrow.NewSchema(fields, &md)-- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {- b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")- b.Field(1).(*arrowArray.StringBuilder).Append("some-future-value")- })-- resp := makeHTTPResponse(data, nil)- result, err := HandleFlatListResponse(resp)- require.NoError(t, err)- require.Len(t, result.Segment.BlobItems, 1)- require.Equal(t, "blob.txt", *result.Segment.BlobItems[0].Name)-}--func TestHandleFlatListResponse_MultiRecordBatch(t *testing.T) {- alloc := memory.DefaultAllocator- md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "3"})- schema := coreSchema(&md)-- var buf bytes.Buffer- w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))-- // First batch: 2 rows- builder := arrowArray.NewRecordBuilder(alloc, schema)- for range 2 {- builder.Field(0).(*arrowArray.StringBuilder).Append("batch1_blob")- for j := 1; j < 11; j++ {- builder.Field(j).AppendNull()- }- }- rec := builder.NewRecordBatch()- require.NoError(t, w.Write(rec))- rec.Release()- builder.Release()-- // Second batch: 1 row- builder = arrowArray.NewRecordBuilder(alloc, schema)- builder.Field(0).(*arrowArray.StringBuilder).Append("batch2_blob")- for j := 1; j < 11; j++ {- builder.Field(j).AppendNull()- }- rec = builder.NewRecordBatch()- require.NoError(t, w.Write(rec))- rec.Release()- builder.Release()-- require.NoError(t, w.Close())-- resp := makeHTTPResponse(buf.Bytes(), nil)- result, err := HandleFlatListResponse(resp)- require.NoError(t, err)- require.Len(t, result.Segment.BlobItems, 3)-}--func TestHandleHierarchyListResponse_SeparatesBlobsAndPrefixes(t *testing.T) {- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})- fields := []arrow.Field{- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},- {Name: "ResourceType", Type: arrow.BinaryTypes.String, Nullable: true},- }- schema := arrow.NewSchema(fields, &md)-- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {- // A regular blob- b.Field(0).(*arrowArray.StringBuilder).Append("folder/file.txt")- b.Field(1).(*arrowArray.StringBuilder).AppendNull()-- // A blob prefix (virtual directory)- b.Field(0).(*arrowArray.StringBuilder).Append("folder/subfolder/")- b.Field(1).(*arrowArray.StringBuilder).Append("blobprefix")- })-- resp := makeHTTPResponse(data, nil)- result, err := HandleHierarchyListResponse(resp)- require.NoError(t, err)-- require.Len(t, result.Segment.BlobItems, 1)- require.Equal(t, "folder/file.txt", *result.Segment.BlobItems[0].Name)-- require.Len(t, result.Segment.BlobPrefixes, 1)- require.Equal(t, "folder/subfolder/", *result.Segment.BlobPrefixes[0].Name)-}--func TestHandleFlatListResponse_ResponseHeaders(t *testing.T) {- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})- fields := []arrow.Field{- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},- }- schema := arrow.NewSchema(fields, &md)-- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {- b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")- })-- resp := makeHTTPResponse(data, map[string]string{- "x-ms-request-id": "req-123",- "x-ms-version": "2026-10-06",- "x-ms-client-request-id": "client-req-456",- "Date": "Mon, 01 Jan 2024 00:00:00 GMT",- })-- result, err := HandleFlatListResponse(resp)- require.NoError(t, err)- require.Equal(t, "req-123", *result.RequestID)- require.Equal(t, "2026-10-06", *result.Version)- require.Equal(t, "client-req-456", *result.ClientRequestID)- require.NotNil(t, result.Date)-}--func TestHandleFlatListResponse_MapFields(t *testing.T) {- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})- fields := []arrow.Field{- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},- {Name: "Metadata", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},- {Name: "Tags", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},- }- schema := arrow.NewSchema(fields, &md)-- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {- b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")-- metaBuilder := b.Field(1).(*arrowArray.MapBuilder)- metaBuilder.Append(true)- metaBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("key1")- metaBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("value1")-- tagBuilder := b.Field(2).(*arrowArray.MapBuilder)- tagBuilder.Append(true)- tagBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("env")- tagBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("prod")- })-- resp := makeHTTPResponse(data, nil)- result, err := HandleFlatListResponse(resp)- require.NoError(t, err)- require.Len(t, result.Segment.BlobItems, 1)-- item := result.Segment.BlobItems[0]- require.NotNil(t, item.Metadata)- require.Equal(t, "value1", *item.Metadata["key1"])-- require.NotNil(t, item.BlobTags)- require.Len(t, item.BlobTags.BlobTagSet, 1)-}--func TestHandleFlatListResponse_BooleanFields(t *testing.T) {- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})- fields := []arrow.Field{- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},- {Name: "Deleted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},- {Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},- {Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},- }- schema := arrow.NewSchema(fields, &md)-- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {- b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")- b.Field(1).(*arrowArray.BooleanBuilder).Append(true)- b.Field(2).(*arrowArray.BooleanBuilder).Append(false)- b.Field(3).(*arrowArray.BooleanBuilder).Append(true)- })-- resp := makeHTTPResponse(data, nil)- result, err := HandleFlatListResponse(resp)- require.NoError(t, err)-- item := result.Segment.BlobItems[0]- require.True(t, *item.Deleted)- require.False(t, *item.IsCurrentVersion)- require.True(t, *item.Properties.ServerEncrypted)-}--func TestHandleFlatListResponse_VersionAndSnapshot(t *testing.T) {- md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})- fields := []arrow.Field{- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},- {Name: "VersionId", Type: arrow.BinaryTypes.String, Nullable: true},- {Name: "Snapshot", Type: arrow.BinaryTypes.String, Nullable: true},- {Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},- }- schema := arrow.NewSchema(fields, &md)-- data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {- b.Field(0).(*arrowArray.StringBuilder).Append("versioned-blob.txt")- b.Field(1).(*arrowArray.StringBuilder).Append("2024-01-01T00:00:00.0000000Z")- b.Field(2).(*arrowArray.StringBuilder).Append("snap-123")- b.Field(3).(*arrowArray.BooleanBuilder).Append(true)- })-- resp := makeHTTPResponse(data, nil)- result, err := HandleFlatListResponse(resp)- require.NoError(t, err)-- item := result.Segment.BlobItems[0]- require.Equal(t, "2024-01-01T00:00:00.0000000Z", *item.VersionID)- require.Equal(t, "snap-123", *item.Snapshot)- require.True(t, *item.IsCurrentVersion)-}diff --git a/backend/azureblob/arrowlist/arrowlist.go b/backend/azureblob/arrowlist/arrowlist.godeleted file mode 100644index a9246adef..000000000--- a/backend/azureblob/arrowlist/arrowlist.go+++ /dev/null@@ -1,259 +0,0 @@-//go:build !plan9 && !js--// Package arrowlist implements the experimental "Blob Listing with Apache-// Arrow" feature on top of the released Azure azblob SDK.-//-// The Azure SDK for Go has support for Arrow listing on the unreleased-// feature/storage/bifrost branch (commit c6fa341ca22b) where the listing-// pagers decode Arrow IPC stream responses into the ordinary ListBlobs-// response types. Until that ships in a tagged azblob release this package-// fills the gap: it exposes the same options and pager interface as the-// experimental SDK, built only on the public API of the released SDK plus-// copies of two small internal auth policies (see sharedkey.go and-// challenge_policy.go).-//-// This package is temporary. When Arrow listing ships in a released azblob-// the whole package should be deleted and its callers pointed back at-// container.Client.NewListBlobsHierarchyPager - the option and response-// types here are deliberately source compatible with the SDK's to make that-// a mechanical change.-package arrowlist--import (- "context"- "errors"- "net/http"- "strconv"- "strings"-- "github.com/Azure/azure-sdk-for-go/sdk/azcore"- "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"- "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"-)--const (- // moduleName and moduleVersion identify this package in the azcore- // telemetry policy's User-Agent fragment.- moduleName = "github.com/rclone/rclone/backend/azureblob/arrowlist"- moduleVersion = "v0.0.1"-- // serviceVersion is the x-ms-version sent with listing requests. Arrow- // responses and the startFrom/endBefore parameters need at least- // 2026-06-06.- serviceVersion = "2026-06-06"-- // tokenScope is the OAuth scope for Azure Storage, as used by the SDK's- // own clients.- tokenScope = "https://storage.azure.com/.default"-)--// ErrEndBeforeXMLFallback is returned by the pager when the server answered-// with XML instead of Arrow while EndBefore was set. The service only honours-// endBefore on the Arrow listing path, so an XML page could silently contain-// blobs outside the requested range and cannot be used.-var ErrEndBeforeXMLFallback = errors.New("arrow listing fell back to XML with endBefore set")--// ListBlobsHierarchyOptions provides the configuration for-// Client.NewListBlobsHierarchyPager.-//-// It mirrors the experimental SDK's container.ListBlobsHierarchyOptions:-// the released SDK's options are embedded (providing Include, Marker,-// MaxResults, Prefix and StartFrom by promotion) with the two fields the-// released SDK lacks added alongside.-type ListBlobsHierarchyOptions struct {- container.ListBlobsHierarchyOptions- // EndBefore limits listing to blobs whose full container path is- // lexically before it (exclusive). Only honoured on Arrow responses.- EndBefore *string- // UseArrowFormat requests the response as an Apache Arrow IPC stream.- UseArrowFormat *bool-}--// ClientOptions contains the optional parameters for client creation.-type ClientOptions struct {- azcore.ClientOptions-}--// Client issues Arrow listing requests to a single container.-type Client struct {- endpoint string- azClient *azcore.Client-}--// newClient creates a Client for the container at containerURL sending-// requests through a pipeline with the given auth policy (which may be nil-// for anonymous or SAS access).-func newClient(containerURL string, authPolicy policy.Policy, options *ClientOptions) (*Client, error) {- if options == nil {- options = &ClientOptions{}- }- plOpts := runtime.PipelineOptions{}- if authPolicy != nil {- plOpts.PerRetry = []policy.Policy{authPolicy}- }- azClient, err := azcore.NewClient(moduleName, moduleVersion, plOpts, &options.ClientOptions)- if err != nil {- return nil, err- }- return &Client{- endpoint: containerURL,- azClient: azClient,- }, nil-}--// NewClient creates a Client authenticating with a token credential.-func NewClient(containerURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {- return newClient(containerURL, NewStorageChallengePolicy(cred, tokenScope, false), options)-}--// NewClientWithNoCredential creates a Client with no authentication, for-// anonymous access or a containerURL carrying a SAS token in its query.-func NewClientWithNoCredential(containerURL string, options *ClientOptions) (*Client, error) {- return newClient(containerURL, nil, options)-}--// NewClientWithSharedKeyCredential creates a Client signing requests with the-// account's shared key.-func NewClientWithSharedKeyCredential(containerURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {- return newClient(containerURL, NewSharedKeyCredPolicy(cred), options)-}--// NewListBlobsHierarchyPager returns a pager over the container's blobs,-// requesting Arrow responses when o.UseArrowFormat is set and returning the-// same response type as container.Client.NewListBlobsHierarchyPager.-//-// If the service does not support Arrow listing (the feature not enabled on-// the account, or an HNS account) it answers with XML, which is decoded-// transparently - unless EndBefore is set, in which case the fetcher returns-// ErrEndBeforeXMLFallback (see there).-func (c *Client) NewListBlobsHierarchyPager(delimiter string, o *ListBlobsHierarchyOptions) *runtime.Pager[container.ListBlobsHierarchyResponse] {- // Copy the options so advancing the Marker doesn't mutate the caller's struct.- opts := ListBlobsHierarchyOptions{}- if o != nil {- opts = *o- }- useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat- return runtime.NewPager(runtime.PagingHandler[container.ListBlobsHierarchyResponse]{- More: func(page container.ListBlobsHierarchyResponse) bool {- return page.NextMarker != nil && len(*page.NextMarker) > 0- },- Fetcher: func(ctx context.Context, page *container.ListBlobsHierarchyResponse) (container.ListBlobsHierarchyResponse, error) {- if page != nil {- opts.Marker = page.NextMarker- }- req, err := c.listCreateRequest(ctx, delimiter, &opts, useArrow)- if err != nil {- return container.ListBlobsHierarchyResponse{}, err- }- resp, err := c.azClient.Pipeline().Do(req)- if err != nil {- return container.ListBlobsHierarchyResponse{}, err- }- if !runtime.HasStatusCode(resp, http.StatusOK) {- return container.ListBlobsHierarchyResponse{}, runtime.NewResponseError(resp)- }- if useArrow && resp.Header.Get("Content-Type") == ArrowContentType {- return HandleHierarchyListResponse(resp)- }- if opts.EndBefore != nil {- return container.ListBlobsHierarchyResponse{}, ErrEndBeforeXMLFallback- }- return handleXMLResponse(resp)- },- })-}--// listCreateRequest builds the ListBlobs request, mirroring the generated-// request builders in the SDK.-func (c *Client) listCreateRequest(ctx context.Context, delimiter string, opts *ListBlobsHierarchyOptions, useArrow bool) (*policy.Request, error) {- req, err := runtime.NewRequest(ctx, http.MethodGet, c.endpoint)- if err != nil {- return nil, err- }- // Reading the existing query first preserves any SAS token in the endpoint.- reqQP := req.Raw().URL.Query()- reqQP.Set("comp", "list")- reqQP.Set("restype", "container")- reqQP.Set("delimiter", delimiter)- if include := formatInclude(opts.Include); include != "" {- reqQP.Set("include", include)- }- if opts.Marker != nil {- reqQP.Set("marker", *opts.Marker)- }- if opts.MaxResults != nil {- reqQP.Set("maxresults", strconv.FormatInt(int64(*opts.MaxResults), 10))- }- if opts.Prefix != nil {- reqQP.Set("prefix", *opts.Prefix)- }- if opts.StartFrom != nil {- reqQP.Set("startFrom", *opts.StartFrom)- }- if useArrow && opts.EndBefore != nil {- reqQP.Set("endBefore", *opts.EndBefore)- }- req.Raw().URL.RawQuery = strings.ReplaceAll(reqQP.Encode(), "+", "%20")- if useArrow {- // Stream the Arrow body rather than buffering it in the pipeline.- runtime.SkipBodyDownload(req)- req.Raw().Header["Accept"] = []string{ArrowAcceptHeader}- } else {- req.Raw().Header["Accept"] = []string{"application/xml"}- }- req.Raw().Header["x-ms-version"] = []string{serviceVersion}- return req, nil-}--// handleXMLResponse decodes an XML ListBlobs response, mirroring the SDK's-// generated ListBlobHierarchySegmentHandleResponse.-func handleXMLResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {- result := container.ListBlobsHierarchyResponse{}- extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)- if err := runtime.UnmarshalAsXML(resp, &result.ListBlobsHierarchySegmentResponse); err != nil {- return container.ListBlobsHierarchyResponse{}, err- }- return result, nil-}--// formatInclude renders the include datasets as the comma separated query-// parameter value, in the same order as the SDK's unexported-// ListBlobsInclude.format.-func formatInclude(l container.ListBlobsInclude) string {- var include []string- if l.Copy {- include = append(include, "copy")- }- if l.Deleted {- include = append(include, "deleted")- }- if l.DeletedWithVersions {- include = append(include, "deletedwithversions")- }- if l.ImmutabilityPolicy {- include = append(include, "immutabilitypolicy")- }- if l.LegalHold {- include = append(include, "legalhold")- }- if l.Metadata {- include = append(include, "metadata")- }- if l.Snapshots {- include = append(include, "snapshots")- }- if l.Tags {- include = append(include, "tags")- }- if l.UncommittedBlobs {- include = append(include, "uncommittedblobs")- }- if l.Versions {- include = append(include, "versions")- }- if l.Permissions {- include = append(include, "permissions")- }- return strings.Join(include, ",")-}diff --git a/backend/azureblob/arrowlist/arrowlist_test.go b/backend/azureblob/arrowlist/arrowlist_test.godeleted file mode 100644index f121aac6b..000000000--- a/backend/azureblob/arrowlist/arrowlist_test.go+++ /dev/null@@ -1,198 +0,0 @@-//go:build !plan9 && !js--package arrowlist--import (- "context"- "encoding/base64"- "net/http"- "net/http/httptest"- "strings"- "testing"-- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"- "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"- "github.com/apache/arrow-go/v18/arrow"- arrowArray "github.com/apache/arrow-go/v18/arrow/array"- "github.com/stretchr/testify/assert"- "github.com/stretchr/testify/require"-)--// arrowPage builds a one row Arrow IPC listing page with the given blob name-// and NextMarker.-func arrowPage(t *testing.T, name, nextMarker string) []byte {- md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{nextMarker, "1"})- fields := []arrow.Field{- {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},- {Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},- }- schema := arrow.NewSchema(fields, &md)- return buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {- b.Field(0).(*arrowArray.StringBuilder).Append(name)- b.Field(1).(*arrowArray.Uint64Builder).Append(42)- })-}--func TestPagerArrow(t *testing.T) {- var requests []*http.Request- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- requests = append(requests, r.Clone(context.Background()))- w.Header().Set("Content-Type", ArrowContentType)- if r.URL.Query().Get("marker") == "" {- _, _ = w.Write(arrowPage(t, "blob1.txt", "marker1"))- } else {- _, _ = w.Write(arrowPage(t, "blob2.txt", ""))- }- }))- defer srv.Close()-- // SAS style query on the endpoint must be preserved in requests.- client, err := NewClientWithNoCredential(srv.URL+"/testcontainer?sv=fakesas", nil)- require.NoError(t, err)-- opts := &ListBlobsHierarchyOptions{- ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{- Include: container.ListBlobsInclude{Metadata: true, Tags: true},- Prefix: new("dir/"),- MaxResults: new(int32(1000)),- StartFrom: new("testcontainer/dir/a"),- },- EndBefore: new("testcontainer/dir/n"),- UseArrowFormat: new(true),- }- pager := client.NewListBlobsHierarchyPager("/", opts)-- var names []string- for pager.More() {- page, err := pager.NextPage(context.Background())- require.NoError(t, err)- assert.Equal(t, ArrowContentType, *page.ContentType)- for _, item := range page.Segment.BlobItems {- names = append(names, *item.Name)- assert.Equal(t, int64(42), *item.Properties.ContentLength)- }- }- assert.Equal(t, []string{"blob1.txt", "blob2.txt"}, names)-- require.Len(t, requests, 2)- q := requests[0].URL.Query()- assert.Equal(t, "list", q.Get("comp"))- assert.Equal(t, "container", q.Get("restype"))- assert.Equal(t, "/", q.Get("delimiter"))- assert.Equal(t, "metadata,tags", q.Get("include"))- assert.Equal(t, "dir/", q.Get("prefix"))- assert.Equal(t, "1000", q.Get("maxresults"))- assert.Equal(t, "testcontainer/dir/a", q.Get("startFrom"))- assert.Equal(t, "testcontainer/dir/n", q.Get("endBefore"))- assert.Equal(t, "fakesas", q.Get("sv"))- assert.Empty(t, q.Get("marker"))- assert.Equal(t, ArrowAcceptHeader, requests[0].Header.Get("Accept"))- assert.Equal(t, serviceVersion, requests[0].Header.Get("x-ms-version"))- assert.Equal(t, "marker1", requests[1].URL.Query().Get("marker"))-- // The caller's options must not have been mutated by paging.- assert.Nil(t, opts.Marker)-}--const xmlListResponse = `<?xml version="1.0" encoding="utf-8"?>-<EnumerationResults ServiceEndpoint="https://example.com/" ContainerName="testcontainer">- <Blobs>- <Blob>- <Name>file.txt</Name>- <Properties>- <Last-Modified>Mon, 01 Jan 2024 00:00:00 GMT</Last-Modified>- <Etag>0x8D1</Etag>- <Content-Length>7</Content-Length>- <BlobType>BlockBlob</BlobType>- </Properties>- </Blob>- <BlobPrefix>- <Name>subdir/</Name>- </BlobPrefix>- </Blobs>- <NextMarker />-</EnumerationResults>`--func TestPagerXMLFallback(t *testing.T) {- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/xml")- _, _ = w.Write([]byte(xmlListResponse))- }))- defer srv.Close()-- client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)- require.NoError(t, err)-- pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{- UseArrowFormat: new(true),- })- page, err := pager.NextPage(context.Background())- require.NoError(t, err)- assert.False(t, pager.More())- assert.Equal(t, "application/xml", *page.ContentType)- require.Len(t, page.Segment.BlobItems, 1)- assert.Equal(t, "file.txt", *page.Segment.BlobItems[0].Name)- assert.Equal(t, int64(7), *page.Segment.BlobItems[0].Properties.ContentLength)- require.Len(t, page.Segment.BlobPrefixes, 1)- assert.Equal(t, "subdir/", *page.Segment.BlobPrefixes[0].Name)-}--func TestPagerXMLFallbackWithEndBefore(t *testing.T) {- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/xml")- _, _ = w.Write([]byte(xmlListResponse))- }))- defer srv.Close()-- client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)- require.NoError(t, err)-- pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{- EndBefore: new("testcontainer/n"),- UseArrowFormat: new(true),- })- _, err = pager.NextPage(context.Background())- require.ErrorIs(t, err, ErrEndBeforeXMLFallback)-}--func TestPagerSharedKeyAuth(t *testing.T) {- var gotAuth, gotDate string- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- gotAuth = r.Header.Get("Authorization")- gotDate = r.Header.Get("x-ms-date")- w.Header().Set("Content-Type", ArrowContentType)- _, _ = w.Write(arrowPage(t, "blob.txt", ""))- }))- defer srv.Close()-- cred, err := NewSharedKeyCredential("testaccount", base64.StdEncoding.EncodeToString([]byte("testkey")))- require.NoError(t, err)- client, err := NewClientWithSharedKeyCredential(srv.URL+"/testcontainer", cred, nil)- require.NoError(t, err)-- pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{- UseArrowFormat: new(true),- })- _, err = pager.NextPage(context.Background())- require.NoError(t, err)- assert.True(t, strings.HasPrefix(gotAuth, "SharedKey testaccount:"), "Authorization = %q", gotAuth)- assert.NotEmpty(t, gotDate)-}--func TestPagerResponseError(t *testing.T) {- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("x-ms-error-code", string(bloberror.ContainerNotFound))- w.WriteHeader(http.StatusNotFound)- }))- defer srv.Close()-- client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)- require.NoError(t, err)-- pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{- UseArrowFormat: new(true),- })- _, err = pager.NextPage(context.Background())- require.Error(t, err)- assert.True(t, bloberror.HasCode(err, bloberror.ContainerNotFound), "expected ContainerNotFound, got %v", err)-}diff --git a/backend/azureblob/arrowlist/challenge_policy.go b/backend/azureblob/arrowlist/challenge_policy.godeleted file mode 100644index 90699c350..000000000--- a/backend/azureblob/arrowlist/challenge_policy.go+++ /dev/null@@ -1,121 +0,0 @@-//go:build !plan9 && !js--// Copyright (c) Microsoft Corporation. All rights reserved.-// Licensed under the MIT License. See License.txt in the project root for license information.-//-// Adapted from internal/shared/challenge_policy.go in the Azure SDK for Go-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0. The-// released SDK does not export its storage bearer-challenge policy, which is-// needed for token authentication with the same semantics as the SDK's own-// clients.--package arrowlist--import (- "errors"- "net/http"- "strings"-- "github.com/Azure/azure-sdk-for-go/sdk/azcore"- "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"- "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"-)--type storageAuthorizer struct {- scopes []string- tenantID string-}--// NewStorageChallengePolicy returns a bearer token policy for cred which-// handles the storage service's authentication challenges.-func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy {- s := storageAuthorizer{scopes: []string{audience}}- return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{- AuthorizationHandler: policy.AuthorizationHandler{- OnRequest: s.onRequest,- OnChallenge: s.onChallenge,- },- InsecureAllowCredentialWithHTTP: allowHTTP,- })-}--func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error {- return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})-}--func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error {- // parse the challenge- err := s.parseChallenge(resp)- if err != nil {- return err- }- // TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841- return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})-}--type challengePolicyError struct {- err error-}--func (c *challengePolicyError) Error() string {- return c.err.Error()-}--func (*challengePolicyError) NonRetriable() {- // marker method-}--func (c *challengePolicyError) Unwrap() error {- return c.err-}--// parses Tenant ID from auth challenge-// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize-func parseTenant(url string) string {- if url == "" {- return ""- }- parts := strings.Split(url, "/")- if len(parts) >= 3 {- tenant := parts[3]- tenant = strings.ReplaceAll(tenant, ",", "")- return tenant- }- return ""-}--func (s *storageAuthorizer) parseChallenge(resp *http.Response) error {- authHeader := resp.Header.Get("WWW-Authenticate")- if authHeader == "" {- return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")}- }-- // Strip down to auth and resource- // Format is "Bearer authorization_uri=\"<site>\" resource_id=\"<site>\""- authHeader = strings.ReplaceAll(authHeader, "Bearer ", "")-- parts := strings.Split(authHeader, " ")-- vals := map[string]string{}- for _, part := range parts {- subParts := strings.Split(part, "=")- if len(subParts) == 2 {- stripped := strings.ReplaceAll(subParts[1], "\"", "")- stripped = strings.TrimSuffix(stripped, ",")- vals[subParts[0]] = stripped- }- }-- s.tenantID = parseTenant(vals["authorization_uri"])-- scope := vals["resource_id"]- if scope == "" {- return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")}- }-- if !strings.HasSuffix(scope, "/.default") {- scope += "/.default"- }- s.scopes = []string{scope}- return nil-}diff --git a/backend/azureblob/arrowlist/sharedkey.go b/backend/azureblob/arrowlist/sharedkey.godeleted file mode 100644index 4044d0265..000000000--- a/backend/azureblob/arrowlist/sharedkey.go+++ /dev/null@@ -1,318 +0,0 @@-//go:build !plan9 && !js--// Copyright (c) Microsoft Corporation. All rights reserved.-// Licensed under the MIT License. See License.txt in the project root for license information.-//-// Adapted from internal/exported/shared_key_credential.go in the Azure SDK-// for Go github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0,-// with the internal header constants inlined. The released SDK does not-// export its shared key signing policy, so a copy is needed to sign the-// Arrow listing requests this package sends through its own pipeline.--package arrowlist--import (- "bytes"- "crypto/hmac"- "crypto/sha256"- "encoding/base64"- "fmt"- "net/http"- "net/url"- "sort"- "strings"- "sync/atomic"- "time"-- "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"- "github.com/rclone/rclone/fs"-)--// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the-// storage account's name and either its primary or secondary key.-func NewSharedKeyCredential(accountName string, accountKey string) (*SharedKeyCredential, error) {- c := SharedKeyCredential{accountName: accountName}- if err := c.SetAccountKey(accountKey); err != nil {- return nil, err- }- return &c, nil-}--// SharedKeyCredential contains an account's name and its primary or secondary key.-type SharedKeyCredential struct {- // Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only- accountName string- accountKey atomic.Value // []byte-}--// AccountName returns the Storage account's name.-func (c *SharedKeyCredential) AccountName() string {- return c.accountName-}--// SetAccountKey replaces the existing account key with the specified account key.-func (c *SharedKeyCredential) SetAccountKey(accountKey string) error {- _bytes, err := base64.StdEncoding.DecodeString(accountKey)- if err != nil {- return fmt.Errorf("decode account key: %w", err)- }- c.accountKey.Store(_bytes)- return nil-}--// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.-func (c *SharedKeyCredential) computeHMACSHA256(message string) (string, error) {- h := hmac.New(sha256.New, c.accountKey.Load().([]byte))- _, err := h.Write([]byte(message))- return base64.StdEncoding.EncodeToString(h.Sum(nil)), err-}--func (c *SharedKeyCredential) buildStringToSign(req *http.Request) (string, error) {- // https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services- headers := req.Header- contentLength := getHeader("Content-Length", headers)- if contentLength == "0" {- contentLength = ""- }-- canonicalizedResource, err := c.buildCanonicalizedResource(req.URL)- if err != nil {- return "", err- }-- stringToSign := strings.Join([]string{- req.Method,- getHeader("Content-Encoding", headers),- getHeader("Content-Language", headers),- contentLength,- getHeader("Content-MD5", headers),- getHeader("Content-Type", headers),- "", // Empty date because x-ms-date is expected (as per web page above)- getHeader("If-Modified-Since", headers),- getHeader("If-Match", headers),- getHeader("If-None-Match", headers),- getHeader("If-Unmodified-Since", headers),- getHeader("Range", headers),- c.buildCanonicalizedHeader(headers),- canonicalizedResource,- }, "\n")- return stringToSign, nil-}--func getHeader(key string, headers map[string][]string) string {- if headers == nil {- return ""- }- if v, ok := headers[key]; ok {- if len(v) > 0 {- return v[0]- }- }-- return ""-}--func getWeightTables() [][]int {- tableLv0 := [...]int{- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,- 0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0xd03, 0xd1a, 0xd1c, 0xd1e,- 0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,- 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,- 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,- 0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,- 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,- 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,- }- tableLv2 := [...]int{- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,- 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,- }- tables := [][]int{tableLv0[:], tableLv2[:]}- return tables-}--// NewHeaderStringComparer performs a multi-level, weight-based comparison of two strings-func compareHeaders(lhs, rhs string, tables [][]int) int {- currLevel, i, j := 0, 0, 0- n := len(tables)- lhsLen := len(lhs)- rhsLen := len(rhs)-- for currLevel < n {- if currLevel == (n-1) && i != j {- if i > j {- return -1- }- if i < j {- return 1- }- return 0- }-- var w1, w2 int-- // Check bounds before accessing lhs[i]- if i < lhsLen {- w1 = tables[currLevel][lhs[i]]- } else {- w1 = 0x1- }-- // Check bounds before accessing rhs[j]- if j < rhsLen {- w2 = tables[currLevel][rhs[j]]- } else {- w2 = 0x1- }-- if w1 == 0x1 && w2 == 0x1 {- i = 0- j = 0- currLevel++- } else if w1 == w2 {- i++- j++- } else if w1 == 0 {- i++- } else if w2 == 0 {- j++- } else {- if w1 < w2 {- return -1- }- if w1 > w2 {- return 1- }- return 0- }- }- return 0-}--func (c *SharedKeyCredential) buildCanonicalizedHeader(headers http.Header) string {- cm := map[string][]string{}- for k, v := range headers {- headerName := strings.TrimSpace(strings.ToLower(k))- if strings.HasPrefix(headerName, "x-ms-") {- cm[headerName] = v // NOTE: the value must not have any whitespace around it.- }- }- if len(cm) == 0 {- return ""- }-- keys := make([]string, 0, len(cm))- for key := range cm {- keys = append(keys, key)- }- tables := getWeightTables()- // Sort the keys using the custom comparator- sort.Slice(keys, func(i, j int) bool {- return compareHeaders(keys[i], keys[j], tables) < 0- })- ch := bytes.NewBufferString("")- for i, key := range keys {- if i > 0 {- ch.WriteRune('\n')- }- ch.WriteString(key)- ch.WriteRune(':')- ch.WriteString(strings.Join(cm[key], ","))- }- return ch.String()-}--func (c *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {- // https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services- cr := bytes.NewBufferString("/")- cr.WriteString(c.accountName)-- if len(u.Path) > 0 {- // Any portion of the CanonicalizedResource string that is derived from- // the resource's URI should be encoded exactly as it is in the URI.- // -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx- cr.WriteString(u.EscapedPath())- } else {- // a slash is required to indicate the root path- cr.WriteString("/")- }-- // params is a map[string][]string; param name is key; params values is []string- params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values- if err != nil {- return "", fmt.Errorf("failed to parse query params: %w", err)- }-- if len(params) > 0 { // There is at least 1 query parameter- var paramNames []string // We use this to sort the parameter key names- for paramName := range params {- paramNames = append(paramNames, paramName) // paramNames must be lowercase- }- sort.Strings(paramNames)-- for _, paramName := range paramNames {- paramValues := params[paramName]- sort.Strings(paramValues)-- // Join the sorted key values separated by ','- // Then prepend "keyName:"; then add this string to the buffer- cr.WriteString("\n" + strings.ToLower(paramName) + ":" + strings.Join(paramValues, ","))- }- }- return cr.String(), nil-}--// ComputeHMACSHA256 is a helper for computing the signed string outside of this package.-func ComputeHMACSHA256(cred *SharedKeyCredential, message string) (string, error) {- return cred.computeHMACSHA256(message)-}--// SharedKeyCredPolicy is a pipeline policy that signs each request with the-// account's shared key.-type SharedKeyCredPolicy struct {- cred *SharedKeyCredential-}--// NewSharedKeyCredPolicy creates a SharedKeyCredPolicy signing with cred.-func NewSharedKeyCredPolicy(cred *SharedKeyCredential) *SharedKeyCredPolicy {- return &SharedKeyCredPolicy{cred: cred}-}--// Do signs the request and sends it via the next policy in the pipeline.-func (s *SharedKeyCredPolicy) Do(req *policy.Request) (*http.Response, error) {- // skip adding the authorization header if no SharedKeyCredential was provided.- // this prevents a panic that might be hard to diagnose and allows testing- // against http endpoints that don't require authentication.- if s.cred == nil {- return req.Next()- }-- if d := getHeader("x-ms-date", req.Raw().Header); d == "" {- req.Raw().Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))- }- stringToSign, err := s.cred.buildStringToSign(req.Raw())- if err != nil {- return nil, err- }- signature, err := s.cred.computeHMACSHA256(stringToSign)- if err != nil {- return nil, err- }- authHeader := strings.Join([]string{"SharedKey ", s.cred.AccountName(), ":", signature}, "")- req.Raw().Header.Set("Authorization", authHeader)-- response, err := req.Next()- if err != nil && response != nil && response.StatusCode == http.StatusForbidden {- // Service failed to authenticate request, log it- fs.Debugf(nil, "azureblob arrowlist: HTTP Forbidden status, String-to-Sign: %q", stringToSign)- }- return response, err-}diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.goindex 642d34d2e..77ecec922 100644--- a/backend/azureblob/azureblob.go+++ b/backend/azureblob/azureblob.go@@ -26,14 +26,12 @@ import ("github.com/Azure/azure-sdk-for-go/sdk/azcore""github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"- "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime""github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob""github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror""github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob""github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container""github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas""github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"- "github.com/rclone/rclone/backend/azureblob/arrowlist""github.com/rclone/rclone/backend/azureblob/auth""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/chunksize"@@ -241,40 +239,38 @@ avoid the time out.`,Advanced: true,}, {Name: "use_arrow_list",- Help: `Use the experimental Apache Arrow listing format.+ Help: `Use the Apache Arrow listing format.If set, directory listings are fetched using the ListBlobs Apache-Arrow response format instead of XML. This can be faster for very-large containers.--This is EXPERIMENTAL and requires the "Blob Listing with Apache Arrow"-preview feature to be enabled on the storage account. It is NOT-supported on accounts with a hierarchical namespace (ADLS Gen2) --those return a 409 error. If the feature is not enabled the server-returns XML and the listing transparently falls back to the normal XML-path (logged at debug level).--Not supported with connection_string auth - falls back to normal-listing.`,+Arrow response format instead of XML. Arrow responses are smaller and+much cheaper to parse, making listings of large containers several+times faster. Combine with "list_parallelism" for the biggest gains.++"Blob Listing with Apache Arrow" is in public preview at Microsoft and+is only supported on flat namespace accounts. On accounts with a+hierarchical namespace (ADLS Gen2), or where the feature is otherwise+unavailable, the server returns XML and the listing transparently+falls back to the normal XML path (logged at debug level).`,Default: false,Advanced: true,- Hide: fs.OptionHideBoth,}, {Name: "list_parallelism",Help: `Number of parallel shards to list a directory with.-EXPERIMENTAL. If set greater than 1, the blob name keyspace of each-directory is split into this many ranges which are listed concurrently-using the Arrow startFrom/endBefore range parameters. This can-dramatically speed up listing containers with millions of objects, for-both recursive (ListR) and single directory listings.+If set greater than 1, the blob name keyspace of each directory is+split into ranges which are listed concurrently using the Arrow+startFrom/endBefore range parameters. This can dramatically speed up+listing containers with millions of objects, for both recursive+(ListR) and single directory listings. Speed keeps improving up to a+parallelism of around 30.This has no effect unless "use_arrow_list" is also set, as Arrow is the-only listing path that supports server-side name ranges. The default of-0 (or 1) lists sequentially.`,+only listing path that supports server-side name ranges. If the+account does not support range listing (e.g. it has a hierarchical+namespace) the listing falls back to sequential. The default of 0 (or+1) lists sequentially.`,Default: 0,Advanced: true,- Hide: fs.OptionHideBoth,}, {Name: "access_tier",Help: `Access tier of blob: hot, cool, cold or archive.@@ -449,10 +445,8 @@ type Fs struct {opt Options // parsed config optionsci *fs.ConfigInfo // global configfeatures *fs.Features // optional features- cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcache and arrowCntSVCcache+ cntSVCcacheMu sync.Mutex // mutex to protect cntSVCcachecntSVCcache map[string]*container.Client // reference to containerClient per container- arrowCntSVCcache map[string]*arrowlist.Client // reference to arrowlist client per container- arrowClientOpts *arrowlist.ClientOptions // client options for the arrowlist clientsarrowXMLFallback atomic.Bool // set once the server answers XML so parallel listing is not retriedsvc *service.Client // client to access azblobcred azcore.TokenCredential // how to generate tokens (may be nil)@@ -670,15 +664,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eci := fs.GetConfig(ctx)f := &Fs{- name: name,- opt: *opt,- ci: ci,- pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),- uploadToken: pacer.NewTokenDispenser(ci.Transfers),- copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),- cache: bucket.NewCache(),- cntSVCcache: make(map[string]*container.Client, 1),- arrowCntSVCcache: make(map[string]*arrowlist.Client, 1),+ name: name,+ opt: *opt,+ ci: ci,+ pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),+ uploadToken: pacer.NewTokenDispenser(ci.Transfers),+ copyToken: pacer.NewTokenDispenser(opt.CopyTotalConcurrency),+ cache: bucket.NewCache(),+ cntSVCcache: make(map[string]*container.Client, 1),}f.publicAccess = container.PublicAccessType(opt.PublicAccess)f.setRoot(root)@@ -725,15 +718,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, ef.usingSharedKeyCred = res.UsingSharedKeyCredf.anonymous = res.Anonymous- // Client options for the arrowlist clients, using the same transport and- // gzip policy as the SDK clients built above.- f.arrowClientOpts = &arrowlist.ClientOptions{- ClientOptions: azcore.ClientOptions{- Transport: auth.Transporter(ctx),- PerCallPolicies: []policy.Policy{setAcceptEncodingGzip{}},- },- }-// if using Container level SAS put the container client into the cacheif opt.SASURL != "" && res.Container != "" {_ = f.cntSVC(res.Container)@@ -772,44 +756,6 @@ func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {return containerClient}-// errArrowAuthUnsupported is returned by arrowCntSVC when the configured-// credentials can't be used for Arrow listing.-var errArrowAuthUnsupported = errors.New("credentials not supported for Arrow listing (connection_string auth is not supported)")--// return the arrowlist client for the container passed in-//-// Returns errArrowAuthUnsupported if the configured credentials can't be-// reused for the arrowlist client's pipeline.-func (f *Fs) arrowCntSVC(containerName string) (client *arrowlist.Client, err error) {- // The container URL includes any SAS token in its query- url := f.cntSVC(containerName).URL()- f.cntSVCcacheMu.Lock()- defer f.cntSVCcacheMu.Unlock()- if client, ok := f.arrowCntSVCcache[containerName]; ok {- return client, nil- }- switch {- case f.usingSharedKeyCred:- // Covers account+key and the emulator (auth fills in Account/Key)- var cred *arrowlist.SharedKeyCredential- cred, err = arrowlist.NewSharedKeyCredential(f.opt.Account, f.opt.Key)- if err == nil {- client, err = arrowlist.NewClientWithSharedKeyCredential(url, cred, f.arrowClientOpts)- }- case f.cred != nil:- client, err = arrowlist.NewClient(url, f.cred, f.arrowClientOpts)- case f.anonymous || f.opt.SASURL != "":- client, err = arrowlist.NewClientWithNoCredential(url, f.arrowClientOpts)- default:- return nil, errArrowAuthUnsupported- }- if err != nil {- return nil, err- }- f.arrowCntSVCcache[containerName] = client- return client, nil-}-// Return an Object from a path//// If it can't be found it returns the error fs.ErrorObjectNotFound.@@ -1186,27 +1132,25 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,delimiter = "/"}- opts := &arrowlist.ListBlobsHierarchyOptions{- ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{- // Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool- Include: container.ListBlobsInclude{- Copy: false,- Metadata: true,- Snapshots: false,- UncommittedBlobs: false,- Deleted: false,- },- Prefix: &directory,- MaxResults: &maxResults,+ opts := &container.ListBlobsHierarchyOptions{+ // Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool+ Include: container.ListBlobsInclude{+ Copy: false,+ Metadata: true,+ Snapshots: false,+ UncommittedBlobs: false,+ Deleted: false,},+ Prefix: &directory,+ MaxResults: &maxResults,}- // Experimental: request the Apache Arrow listing format. The arrowlist- // pager requests an Arrow IPC stream and decodes it, falling back to XML- // if the account doesn't have Arrow listing enabled. Skip the- // maxResults==1 probe (isEmpty) which doesn't benefit.+ // Request the Apache Arrow listing format. The SDK pager requests an+ // Arrow IPC stream and decodes it, falling back to XML if the account+ // doesn't have Arrow listing enabled. Skip the maxResults==1 probe+ // (isEmpty) which doesn't benefit.useArrow := f.opt.UseArrowList && maxResults != 1if useArrow {- opts.UseArrowFormat = new(true)+ opts.ResponseFormat = container.StorageResponseFormatArrow}var foundItems int@@ -1237,25 +1181,11 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,// listBlobsPager runs the hierarchy listing described by opts, calling fn for// each blob and subdirectory, and returns the number of raw items seen.// delimiter selects flat (recurse) vs hierarchical listing. If opts requests-// the Apache Arrow format the listing goes through the arrowlist pager-// (falling back to the SDK pager if the credentials don't support it); if the-// service then answers with XML (Arrow listing not enabled) a debug message-// is logged.-func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {- useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat- var pager *runtime.Pager[container.ListBlobsHierarchyResponse]- if useArrow {- arrowSVC, err := f.arrowCntSVC(containerName)- if err != nil {- fs.Debugf(f, "Not using Arrow listing: %v", err)- useArrow = false- } else {- pager = arrowSVC.NewListBlobsHierarchyPager(delimiter, opts)- }- }- if pager == nil {- pager = f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &opts.ListBlobsHierarchyOptions)- }+// the Apache Arrow format and the service answers with XML (Arrow listing+// not enabled) a debug message is logged.+func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {+ useArrow := opts.ResponseFormat == container.StorageResponseFormatArrow+ pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, opts)checkedArrow := falsefor pager.More() {var response container.ListBlobsHierarchyResponse@@ -1271,7 +1201,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi// but not accelerated.if useArrow && !checkedArrow && err == nil {checkedArrow = true- if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowlist.ArrowContentType) {+ if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowContentType) {fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")}}diff --git a/go.mod b/go.modindex d344a18d3..cbdca57db 100644--- a/go.mod+++ b/go.mod@@ -6,9 +6,9 @@ godebug x509negativeserial=1require (bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5- github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0+ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0- github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0+ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0github.com/Azure/go-ntlmssp v0.1.1github.com/FilenCloudDienste/filen-sdk-go v0.0.39@@ -21,7 +21,7 @@ require (github.com/adrg/xdg v0.5.3github.com/anacrolix/dms v1.7.2github.com/anacrolix/log v0.17.0- github.com/apache/arrow-go/v18 v18.7.0+ github.com/apache/arrow-go/v18 v18.7.0 // indirectgithub.com/atotto/clipboard v0.1.4github.com/aws/aws-sdk-go-v2 v1.43.7github.com/aws/aws-sdk-go-v2/config v1.32.30@@ -52,7 +52,7 @@ require (github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bbgithub.com/josephspurrier/goversioninfo v1.7.0github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004- github.com/klauspost/compress v1.19.0+ github.com/klauspost/compress v1.19.2github.com/koofr/go-httpclient v0.0.0-20240520111329-e20f8f203988github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6github.com/lanrat/extsort v1.4.2@@ -82,7 +82,7 @@ require (github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966github.com/spf13/cobra v1.10.2github.com/spf13/pflag v1.0.10- github.com/stretchr/testify v1.11.1+ github.com/stretchr/testify v1.12.1github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04github.com/unknwon/goconfig v1.0.0github.com/willscott/go-nfs v0.0.4@@ -115,7 +115,7 @@ require (cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirectcloud.google.com/go/compute/metadata v0.9.0 // indirectgithub.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect- github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect+ github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirectgithub.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirectgithub.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirectgithub.com/ProtonMail/go-srp v0.0.7 // indirect@@ -157,7 +157,6 @@ require (github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirectgithub.com/creasty/defaults v1.8.0 // indirectgithub.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect- github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirectgithub.com/dromara/dongle v1.0.1 // indirectgithub.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirectgithub.com/dustin/go-humanize v1.0.1 // indirect@@ -222,7 +221,7 @@ require (github.com/panjf2000/ants/v2 v2.12.1 // indirectgithub.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirectgithub.com/philhofer/fwd v1.2.0 // indirect- github.com/pierrec/lz4/v4 v4.1.27 // indirect+ github.com/pierrec/lz4/v4 v4.1.28 // indirectgithub.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirectgithub.com/pkg/errors v0.9.1 // indirectgithub.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect@@ -258,11 +257,11 @@ require (go.opentelemetry.io/otel/metric v1.45.0 // indirectgo.opentelemetry.io/otel/trace v1.45.0 // indirectgo.yaml.in/yaml/v2 v2.4.4 // indirect- go.yaml.in/yaml/v3 v3.0.4 // indirect+ go.yaml.in/yaml/v3 v3.0.5 // indirectgo4.org v0.0.0-20260112195520-a5071408f32f // indirect- golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 // indirect+ golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 // indirectgolang.org/x/image v0.45.0 // indirect- golang.org/x/tools v0.48.0 // indirect+ golang.org/x/tools v0.49.0 // indirectgoogle.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirectgoogle.golang.org/grpc v1.85.0-dev.0.20260825072537-93e31b48545e // indirectgoogle.golang.org/protobuf v1.36.12 // indirectdiff --git a/go.sum b/go.sumindex f27afaefd..9fc4ff90e 100644--- a/go.sum+++ b/go.sum@@ -6,8 +6,8 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIicloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE=-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM=+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo=+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU=github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU=github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA=github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs=@@ -16,16 +16,16 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6Xugithub.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw=-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y=+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1 h1:gkBLVmB3Z/HnGP/Jo4o12/RDpi0agnKav6sCKsX5Vu0=+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.1/go.mod h1:e3/1P5K+jIUi9JevDRklq/tFeTvbBb75bNAjU4xd31w=github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0 h1:cuiKf1UVyWHu+XSQghPZR/qEF43JIcuk2CDqMlPiT6M=github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.7.0/go.mod h1:9JSyvgXLPAOC4jfhgZg58XeU2FJHsGmxuYSclCFQ4ZY=github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=-github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=-github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=+github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI=+github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=github.com/FilenCloudDienste/filen-sdk-go v0.0.39 h1:tgV5jYL6dsXop9TpDTIQU6UwJjws122HrwskaEE/igY=github.com/FilenCloudDienste/filen-sdk-go v0.0.39/go.mod h1:0cBhKXQg49XbKZZfk5TCDa3sVLP+xMxZTWL+7KY0XR0=github.com/Files-com/files-sdk-go/v3 v3.3.194 h1:dtOFxSTWWRpkmvXa6ycNiw8dVDu1wkgzcXyVV1VafNc=@@ -184,8 +184,6 @@ github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 h1:tLWBZgPggithub.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA=github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=github.com/diskfs/go-diskfs v1.9.4 h1:0j2d7eG4IjyxL6+ChWbDPocdBCF6HQ4HBWU2WDYWVnc=github.com/diskfs/go-diskfs v1.9.4/go.mod h1:TePJORO83Adh5pb2SqsxAwaP0fofFxKLkxctiS/9OQc=github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=@@ -357,8 +355,8 @@ github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwgithub.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=-github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=-github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=+github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=+github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=@@ -458,8 +456,8 @@ github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw=github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI=github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=-github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk=-github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=+github.com/pierrec/lz4/v4 v4.1.28 h1:pPEPwRJ4kybBTfGt28q7lQsRJQHhC08axprdLD5Ppio=+github.com/pierrec/lz4/v4 v4.1.28/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=@@ -552,8 +550,9 @@ github.com/stangelandcl/ppmd v0.1.1/go.mod h1:Rrv7M+/2P5jYr/GMLhBl7Ug3uJ1bUiVzr5github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=-github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=+github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=+github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=@@ -564,8 +563,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=+github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=+github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04 h1:s30A8dMuZ55lUOi5xUTh1hlfLqHpVBsRMfpk2iKCayk=github.com/t3rm1n4l/go-mega v0.0.0-20260717075258-c6acd6a5bd04/go.mod h1:BF/l2jNyK+2h/BJZ7VLMAz6m/IWjA2F67gTjV1C/+Bo=github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=@@ -641,8 +641,9 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=+go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=+go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw=go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0=goftp.io/server/v2 v2.0.3 h1:iz6Gxj7f2SFQVxrj0s1is+gueE6O9yTc+Ab0vtQ6Zn4=@@ -660,8 +661,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=-golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g=-golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q=+golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297 h1:YXnL44eJ77R+ji4/ooy8UsXIhz+lbi2Qgdlc8iRN0gY=+golang.org/x/exp v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:Mkmymgv+uMpSQ/XxJ/7GpdrdYoqm3u72jEbpCLiJmNk=golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=golang.org/x/mobile v0.0.0-20260709172247-6129f5bee9d5 h1:Mn1OzFmF0ZKX/ZayHz/UdnWHufPp1wlD9lZ5U8LRDFY=@@ -744,8 +745,8 @@ golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fgolang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=-golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=-golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=+golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=+golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=