"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
azureblob: add arrowlist package for Apache Arrow blob listing
Nick Craig-Wood267f1560d2d7
diff --git a/backend/azureblob/arrowlist/arrow.go b/backend/azureblob/arrowlist/arrow.gonew file mode 100644index 000000000..f5b0e7b5d--- /dev/null+++ b/backend/azureblob/arrowlist/arrow.go@@ -0,0 +1,421 @@+//go:build !plan9 && !solaris && !js++// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License. See License.txt in the project root for license information.+//+// Adapted from internal/arrow/arrow.go in the Azure SDK for Go+// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit+// c6fa341ca22b (branch feature/storage/bifrost), retyped onto the SDK's+// public type aliases so it compiles against the released azblob module.++package arrowlist++import (+ "encoding/base64"+ "fmt"+ "io"+ "net/http"+ "strconv"+ "time"++ "github.com/Azure/azure-sdk-for-go/sdk/azcore"+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"+ "github.com/apache/arrow-go/v18/arrow"+ "github.com/apache/arrow-go/v18/arrow/array"+ "github.com/apache/arrow-go/v18/arrow/ipc"+ "github.com/apache/arrow-go/v18/arrow/memory"+)++const (+ // ArrowContentType is the Content-Type for Apache Arrow IPC stream responses.+ ArrowContentType = "application/vnd.apache.arrow.stream"++ // ArrowAcceptHeader is the Accept header value to request Arrow format.+ ArrowAcceptHeader = ArrowContentType++ // resourceTypeBlobPrefix is the ResourceType value that identifies a virtual directory prefix+ // in hierarchy listing responses. In XML, prefixes are separate <BlobPrefix> elements;+ // in Arrow, all rows share the same schema and are distinguished by this field value.+ resourceTypeBlobPrefix = "blobprefix"+)++// HandleFlatListResponse parses an Arrow IPC stream response into a ContainerClientListBlobFlatSegmentResponse.+func HandleFlatListResponse(resp *http.Response) (container.ListBlobsFlatResponse, error) {+ result := container.ListBlobsFlatResponse{}++ // Extract response headers+ extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)++ // Parse Arrow IPC stream+ items, nextMarker, err := parseArrowStream(resp.Body)+ if err != nil {+ return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)+ }++ result.ListBlobsFlatSegmentResponse = container.ListBlobsFlatSegmentResponse{+ Segment: &container.BlobFlatListSegment{+ BlobItems: items,+ },+ NextMarker: nextMarker,+ }++ return result, nil+}++// HandleHierarchyListResponse parses an Arrow IPC stream response into a ContainerClientListBlobHierarchySegmentResponse.+func HandleHierarchyListResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {+ result := container.ListBlobsHierarchyResponse{}++ extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)++ items, nextMarker, err := parseArrowStream(resp.Body)+ if err != nil {+ return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)+ }++ // Separate blob items from blob prefixes based on ResourceType field.+ var blobItems []*container.BlobItem+ var blobPrefixes []*container.BlobPrefix+ for _, item := range items {+ if item.Properties != nil && item.Properties.ResourceType != nil && *item.Properties.ResourceType == resourceTypeBlobPrefix {+ blobPrefixes = append(blobPrefixes, &container.BlobPrefix{+ Name: item.Name,+ Properties: item.Properties,+ })+ } else {+ blobItems = append(blobItems, item)+ }+ }++ result.ListBlobsHierarchySegmentResponse = container.ListBlobsHierarchySegmentResponse{+ Segment: &container.BlobHierarchyListSegment{+ BlobItems: blobItems,+ BlobPrefixes: blobPrefixes,+ },+ NextMarker: nextMarker,+ }++ return result, nil+}++func extractResponseHeaders(resp *http.Response, clientRequestID, contentType **string, date **time.Time, requestID, version **string) {+ if val := resp.Header.Get("x-ms-client-request-id"); val != "" {+ *clientRequestID = &val+ }+ if val := resp.Header.Get("Content-Type"); val != "" {+ *contentType = &val+ }+ if val := resp.Header.Get("Date"); val != "" {+ if t, err := time.Parse(time.RFC1123, val); err == nil {+ *date = &t+ }+ }+ if val := resp.Header.Get("x-ms-request-id"); val != "" {+ *requestID = &val+ }+ if val := resp.Header.Get("x-ms-version"); val != "" {+ *version = &val+ }+}++// parseArrowStream reads an Arrow IPC stream and converts it to BlobItems.+func parseArrowStream(body io.Reader) ([]*container.BlobItem, *string, error) {+ reader, err := ipc.NewReader(body, ipc.WithAllocator(memory.DefaultAllocator))+ if err != nil {+ return nil, nil, fmt.Errorf("failed to create Arrow IPC reader: %w", err)+ }+ defer reader.Release()++ // Extract NextMarker from schema metadata+ var nextMarker *string+ var numRecords int+ if md := reader.Schema().Metadata(); md.Len() > 0 {+ if idx := md.FindKey("NextMarker"); idx >= 0 {+ val := md.Values()[idx]+ if val != "" {+ nextMarker = &val+ }+ }+ if idx := md.FindKey("NumberOfRecords"); idx >= 0 {+ if n, err := strconv.Atoi(md.Values()[idx]); err == nil {+ numRecords = n+ }+ }+ }++ // Build column index for the schema+ schema := reader.Schema()+ colIndex := buildColumnIndex(schema)++ // Pre-allocate if we know the count+ items := make([]*container.BlobItem, 0, numRecords)++ // Iterate over record batches+ for reader.Next() {+ rec := reader.RecordBatch()+ rows := int(rec.NumRows())++ for row := 0; row < rows; row++ {+ item := &container.BlobItem{+ Properties: &container.BlobProperties{},+ }+ populateBlobItem(item, rec, row, colIndex)+ items = append(items, item)+ }+ }++ if err := reader.Err(); err != nil {+ return nil, nil, fmt.Errorf("error reading Arrow record batches: %w", err)+ }++ return items, nextMarker, nil+}++// columnIndex maps column names to their indices in the record batch for O(1) lookup.+type columnIndex map[string]int++func buildColumnIndex(schema *arrow.Schema) columnIndex {+ idx := make(columnIndex, len(schema.Fields()))+ for i, f := range schema.Fields() {+ idx[f.Name] = i+ }+ return idx+}++// populateBlobItem fills a BlobItem from a single row across all columns.+func populateBlobItem(item *container.BlobItem, rec arrow.RecordBatch, row int, colIdx columnIndex) {+ // Top-level BlobItem fields+ item.Name = getStringField(rec, row, colIdx, "Name")+ item.Deleted = getBoolField(rec, row, colIdx, "Deleted")+ item.Snapshot = getStringField(rec, row, colIdx, "Snapshot")+ item.VersionID = getStringField(rec, row, colIdx, "VersionId")+ item.IsCurrentVersion = getBoolField(rec, row, colIdx, "IsCurrentVersion")+ item.HasVersionsOnly = getBoolField(rec, row, colIdx, "HasVersionsOnly")++ // Metadata maps+ item.Metadata = getMapField(rec, row, colIdx, "Metadata")+ item.OrMetadata = getMapField(rec, row, colIdx, "OrMetadata")++ // Tags+ if tags := getMapField(rec, row, colIdx, "Tags"); tags != nil {+ var blobTags []*container.BlobTag+ for k, v := range tags {+ key := k+ blobTags = append(blobTags, &container.BlobTag{Key: &key, Value: v})+ }+ item.BlobTags = &container.BlobTags{BlobTagSet: blobTags}+ }++ // BlobProperties fields+ p := item.Properties+ p.CreationTime = getTimestampField(rec, row, colIdx, "Creation-Time")+ p.LastModified = getTimestampField(rec, row, colIdx, "Last-Modified")+ p.ETag = getETagField(rec, row, colIdx, "Etag")+ p.ContentLength = getInt64Field(rec, row, colIdx, "Content-Length")+ p.ContentType = getStringField(rec, row, colIdx, "Content-Type")+ p.ContentEncoding = getStringField(rec, row, colIdx, "Content-Encoding")+ p.ContentLanguage = getStringField(rec, row, colIdx, "Content-Language")+ p.ContentDisposition = getStringField(rec, row, colIdx, "Content-Disposition")+ p.CacheControl = getStringField(rec, row, colIdx, "Cache-Control")+ p.ContentMD5 = getBase64BytesField(rec, row, colIdx, "Content-MD5")+ p.BlobType = getEnumField[container.BlobType](rec, row, colIdx, "BlobType")+ p.AccessTier = getEnumField[container.AccessTier](rec, row, colIdx, "AccessTier")+ p.AccessTierInferred = getBoolField(rec, row, colIdx, "AccessTierInferred")+ p.AccessTierChangeTime = getTimestampField(rec, row, colIdx, "AccessTierChangeTime")+ p.LeaseState = getEnumField[lease.StateType](rec, row, colIdx, "LeaseState")+ p.LeaseStatus = getEnumField[lease.StatusType](rec, row, colIdx, "LeaseStatus")+ p.LeaseDuration = getEnumField[lease.DurationType](rec, row, colIdx, "LeaseDuration")+ p.ServerEncrypted = getBoolField(rec, row, colIdx, "ServerEncrypted")+ p.CustomerProvidedKeySHA256 = getStringField(rec, row, colIdx, "CustomerProvidedKeySha256")+ p.EncryptionScope = getStringField(rec, row, colIdx, "EncryptionScope")+ p.IncrementalCopy = getBoolField(rec, row, colIdx, "IncrementalCopy")+ p.IsSealed = getBoolField(rec, row, colIdx, "Sealed")+ p.ArchiveStatus = getEnumField[container.ArchiveStatus](rec, row, colIdx, "ArchiveStatus")+ p.RehydratePriority = getEnumField[blob.RehydratePriority](rec, row, colIdx, "RehydratePriority")+ p.CopyID = getStringField(rec, row, colIdx, "CopyId")+ p.CopyStatus = getEnumField[blob.CopyStatusType](rec, row, colIdx, "CopyStatus")+ p.CopySource = getStringField(rec, row, colIdx, "CopySource")+ p.CopyProgress = getStringField(rec, row, colIdx, "CopyProgress")+ p.CopyCompletionTime = getTimestampField(rec, row, colIdx, "CopyCompletionTime")+ p.CopyStatusDescription = getStringField(rec, row, colIdx, "CopyStatusDescription")+ p.DestinationSnapshot = getStringField(rec, row, colIdx, "CopyDestinationSnapshot")+ p.ImmutabilityPolicyExpiresOn = getTimestampField(rec, row, colIdx, "ImmutabilityPolicyUntilDate")+ p.ImmutabilityPolicyMode = getEnumField[container.ImmutabilityPolicyMode](rec, row, colIdx, "ImmutabilityPolicyMode")+ p.LegalHold = getBoolField(rec, row, colIdx, "LegalHold")+ p.DeletedTime = getTimestampField(rec, row, colIdx, "DeletedTime")+ p.RemainingRetentionDays = getInt32Field(rec, row, colIdx, "RemainingRetentionDays")+ p.LastAccessedOn = getTimestampField(rec, row, colIdx, "LastAccessTime")+ p.TagCount = getInt32Field(rec, row, colIdx, "TagCount")+ p.BlobSequenceNumber = getInt64Field(rec, row, colIdx, "x-ms-blob-sequence-number")+ p.ResourceType = getStringField(rec, row, colIdx, "ResourceType")++ // Content-CRC64 is a string field in Arrow but not directly mapped to BlobProperties.+ // The field exists in the Arrow schema but has no corresponding field in the Go SDK BlobProperties.+ // It is silently ignored for forward compatibility.+}++// Field accessor helpers that safely handle missing columns and null values.++func getStringField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *string {+ idx, ok := colIdx[name]+ if !ok {+ return nil+ }+ col := rec.Column(idx)+ if col.IsNull(row) {+ return nil+ }+ if arr, ok := col.(*array.String); ok {+ val := arr.Value(row)+ return &val+ }+ return nil+}++func getBoolField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *bool {+ idx, ok := colIdx[name]+ if !ok {+ return nil+ }+ col := rec.Column(idx)+ if col.IsNull(row) {+ return nil+ }+ if arr, ok := col.(*array.Boolean); ok {+ val := arr.Value(row)+ return &val+ }+ return nil+}++func getTimestampField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *time.Time {+ idx, ok := colIdx[name]+ if !ok {+ return nil+ }+ col := rec.Column(idx)+ if col.IsNull(row) {+ return nil+ }+ if arr, ok := col.(*array.Timestamp); ok {+ val := arr.Value(row)+ // Arrow timestamps are stored as int64 with a unit; convert to time.Time+ dt := val.ToTime(arr.DataType().(*arrow.TimestampType).Unit)+ return &dt+ }+ return nil+}++func getInt64Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int64 {+ idx, ok := colIdx[name]+ if !ok {+ return nil+ }+ col := rec.Column(idx)+ if col.IsNull(row) {+ return nil+ }+ if arr, ok := col.(*array.Uint64); ok {+ val := int64(arr.Value(row))+ return &val+ }+ if arr, ok := col.(*array.Int64); ok {+ val := arr.Value(row)+ return &val+ }+ return nil+}++func getInt32Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int32 {+ idx, ok := colIdx[name]+ if !ok {+ return nil+ }+ col := rec.Column(idx)+ if col.IsNull(row) {+ return nil+ }+ if arr, ok := col.(*array.Uint64); ok {+ val := int32(arr.Value(row))+ return &val+ }+ if arr, ok := col.(*array.Int32); ok {+ val := arr.Value(row)+ return &val+ }+ return nil+}++func getETagField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *azcore.ETag {+ s := getStringField(rec, row, colIdx, name)+ if s == nil {+ return nil+ }+ etag := azcore.ETag(*s)+ return &etag+}++func getBase64BytesField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) []byte {+ s := getStringField(rec, row, colIdx, name)+ if s == nil {+ return nil+ }+ decoded, err := base64.StdEncoding.DecodeString(*s)+ if err != nil {+ return nil+ }+ return decoded+}++// getEnumField returns a pointer to a string-typed enum value.+type stringEnum interface {+ ~string+}++func getEnumField[T stringEnum](rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *T {+ s := getStringField(rec, row, colIdx, name)+ if s == nil {+ return nil+ }+ val := T(*s)+ return &val+}++func getMapField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) map[string]*string {+ idx, ok := colIdx[name]+ if !ok {+ return nil+ }+ col := rec.Column(idx)+ if col.IsNull(row) {+ return nil+ }+ mapArr, ok := col.(*array.Map)+ if !ok {+ return nil+ }++ // Get the offsets for this row's map entries+ start := int(mapArr.Offsets()[row])+ end := int(mapArr.Offsets()[row+1])+ if start == end {+ return nil+ }++ keys := mapArr.Keys().(*array.String)+ values := mapArr.Items().(*array.String)++ result := make(map[string]*string, end-start)+ for i := start; i < end; i++ {+ k := keys.Value(i)+ if values.IsNull(i) {+ result[k] = nil+ } else {+ v := values.Value(i)+ result[k] = &v+ }+ }+ return result+}diff --git a/backend/azureblob/arrowlist/arrow_test.go b/backend/azureblob/arrowlist/arrow_test.gonew file mode 100644index 000000000..a0000f2a4--- /dev/null+++ b/backend/azureblob/arrowlist/arrow_test.go@@ -0,0 +1,374 @@+//go:build !plan9 && !solaris && !js++// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License. See License.txt in the project root for license information.+//+// Adapted from internal/arrow/arrow_test.go in the Azure SDK for Go+// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit+// c6fa341ca22b (branch feature/storage/bifrost).++package arrowlist++import (+ "bytes"+ "io"+ "net/http"+ "testing"++ "github.com/apache/arrow-go/v18/arrow"+ arrowArray "github.com/apache/arrow-go/v18/arrow/array"+ "github.com/apache/arrow-go/v18/arrow/ipc"+ "github.com/apache/arrow-go/v18/arrow/memory"+ "github.com/stretchr/testify/require"+)++// coreSchema returns a basic Arrow schema for testing with common blob fields.+func coreSchema(md *arrow.Metadata) *arrow.Schema {+ fields := []arrow.Field{+ {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},+ {Name: "Creation-Time", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},+ {Name: "Last-Modified", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},+ {Name: "BlobType", Type: arrow.BinaryTypes.String, Nullable: true},+ {Name: "Etag", Type: arrow.BinaryTypes.String, Nullable: true},+ {Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},+ {Name: "Content-Type", Type: arrow.BinaryTypes.String, Nullable: true},+ {Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},+ {Name: "AccessTier", Type: arrow.BinaryTypes.String, Nullable: true},+ {Name: "LeaseState", Type: arrow.BinaryTypes.String, Nullable: true},+ {Name: "LeaseStatus", Type: arrow.BinaryTypes.String, Nullable: true},+ }+ return arrow.NewSchema(fields, md)+}++// buildArrowStream constructs a synthetic Arrow IPC stream from a schema and a populate function.+func buildArrowStream(t *testing.T, schema *arrow.Schema, populate func(builder *arrowArray.RecordBuilder)) []byte {+ t.Helper()+ alloc := memory.DefaultAllocator++ var buf bytes.Buffer+ w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))++ builder := arrowArray.NewRecordBuilder(alloc, schema)+ defer builder.Release()++ populate(builder)++ rec := builder.NewRecordBatch()+ defer rec.Release()++ require.NoError(t, w.Write(rec))+ require.NoError(t, w.Close())++ return buf.Bytes()+}++func makeHTTPResponse(data []byte, headers map[string]string) *http.Response {+ h := http.Header{}+ h.Set("Content-Type", ArrowContentType)+ for k, v := range headers {+ h.Set(k, v)+ }+ return &http.Response{+ StatusCode: http.StatusOK,+ Header: h,+ Body: io.NopCloser(bytes.NewReader(data)),+ }+}++func TestHandleFlatListResponse_BasicParsing(t *testing.T) {+ md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"marker123", "2"})+ schema := coreSchema(&md)++ data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {+ b.Field(0).(*arrowArray.StringBuilder).Append("blob1.txt")+ b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))+ b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))+ b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")+ b.Field(4).(*arrowArray.StringBuilder).Append("0x1234")+ b.Field(5).(*arrowArray.Uint64Builder).Append(1024)+ b.Field(6).(*arrowArray.StringBuilder).Append("application/octet-stream")+ b.Field(7).(*arrowArray.BooleanBuilder).Append(true)+ b.Field(8).(*arrowArray.StringBuilder).Append("Hot")+ b.Field(9).(*arrowArray.StringBuilder).Append("available")+ b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")++ b.Field(0).(*arrowArray.StringBuilder).Append("blob2.txt")+ b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))+ b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))+ b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")+ b.Field(4).(*arrowArray.StringBuilder).Append("0x5678")+ b.Field(5).(*arrowArray.Uint64Builder).Append(2048)+ b.Field(6).(*arrowArray.StringBuilder).Append("text/plain")+ b.Field(7).(*arrowArray.BooleanBuilder).Append(false)+ b.Field(8).(*arrowArray.StringBuilder).Append("Cool")+ b.Field(9).(*arrowArray.StringBuilder).Append("available")+ b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")+ })++ resp := makeHTTPResponse(data, nil)+ result, err := HandleFlatListResponse(resp)+ require.NoError(t, err)++ require.NotNil(t, result.NextMarker)+ require.Equal(t, "marker123", *result.NextMarker)+ require.Len(t, result.Segment.BlobItems, 2)++ item := result.Segment.BlobItems[0]+ require.Equal(t, "blob1.txt", *item.Name)+ require.Equal(t, "BlockBlob", string(*item.Properties.BlobType))+ require.Equal(t, int64(1024), *item.Properties.ContentLength)+ require.Equal(t, "Hot", string(*item.Properties.AccessTier))+}++func TestHandleFlatListResponse_EmptyRecordBatch(t *testing.T) {+ md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "0"})+ schema := coreSchema(&md)++ data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {+ // No rows added+ })++ resp := makeHTTPResponse(data, nil)+ result, err := HandleFlatListResponse(resp)+ require.NoError(t, err)+ require.Nil(t, result.NextMarker)+ require.Empty(t, result.Segment.BlobItems)+}++func TestHandleFlatListResponse_NullFields(t *testing.T) {+ md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})+ schema := coreSchema(&md)++ data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {+ b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")+ b.Field(1).(*arrowArray.TimestampBuilder).AppendNull()+ b.Field(2).(*arrowArray.TimestampBuilder).AppendNull()+ b.Field(3).(*arrowArray.StringBuilder).AppendNull()+ b.Field(4).(*arrowArray.StringBuilder).AppendNull()+ b.Field(5).(*arrowArray.Uint64Builder).AppendNull()+ b.Field(6).(*arrowArray.StringBuilder).AppendNull()+ b.Field(7).(*arrowArray.BooleanBuilder).AppendNull()+ b.Field(8).(*arrowArray.StringBuilder).AppendNull()+ b.Field(9).(*arrowArray.StringBuilder).AppendNull()+ b.Field(10).(*arrowArray.StringBuilder).AppendNull()+ })++ resp := makeHTTPResponse(data, nil)+ result, err := HandleFlatListResponse(resp)+ require.NoError(t, err)+ require.Len(t, result.Segment.BlobItems, 1)++ item := result.Segment.BlobItems[0]+ require.Equal(t, "blob.txt", *item.Name)+ require.Nil(t, item.Properties.CreationTime)+ require.Nil(t, item.Properties.BlobType)+ require.Nil(t, item.Properties.ETag)+ require.Nil(t, item.Properties.ContentLength)+ require.Nil(t, item.Properties.ContentType)+ require.Nil(t, item.Properties.ServerEncrypted)+}++func TestHandleFlatListResponse_UnknownColumnsIgnored(t *testing.T) {+ md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})+ fields := []arrow.Field{+ {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},+ {Name: "FutureField", Type: arrow.BinaryTypes.String, Nullable: true},+ }+ schema := arrow.NewSchema(fields, &md)++ data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {+ b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")+ b.Field(1).(*arrowArray.StringBuilder).Append("some-future-value")+ })++ resp := makeHTTPResponse(data, nil)+ result, err := HandleFlatListResponse(resp)+ require.NoError(t, err)+ require.Len(t, result.Segment.BlobItems, 1)+ require.Equal(t, "blob.txt", *result.Segment.BlobItems[0].Name)+}++func TestHandleFlatListResponse_MultiRecordBatch(t *testing.T) {+ alloc := memory.DefaultAllocator+ md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "3"})+ schema := coreSchema(&md)++ var buf bytes.Buffer+ w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))++ // First batch: 2 rows+ builder := arrowArray.NewRecordBuilder(alloc, schema)+ for i := 0; i < 2; i++ {+ builder.Field(0).(*arrowArray.StringBuilder).Append("batch1_blob")+ for j := 1; j < 11; j++ {+ builder.Field(j).AppendNull()+ }+ }+ rec := builder.NewRecordBatch()+ require.NoError(t, w.Write(rec))+ rec.Release()+ builder.Release()++ // Second batch: 1 row+ builder = arrowArray.NewRecordBuilder(alloc, schema)+ builder.Field(0).(*arrowArray.StringBuilder).Append("batch2_blob")+ for j := 1; j < 11; j++ {+ builder.Field(j).AppendNull()+ }+ rec = builder.NewRecordBatch()+ require.NoError(t, w.Write(rec))+ rec.Release()+ builder.Release()++ require.NoError(t, w.Close())++ resp := makeHTTPResponse(buf.Bytes(), nil)+ result, err := HandleFlatListResponse(resp)+ require.NoError(t, err)+ require.Len(t, result.Segment.BlobItems, 3)+}++func TestHandleHierarchyListResponse_SeparatesBlobsAndPrefixes(t *testing.T) {+ md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})+ fields := []arrow.Field{+ {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},+ {Name: "ResourceType", Type: arrow.BinaryTypes.String, Nullable: true},+ }+ schema := arrow.NewSchema(fields, &md)++ data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {+ // A regular blob+ b.Field(0).(*arrowArray.StringBuilder).Append("folder/file.txt")+ b.Field(1).(*arrowArray.StringBuilder).AppendNull()++ // A blob prefix (virtual directory)+ b.Field(0).(*arrowArray.StringBuilder).Append("folder/subfolder/")+ b.Field(1).(*arrowArray.StringBuilder).Append("blobprefix")+ })++ resp := makeHTTPResponse(data, nil)+ result, err := HandleHierarchyListResponse(resp)+ require.NoError(t, err)++ require.Len(t, result.Segment.BlobItems, 1)+ require.Equal(t, "folder/file.txt", *result.Segment.BlobItems[0].Name)++ require.Len(t, result.Segment.BlobPrefixes, 1)+ require.Equal(t, "folder/subfolder/", *result.Segment.BlobPrefixes[0].Name)+}++func TestHandleFlatListResponse_ResponseHeaders(t *testing.T) {+ md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})+ fields := []arrow.Field{+ {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},+ }+ schema := arrow.NewSchema(fields, &md)++ data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {+ b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")+ })++ resp := makeHTTPResponse(data, map[string]string{+ "x-ms-request-id": "req-123",+ "x-ms-version": "2026-10-06",+ "x-ms-client-request-id": "client-req-456",+ "Date": "Mon, 01 Jan 2024 00:00:00 GMT",+ })++ result, err := HandleFlatListResponse(resp)+ require.NoError(t, err)+ require.Equal(t, "req-123", *result.RequestID)+ require.Equal(t, "2026-10-06", *result.Version)+ require.Equal(t, "client-req-456", *result.ClientRequestID)+ require.NotNil(t, result.Date)+}++func TestHandleFlatListResponse_MapFields(t *testing.T) {+ md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})+ fields := []arrow.Field{+ {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},+ {Name: "Metadata", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},+ {Name: "Tags", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},+ }+ schema := arrow.NewSchema(fields, &md)++ data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {+ b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")++ metaBuilder := b.Field(1).(*arrowArray.MapBuilder)+ metaBuilder.Append(true)+ metaBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("key1")+ metaBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("value1")++ tagBuilder := b.Field(2).(*arrowArray.MapBuilder)+ tagBuilder.Append(true)+ tagBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("env")+ tagBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("prod")+ })++ resp := makeHTTPResponse(data, nil)+ result, err := HandleFlatListResponse(resp)+ require.NoError(t, err)+ require.Len(t, result.Segment.BlobItems, 1)++ item := result.Segment.BlobItems[0]+ require.NotNil(t, item.Metadata)+ require.Equal(t, "value1", *item.Metadata["key1"])++ require.NotNil(t, item.BlobTags)+ require.Len(t, item.BlobTags.BlobTagSet, 1)+}++func TestHandleFlatListResponse_BooleanFields(t *testing.T) {+ md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})+ fields := []arrow.Field{+ {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},+ {Name: "Deleted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},+ {Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},+ {Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},+ }+ schema := arrow.NewSchema(fields, &md)++ data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {+ b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")+ b.Field(1).(*arrowArray.BooleanBuilder).Append(true)+ b.Field(2).(*arrowArray.BooleanBuilder).Append(false)+ b.Field(3).(*arrowArray.BooleanBuilder).Append(true)+ })++ resp := makeHTTPResponse(data, nil)+ result, err := HandleFlatListResponse(resp)+ require.NoError(t, err)++ item := result.Segment.BlobItems[0]+ require.True(t, *item.Deleted)+ require.False(t, *item.IsCurrentVersion)+ require.True(t, *item.Properties.ServerEncrypted)+}++func TestHandleFlatListResponse_VersionAndSnapshot(t *testing.T) {+ md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})+ fields := []arrow.Field{+ {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},+ {Name: "VersionId", Type: arrow.BinaryTypes.String, Nullable: true},+ {Name: "Snapshot", Type: arrow.BinaryTypes.String, Nullable: true},+ {Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},+ }+ schema := arrow.NewSchema(fields, &md)++ data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {+ b.Field(0).(*arrowArray.StringBuilder).Append("versioned-blob.txt")+ b.Field(1).(*arrowArray.StringBuilder).Append("2024-01-01T00:00:00.0000000Z")+ b.Field(2).(*arrowArray.StringBuilder).Append("snap-123")+ b.Field(3).(*arrowArray.BooleanBuilder).Append(true)+ })++ resp := makeHTTPResponse(data, nil)+ result, err := HandleFlatListResponse(resp)+ require.NoError(t, err)++ item := result.Segment.BlobItems[0]+ require.Equal(t, "2024-01-01T00:00:00.0000000Z", *item.VersionID)+ require.Equal(t, "snap-123", *item.Snapshot)+ require.True(t, *item.IsCurrentVersion)+}diff --git a/backend/azureblob/arrowlist/arrowlist.go b/backend/azureblob/arrowlist/arrowlist.gonew file mode 100644index 000000000..d80a9d47f--- /dev/null+++ b/backend/azureblob/arrowlist/arrowlist.go@@ -0,0 +1,259 @@+//go:build !plan9 && !solaris && !js++// Package arrowlist implements the experimental "Blob Listing with Apache+// Arrow" feature on top of the released Azure azblob SDK.+//+// The Azure SDK for Go has support for Arrow listing on the unreleased+// feature/storage/bifrost branch (commit c6fa341ca22b) where the listing+// pagers decode Arrow IPC stream responses into the ordinary ListBlobs+// response types. Until that ships in a tagged azblob release this package+// fills the gap: it exposes the same options and pager interface as the+// experimental SDK, built only on the public API of the released SDK plus+// copies of two small internal auth policies (see sharedkey.go and+// challenge_policy.go).+//+// This package is temporary. When Arrow listing ships in a released azblob+// the whole package should be deleted and its callers pointed back at+// container.Client.NewListBlobsHierarchyPager - the option and response+// types here are deliberately source compatible with the SDK's to make that+// a mechanical change.+package arrowlist++import (+ "context"+ "errors"+ "net/http"+ "strconv"+ "strings"++ "github.com/Azure/azure-sdk-for-go/sdk/azcore"+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"+)++const (+ // moduleName and moduleVersion identify this package in the azcore+ // telemetry policy's User-Agent fragment.+ moduleName = "github.com/rclone/rclone/backend/azureblob/arrowlist"+ moduleVersion = "v0.0.1"++ // serviceVersion is the x-ms-version sent with listing requests. Arrow+ // responses and the startFrom/endBefore parameters need at least+ // 2026-06-06.+ serviceVersion = "2026-06-06"++ // tokenScope is the OAuth scope for Azure Storage, as used by the SDK's+ // own clients.+ tokenScope = "https://storage.azure.com/.default"+)++// ErrEndBeforeXMLFallback is returned by the pager when the server answered+// with XML instead of Arrow while EndBefore was set. The service only honours+// endBefore on the Arrow listing path, so an XML page could silently contain+// blobs outside the requested range and cannot be used.+var ErrEndBeforeXMLFallback = errors.New("arrow listing fell back to XML with endBefore set")++// ListBlobsHierarchyOptions provides the configuration for+// Client.NewListBlobsHierarchyPager.+//+// It mirrors the experimental SDK's container.ListBlobsHierarchyOptions:+// the released SDK's options are embedded (providing Include, Marker,+// MaxResults, Prefix and StartFrom by promotion) with the two fields the+// released SDK lacks added alongside.+type ListBlobsHierarchyOptions struct {+ container.ListBlobsHierarchyOptions+ // EndBefore limits listing to blobs whose full container path is+ // lexically before it (exclusive). Only honoured on Arrow responses.+ EndBefore *string+ // UseArrowFormat requests the response as an Apache Arrow IPC stream.+ UseArrowFormat *bool+}++// ClientOptions contains the optional parameters for client creation.+type ClientOptions struct {+ azcore.ClientOptions+}++// Client issues Arrow listing requests to a single container.+type Client struct {+ endpoint string+ azClient *azcore.Client+}++// newClient creates a Client for the container at containerURL sending+// requests through a pipeline with the given auth policy (which may be nil+// for anonymous or SAS access).+func newClient(containerURL string, authPolicy policy.Policy, options *ClientOptions) (*Client, error) {+ if options == nil {+ options = &ClientOptions{}+ }+ plOpts := runtime.PipelineOptions{}+ if authPolicy != nil {+ plOpts.PerRetry = []policy.Policy{authPolicy}+ }+ azClient, err := azcore.NewClient(moduleName, moduleVersion, plOpts, &options.ClientOptions)+ if err != nil {+ return nil, err+ }+ return &Client{+ endpoint: containerURL,+ azClient: azClient,+ }, nil+}++// NewClient creates a Client authenticating with a token credential.+func NewClient(containerURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {+ return newClient(containerURL, NewStorageChallengePolicy(cred, tokenScope, false), options)+}++// NewClientWithNoCredential creates a Client with no authentication, for+// anonymous access or a containerURL carrying a SAS token in its query.+func NewClientWithNoCredential(containerURL string, options *ClientOptions) (*Client, error) {+ return newClient(containerURL, nil, options)+}++// NewClientWithSharedKeyCredential creates a Client signing requests with the+// account's shared key.+func NewClientWithSharedKeyCredential(containerURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {+ return newClient(containerURL, NewSharedKeyCredPolicy(cred), options)+}++// NewListBlobsHierarchyPager returns a pager over the container's blobs,+// requesting Arrow responses when o.UseArrowFormat is set and returning the+// same response type as container.Client.NewListBlobsHierarchyPager.+//+// If the service does not support Arrow listing (the feature not enabled on+// the account, or an HNS account) it answers with XML, which is decoded+// transparently - unless EndBefore is set, in which case the fetcher returns+// ErrEndBeforeXMLFallback (see there).+func (c *Client) NewListBlobsHierarchyPager(delimiter string, o *ListBlobsHierarchyOptions) *runtime.Pager[container.ListBlobsHierarchyResponse] {+ // Copy the options so advancing the Marker doesn't mutate the caller's struct.+ opts := ListBlobsHierarchyOptions{}+ if o != nil {+ opts = *o+ }+ useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat+ return runtime.NewPager(runtime.PagingHandler[container.ListBlobsHierarchyResponse]{+ More: func(page container.ListBlobsHierarchyResponse) bool {+ return page.NextMarker != nil && len(*page.NextMarker) > 0+ },+ Fetcher: func(ctx context.Context, page *container.ListBlobsHierarchyResponse) (container.ListBlobsHierarchyResponse, error) {+ if page != nil {+ opts.Marker = page.NextMarker+ }+ req, err := c.listCreateRequest(ctx, delimiter, &opts, useArrow)+ if err != nil {+ return container.ListBlobsHierarchyResponse{}, err+ }+ resp, err := c.azClient.Pipeline().Do(req)+ if err != nil {+ return container.ListBlobsHierarchyResponse{}, err+ }+ if !runtime.HasStatusCode(resp, http.StatusOK) {+ return container.ListBlobsHierarchyResponse{}, runtime.NewResponseError(resp)+ }+ if useArrow && resp.Header.Get("Content-Type") == ArrowContentType {+ return HandleHierarchyListResponse(resp)+ }+ if opts.EndBefore != nil {+ return container.ListBlobsHierarchyResponse{}, ErrEndBeforeXMLFallback+ }+ return handleXMLResponse(resp)+ },+ })+}++// listCreateRequest builds the ListBlobs request, mirroring the generated+// request builders in the SDK.+func (c *Client) listCreateRequest(ctx context.Context, delimiter string, opts *ListBlobsHierarchyOptions, useArrow bool) (*policy.Request, error) {+ req, err := runtime.NewRequest(ctx, http.MethodGet, c.endpoint)+ if err != nil {+ return nil, err+ }+ // Reading the existing query first preserves any SAS token in the endpoint.+ reqQP := req.Raw().URL.Query()+ reqQP.Set("comp", "list")+ reqQP.Set("restype", "container")+ reqQP.Set("delimiter", delimiter)+ if include := formatInclude(opts.Include); include != "" {+ reqQP.Set("include", include)+ }+ if opts.Marker != nil {+ reqQP.Set("marker", *opts.Marker)+ }+ if opts.MaxResults != nil {+ reqQP.Set("maxresults", strconv.FormatInt(int64(*opts.MaxResults), 10))+ }+ if opts.Prefix != nil {+ reqQP.Set("prefix", *opts.Prefix)+ }+ if opts.StartFrom != nil {+ reqQP.Set("startFrom", *opts.StartFrom)+ }+ if useArrow && opts.EndBefore != nil {+ reqQP.Set("endBefore", *opts.EndBefore)+ }+ req.Raw().URL.RawQuery = strings.ReplaceAll(reqQP.Encode(), "+", "%20")+ if useArrow {+ // Stream the Arrow body rather than buffering it in the pipeline.+ runtime.SkipBodyDownload(req)+ req.Raw().Header["Accept"] = []string{ArrowAcceptHeader}+ } else {+ req.Raw().Header["Accept"] = []string{"application/xml"}+ }+ req.Raw().Header["x-ms-version"] = []string{serviceVersion}+ return req, nil+}++// handleXMLResponse decodes an XML ListBlobs response, mirroring the SDK's+// generated ListBlobHierarchySegmentHandleResponse.+func handleXMLResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {+ result := container.ListBlobsHierarchyResponse{}+ extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)+ if err := runtime.UnmarshalAsXML(resp, &result.ListBlobsHierarchySegmentResponse); err != nil {+ return container.ListBlobsHierarchyResponse{}, err+ }+ return result, nil+}++// formatInclude renders the include datasets as the comma separated query+// parameter value, in the same order as the SDK's unexported+// ListBlobsInclude.format.+func formatInclude(l container.ListBlobsInclude) string {+ var include []string+ if l.Copy {+ include = append(include, "copy")+ }+ if l.Deleted {+ include = append(include, "deleted")+ }+ if l.DeletedWithVersions {+ include = append(include, "deletedwithversions")+ }+ if l.ImmutabilityPolicy {+ include = append(include, "immutabilitypolicy")+ }+ if l.LegalHold {+ include = append(include, "legalhold")+ }+ if l.Metadata {+ include = append(include, "metadata")+ }+ if l.Snapshots {+ include = append(include, "snapshots")+ }+ if l.Tags {+ include = append(include, "tags")+ }+ if l.UncommittedBlobs {+ include = append(include, "uncommittedblobs")+ }+ if l.Versions {+ include = append(include, "versions")+ }+ if l.Permissions {+ include = append(include, "permissions")+ }+ return strings.Join(include, ",")+}diff --git a/backend/azureblob/arrowlist/arrowlist_test.go b/backend/azureblob/arrowlist/arrowlist_test.gonew file mode 100644index 000000000..d74389284--- /dev/null+++ b/backend/azureblob/arrowlist/arrowlist_test.go@@ -0,0 +1,199 @@+//go:build !plan9 && !solaris && !js++package arrowlist++import (+ "context"+ "encoding/base64"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"+ "github.com/apache/arrow-go/v18/arrow"+ arrowArray "github.com/apache/arrow-go/v18/arrow/array"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// arrowPage builds a one row Arrow IPC listing page with the given blob name+// and NextMarker.+func arrowPage(t *testing.T, name, nextMarker string) []byte {+ md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{nextMarker, "1"})+ fields := []arrow.Field{+ {Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},+ {Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},+ }+ schema := arrow.NewSchema(fields, &md)+ return buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {+ b.Field(0).(*arrowArray.StringBuilder).Append(name)+ b.Field(1).(*arrowArray.Uint64Builder).Append(42)+ })+}++func TestPagerArrow(t *testing.T) {+ var requests []*http.Request+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests = append(requests, r.Clone(context.Background()))+ w.Header().Set("Content-Type", ArrowContentType)+ if r.URL.Query().Get("marker") == "" {+ _, _ = w.Write(arrowPage(t, "blob1.txt", "marker1"))+ } else {+ _, _ = w.Write(arrowPage(t, "blob2.txt", ""))+ }+ }))+ defer srv.Close()++ // SAS style query on the endpoint must be preserved in requests.+ client, err := NewClientWithNoCredential(srv.URL+"/testcontainer?sv=fakesas", nil)+ require.NoError(t, err)++ opts := &ListBlobsHierarchyOptions{+ ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{+ Include: container.ListBlobsInclude{Metadata: true, Tags: true},+ Prefix: to.Ptr("dir/"),+ MaxResults: to.Ptr(int32(1000)),+ StartFrom: to.Ptr("testcontainer/dir/a"),+ },+ EndBefore: to.Ptr("testcontainer/dir/n"),+ UseArrowFormat: to.Ptr(true),+ }+ pager := client.NewListBlobsHierarchyPager("/", opts)++ var names []string+ for pager.More() {+ page, err := pager.NextPage(context.Background())+ require.NoError(t, err)+ assert.Equal(t, ArrowContentType, *page.ContentType)+ for _, item := range page.Segment.BlobItems {+ names = append(names, *item.Name)+ assert.Equal(t, int64(42), *item.Properties.ContentLength)+ }+ }+ assert.Equal(t, []string{"blob1.txt", "blob2.txt"}, names)++ require.Len(t, requests, 2)+ q := requests[0].URL.Query()+ assert.Equal(t, "list", q.Get("comp"))+ assert.Equal(t, "container", q.Get("restype"))+ assert.Equal(t, "/", q.Get("delimiter"))+ assert.Equal(t, "metadata,tags", q.Get("include"))+ assert.Equal(t, "dir/", q.Get("prefix"))+ assert.Equal(t, "1000", q.Get("maxresults"))+ assert.Equal(t, "testcontainer/dir/a", q.Get("startFrom"))+ assert.Equal(t, "testcontainer/dir/n", q.Get("endBefore"))+ assert.Equal(t, "fakesas", q.Get("sv"))+ assert.Empty(t, q.Get("marker"))+ assert.Equal(t, ArrowAcceptHeader, requests[0].Header.Get("Accept"))+ assert.Equal(t, serviceVersion, requests[0].Header.Get("x-ms-version"))+ assert.Equal(t, "marker1", requests[1].URL.Query().Get("marker"))++ // The caller's options must not have been mutated by paging.+ assert.Nil(t, opts.Marker)+}++const xmlListResponse = `<?xml version="1.0" encoding="utf-8"?>+<EnumerationResults ServiceEndpoint="https://example.com/" ContainerName="testcontainer">+ <Blobs>+ <Blob>+ <Name>file.txt</Name>+ <Properties>+ <Last-Modified>Mon, 01 Jan 2024 00:00:00 GMT</Last-Modified>+ <Etag>0x8D1</Etag>+ <Content-Length>7</Content-Length>+ <BlobType>BlockBlob</BlobType>+ </Properties>+ </Blob>+ <BlobPrefix>+ <Name>subdir/</Name>+ </BlobPrefix>+ </Blobs>+ <NextMarker />+</EnumerationResults>`++func TestPagerXMLFallback(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/xml")+ _, _ = w.Write([]byte(xmlListResponse))+ }))+ defer srv.Close()++ client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)+ require.NoError(t, err)++ pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{+ UseArrowFormat: to.Ptr(true),+ })+ page, err := pager.NextPage(context.Background())+ require.NoError(t, err)+ assert.False(t, pager.More())+ assert.Equal(t, "application/xml", *page.ContentType)+ require.Len(t, page.Segment.BlobItems, 1)+ assert.Equal(t, "file.txt", *page.Segment.BlobItems[0].Name)+ assert.Equal(t, int64(7), *page.Segment.BlobItems[0].Properties.ContentLength)+ require.Len(t, page.Segment.BlobPrefixes, 1)+ assert.Equal(t, "subdir/", *page.Segment.BlobPrefixes[0].Name)+}++func TestPagerXMLFallbackWithEndBefore(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/xml")+ _, _ = w.Write([]byte(xmlListResponse))+ }))+ defer srv.Close()++ client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)+ require.NoError(t, err)++ pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{+ EndBefore: to.Ptr("testcontainer/n"),+ UseArrowFormat: to.Ptr(true),+ })+ _, err = pager.NextPage(context.Background())+ require.ErrorIs(t, err, ErrEndBeforeXMLFallback)+}++func TestPagerSharedKeyAuth(t *testing.T) {+ var gotAuth, gotDate string+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotAuth = r.Header.Get("Authorization")+ gotDate = r.Header.Get("x-ms-date")+ w.Header().Set("Content-Type", ArrowContentType)+ _, _ = w.Write(arrowPage(t, "blob.txt", ""))+ }))+ defer srv.Close()++ cred, err := NewSharedKeyCredential("testaccount", base64.StdEncoding.EncodeToString([]byte("testkey")))+ require.NoError(t, err)+ client, err := NewClientWithSharedKeyCredential(srv.URL+"/testcontainer", cred, nil)+ require.NoError(t, err)++ pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{+ UseArrowFormat: to.Ptr(true),+ })+ _, err = pager.NextPage(context.Background())+ require.NoError(t, err)+ assert.True(t, strings.HasPrefix(gotAuth, "SharedKey testaccount:"), "Authorization = %q", gotAuth)+ assert.NotEmpty(t, gotDate)+}++func TestPagerResponseError(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("x-ms-error-code", string(bloberror.ContainerNotFound))+ w.WriteHeader(http.StatusNotFound)+ }))+ defer srv.Close()++ client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)+ require.NoError(t, err)++ pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{+ UseArrowFormat: to.Ptr(true),+ })+ _, err = pager.NextPage(context.Background())+ require.Error(t, err)+ assert.True(t, bloberror.HasCode(err, bloberror.ContainerNotFound), "expected ContainerNotFound, got %v", err)+}diff --git a/backend/azureblob/arrowlist/challenge_policy.go b/backend/azureblob/arrowlist/challenge_policy.gonew file mode 100644index 000000000..df6698ddc--- /dev/null+++ b/backend/azureblob/arrowlist/challenge_policy.go@@ -0,0 +1,121 @@+//go:build !plan9 && !solaris && !js++// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License. See License.txt in the project root for license information.+//+// Adapted from internal/shared/challenge_policy.go in the Azure SDK for Go+// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0. The+// released SDK does not export its storage bearer-challenge policy, which is+// needed for token authentication with the same semantics as the SDK's own+// clients.++package arrowlist++import (+ "errors"+ "net/http"+ "strings"++ "github.com/Azure/azure-sdk-for-go/sdk/azcore"+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"+)++type storageAuthorizer struct {+ scopes []string+ tenantID string+}++// NewStorageChallengePolicy returns a bearer token policy for cred which+// handles the storage service's authentication challenges.+func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy {+ s := storageAuthorizer{scopes: []string{audience}}+ return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{+ AuthorizationHandler: policy.AuthorizationHandler{+ OnRequest: s.onRequest,+ OnChallenge: s.onChallenge,+ },+ InsecureAllowCredentialWithHTTP: allowHTTP,+ })+}++func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error {+ return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})+}++func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error {+ // parse the challenge+ err := s.parseChallenge(resp)+ if err != nil {+ return err+ }+ // TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841+ return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})+}++type challengePolicyError struct {+ err error+}++func (c *challengePolicyError) Error() string {+ return c.err.Error()+}++func (*challengePolicyError) NonRetriable() {+ // marker method+}++func (c *challengePolicyError) Unwrap() error {+ return c.err+}++// parses Tenant ID from auth challenge+// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize+func parseTenant(url string) string {+ if url == "" {+ return ""+ }+ parts := strings.Split(url, "/")+ if len(parts) >= 3 {+ tenant := parts[3]+ tenant = strings.ReplaceAll(tenant, ",", "")+ return tenant+ }+ return ""+}++func (s *storageAuthorizer) parseChallenge(resp *http.Response) error {+ authHeader := resp.Header.Get("WWW-Authenticate")+ if authHeader == "" {+ return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")}+ }++ // Strip down to auth and resource+ // Format is "Bearer authorization_uri=\"<site>\" resource_id=\"<site>\""+ authHeader = strings.ReplaceAll(authHeader, "Bearer ", "")++ parts := strings.Split(authHeader, " ")++ vals := map[string]string{}+ for _, part := range parts {+ subParts := strings.Split(part, "=")+ if len(subParts) == 2 {+ stripped := strings.ReplaceAll(subParts[1], "\"", "")+ stripped = strings.TrimSuffix(stripped, ",")+ vals[subParts[0]] = stripped+ }+ }++ s.tenantID = parseTenant(vals["authorization_uri"])++ scope := vals["resource_id"]+ if scope == "" {+ return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")}+ }++ if !strings.HasSuffix(scope, "/.default") {+ scope += "/.default"+ }+ s.scopes = []string{scope}+ return nil+}diff --git a/backend/azureblob/arrowlist/sharedkey.go b/backend/azureblob/arrowlist/sharedkey.gonew file mode 100644index 000000000..bbee8b8a4--- /dev/null+++ b/backend/azureblob/arrowlist/sharedkey.go@@ -0,0 +1,318 @@+//go:build !plan9 && !solaris && !js++// Copyright (c) Microsoft Corporation. All rights reserved.+// Licensed under the MIT License. See License.txt in the project root for license information.+//+// Adapted from internal/exported/shared_key_credential.go in the Azure SDK+// for Go github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0,+// with the internal header constants inlined. The released SDK does not+// export its shared key signing policy, so a copy is needed to sign the+// Arrow listing requests this package sends through its own pipeline.++package arrowlist++import (+ "bytes"+ "crypto/hmac"+ "crypto/sha256"+ "encoding/base64"+ "fmt"+ "net/http"+ "net/url"+ "sort"+ "strings"+ "sync/atomic"+ "time"++ "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"+ "github.com/rclone/rclone/fs"+)++// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the+// storage account's name and either its primary or secondary key.+func NewSharedKeyCredential(accountName string, accountKey string) (*SharedKeyCredential, error) {+ c := SharedKeyCredential{accountName: accountName}+ if err := c.SetAccountKey(accountKey); err != nil {+ return nil, err+ }+ return &c, nil+}++// SharedKeyCredential contains an account's name and its primary or secondary key.+type SharedKeyCredential struct {+ // Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only+ accountName string+ accountKey atomic.Value // []byte+}++// AccountName returns the Storage account's name.+func (c *SharedKeyCredential) AccountName() string {+ return c.accountName+}++// SetAccountKey replaces the existing account key with the specified account key.+func (c *SharedKeyCredential) SetAccountKey(accountKey string) error {+ _bytes, err := base64.StdEncoding.DecodeString(accountKey)+ if err != nil {+ return fmt.Errorf("decode account key: %w", err)+ }+ c.accountKey.Store(_bytes)+ return nil+}++// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.+func (c *SharedKeyCredential) computeHMACSHA256(message string) (string, error) {+ h := hmac.New(sha256.New, c.accountKey.Load().([]byte))+ _, err := h.Write([]byte(message))+ return base64.StdEncoding.EncodeToString(h.Sum(nil)), err+}++func (c *SharedKeyCredential) buildStringToSign(req *http.Request) (string, error) {+ // https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services+ headers := req.Header+ contentLength := getHeader("Content-Length", headers)+ if contentLength == "0" {+ contentLength = ""+ }++ canonicalizedResource, err := c.buildCanonicalizedResource(req.URL)+ if err != nil {+ return "", err+ }++ stringToSign := strings.Join([]string{+ req.Method,+ getHeader("Content-Encoding", headers),+ getHeader("Content-Language", headers),+ contentLength,+ getHeader("Content-MD5", headers),+ getHeader("Content-Type", headers),+ "", // Empty date because x-ms-date is expected (as per web page above)+ getHeader("If-Modified-Since", headers),+ getHeader("If-Match", headers),+ getHeader("If-None-Match", headers),+ getHeader("If-Unmodified-Since", headers),+ getHeader("Range", headers),+ c.buildCanonicalizedHeader(headers),+ canonicalizedResource,+ }, "\n")+ return stringToSign, nil+}++func getHeader(key string, headers map[string][]string) string {+ if headers == nil {+ return ""+ }+ if v, ok := headers[key]; ok {+ if len(v) > 0 {+ return v[0]+ }+ }++ return ""+}++func getWeightTables() [][]int {+ tableLv0 := [...]int{+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,+ 0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0xd03, 0xd1a, 0xd1c, 0xd1e,+ 0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,+ 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,+ 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,+ 0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,+ 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,+ 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,+ }+ tableLv2 := [...]int{+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,+ }+ tables := [][]int{tableLv0[:], tableLv2[:]}+ return tables+}++// NewHeaderStringComparer performs a multi-level, weight-based comparison of two strings+func compareHeaders(lhs, rhs string, tables [][]int) int {+ currLevel, i, j := 0, 0, 0+ n := len(tables)+ lhsLen := len(lhs)+ rhsLen := len(rhs)++ for currLevel < n {+ if currLevel == (n-1) && i != j {+ if i > j {+ return -1+ }+ if i < j {+ return 1+ }+ return 0+ }++ var w1, w2 int++ // Check bounds before accessing lhs[i]+ if i < lhsLen {+ w1 = tables[currLevel][lhs[i]]+ } else {+ w1 = 0x1+ }++ // Check bounds before accessing rhs[j]+ if j < rhsLen {+ w2 = tables[currLevel][rhs[j]]+ } else {+ w2 = 0x1+ }++ if w1 == 0x1 && w2 == 0x1 {+ i = 0+ j = 0+ currLevel+++ } else if w1 == w2 {+ i+++ j+++ } else if w1 == 0 {+ i+++ } else if w2 == 0 {+ j+++ } else {+ if w1 < w2 {+ return -1+ }+ if w1 > w2 {+ return 1+ }+ return 0+ }+ }+ return 0+}++func (c *SharedKeyCredential) buildCanonicalizedHeader(headers http.Header) string {+ cm := map[string][]string{}+ for k, v := range headers {+ headerName := strings.TrimSpace(strings.ToLower(k))+ if strings.HasPrefix(headerName, "x-ms-") {+ cm[headerName] = v // NOTE: the value must not have any whitespace around it.+ }+ }+ if len(cm) == 0 {+ return ""+ }++ keys := make([]string, 0, len(cm))+ for key := range cm {+ keys = append(keys, key)+ }+ tables := getWeightTables()+ // Sort the keys using the custom comparator+ sort.Slice(keys, func(i, j int) bool {+ return compareHeaders(keys[i], keys[j], tables) < 0+ })+ ch := bytes.NewBufferString("")+ for i, key := range keys {+ if i > 0 {+ ch.WriteRune('\n')+ }+ ch.WriteString(key)+ ch.WriteRune(':')+ ch.WriteString(strings.Join(cm[key], ","))+ }+ return ch.String()+}++func (c *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {+ // https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services+ cr := bytes.NewBufferString("/")+ cr.WriteString(c.accountName)++ if len(u.Path) > 0 {+ // Any portion of the CanonicalizedResource string that is derived from+ // the resource's URI should be encoded exactly as it is in the URI.+ // -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx+ cr.WriteString(u.EscapedPath())+ } else {+ // a slash is required to indicate the root path+ cr.WriteString("/")+ }++ // params is a map[string][]string; param name is key; params values is []string+ params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values+ if err != nil {+ return "", fmt.Errorf("failed to parse query params: %w", err)+ }++ if len(params) > 0 { // There is at least 1 query parameter+ var paramNames []string // We use this to sort the parameter key names+ for paramName := range params {+ paramNames = append(paramNames, paramName) // paramNames must be lowercase+ }+ sort.Strings(paramNames)++ for _, paramName := range paramNames {+ paramValues := params[paramName]+ sort.Strings(paramValues)++ // Join the sorted key values separated by ','+ // Then prepend "keyName:"; then add this string to the buffer+ cr.WriteString("\n" + strings.ToLower(paramName) + ":" + strings.Join(paramValues, ","))+ }+ }+ return cr.String(), nil+}++// ComputeHMACSHA256 is a helper for computing the signed string outside of this package.+func ComputeHMACSHA256(cred *SharedKeyCredential, message string) (string, error) {+ return cred.computeHMACSHA256(message)+}++// SharedKeyCredPolicy is a pipeline policy that signs each request with the+// account's shared key.+type SharedKeyCredPolicy struct {+ cred *SharedKeyCredential+}++// NewSharedKeyCredPolicy creates a SharedKeyCredPolicy signing with cred.+func NewSharedKeyCredPolicy(cred *SharedKeyCredential) *SharedKeyCredPolicy {+ return &SharedKeyCredPolicy{cred: cred}+}++// Do signs the request and sends it via the next policy in the pipeline.+func (s *SharedKeyCredPolicy) Do(req *policy.Request) (*http.Response, error) {+ // skip adding the authorization header if no SharedKeyCredential was provided.+ // this prevents a panic that might be hard to diagnose and allows testing+ // against http endpoints that don't require authentication.+ if s.cred == nil {+ return req.Next()+ }++ if d := getHeader("x-ms-date", req.Raw().Header); d == "" {+ req.Raw().Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))+ }+ stringToSign, err := s.cred.buildStringToSign(req.Raw())+ if err != nil {+ return nil, err+ }+ signature, err := s.cred.computeHMACSHA256(stringToSign)+ if err != nil {+ return nil, err+ }+ authHeader := strings.Join([]string{"SharedKey ", s.cred.AccountName(), ":", signature}, "")+ req.Raw().Header.Set("Authorization", authHeader)++ response, err := req.Next()+ if err != nil && response != nil && response.StatusCode == http.StatusForbidden {+ // Service failed to authenticate request, log it+ fs.Debugf(nil, "azureblob arrowlist: HTTP Forbidden status, String-to-Sign: %q", stringToSign)+ }+ return response, err+}diff --git a/backend/azureblob/auth/auth.go b/backend/azureblob/auth/auth.goindex bbea6a4fb..5fa74ceb8 100644--- a/backend/azureblob/auth/auth.go+++ b/backend/azureblob/auth/auth.go@@ -297,6 +297,13 @@ func (tr transporter) Do(req *http.Request) (*http.Response, error) {return tr.RoundTripper.RoundTrip(req)}+// Transporter returns the policy.Transporter rclone uses for Azure SDK+// clients (an fshttp based transport with the APN user agent), for callers+// which build their own azcore pipelines.+func Transporter(ctx context.Context) policy.Transporter {+ return newTransporter(ctx)+}+// NewClientOpts should be passed to configure NewClienttype NewClientOpts[Client, ClientOptions, SharedKeyCredential any] struct {DefaultBaseURL string // Base URL, eg blob.core.windows.netdiff --git a/go.mod b/go.modindex 7034c3986..50a64b8e5 100644--- a/go.mod+++ b/go.mod@@ -21,6 +21,7 @@ require (github.com/adrg/xdg v0.5.3github.com/anacrolix/dms v1.7.2github.com/anacrolix/log v0.17.0+ github.com/apache/arrow-go/v18 v18.7.0github.com/atotto/clipboard v0.1.4github.com/aws/aws-sdk-go-v2 v1.42.1github.com/aws/aws-sdk-go-v2/config v1.32.30@@ -177,9 +178,11 @@ require (github.com/go-playground/validator/v10 v10.30.3 // indirectgithub.com/go-resty/resty/v2 v2.17.2 // indirectgithub.com/go-viper/mapstructure/v2 v2.5.0 // indirect+ github.com/goccy/go-json v0.10.6 // indirectgithub.com/gofrs/flock v0.13.0 // indirectgithub.com/gogo/protobuf v1.3.2 // indirectgithub.com/google/btree v1.1.3 // indirect+ github.com/google/flatbuffers v25.12.19+incompatible // indirectgithub.com/google/s2a-go v0.1.9 // indirectgithub.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirectgithub.com/googleapis/gax-go/v2 v2.22.0 // indirectdiff --git a/go.sum b/go.sumindex 20b902b2b..883cb8c18 100644--- a/go.sum+++ b/go.sum@@ -79,6 +79,10 @@ github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtngithub.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg=github.com/andybalholm/cascadia v1.3.4/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=+github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY=+github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A=+github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ=+github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs=github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er+LKDHDUfU5tRvFwxH0LjPpZN8OoG8Ll+liGU=github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc/go.mod h1:w648aMHEgFYS6xb0KVMMtZ2uMeemhiKCuD2vj6gY52A=github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=@@ -266,8 +270,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vgithub.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=-github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=-github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=+github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=+github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=@@ -280,6 +284,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWekgithub.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=+github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs=+github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=