"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
azureblob: fix directories on storage accounts with hierarchical namespace - fixes #7047 #9577
Nick Craig-Wood93c2787dc8a6
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.goindex 77ecec922..03894e0f5 100644--- a/backend/azureblob/azureblob.go+++ b/backend/azureblob/azureblob.go@@ -19,6 +19,7 @@ import ("path""slices""sort"+ "strconv""strings""sync""sync/atomic"@@ -364,7 +365,22 @@ creates an empty object ending with "/", to persist the folder.This object also has the metadata "` + dirMetaKey + ` = ` + dirMetaValue + `" to conform tothe Microsoft standard.++This is not needed on storage accounts with a hierarchical namespace+(see the hns option) as these always support empty directories.`,+ }, {+ Name: "hns",+ Default: fs.Tristate{},+ Advanced: true,+ Help: `Set if the storage account has a hierarchical namespace.++Storage accounts with hierarchical namespace enabled (Azure Data Lake+Storage Gen2) have real directories which can be empty.++If this is unset then rclone will read it from the storage account+and save it in the config file.+`,}, {Name: "no_check_container",Help: `If set, don't attempt to check the container exists or create it.@@ -432,6 +448,7 @@ type Options struct {Enc encoder.MultiEncoder `config:"encoding"`PublicAccess string `config:"public_access"`DirectoryMarkers bool `config:"directory_markers"`+ HNS fs.Tristate `config:"hns"`NoCheckContainer bool `config:"no_check_container"`NoHeadObject bool `config:"no_head_object"`DeleteSnapshots string `config:"delete_snapshots"`@@ -455,6 +472,7 @@ type Fs struct {rootContainer string // container part of root (if any)rootDirectory string // directory part of root (if any)isLimited bool // if limited to one container+ hns bool // if the storage account has a hierarchical namespacecache *bucket.Cache // cache for container creation statuspacer *fs.Pacer // To pace and retry the API callsuploadToken *pacer.TokenDispenser // control concurrency@@ -521,7 +539,7 @@ func parsePath(path string) (root string) {// relative to f.rootfunc (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))- if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {+ if f.dirMarkers() && strings.HasSuffix(containerPath, "//") {containerPath = containerPath[:len(containerPath)-1]}return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)@@ -688,10 +706,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, eServerSideAcrossConfigs: true,DoubleSlash: true,}).Fill(ctx, f)- if opt.DirectoryMarkers {- f.features.CanHaveEmptyDirectories = true- fs.Debugf(f, "Using directory markers")- }conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{DefaultBaseURL: storageDefaultBaseURL,@@ -724,6 +738,28 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, ef.isLimited = true}+ if opt.HNS.Valid {+ f.hns = opt.HNS.Value+ } else if !f.anonymous {+ containerName := f.rootContainer+ if containerName == "" {+ containerName = res.Container+ }+ f.hns, err = f.readHNS(ctx, containerName)+ if err != nil {+ fs.Debugf(f, "Failed to read whether storage account has a hierarchical namespace - assuming not: %v", err)+ } else {+ m.Set("hns", strconv.FormatBool(f.hns))+ }+ }+ if f.hns {+ fs.Debugf(f, "Storage account has a hierarchical namespace")+ }+ if f.dirMarkers() {+ f.features.CanHaveEmptyDirectories = true+ fs.Debugf(f, "Using directory markers")+ }+if f.rootContainer != "" && f.rootDirectory != "" {// Check to see if the (container,directory) is actually an existing fileoldRoot := f.root@@ -744,6 +780,39 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, ereturn f, nil}+// readHNS reads whether the storage account has a hierarchical namespace+//+// If containerName is set then the container client is used which works+// with credentials limited to that container.+func (f *Fs) readHNS(ctx context.Context, containerName string) (hns bool, err error) {+ var isHNS *bool+ err = f.pacer.Call(func() (bool, error) {+ var err error+ if containerName != "" {+ var resp container.GetAccountInfoResponse+ resp, err = f.cntSVC(containerName).GetAccountInfo(ctx, nil)+ isHNS = resp.IsHierarchicalNamespaceEnabled+ } else {+ var resp service.GetAccountInfoResponse+ resp, err = f.svc.GetAccountInfo(ctx, nil)+ isHNS = resp.IsHierarchicalNamespaceEnabled+ }+ return f.shouldRetry(ctx, err)+ })+ if err != nil {+ return false, err+ }+ if isHNS == nil {+ return false, errors.New("not returned by server")+ }+ return *isHNS, nil+}++// dirMarkers returns true if directories are persisted with directory markers+func (f *Fs) dirMarkers() bool {+ return f.opt.DirectoryMarkers || f.hns+}+// return the container client for the container passed infunc (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {f.cntSVCcacheMu.Lock()@@ -1095,18 +1164,33 @@ func isDirectoryMarker(size int64, metadata map[string]*string, remote string) bif endsWithSlash || remote == "" {return true}- // Note that metadata with hdi_isfolder = true seems to be a- // defacto standard for marking blobs as directories.- // Note also that the metadata hasn't been normalised to lower case yet- for k, v := range metadata {- if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {- return true- }+ return hasDirMetadata(metadata)+ }+ return false+}++// Returns whether the metadata marks the blob as a directory+func hasDirMetadata(metadata map[string]*string) bool {+ // Note that metadata with hdi_isfolder = true seems to be a+ // defacto standard for marking blobs as directories.+ // Note also that the metadata hasn't been normalised to lower case yet+ for k, v := range metadata {+ if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {+ return true}}return false}+// Returns whether props are for a directory on a storage account with+// a hierarchical namespace+//+// These ignore a trailing "/" on the name so a file can be returned+// when looking for a directory.+func isHNSDirectory(props *blob.GetPropertiesResponse) bool {+ return props.ContentLength != nil && *props.ContentLength == 0 && hasDirMetadata(props.Metadata)+}+// listFn is called from list to handle an objecttype listFn func(remote string, object *container.BlobItem, isDirectory bool) error@@ -1165,15 +1249,18 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,if err != nil {return err}- if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {+ if f.dirMarkers() && foundItems == 0 && directory != "" {// Determine whether the directory exists or not by whether it has a marker- _, err := f.readMetaData(ctx, containerName, directory)+ props, err := f.readMetaData(ctx, containerName, directory)if err != nil {if err == fs.ErrorObjectNotFound {return fs.ErrorDirNotFound}return err}+ if f.hns && !isHNSDirectory(props) {+ return fs.ErrorDirNotFound+ }}return nil}@@ -1537,7 +1624,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt// Create directory marker file and parentsfunc (f *Fs) createDirectoryMarker(ctx context.Context, container, dir string) error {- if !f.opt.DirectoryMarkers || container == "" {+ if !f.dirMarkers() || container == "" {return nil}@@ -1594,6 +1681,11 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {// mkdirParent creates the parent bucket/directory if it doesn't existfunc (f *Fs) mkdirParent(ctx context.Context, remote string) error {+ if f.hns {+ // Parent directories are created by the upload+ container, _ := f.split(remote)+ return f.makeContainer(ctx, container)+ }remote, _ = strings.CutSuffix(remote, "/")dir := path.Dir(remote)if dir == "/" || dir == "." {@@ -1699,6 +1791,9 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {// Returns an error if it isn't emptyfunc (f *Fs) Rmdir(ctx context.Context, dir string) error {container, directory := f.split(dir)+ if f.hns && container != "" && directory != "" {+ return f.rmdirHNS(ctx, container, directory)+ }// Remove directory marker fileif f.opt.DirectoryMarkers && container != "" && directory != "" {o := &Object{@@ -1721,6 +1816,39 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {return f.deleteContainer(ctx, container)}+// rmdirHNS removes the directory on a storage account with a+// hierarchical namespace+//+// The directory must be deleted without a trailing "/" and the server+// refuses to delete it if it isn't empty.+func (f *Fs) rmdirHNS(ctx context.Context, container, directory string) error {+ directory, _ = strings.CutSuffix(directory, "/")+ // Check it is a directory as deleting a file would succeed+ props, err := f.readMetaData(ctx, container, directory)+ if err == fs.ErrorObjectNotFound {+ return fs.ErrorDirNotFound+ } else if err != nil {+ return err+ }+ if !isHNSDirectory(props) {+ return fs.ErrorIsFile+ }+ blb := f.getBlobSVC(container, directory)+ err = f.pacer.Call(func() (bool, error) {+ _, err := blb.Delete(ctx, nil)+ return f.shouldRetry(ctx, err)+ })+ if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {+ if storageErr.ErrorCode == "DirectoryIsNotEmpty" {+ return fs.ErrorDirectoryNotEmpty+ }+ if storageErr.StatusCode == http.StatusNotFound {+ return fs.ErrorDirNotFound+ }+ }+ return err+}+// Precision of the remotefunc (f *Fs) Precision() time.Duration {return time.Nanosecond