Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

azureblob: fix directories on storage accounts with hierarchical namespace - fixes #7047 #9577

Nick Craig-Wood93c2787dc8a6
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 77ecec922..03894e0f5 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -19,6 +19,7 @@ import (
"path"
"slices"
"sort"
+ "strconv"
"strings"
"sync"
"sync/atomic"
@@ -364,7 +365,22 @@ creates an empty object ending with "/", to persist the folder.
This object also has the metadata "` + dirMetaKey + ` = ` + dirMetaValue + `" to conform to
the Microsoft standard.
+
+This is not needed on storage accounts with a hierarchical namespace
+(see the hns option) as these always support empty directories.
`,
+ }, {
+ Name: "hns",
+ Default: fs.Tristate{},
+ Advanced: true,
+ Help: `Set if the storage account has a hierarchical namespace.
+
+Storage accounts with hierarchical namespace enabled (Azure Data Lake
+Storage Gen2) have real directories which can be empty.
+
+If this is unset then rclone will read it from the storage account
+and save it in the config file.
+`,
}, {
Name: "no_check_container",
Help: `If set, don't attempt to check the container exists or create it.
@@ -432,6 +448,7 @@ type Options struct {
Enc encoder.MultiEncoder `config:"encoding"`
PublicAccess string `config:"public_access"`
DirectoryMarkers bool `config:"directory_markers"`
+ HNS fs.Tristate `config:"hns"`
NoCheckContainer bool `config:"no_check_container"`
NoHeadObject bool `config:"no_head_object"`
DeleteSnapshots string `config:"delete_snapshots"`
@@ -455,6 +472,7 @@ type Fs struct {
rootContainer string // container part of root (if any)
rootDirectory string // directory part of root (if any)
isLimited bool // if limited to one container
+ hns bool // if the storage account has a hierarchical namespace
cache *bucket.Cache // cache for container creation status
pacer *fs.Pacer // To pace and retry the API calls
uploadToken *pacer.TokenDispenser // control concurrency
@@ -521,7 +539,7 @@ func parsePath(path string) (root string) {
// relative to f.root
func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
- if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
+ if f.dirMarkers() && strings.HasSuffix(containerPath, "//") {
containerPath = containerPath[:len(containerPath)-1]
}
return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
@@ -688,10 +706,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ServerSideAcrossConfigs: true,
DoubleSlash: true,
}).Fill(ctx, f)
- if opt.DirectoryMarkers {
- f.features.CanHaveEmptyDirectories = true
- fs.Debugf(f, "Using directory markers")
- }
conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
DefaultBaseURL: storageDefaultBaseURL,
@@ -724,6 +738,28 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
f.isLimited = true
}
+ if opt.HNS.Valid {
+ f.hns = opt.HNS.Value
+ } else if !f.anonymous {
+ containerName := f.rootContainer
+ if containerName == "" {
+ containerName = res.Container
+ }
+ f.hns, err = f.readHNS(ctx, containerName)
+ if err != nil {
+ fs.Debugf(f, "Failed to read whether storage account has a hierarchical namespace - assuming not: %v", err)
+ } else {
+ m.Set("hns", strconv.FormatBool(f.hns))
+ }
+ }
+ if f.hns {
+ fs.Debugf(f, "Storage account has a hierarchical namespace")
+ }
+ if f.dirMarkers() {
+ f.features.CanHaveEmptyDirectories = true
+ fs.Debugf(f, "Using directory markers")
+ }
+
if f.rootContainer != "" && f.rootDirectory != "" {
// Check to see if the (container,directory) is actually an existing file
oldRoot := f.root
@@ -744,6 +780,39 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return f, nil
}
+// readHNS reads whether the storage account has a hierarchical namespace
+//
+// If containerName is set then the container client is used which works
+// with credentials limited to that container.
+func (f *Fs) readHNS(ctx context.Context, containerName string) (hns bool, err error) {
+ var isHNS *bool
+ err = f.pacer.Call(func() (bool, error) {
+ var err error
+ if containerName != "" {
+ var resp container.GetAccountInfoResponse
+ resp, err = f.cntSVC(containerName).GetAccountInfo(ctx, nil)
+ isHNS = resp.IsHierarchicalNamespaceEnabled
+ } else {
+ var resp service.GetAccountInfoResponse
+ resp, err = f.svc.GetAccountInfo(ctx, nil)
+ isHNS = resp.IsHierarchicalNamespaceEnabled
+ }
+ return f.shouldRetry(ctx, err)
+ })
+ if err != nil {
+ return false, err
+ }
+ if isHNS == nil {
+ return false, errors.New("not returned by server")
+ }
+ return *isHNS, nil
+}
+
+// dirMarkers returns true if directories are persisted with directory markers
+func (f *Fs) dirMarkers() bool {
+ return f.opt.DirectoryMarkers || f.hns
+}
+
// return the container client for the container passed in
func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
f.cntSVCcacheMu.Lock()
@@ -1095,18 +1164,33 @@ func isDirectoryMarker(size int64, metadata map[string]*string, remote string) b
if endsWithSlash || remote == "" {
return true
}
- // Note that metadata with hdi_isfolder = true seems to be a
- // defacto standard for marking blobs as directories.
- // Note also that the metadata hasn't been normalised to lower case yet
- for k, v := range metadata {
- if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
- return true
- }
+ return hasDirMetadata(metadata)
+ }
+ return false
+}
+
+// Returns whether the metadata marks the blob as a directory
+func hasDirMetadata(metadata map[string]*string) bool {
+ // Note that metadata with hdi_isfolder = true seems to be a
+ // defacto standard for marking blobs as directories.
+ // Note also that the metadata hasn't been normalised to lower case yet
+ for k, v := range metadata {
+ if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
+ return true
}
}
return false
}
+// Returns whether props are for a directory on a storage account with
+// a hierarchical namespace
+//
+// These ignore a trailing "/" on the name so a file can be returned
+// when looking for a directory.
+func isHNSDirectory(props *blob.GetPropertiesResponse) bool {
+ return props.ContentLength != nil && *props.ContentLength == 0 && hasDirMetadata(props.Metadata)
+}
+
// listFn is called from list to handle an object
type listFn func(remote string, object *container.BlobItem, isDirectory bool) error
@@ -1165,15 +1249,18 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
if err != nil {
return err
}
- if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
+ if f.dirMarkers() && foundItems == 0 && directory != "" {
// Determine whether the directory exists or not by whether it has a marker
- _, err := f.readMetaData(ctx, containerName, directory)
+ props, err := f.readMetaData(ctx, containerName, directory)
if err != nil {
if err == fs.ErrorObjectNotFound {
return fs.ErrorDirNotFound
}
return err
}
+ if f.hns && !isHNSDirectory(props) {
+ return fs.ErrorDirNotFound
+ }
}
return nil
}
@@ -1537,7 +1624,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
// Create directory marker file and parents
func (f *Fs) createDirectoryMarker(ctx context.Context, container, dir string) error {
- if !f.opt.DirectoryMarkers || container == "" {
+ if !f.dirMarkers() || container == "" {
return nil
}
@@ -1594,6 +1681,11 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// mkdirParent creates the parent bucket/directory if it doesn't exist
func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
+ if f.hns {
+ // Parent directories are created by the upload
+ container, _ := f.split(remote)
+ return f.makeContainer(ctx, container)
+ }
remote, _ = strings.CutSuffix(remote, "/")
dir := path.Dir(remote)
if dir == "/" || dir == "." {
@@ -1699,6 +1791,9 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
// Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
container, directory := f.split(dir)
+ if f.hns && container != "" && directory != "" {
+ return f.rmdirHNS(ctx, container, directory)
+ }
// Remove directory marker file
if f.opt.DirectoryMarkers && container != "" && directory != "" {
o := &Object{
@@ -1721,6 +1816,39 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
return f.deleteContainer(ctx, container)
}
+// rmdirHNS removes the directory on a storage account with a
+// hierarchical namespace
+//
+// The directory must be deleted without a trailing "/" and the server
+// refuses to delete it if it isn't empty.
+func (f *Fs) rmdirHNS(ctx context.Context, container, directory string) error {
+ directory, _ = strings.CutSuffix(directory, "/")
+ // Check it is a directory as deleting a file would succeed
+ props, err := f.readMetaData(ctx, container, directory)
+ if err == fs.ErrorObjectNotFound {
+ return fs.ErrorDirNotFound
+ } else if err != nil {
+ return err
+ }
+ if !isHNSDirectory(props) {
+ return fs.ErrorIsFile
+ }
+ blb := f.getBlobSVC(container, directory)
+ err = f.pacer.Call(func() (bool, error) {
+ _, err := blb.Delete(ctx, nil)
+ return f.shouldRetry(ctx, err)
+ })
+ if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
+ if storageErr.ErrorCode == "DirectoryIsNotEmpty" {
+ return fs.ErrorDirectoryNotEmpty
+ }
+ if storageErr.StatusCode == http.StatusNotFound {
+ return fs.ErrorDirNotFound
+ }
+ }
+ return err
+}
+
// Precision of the remote
func (f *Fs) Precision() time.Duration {
return time.Nanosecond