"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
internxt: fix lookups of files starting with a dot and dropped uploads
jzunigax277ec281072b5
diff --git a/backend/internxt/internxt.go b/backend/internxt/internxt.goindex ed5b8682c..89f144c64 100644--- a/backend/internxt/internxt.go+++ b/backend/internxt/internxt.go@@ -556,43 +556,86 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (string, error)return resp.UUID, nil}-// splitNameExt splits a basename into the (name, ext) shape+// splitNameExt splits a basename into the (plainName, type) pair that Internxt+// stores a file under.func splitNameExt(baseName string) (name, ext string) {- name = strings.TrimSuffix(baseName, path.Ext(baseName))- ext = strings.TrimPrefix(path.Ext(baseName), ".")- return+ ext = path.Ext(baseName)+ if ext == baseName {+ return baseName, ""+ }+ return strings.TrimSuffix(baseName, ext), strings.TrimPrefix(ext, ".")+}++// joinNameExt reassembles the basename a stored (plainName, type) pair+// displays as, the way List builds its entries.+func joinNameExt(name, ext string) string {+ if ext == "" {+ return name+ }+ return name + "." + ext+}++// legacyNameExt splits a basename at the final dot, the way this backend used+// to before it adopted the convention the other clients share. It differs from+// splitNameExt only for a name whose sole dot leads it: ".bashrc" splits into+// an empty name of type "bashrc".+func legacyNameExt(baseName string) (name, ext string) {+ return strings.TrimSuffix(baseName, path.Ext(baseName)), strings.TrimPrefix(path.Ext(baseName), ".")+}++// existenceCheck builds a lookup criterion for one (plainName, type) spelling.+func existenceCheck(name, ext string) files.FileExistenceCheck {+ return files.FileExistenceCheck{+ PlainName: name,+ Type: ext,+ OriginalFile: struct{}{},+ }}// findFile looks up a single file by name within directoryID. Returns// (nil, nil) when the file does not exist; surfaces transport/API errors.func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.File, error) {- name, ext := splitNameExt(f.opt.Encoding.FromStandardName(leaf))+ encodedName := f.opt.Encoding.FromStandardName(leaf)+ name, ext := splitNameExt(encodedName)+ checks := []files.FileExistenceCheck{existenceCheck(name, ext)}+ if encodedName != name {+ checks = append(checks, existenceCheck(encodedName, ""))+ }+ if legacyName, legacyExt := legacyNameExt(encodedName); legacyExt != "" && legacyName != name {+ checks = append(checks, existenceCheck(legacyName, legacyExt))+ }var checkResult *files.CheckFilesExistenceResponseerr := f.pacer.Call(func() (bool, error) {var err error- checkResult, err = files.CheckFilesExistence(ctx, f.cfg, directoryID, []files.FileExistenceCheck{{- PlainName: name,- Type: ext,- OriginalFile: struct{}{},- }})+ checkResult, err = files.CheckFilesExistence(ctx, f.cfg, directoryID, checks)+ if isNotFoundError(err) {+ return true, err+ }return f.shouldRetry(ctx, err)})if err != nil {return nil, err}- if len(checkResult.Files) == 0 || !checkResult.Files[0].FileExists() {- return nil, nil++ var uuid string+ for _, result := range checkResult.Files {+ if !result.FileExists() || result.UUID == "" {+ continue+ }+ if joinNameExt(result.PlainName, result.Type) == encodedName {+ uuid = result.UUID+ break+ }}- result := checkResult.Files[0]- if result.Type != ext || result.UUID == "" {+ if uuid == "" {return nil, nil}var fileMeta *files.FileMetaerr = f.pacer.Call(func() (bool, error) {var err error- fileMeta, err = files.GetFileMeta(ctx, f.cfg, result.UUID)+ fileMeta, err = files.GetFileMeta(ctx, f.cfg, uuid)return f.shouldRetry(ctx, err)})if err != nil {@@ -680,8 +723,10 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .return nil, err}- // Best-effort existence check- existingFile, _ := f.findFile(ctx, leaf, directoryID)+ existingFile, err := f.findFile(ctx, leaf, directoryID)+ if err != nil {+ return nil, err+ }// Create object - if file exists, populate it with existing metadatao := &Object{@@ -850,10 +895,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,return nil, err}- // Parse name and extension from the leaf- baseName := f.opt.Encoding.FromStandardName(leaf)- newName := strings.TrimSuffix(baseName, path.Ext(baseName))- newType := strings.TrimPrefix(path.Ext(baseName), ".")+ newName, newType := splitNameExt(f.opt.Encoding.FromStandardName(leaf))// Move the file server-sideerr = f.pacer.Call(func() (bool, error) {@@ -1040,6 +1082,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opreturn o.f.tooLargeError(remote, tooLarge)}+ if err != nil && isStaleReadError(err, backupUUID != "") {+ o.restoreBackupFile(ctx, backupUUID, origName, origType)+ return fserrors.RetryError(err)+ }+if err != nil {meta, err = o.recoverFromTimeoutConflict(ctx, err, remote, dirID)}@@ -1096,6 +1143,19 @@ func isTimeoutError(err error) bool {return false}+// isNotFoundError reports whether err is a 404 from the API.+func isNotFoundError(err error) bool {+ var httpErr *sdkerrors.HTTPError+ return errors.As(err, &httpErr) && httpErr.StatusCode() == 404+}++// isStaleReadError reports whether err is the API contradicting a write this+// backend has already made, which it does because lookups are served by read+// replicas that lag behind the primary.+func isStaleReadError(err error, renamed bool) bool {+ return isNotFoundError(err) || (renamed && isConflictError(err))+}+// isConflictError checks if an error indicates a file conflict (409)func isConflictError(err error) bool {errMsg := err.Error()diff --git a/backend/internxt/internxt_internal_test.go b/backend/internxt/internxt_internal_test.gonew file mode 100644index 000000000..fe795cf59--- /dev/null+++ b/backend/internxt/internxt_internal_test.go@@ -0,0 +1,167 @@+package internxt++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/internxt/rclone-adapter/config"+ "github.com/internxt/rclone-adapter/endpoints"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/encoder"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// storedFile is a file as the service holds it: a (plainName, type) pair+// split however the client that uploaded it chose to split it.+type storedFile struct {+ uuid string+ plainName string+ fileType string+}++// wireType renders a stored type the way the service does: a file with no+// extension comes back as JSON null, not an empty string.+func wireType(fileType string) any {+ if fileType == "" {+ return nil+ }+ return fileType+}++// existenceServer answers the existence check and file meta endpoints from+// files, matching plainName exactly and only constraining type when the+// criterion supplies one, as the service does.+func existenceServer(t *testing.T, stored []storedFile) *httptest.Server {+ t.Helper()+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ if strings.HasSuffix(r.URL.Path, "/meta") {+ uuid := strings.Split(strings.TrimPrefix(r.URL.Path, "/drive/files/"), "/")[0]+ for _, f := range stored {+ if f.uuid == uuid {+ require.NoError(t, json.NewEncoder(w).Encode(map[string]any{+ "uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),+ }))+ return+ }+ }+ w.WriteHeader(http.StatusNotFound)+ return+ }+ var req struct {+ Files []struct {+ PlainName string `json:"plainName"`+ Type string `json:"type"`+ } `json:"files"`+ }+ require.NoError(t, json.NewDecoder(r.Body).Decode(&req))+ var existent []map[string]any+ for _, f := range stored {+ for _, c := range req.Files {+ if c.PlainName == f.plainName && (c.Type == "" || c.Type == f.fileType) {+ existent = append(existent, map[string]any{+ "exists": true, "status": "EXISTS",+ "uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),+ })+ break+ }+ }+ }+ require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"existentFiles": existent}))+ }))+}++func testFs(t *testing.T, server *httptest.Server) *Fs {+ t.Helper()+ f := &Fs{+ opt: Options{Encoding: encoder.EncodeInvalidUtf8},+ cfg: &config.Config{+ HTTPClient: server.Client(),+ Endpoints: endpoints.NewConfig(server.URL),+ },+ }+ f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))+ return f+}++func TestSplitJoinNameExt(t *testing.T) {+ for _, test := range []struct {+ baseName string+ name string+ ext string+ }{+ {"foo.txt", "foo", "txt"},+ {"foo.tar.gz", "foo.tar", "gz"},+ {"README", "README", ""},+ {".bashrc", ".bashrc", ""},+ {"..hidden", ".", "hidden"},+ {"", "", ""},+ } {+ name, ext := splitNameExt(test.baseName)+ assert.Equal(t, test.name, name, test.baseName)+ assert.Equal(t, test.ext, ext, test.baseName)+ assert.Equal(t, test.baseName, joinNameExt(name, ext), test.baseName)+ }+}++func TestFindFile(t *testing.T) {+ for _, test := range []struct {+ what string+ stored []storedFile+ leaf string+ wantUUID string+ }{{+ what: "split as rclone splits it",+ stored: []storedFile{{"uuid-1", "foo.tar", "gz"}},+ leaf: "foo.tar.gz",+ wantUUID: "uuid-1",+ }, {+ what: "whole name stored as plainName by another client",+ stored: []storedFile{{"uuid-2", "foo.tar.gz", ""}},+ leaf: "foo.tar.gz",+ wantUUID: "uuid-2",+ }, {+ what: "extensionless name is not confused with an extended one",+ stored: []storedFile{{"uuid-3", "README", "md"}, {"uuid-4", "README", ""}},+ leaf: "README",+ wantUUID: "uuid-4",+ }, {+ what: "dotfile stored the way the web client splits it",+ stored: []storedFile{{"uuid-6", ".bashrc", ""}},+ leaf: ".bashrc",+ wantUUID: "uuid-6",+ }, {+ what: "dotfile stored the way older rclone versions split it",+ stored: []storedFile{{"uuid-7", "", "bashrc"}},+ leaf: ".bashrc",+ wantUUID: "uuid-7",+ }, {+ what: "trailing dot kept in plainName by a rename",+ stored: []storedFile{{"uuid-8", "trailing.", ""}},+ leaf: "trailing.",+ wantUUID: "uuid-8",+ }, {+ what: "missing file",+ stored: []storedFile{{"uuid-5", "other", "txt"}},+ leaf: "foo.txt",+ }} {+ t.Run(test.what, func(t *testing.T) {+ server := existenceServer(t, test.stored)+ defer server.Close()+ file, err := testFs(t, server).findFile(context.Background(), test.leaf, "dir-uuid")+ require.NoError(t, err)+ if test.wantUUID == "" {+ assert.Nil(t, file)+ return+ }+ require.NotNil(t, file)+ assert.Equal(t, test.wantUUID, file.UUID)+ })+ }+}diff --git a/backend/internxt/upload.go b/backend/internxt/upload.goindex 4d1665b3e..caadcbe25 100644--- a/backend/internxt/upload.go+++ b/backend/internxt/upload.go@@ -7,7 +7,6 @@ import ("io""path""sort"- "strings""sync""github.com/internxt/rclone-adapter/buckets"@@ -263,9 +262,7 @@ func (w *internxtChunkWriter) Close(ctx context.Context) error {}// Create file metadata in Internxt Drive- baseName := w.f.opt.Encoding.FromStandardName(path.Base(w.remote))- name := strings.TrimSuffix(baseName, path.Ext(baseName))- ext := strings.TrimPrefix(path.Ext(baseName), ".")+ name, ext := splitNameExt(w.f.opt.Encoding.FromStandardName(path.Base(w.remote)))var meta *buckets.CreateMetaResponseerr = w.f.pacer.Call(func() (bool, error) {diff --git a/go.mod b/go.modindex 9bea5db8f..d344a18d3 100644--- a/go.mod+++ b/go.mod@@ -47,7 +47,7 @@ require (github.com/golang-jwt/jwt/v5 v5.3.1github.com/google/uuid v1.6.0github.com/hanwen/go-fuse/v2 v2.10.1- github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2+ github.com/internxt/rclone-adapter v0.0.0-20260904121407-404a75f3d031github.com/jcmturner/gokrb5/v8 v8.4.4github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bbgithub.com/josephspurrier/goversioninfo v1.7.0diff --git a/go.sum b/go.sumindex 8512fa045..f27afaefd 100644--- a/go.sum+++ b/go.sum@@ -326,8 +326,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFsgithub.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=-github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2 h1:ZeebPK9Bnpy40uTuneEeVMYaMMerol3qsmIvzD0EIAk=-github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2/go.mod h1:4jGLEnNHyWOVSGn89IeWUVqlKCEitOM3D32/XypGy/o=+github.com/internxt/rclone-adapter v0.0.0-20260904121407-404a75f3d031 h1:l0apMNjUmo3E9wtiveXKkBL1gpvWb7OUawnn9ZsOS2I=+github.com/internxt/rclone-adapter v0.0.0-20260904121407-404a75f3d031/go.mod h1:4jGLEnNHyWOVSGn89IeWUVqlKCEitOM3D32/XypGy/o=github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=