Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

internxt: fix lookups of files starting with a dot and dropped uploads

jzunigax277ec281072b5
diff --git a/backend/internxt/internxt.go b/backend/internxt/internxt.go
index ed5b8682c..89f144c64 100644
--- a/backend/internxt/internxt.go
+++ b/backend/internxt/internxt.go
@@ -556,43 +556,86 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (string, error)
return resp.UUID, nil
}
-// splitNameExt splits a basename into the (name, ext) shape
+// splitNameExt splits a basename into the (plainName, type) pair that Internxt
+// stores a file under.
func splitNameExt(baseName string) (name, ext string) {
- name = strings.TrimSuffix(baseName, path.Ext(baseName))
- ext = strings.TrimPrefix(path.Ext(baseName), ".")
- return
+ ext = path.Ext(baseName)
+ if ext == baseName {
+ return baseName, ""
+ }
+ return strings.TrimSuffix(baseName, ext), strings.TrimPrefix(ext, ".")
+}
+
+// joinNameExt reassembles the basename a stored (plainName, type) pair
+// displays as, the way List builds its entries.
+func joinNameExt(name, ext string) string {
+ if ext == "" {
+ return name
+ }
+ return name + "." + ext
+}
+
+// legacyNameExt splits a basename at the final dot, the way this backend used
+// to before it adopted the convention the other clients share. It differs from
+// splitNameExt only for a name whose sole dot leads it: ".bashrc" splits into
+// an empty name of type "bashrc".
+func legacyNameExt(baseName string) (name, ext string) {
+ return strings.TrimSuffix(baseName, path.Ext(baseName)), strings.TrimPrefix(path.Ext(baseName), ".")
+}
+
+// existenceCheck builds a lookup criterion for one (plainName, type) spelling.
+func existenceCheck(name, ext string) files.FileExistenceCheck {
+ return files.FileExistenceCheck{
+ PlainName: name,
+ Type: ext,
+ OriginalFile: struct{}{},
+ }
}
// findFile looks up a single file by name within directoryID. Returns
// (nil, nil) when the file does not exist; surfaces transport/API errors.
func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
- name, ext := splitNameExt(f.opt.Encoding.FromStandardName(leaf))
+ encodedName := f.opt.Encoding.FromStandardName(leaf)
+ name, ext := splitNameExt(encodedName)
+ checks := []files.FileExistenceCheck{existenceCheck(name, ext)}
+ if encodedName != name {
+ checks = append(checks, existenceCheck(encodedName, ""))
+ }
+ if legacyName, legacyExt := legacyNameExt(encodedName); legacyExt != "" && legacyName != name {
+ checks = append(checks, existenceCheck(legacyName, legacyExt))
+ }
var checkResult *files.CheckFilesExistenceResponse
err := f.pacer.Call(func() (bool, error) {
var err error
- checkResult, err = files.CheckFilesExistence(ctx, f.cfg, directoryID, []files.FileExistenceCheck{{
- PlainName: name,
- Type: ext,
- OriginalFile: struct{}{},
- }})
+ checkResult, err = files.CheckFilesExistence(ctx, f.cfg, directoryID, checks)
+ if isNotFoundError(err) {
+ return true, err
+ }
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
- if len(checkResult.Files) == 0 || !checkResult.Files[0].FileExists() {
- return nil, nil
+
+ var uuid string
+ for _, result := range checkResult.Files {
+ if !result.FileExists() || result.UUID == "" {
+ continue
+ }
+ if joinNameExt(result.PlainName, result.Type) == encodedName {
+ uuid = result.UUID
+ break
+ }
}
- result := checkResult.Files[0]
- if result.Type != ext || result.UUID == "" {
+ if uuid == "" {
return nil, nil
}
var fileMeta *files.FileMeta
err = f.pacer.Call(func() (bool, error) {
var err error
- fileMeta, err = files.GetFileMeta(ctx, f.cfg, result.UUID)
+ fileMeta, err = files.GetFileMeta(ctx, f.cfg, uuid)
return f.shouldRetry(ctx, err)
})
if err != nil {
@@ -680,8 +723,10 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
return nil, err
}
- // Best-effort existence check
- existingFile, _ := f.findFile(ctx, leaf, directoryID)
+ existingFile, err := f.findFile(ctx, leaf, directoryID)
+ if err != nil {
+ return nil, err
+ }
// Create object - if file exists, populate it with existing metadata
o := &Object{
@@ -850,10 +895,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
return nil, err
}
- // Parse name and extension from the leaf
- baseName := f.opt.Encoding.FromStandardName(leaf)
- newName := strings.TrimSuffix(baseName, path.Ext(baseName))
- newType := strings.TrimPrefix(path.Ext(baseName), ".")
+ newName, newType := splitNameExt(f.opt.Encoding.FromStandardName(leaf))
// Move the file server-side
err = f.pacer.Call(func() (bool, error) {
@@ -1040,6 +1082,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return o.f.tooLargeError(remote, tooLarge)
}
+ if err != nil && isStaleReadError(err, backupUUID != "") {
+ o.restoreBackupFile(ctx, backupUUID, origName, origType)
+ return fserrors.RetryError(err)
+ }
+
if err != nil {
meta, err = o.recoverFromTimeoutConflict(ctx, err, remote, dirID)
}
@@ -1096,6 +1143,19 @@ func isTimeoutError(err error) bool {
return false
}
+// isNotFoundError reports whether err is a 404 from the API.
+func isNotFoundError(err error) bool {
+ var httpErr *sdkerrors.HTTPError
+ return errors.As(err, &httpErr) && httpErr.StatusCode() == 404
+}
+
+// isStaleReadError reports whether err is the API contradicting a write this
+// backend has already made, which it does because lookups are served by read
+// replicas that lag behind the primary.
+func isStaleReadError(err error, renamed bool) bool {
+ return isNotFoundError(err) || (renamed && isConflictError(err))
+}
+
// isConflictError checks if an error indicates a file conflict (409)
func isConflictError(err error) bool {
errMsg := err.Error()
diff --git a/backend/internxt/internxt_internal_test.go b/backend/internxt/internxt_internal_test.go
new file mode 100644
index 000000000..fe795cf59
--- /dev/null
+++ b/backend/internxt/internxt_internal_test.go
@@ -0,0 +1,167 @@
+package internxt
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/internxt/rclone-adapter/config"
+ "github.com/internxt/rclone-adapter/endpoints"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/lib/encoder"
+ "github.com/rclone/rclone/lib/pacer"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// storedFile is a file as the service holds it: a (plainName, type) pair
+// split however the client that uploaded it chose to split it.
+type storedFile struct {
+ uuid string
+ plainName string
+ fileType string
+}
+
+// wireType renders a stored type the way the service does: a file with no
+// extension comes back as JSON null, not an empty string.
+func wireType(fileType string) any {
+ if fileType == "" {
+ return nil
+ }
+ return fileType
+}
+
+// existenceServer answers the existence check and file meta endpoints from
+// files, matching plainName exactly and only constraining type when the
+// criterion supplies one, as the service does.
+func existenceServer(t *testing.T, stored []storedFile) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if strings.HasSuffix(r.URL.Path, "/meta") {
+ uuid := strings.Split(strings.TrimPrefix(r.URL.Path, "/drive/files/"), "/")[0]
+ for _, f := range stored {
+ if f.uuid == uuid {
+ require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
+ "uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),
+ }))
+ return
+ }
+ }
+ w.WriteHeader(http.StatusNotFound)
+ return
+ }
+ var req struct {
+ Files []struct {
+ PlainName string `json:"plainName"`
+ Type string `json:"type"`
+ } `json:"files"`
+ }
+ require.NoError(t, json.NewDecoder(r.Body).Decode(&req))
+ var existent []map[string]any
+ for _, f := range stored {
+ for _, c := range req.Files {
+ if c.PlainName == f.plainName && (c.Type == "" || c.Type == f.fileType) {
+ existent = append(existent, map[string]any{
+ "exists": true, "status": "EXISTS",
+ "uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),
+ })
+ break
+ }
+ }
+ }
+ require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"existentFiles": existent}))
+ }))
+}
+
+func testFs(t *testing.T, server *httptest.Server) *Fs {
+ t.Helper()
+ f := &Fs{
+ opt: Options{Encoding: encoder.EncodeInvalidUtf8},
+ cfg: &config.Config{
+ HTTPClient: server.Client(),
+ Endpoints: endpoints.NewConfig(server.URL),
+ },
+ }
+ f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
+ return f
+}
+
+func TestSplitJoinNameExt(t *testing.T) {
+ for _, test := range []struct {
+ baseName string
+ name string
+ ext string
+ }{
+ {"foo.txt", "foo", "txt"},
+ {"foo.tar.gz", "foo.tar", "gz"},
+ {"README", "README", ""},
+ {".bashrc", ".bashrc", ""},
+ {"..hidden", ".", "hidden"},
+ {"", "", ""},
+ } {
+ name, ext := splitNameExt(test.baseName)
+ assert.Equal(t, test.name, name, test.baseName)
+ assert.Equal(t, test.ext, ext, test.baseName)
+ assert.Equal(t, test.baseName, joinNameExt(name, ext), test.baseName)
+ }
+}
+
+func TestFindFile(t *testing.T) {
+ for _, test := range []struct {
+ what string
+ stored []storedFile
+ leaf string
+ wantUUID string
+ }{{
+ what: "split as rclone splits it",
+ stored: []storedFile{{"uuid-1", "foo.tar", "gz"}},
+ leaf: "foo.tar.gz",
+ wantUUID: "uuid-1",
+ }, {
+ what: "whole name stored as plainName by another client",
+ stored: []storedFile{{"uuid-2", "foo.tar.gz", ""}},
+ leaf: "foo.tar.gz",
+ wantUUID: "uuid-2",
+ }, {
+ what: "extensionless name is not confused with an extended one",
+ stored: []storedFile{{"uuid-3", "README", "md"}, {"uuid-4", "README", ""}},
+ leaf: "README",
+ wantUUID: "uuid-4",
+ }, {
+ what: "dotfile stored the way the web client splits it",
+ stored: []storedFile{{"uuid-6", ".bashrc", ""}},
+ leaf: ".bashrc",
+ wantUUID: "uuid-6",
+ }, {
+ what: "dotfile stored the way older rclone versions split it",
+ stored: []storedFile{{"uuid-7", "", "bashrc"}},
+ leaf: ".bashrc",
+ wantUUID: "uuid-7",
+ }, {
+ what: "trailing dot kept in plainName by a rename",
+ stored: []storedFile{{"uuid-8", "trailing.", ""}},
+ leaf: "trailing.",
+ wantUUID: "uuid-8",
+ }, {
+ what: "missing file",
+ stored: []storedFile{{"uuid-5", "other", "txt"}},
+ leaf: "foo.txt",
+ }} {
+ t.Run(test.what, func(t *testing.T) {
+ server := existenceServer(t, test.stored)
+ defer server.Close()
+ file, err := testFs(t, server).findFile(context.Background(), test.leaf, "dir-uuid")
+ require.NoError(t, err)
+ if test.wantUUID == "" {
+ assert.Nil(t, file)
+ return
+ }
+ require.NotNil(t, file)
+ assert.Equal(t, test.wantUUID, file.UUID)
+ })
+ }
+}
diff --git a/backend/internxt/upload.go b/backend/internxt/upload.go
index 4d1665b3e..caadcbe25 100644
--- a/backend/internxt/upload.go
+++ b/backend/internxt/upload.go
@@ -7,7 +7,6 @@ import (
"io"
"path"
"sort"
- "strings"
"sync"
"github.com/internxt/rclone-adapter/buckets"
@@ -263,9 +262,7 @@ func (w *internxtChunkWriter) Close(ctx context.Context) error {
}
// Create file metadata in Internxt Drive
- baseName := w.f.opt.Encoding.FromStandardName(path.Base(w.remote))
- name := strings.TrimSuffix(baseName, path.Ext(baseName))
- ext := strings.TrimPrefix(path.Ext(baseName), ".")
+ name, ext := splitNameExt(w.f.opt.Encoding.FromStandardName(path.Base(w.remote)))
var meta *buckets.CreateMetaResponse
err = w.f.pacer.Call(func() (bool, error) {
diff --git a/go.mod b/go.mod
index 9bea5db8f..d344a18d3 100644
--- a/go.mod
+++ b/go.mod
@@ -47,7 +47,7 @@ require (
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/uuid v1.6.0
github.com/hanwen/go-fuse/v2 v2.10.1
- github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2
+ github.com/internxt/rclone-adapter v0.0.0-20260904121407-404a75f3d031
github.com/jcmturner/gokrb5/v8 v8.4.4
github.com/jlaffaye/ftp v0.2.1-0.20251026020404-6602e981a1bb
github.com/josephspurrier/goversioninfo v1.7.0
diff --git a/go.sum b/go.sum
index 8512fa045..f27afaefd 100644
--- a/go.sum
+++ b/go.sum
@@ -326,8 +326,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2 h1:ZeebPK9Bnpy40uTuneEeVMYaMMerol3qsmIvzD0EIAk=
-github.com/internxt/rclone-adapter v0.0.0-20260708165336-dd6561bacfa2/go.mod h1:4jGLEnNHyWOVSGn89IeWUVqlKCEitOM3D32/XypGy/o=
+github.com/internxt/rclone-adapter v0.0.0-20260904121407-404a75f3d031 h1:l0apMNjUmo3E9wtiveXKkBL1gpvWb7OUawnn9ZsOS2I=
+github.com/internxt/rclone-adapter v0.0.0-20260904121407-404a75f3d031/go.mod h1:4jGLEnNHyWOVSGn89IeWUVqlKCEitOM3D32/XypGy/o=
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=