"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
// Package internxt provides an interface to Internxt's Drive API
package internxt
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"path"
"strings"
"sync"
"time"
"github.com/internxt/rclone-adapter/auth"
"github.com/internxt/rclone-adapter/buckets"
config "github.com/internxt/rclone-adapter/config"
sdkerrors "github.com/internxt/rclone-adapter/errors"
"github.com/internxt/rclone-adapter/files"
"github.com/internxt/rclone-adapter/folders"
"github.com/internxt/rclone-adapter/users"
"github.com/rclone/rclone/fs"
rclone_config "github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
)
const (
minSleep = 10 * time.Millisecond
maxSleep = 2 * time.Second
decayConstant = 2
maxUploadParts = 10000
minChunkSize = fs.SizeSuffix(5 * 1024 * 1024)
minUploadCutoff = fs.SizeSuffix(100 * 1024 * 1024)
defaultUploadCutoff = fs.SizeSuffix(100 * 1024 * 1024)
maxUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
)
// shouldRetry determines if an error should be retried.
// On 401, it attempts to re-authorize before retrying.
// On 429, it honours the server's rate limit retry delay.
func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
if fserrors.ContextError(ctx, &err) {
return false, err
}
if httpErr, ok := errors.AsType[*sdkerrors.HTTPError](err); ok {
switch httpErr.StatusCode() {
case 401:
if !f.authFailed {
authErr := f.reAuthorize(ctx)
if authErr != nil {
fs.Debugf(f, "Re-authorization failed: %v", authErr)
return false, err
}
return true, err
}
return false, err
case 429:
delay := httpErr.RetryAfter()
if delay <= 0 {
delay = time.Second
}
return true, pacer.RetryAfterError(err, delay)
}
}
return fserrors.ShouldRetry(err), err
}
// Register with Fs
func init() {
fs.Register(&fs.RegInfo{
Name: "internxt",
Description: "Internxt Drive",
NewFs: NewFs,
Config: Config,
Options: []fs.Option{{
Name: "email",
Help: "Email of your Internxt account.",
Required: true,
Sensitive: true,
}, {
Name: "pass",
Help: "Password.",
Required: true,
IsPassword: true,
}, {
Name: "mnemonic",
Help: "Mnemonic (internal use only)",
Required: false,
Advanced: true,
Sensitive: true,
Hide: fs.OptionHideBoth,
}, {
Name: "skip_hash_validation",
Default: true,
Advanced: true,
Help: "Skip hash validation when downloading files.\n\nBy default, hash validation is disabled. Set this to false to enable validation.",
}, {
Name: "upload_concurrency",
Help: "Concurrency for multipart uploads.\n\nThis is the number of chunks of the same file that are uploaded concurrently.\n\nNote that each chunk is buffered in memory.",
Default: 4,
Advanced: true,
}, {
Name: "upload_cutoff",
Help: "Cutoff for switching to multipart upload.\n\nAny files larger than this will be uploaded in chunks of chunk_size.\nThe minimum is 100 MiB and the maximum is 5 GiB.",
Default: defaultUploadCutoff,
Advanced: true,
}, {
Name: "chunk_size",
Help: "Chunk size for multipart uploads.\n\nFiles larger than upload_cutoff will be uploaded in chunks of this size.\n\nMemory usage is approximately chunk_size * upload_concurrency.",
Default: fs.SizeSuffix(30 * 1024 * 1024),
Advanced: true,
}, {
Name: rclone_config.ConfigEncoding,
Help: rclone_config.ConfigEncodingHelp,
Advanced: true,
Default: encoder.EncodeInvalidUtf8 |
encoder.EncodeSlash |
encoder.EncodeBackSlash |
encoder.EncodeRightPeriod |
encoder.EncodeDot |
encoder.EncodeCrLf,
}},
})
}
// Config configures the Internxt remote by performing login
func Config(ctx context.Context, name string, m configmap.Mapper, configIn fs.ConfigIn) (*fs.ConfigOut, error) {
email, _ := m.Get("email")
if email == "" {
return nil, errors.New("email is required")
}
pass, _ := m.Get("pass")
if pass != "" {
var err error
pass, err = obscure.Reveal(pass)
if err != nil {
return nil, fmt.Errorf("couldn't decrypt password: %w", err)
}
}
cfg := config.NewDefaultToken("")
cfg.HTTPClient = fshttp.NewClient(ctx)
switch configIn.State {
case "":
// Check if 2FA is required
loginResp, err := auth.Login(ctx, cfg, email)
if err != nil {
return nil, fmt.Errorf("failed to check login requirements: %w", err)
}
if loginResp.TFA {
return fs.ConfigInput("2fa", "config_2fa", "Two-factor authentication code")
}
// No 2FA required, do login directly
return fs.ConfigGoto("login")
case "2fa":
twoFA := configIn.Result
if twoFA == "" {
return fs.ConfigError("", "2FA code is required")
}
m.Set("2fa_code", twoFA)
return fs.ConfigGoto("login")
case "login":
twoFA, _ := m.Get("2fa_code")
loginResp, err := auth.DoLogin(ctx, cfg, email, pass, twoFA)
if err != nil {
return nil, fmt.Errorf("login failed: %w", err)
}
// Store mnemonic (obscured)
m.Set("mnemonic", obscure.MustObscure(loginResp.User.Mnemonic))
// Store token
oauthToken, err := jwtToOAuth2Token(loginResp.NewToken)
if err != nil {
return nil, fmt.Errorf("failed to parse token: %w", err)
}
err = oauthutil.PutToken(name, m, oauthToken, true)
if err != nil {
return nil, fmt.Errorf("failed to save token: %w", err)
}
// Clear temporary 2FA code
m.Set("2fa_code", "")
return nil, nil
}
return nil, fmt.Errorf("unknown state %q", configIn.State)
}
// Options defines the configuration for this backend
type Options struct {
Email string `config:"email"`
Pass string `config:"pass"`
TwoFA string `config:"2fa"`
Mnemonic string `config:"mnemonic"`
SkipHashValidation bool `config:"skip_hash_validation"`
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
ChunkSize fs.SizeSuffix `config:"chunk_size"`
UploadConcurrency int `config:"upload_concurrency"`
Encoding encoder.MultiEncoder `config:"encoding"`
}
// Fs represents an Internxt remote
type Fs struct {
name string
root string
opt Options
m configmap.Mapper
dirCache *dircache.DirCache
cfg *config.Config
features *fs.Features
pacer *fs.Pacer
tokenRenewer *oauthutil.Renew
bridgeUser string
userID string
authMu sync.Mutex
authFailed bool
}
// Object holds the data for a remote file object
type Object struct {
f *Fs
remote string
id string
uuid string
size int64
modTime time.Time
}
// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string { return f.name }
// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string { return f.root }
// String converts this Fs to a string
func (f *Fs) String() string { return fmt.Sprintf("Internxt root '%s'", f.root) }
// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
return f.features
}
// DirCacheFlush resets the directory cache
func (f *Fs) DirCacheFlush() {
f.dirCache.ResetRoot()
}
// Hashes returns type of hashes supported by Internxt
func (f *Fs) Hashes() hash.Set {
return hash.NewHashSet()
}
// Precision return the precision of this Fs
func (f *Fs) Precision() time.Duration {
return fs.ModTimeNotSupported
}
// NewFs constructs an Fs from the path
func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
opt := new(Options)
if err := configstruct.Set(m, opt); err != nil {
return nil, err
}
if err := checkUploadChunkSize(opt.ChunkSize); err != nil {
return nil, fmt.Errorf("internxt: chunk size: %w", err)
}
if err := checkUploadCutoff(opt.UploadCutoff); err != nil {
return nil, fmt.Errorf("internxt: upload cutoff: %w", err)
}
if opt.Mnemonic == "" {
return nil, errors.New("mnemonic is required - please run: rclone config reconnect " + name + ":")
}
var err error
opt.Mnemonic, err = obscure.Reveal(opt.Mnemonic)
if err != nil {
return nil, fmt.Errorf("couldn't decrypt mnemonic: %w", err)
}
oauthToken, err := oauthutil.GetToken(name, m)
if err != nil {
return nil, fmt.Errorf("failed to get token - please run: rclone config reconnect %s: - %w", name, err)
}
oauthConfig := &oauthutil.Config{
TokenURL: "https://gateway.internxt.com/drive/users/refresh",
}
_, ts, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
if err != nil {
return nil, fmt.Errorf("failed to create oauth client: %w", err)
}
cfg := config.NewDefaultToken(oauthToken.AccessToken)
cfg.Mnemonic = opt.Mnemonic
cfg.SkipHashValidation = opt.SkipHashValidation
cfg.HTTPClient = fshttp.NewClient(ctx)
f := &Fs{
name: name,
root: strings.Trim(root, "/"),
opt: *opt,
m: m,
cfg: cfg,
}
f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
var userInfo *userInfo
const maxRetries = 3
for attempt := 1; attempt <= maxRetries; attempt++ {
userInfo, err = getUserInfo(ctx, &userInfoConfig{Token: f.cfg.Token})
if err == nil {
break
}
var httpErr *sdkerrors.HTTPError
if errors.As(err, &httpErr) && httpErr.StatusCode() == 401 {
fs.Debugf(f, "getUserInfo returned 401, attempting re-auth")
authErr := f.refreshOrReLogin(ctx)
if authErr != nil {
return nil, fmt.Errorf("failed to fetch user info (re-auth failed: %w): %w", authErr, err)
}
userInfo, err = getUserInfo(ctx, &userInfoConfig{Token: f.cfg.Token})
if err == nil {
break
}
return nil, fmt.Errorf("failed to fetch user info after re-auth: %w", err)
}
if fserrors.ShouldRetry(err) && attempt < maxRetries {
fs.Debugf(f, "getUserInfo transient error (attempt %d/%d): %v", attempt, maxRetries, err)
time.Sleep(time.Duration(attempt) * time.Second)
continue
}
return nil, fmt.Errorf("failed to fetch user info: %w", err)
}
f.cfg.RootFolderID = userInfo.RootFolderID
f.cfg.Bucket = userInfo.Bucket
f.cfg.BasicAuthHeader = computeBasicAuthHeader(userInfo.BridgeUser, userInfo.UserID)
f.bridgeUser = userInfo.BridgeUser
f.userID = userInfo.UserID
// The refresh endpoint rotates the token on every successful call.
// Persist the rotated token so routine use keeps the stored token
// current; otherwise it keeps its original expiry and accounts that
// cannot re-login non-interactively (2FA) eventually strand.
if userInfo.NewToken != "" {
if rotated, rotErr := jwtToOAuth2Token(userInfo.NewToken); rotErr != nil {
fs.Debugf(f, "Not adopting rotated token from user info: %v", rotErr)
} else {
// Use the rotated token for this session even if saving it
// fails; persistence is best-effort.
f.cfg.Token = userInfo.NewToken
if putErr := oauthutil.PutToken(name, m, rotated, false); putErr != nil {
fs.Debugf(f, "Failed to save rotated token from user info: %v", putErr)
} else {
fs.Debugf(f, "Persisted rotated token from user info, expiry: %v", rotated.Expiry)
}
}
}
f.features = (&fs.Features{
CanHaveEmptyDirectories: true,
}).Fill(ctx, f)
if ts != nil {
f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
f.authMu.Lock()
defer f.authMu.Unlock()
return f.refreshOrReLogin(ctx)
})
f.tokenRenewer.Start()
}
f.dirCache = dircache.New(f.root, cfg.RootFolderID, f)
err = f.dirCache.FindRoot(ctx, false)
if err != nil {
// Assume it might be a file
newRoot, remote := dircache.SplitPath(f.root)
tempF := &Fs{
name: f.name,
root: newRoot,
opt: f.opt,
m: f.m,
cfg: f.cfg,
features: f.features,
pacer: f.pacer,
tokenRenewer: f.tokenRenewer,
bridgeUser: f.bridgeUser,
userID: f.userID,
}
tempF.dirCache = dircache.New(newRoot, f.cfg.RootFolderID, tempF)
err = tempF.dirCache.FindRoot(ctx, false)
if err != nil {
return f, nil
}
_, err := tempF.NewObject(ctx, remote)
if err != nil {
if err == fs.ErrorObjectNotFound {
return f, nil
}
return nil, err
}
f.dirCache = tempF.dirCache
f.root = tempF.root
return f, fs.ErrorIsFile
}
return f, nil
}
// Mkdir creates a new directory
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
id, err := f.dirCache.FindDir(ctx, dir, true)
if err != nil {
return err
}
f.dirCache.Put(dir, id)
return nil
}
// Rmdir removes a directory
// Returns an error if it isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
root := path.Join(f.root, dir)
if root == "" {
return errors.New("cannot remove root directory")
}
id, err := f.dirCache.FindDir(ctx, dir, false)
if err != nil {
return fs.ErrorDirNotFound
}
// Check if directory is empty
var childFolders []folders.Folder
err = f.pacer.Call(func() (bool, error) {
var err error
childFolders, err = folders.ListAllFolders(ctx, f.cfg, id)
return f.shouldRetry(ctx, err)
})
if err != nil {
return err
}
for _, e := range childFolders {
if !recent.stale(e.UUID, id, e.PlainName) {
return fs.ErrorDirectoryNotEmpty
}
}
var childFiles []folders.File
err = f.pacer.Call(func() (bool, error) {
var err error
childFiles, err = folders.ListAllFiles(ctx, f.cfg, id)
return f.shouldRetry(ctx, err)
})
if err != nil {
return err
}
for _, e := range childFiles {
if !recent.stale(e.UUID, id, joinNameExt(e.PlainName, e.Type)) {
return fs.ErrorDirectoryNotEmpty
}
}
// Delete the directory
err = f.pacer.Call(func() (bool, error) {
err := folders.DeleteFolder(ctx, f.cfg, id)
if err != nil && strings.Contains(err.Error(), "404") {
return false, fs.ErrorDirNotFound
}
return f.shouldRetry(ctx, err)
})
if err != nil {
return err
}
recent.deleted(id)
f.dirCache.FlushDir(dir)
return nil
}
// FindLeaf looks for a sub‑folder named `leaf` under the Internxt folder `pathID`.
// If found, it returns its UUID and true. If not found, returns "", false.
func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (string, bool, error) {
var entries []folders.Folder
err := f.pacer.Call(func() (bool, error) {
var err error
entries, err = folders.ListAllFolders(ctx, f.cfg, pathID)
return f.shouldRetry(ctx, err)
})
if err != nil {
return "", false, err
}
for _, e := range entries {
if recent.stale(e.UUID, pathID, e.PlainName) {
continue
}
if f.opt.Encoding.ToStandardName(e.PlainName) == leaf {
return e.UUID, true, nil
}
}
return "", false, nil
}
// CreateDir creates a new directory
func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (string, error) {
request := folders.CreateFolderRequest{
PlainName: f.opt.Encoding.FromStandardName(leaf),
ParentFolderUUID: pathID,
ModificationTime: time.Now().UTC().Format(time.RFC3339),
}
var resp *folders.Folder
err := f.pacer.CallNoRetry(func() (bool, error) {
var err error
resp, err = folders.CreateFolder(ctx, f.cfg, request)
return f.shouldRetry(ctx, err)
})
if err != nil {
// If folder already exists (409 conflict), try to find it
if strings.Contains(err.Error(), "409") || strings.Contains(err.Error(), "Conflict") {
existingID, found, findErr := f.FindLeaf(ctx, pathID, leaf)
if findErr == nil && found {
fs.Debugf(f, "Folder %q already exists in %q, using existing UUID: %s", leaf, pathID, existingID)
return existingID, nil
}
}
return "", fmt.Errorf("can't create folder, %w", err)
}
return resp.UUID, nil
}
// splitNameExt splits a basename into the (plainName, type) pair that Internxt
// stores a file under.
func splitNameExt(baseName string) (name, ext string) {
ext = path.Ext(baseName)
if ext == baseName {
return baseName, ""
}
return strings.TrimSuffix(baseName, ext), strings.TrimPrefix(ext, ".")
}
// joinNameExt reassembles the basename a stored (plainName, type) pair
// displays as, the way List builds its entries.
func joinNameExt(name, ext string) string {
if ext == "" {
return name
}
return name + "." + ext
}
// legacyNameExt splits a basename at the final dot, the way this backend used
// to before it adopted the convention the other clients share. It differs from
// splitNameExt only for a name whose sole dot leads it: ".bashrc" splits into
// an empty name of type "bashrc".
func legacyNameExt(baseName string) (name, ext string) {
return strings.TrimSuffix(baseName, path.Ext(baseName)), strings.TrimPrefix(path.Ext(baseName), ".")
}
// existenceCheck builds a lookup criterion for one (plainName, type) spelling.
func existenceCheck(name, ext string) files.FileExistenceCheck {
return files.FileExistenceCheck{
PlainName: name,
Type: ext,
OriginalFile: struct{}{},
}
}
// staleWindow is how long the API may carry on returning an item from
// where it was before this process moved or deleted it.
const staleWindow = time.Minute
// location is where this process last put an item. An empty parentUUID
// means the item was deleted.
type location struct {
parentUUID string
name string // encoded name, including any extension
expires time.Time
}
// recentChanges holds the files and folders this process has moved or
// deleted recently, keyed by UUID.
//
// The API serves lookups and listings from read replicas which lag behind
// writes, so an item can be returned from its old location for a while after
// it was moved or deleted. Acting on such a stale entry can be destructive,
// eg overwriting it would delete the file from where it was moved to.
type recentChanges struct {
mu sync.Mutex
items map[string]location
lastPrune time.Time
}
// recent is shared by all the Fs in the process as UUIDs are global.
var recent = recentChanges{items: map[string]location{}}
// set records the new location of uuid, pruning expired entries.
func (r *recentChanges) set(uuid string, loc location) {
r.mu.Lock()
defer r.mu.Unlock()
now := time.Now()
if now.Sub(r.lastPrune) > staleWindow {
for k, v := range r.items {
if now.After(v.expires) {
delete(r.items, k)
}
}
r.lastPrune = now
}
loc.expires = now.Add(staleWindow)
r.items[uuid] = loc
}
// moved records that uuid was moved to name in parentUUID.
func (r *recentChanges) moved(uuid, parentUUID, name string) {
r.set(uuid, location{parentUUID: parentUUID, name: name})
}
// deleted records that uuid was deleted.
func (r *recentChanges) deleted(uuid string) {
r.set(uuid, location{})
}
// stale reports whether an API result saying uuid is called name in
// parentUUID is out of date because of a change made by this process.
func (r *recentChanges) stale(uuid, parentUUID, name string) bool {
r.mu.Lock()
defer r.mu.Unlock()
loc, ok := r.items[uuid]
if !ok || time.Now().After(loc.expires) {
return false
}
return loc.parentUUID != parentUUID || loc.name != name
}
// findFile looks up a single file by name within directoryID. Returns
// (nil, nil) when the file does not exist; surfaces transport/API errors.
func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
encodedName := f.opt.Encoding.FromStandardName(leaf)
name, ext := splitNameExt(encodedName)
checks := []files.FileExistenceCheck{existenceCheck(name, ext)}
if encodedName != name {
checks = append(checks, existenceCheck(encodedName, ""))
}
if legacyName, legacyExt := legacyNameExt(encodedName); legacyExt != "" && legacyName != name {
checks = append(checks, existenceCheck(legacyName, legacyExt))
}
var checkResult *files.CheckFilesExistenceResponse
err := f.pacer.Call(func() (bool, error) {
var err error
checkResult, err = files.CheckFilesExistence(ctx, f.cfg, directoryID, checks)
if isNotFoundError(err) {
return true, err
}
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
var uuid string
for _, result := range checkResult.Files {
if !result.FileExists() || result.UUID == "" {
continue
}
if recent.stale(result.UUID, directoryID, joinNameExt(result.PlainName, result.Type)) {
fs.Debugf(f, "Ignoring stale lookup of %q (UUID: %s) which was moved or deleted", leaf, result.UUID)
continue
}
if joinNameExt(result.PlainName, result.Type) == encodedName {
uuid = result.UUID
break
}
}
if uuid == "" {
return nil, nil
}
var fileMeta *files.FileMeta
err = f.pacer.Call(func() (bool, error) {
var err error
fileMeta, err = files.GetFileMeta(ctx, f.cfg, uuid)
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
if fileMeta == nil {
return nil, nil
}
return convertFileMetaToFile(fileMeta), nil
}
// convertFileMetaToFile converts files.FileMeta to folders.File
func convertFileMetaToFile(meta *files.FileMeta) *folders.File {
// FileMeta and folders.File have compatible structures
return &folders.File{
ID: meta.ID,
UUID: meta.UUID,
FileID: meta.FileID,
PlainName: meta.PlainName,
Type: meta.Type,
Size: meta.Size,
Bucket: meta.Bucket,
FolderUUID: meta.FolderUUID,
EncryptVersion: meta.EncryptVersion,
ModificationTime: meta.ModificationTime,
}
}
// List lists a directory
func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
dirID, err := f.dirCache.FindDir(ctx, dir, false)
if err != nil {
return nil, err
}
var out fs.DirEntries
var foldersList []folders.Folder
err = f.pacer.Call(func() (bool, error) {
var err error
foldersList, err = folders.ListAllFolders(ctx, f.cfg, dirID)
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
for _, e := range foldersList {
if recent.stale(e.UUID, dirID, e.PlainName) {
continue
}
remote := path.Join(dir, f.opt.Encoding.ToStandardName(e.PlainName))
out = append(out, fs.NewDir(remote, e.ModificationTime))
}
var filesList []folders.File
err = f.pacer.Call(func() (bool, error) {
var err error
filesList, err = folders.ListAllFiles(ctx, f.cfg, dirID)
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
for _, e := range filesList {
remote := e.PlainName
if len(e.Type) > 0 {
remote += "." + e.Type
}
if recent.stale(e.UUID, dirID, remote) {
continue
}
remote = path.Join(dir, f.opt.Encoding.ToStandardName(remote))
out = append(out, newObjectWithFile(f, remote, &e))
}
return out, nil
}
// Put uploads a file
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
remote := src.Remote()
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
if err != nil {
if err == fs.ErrorDirNotFound {
o := &Object{
f: f,
remote: remote,
size: src.Size(),
modTime: src.ModTime(ctx),
}
return o, o.Update(ctx, in, src, options...)
}
return nil, err
}
existingFile, err := f.findFile(ctx, leaf, directoryID)
if err != nil {
return nil, err
}
// Create object - if file exists, populate it with existing metadata
o := &Object{
f: f,
remote: remote,
size: src.Size(),
modTime: src.ModTime(ctx),
}
if existingFile != nil {
// File exists - populate object with existing metadata
size, _ := existingFile.Size.Int64()
o.id = existingFile.FileID
o.uuid = existingFile.UUID
o.size = size
o.modTime = existingFile.ModificationTime
}
return o, o.Update(ctx, in, src, options...)
}
// Remove removes an object
func (f *Fs) Remove(ctx context.Context, remote string) error {
obj, err := f.NewObject(ctx, remote)
if err == nil {
if err := obj.Remove(ctx); err != nil {
return err
}
parent := path.Dir(remote)
f.dirCache.FlushDir(parent)
return nil
}
dirID, err := f.dirCache.FindDir(ctx, remote, false)
if err != nil {
return err
}
err = f.pacer.Call(func() (bool, error) {
err := folders.DeleteFolder(ctx, f.cfg, dirID)
return f.shouldRetry(ctx, err)
})
if err != nil {
return err
}
recent.deleted(dirID)
f.dirCache.FlushDir(remote)
return nil
}
// NewObject creates a new object by looking up a single file's metadata.
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
if err != nil {
if err == fs.ErrorDirNotFound {
return nil, fs.ErrorObjectNotFound
}
return nil, err
}
file, err := f.findFile(ctx, leaf, directoryID)
if err != nil {
return nil, err
}
if file == nil {
return nil, fs.ErrorObjectNotFound
}
return newObjectWithFile(f, remote, file), nil
}
// newObjectWithFile returns a new object by file info
func newObjectWithFile(f *Fs, remote string, file *folders.File) fs.Object {
size, _ := file.Size.Int64()
return &Object{
f: f,
remote: remote,
id: file.FileID,
uuid: file.UUID,
size: size,
modTime: file.ModificationTime,
}
}
// Fs returns the parent Fs
func (o *Object) Fs() fs.Info {
return o.f
}
// String returns the remote path
func (o *Object) String() string {
return o.remote
}
// Remote returns the remote path
func (o *Object) Remote() string {
return o.remote
}
// Size is the file length
func (o *Object) Size() int64 {
return o.size
}
// ModTime is the last modified time (read-only)
func (o *Object) ModTime(ctx context.Context) time.Time {
return o.modTime
}
// Hash returns the hash value (not implemented)
func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
return "", hash.ErrUnsupported
}
// Storable returns if this object is storable
func (o *Object) Storable() bool {
return true
}
// SetModTime sets the modified time
func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
return fs.ErrorCantSetModTime
}
// About gets quota information
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
var internxtLimit *users.LimitResponse
err := f.pacer.Call(func() (bool, error) {
var err error
internxtLimit, err = users.GetLimit(ctx, f.cfg)
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
var internxtUsage *users.UsageResponse
err = f.pacer.Call(func() (bool, error) {
var err error
internxtUsage, err = users.GetUsage(ctx, f.cfg)
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
usage := &fs.Usage{
Used: fs.NewUsageValue(internxtUsage.Drive),
}
usage.Total = fs.NewUsageValue(internxtLimit.MaxSpaceBytes)
usage.Free = fs.NewUsageValue(*usage.Total - *usage.Used)
return usage, nil
}
// Move moves a file to a new location
func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
srcObj, ok := src.(*Object)
if !ok {
return nil, fs.ErrorCantMove
}
if srcObj.uuid == "" {
return nil, fs.ErrorCantMove
}
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
if err != nil {
return nil, err
}
newName, newType := splitNameExt(f.opt.Encoding.FromStandardName(leaf))
// Move the file server-side
err = f.pacer.Call(func() (bool, error) {
err := files.MoveFile(ctx, f.cfg, srcObj.uuid, directoryID, newName, newType)
// The move is checked against lagging read replicas so it can
// fail with 404 if the destination directory was just created
// or 409 if a file at the destination was just deleted.
if err != nil && isStaleReadError(err, true) {
return true, err
}
return f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
recent.moved(srcObj.uuid, directoryID, joinNameExt(newName, newType))
dstObj := &Object{
f: f,
remote: remote,
id: srcObj.id,
uuid: srcObj.uuid,
size: srcObj.size,
modTime: srcObj.modTime,
}
return dstObj, nil
}
// DirMove moves src, srcRemote to this remote at dstRemote
// using server-side move operations.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
srcFs, ok := src.(*Fs)
if !ok {
return fs.ErrorCantDirMove
}
srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
if err != nil {
return err
}
encodedLeaf := f.opt.Encoding.FromStandardName(dstLeaf)
err = f.pacer.Call(func() (bool, error) {
err := folders.MoveFolder(ctx, f.cfg, srcID, dstDirectoryID, encodedLeaf)
// Moving a large folder can outlast the gateway timeout (520/502)
// yet still complete, in which case the retry reports this.
if err != nil && isConflictError(err) && strings.Contains(err.Error(), "already moved to that location") {
return false, nil
}
return f.shouldRetry(ctx, err)
})
if err != nil {
return err
}
recent.moved(srcID, dstDirectoryID, encodedLeaf)
srcFs.dirCache.FlushDir(srcRemote)
return nil
}
// Shutdown the backend, closing any background tasks and any cached
// connections.
func (f *Fs) Shutdown(ctx context.Context) error {
buckets.WaitForPendingThumbnails()
if f.tokenRenewer != nil {
f.tokenRenewer.Shutdown()
}
return nil
}
// Open opens a file for streaming
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
fs.FixRangeOption(options, o.size)
rangeValue := ""
for _, option := range options {
switch option.(type) {
case *fs.RangeOption, *fs.SeekOption:
_, rangeValue = option.Header()
}
}
if o.size == 0 {
return io.NopCloser(bytes.NewReader(nil)), nil
}
var stream io.ReadCloser
err := o.f.pacer.Call(func() (bool, error) {
var err error
stream, err = buckets.DownloadFileStream(ctx, o.f.cfg, o.id, rangeValue)
return o.f.shouldRetry(ctx, err)
})
if err != nil {
return nil, err
}
return stream, nil
}
// Update updates an existing file or creates a new one
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
remote := o.remote
origName, origType := splitNameExt(path.Base(remote))
// Create directory if it doesn't exist
_, dirID, err := o.f.dirCache.FindPath(ctx, remote, true)
if err != nil {
return err
}
// rename based rollback pattern
// old file is preserved until new upload succeeds
var backupUUID string
var backupName, backupType string
oldUUID := o.uuid
// Step 1: If file exists, rename to backup (preserves old file during upload)
if oldUUID != "" {
name, ext := splitNameExt(path.Base(remote))
backupSuffix := fmt.Sprintf(".rclone-backup-%s", random.String(8))
backupName = o.f.opt.Encoding.FromStandardName(name + backupSuffix)
backupType = ext
// Rename existing file to backup name
err = o.f.pacer.Call(func() (bool, error) {
err := files.RenameFile(ctx, o.f.cfg, oldUUID, backupName, backupType)
if err != nil {
// Handle 409 Conflict: Treat as success.
var httpErr *sdkerrors.HTTPError
if errors.As(err, &httpErr) && httpErr.StatusCode() == 409 {
return false, nil
}
}
return o.f.shouldRetry(ctx, err)
})
if err != nil {
return fmt.Errorf("failed to rename existing file to backup: %w", err)
}
backupUUID = oldUUID
fs.Debugf(o.f, "Renamed existing file %s to backup %s.%s (UUID: %s)", remote, backupName, backupType, backupUUID)
}
size := src.Size()
var meta *buckets.CreateMetaResponse
if size < 0 || size >= int64(o.f.opt.UploadCutoff) {
chunkWriter, uploadErr := multipart.UploadMultipart(ctx, src, in, multipart.UploadMultipartOptions{
Open: o.f,
OpenOptions: options,
})
if uploadErr != nil {
if isEmptyFileLimitError(uploadErr) {
o.restoreBackupFile(ctx, backupUUID, origName, origType)
return fs.ErrorCantUploadEmptyFiles
}
if tooLarge := fileTooLargeError(uploadErr); tooLarge != nil {
o.restoreBackupFile(ctx, backupUUID, origName, origType)
return o.f.tooLargeError(remote, tooLarge)
}
o.restoreBackupFile(ctx, backupUUID, origName, origType)
return uploadErr
}
w := chunkWriter.(*internxtChunkWriter)
meta = w.meta
} else {
// Use single-part upload for small files
err = o.f.pacer.CallNoRetry(func() (bool, error) {
var err error
meta, err = buckets.UploadFileStreamAuto(ctx,
o.f.cfg,
dirID,
o.f.opt.Encoding.FromStandardName(path.Base(remote)),
in,
size,
src.ModTime(ctx),
)
return o.f.shouldRetry(ctx, err)
})
if err != nil && isEmptyFileLimitError(err) {
o.restoreBackupFile(ctx, backupUUID, origName, origType)
return fs.ErrorCantUploadEmptyFiles
}
if tooLarge := fileTooLargeError(err); tooLarge != nil {
o.restoreBackupFile(ctx, backupUUID, origName, origType)
return o.f.tooLargeError(remote, tooLarge)
}
if err != nil && isStaleReadError(err, backupUUID != "") {
o.restoreBackupFile(ctx, backupUUID, origName, origType)
return fserrors.RetryError(err)
}
if err != nil {
meta, err = o.recoverFromTimeoutConflict(ctx, err, remote, dirID)
}
if err != nil {
o.restoreBackupFile(ctx, backupUUID, origName, origType)
return err
}
}
// Update object metadata
o.uuid = meta.UUID
o.id = meta.FileID
o.size = src.Size()
o.remote = remote
// Step 3: Upload succeeded - delete the backup file
if backupUUID != "" {
fs.Debugf(o.f, "Upload succeeded, deleting backup file %s.%s (UUID: %s)", backupName, backupType, backupUUID)
err := o.f.pacer.Call(func() (bool, error) {
err := files.DeleteFile(ctx, o.f.cfg, backupUUID)
if err != nil {
if httpErr, ok := errors.AsType[*sdkerrors.HTTPError](err); ok {
// Treat 404 (Not Found) and 204 (No Content) as success
switch httpErr.StatusCode() {
case 404, 204:
return false, nil
}
}
}
return o.f.shouldRetry(ctx, err)
})
if err != nil {
fs.Errorf(o.f, "Failed to delete backup file %s.%s (UUID: %s): %v. This may leave an orphaned backup file.",
backupName, backupType, backupUUID, err)
// Don't fail the upload just because backup deletion failed
} else {
recent.deleted(backupUUID)
fs.Debugf(o.f, "Successfully deleted backup file")
}
}
return nil
}
// isTimeoutError checks if an error is a timeout using proper error type checking
func isTimeoutError(err error) bool {
if errors.Is(err, context.DeadlineExceeded) {
return true
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
return true
}
return false
}
// isNotFoundError reports whether err is a 404 from the API.
func isNotFoundError(err error) bool {
var httpErr *sdkerrors.HTTPError
return errors.As(err, &httpErr) && httpErr.StatusCode() == 404
}
// isStaleReadError reports whether err is the API contradicting a write this
// backend has already made, which it does because lookups are served by read
// replicas that lag behind the primary.
func isStaleReadError(err error, renamed bool) bool {
return isNotFoundError(err) || (renamed && isConflictError(err))
}
// isConflictError checks if an error indicates a file conflict (409)
func isConflictError(err error) bool {
errMsg := err.Error()
return strings.Contains(errMsg, "409") ||
strings.Contains(errMsg, "Conflict") ||
strings.Contains(errMsg, "already exists")
}
func isEmptyFileLimitError(err error) bool {
errMsg := strings.ToLower(err.Error())
return strings.Contains(errMsg, "can not have more empty files") ||
strings.Contains(errMsg, "cannot have more empty files") ||
strings.Contains(errMsg, "you can not have empty files")
}
// fileTooLargeError extracts the SDK's FileTooLargeError from a wrapped error
// chain, returning it (or nil) so callers can branch on the size limit.
func fileTooLargeError(err error) *sdkerrors.FileTooLargeError {
if tooLarge, ok := errors.AsType[*sdkerrors.FileTooLargeError](err); ok {
return tooLarge
}
return nil
}
// tooLargeError formats a per-file, non-retryable error for the sync engine.
// fserrors.NoRetryError signals "skip this file but continue the sync."
func (f *Fs) tooLargeError(remote string, tooLarge *sdkerrors.FileTooLargeError) error {
return fserrors.NoRetryError(fmt.Errorf("%s: file size %s exceeds account upload limit of %s",
remote,
fs.SizeSuffix(tooLarge.Size),
fs.SizeSuffix(tooLarge.MaxSize)))
}
// recoverFromTimeoutConflict attempts to recover from a timeout or conflict error
func (o *Object) recoverFromTimeoutConflict(ctx context.Context, uploadErr error, remote, dirID string) (*buckets.CreateMetaResponse, error) {
if !isTimeoutError(uploadErr) && !isConflictError(uploadErr) {
return nil, uploadErr
}
baseName := path.Base(remote)
existingFile, err := o.f.findFile(ctx, baseName, dirID)
if err != nil || existingFile == nil {
return nil, uploadErr
}
name, ext := splitNameExt(baseName)
o.id = existingFile.FileID
return &buckets.CreateMetaResponse{
UUID: existingFile.UUID,
FileID: existingFile.FileID,
Name: name,
PlainName: name,
Type: ext,
Size: existingFile.Size,
}, nil
}
// restoreBackupFile restores a backup file after upload failure
func (o *Object) restoreBackupFile(ctx context.Context, backupUUID, origName, origType string) {
if backupUUID == "" {
return
}
_ = o.f.pacer.Call(func() (bool, error) {
err := files.RenameFile(ctx, o.f.cfg, backupUUID,
o.f.opt.Encoding.FromStandardName(origName), origType)
return o.f.shouldRetry(ctx, err)
})
}
// Remove deletes a file
func (o *Object) Remove(ctx context.Context) error {
err := o.f.pacer.Call(func() (bool, error) {
err := files.DeleteFile(ctx, o.f.cfg, o.uuid)
return o.f.shouldRetry(ctx, err)
})
if err != nil {
return err
}
recent.deleted(o.uuid)
return nil
}