"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
internetarchive: add --internetarchive-metadata="key=value" for setting item metadata
Corentin Barreaudbb21165d4a3
diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.goindex 8718401ec..ec6db9684 100644--- a/backend/internetarchive/internetarchive.go+++ b/backend/internetarchive/internetarchive.go@@ -151,6 +151,19 @@ Owner is able to add custom keys. Metadata feature grabs all the keys includingHelp: "Host of InternetArchive Frontend.\n\nLeave blank for default value.",Default: "https://archive.org",Advanced: true,+ }, {+ Name: "item_metadata",+ Help: `Metadata to be set on the IA item, this is different from file-level metadata that can be set using --metadata-set.+Format is key=value and the 'x-archive-meta-' prefix is automatically added.`,+ Default: []string{},+ Hide: fs.OptionHideConfigurator,+ Advanced: true,+ }, {+ Name: "item_derive",+ Help: `Whether to trigger derive on the IA item or not. If set to false, the item will not be derived by IA upon upload.+The derive process produces a number of secondary files from an upload to make an upload more usable on the web.+Setting this to false is useful for uploading files that are already in a format that IA can display or reduce burden on IA's infrastructure.`,+ Default: true,}, {Name: "disable_checksum",Help: `Don't ask the server to test against MD5 checksum calculated by rclone.@@ -201,6 +214,8 @@ type Options struct {Endpoint string `config:"endpoint"`FrontEndpoint string `config:"front_endpoint"`DisableChecksum bool `config:"disable_checksum"`+ ItemMetadata []string `config:"item_metadata"`+ ItemDerive bool `config:"item_derive"`WaitArchive fs.Duration `config:"wait_archive"`Enc encoder.MultiEncoder `config:"encoding"`}@@ -790,17 +805,23 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op"x-amz-filemeta-rclone-update-track": updateTracker,// we add some more headers for intuitive actions- "x-amz-auto-make-bucket": "1", // create an item if does not exist, do nothing if already- "x-archive-auto-make-bucket": "1", // same as above in IAS3 original way- "x-archive-keep-old-version": "0", // do not keep old versions (a.k.a. trashes in other clouds)- "x-archive-meta-mediatype": "data", // mark media type of the uploading file as "data"- "x-archive-queue-derive": "0", // skip derivation process (e.g. encoding to smaller files, OCR on PDFs)- "x-archive-cascade-delete": "1", // enable "cascate delete" (delete all derived files in addition to the file itself)+ "x-amz-auto-make-bucket": "1", // create an item if does not exist, do nothing if already+ "x-archive-auto-make-bucket": "1", // same as above in IAS3 original way+ "x-archive-keep-old-version": "0", // do not keep old versions (a.k.a. trashes in other clouds)+ "x-archive-cascade-delete": "1", // enable "cascate delete" (delete all derived files in addition to the file itself)}+if size >= 0 {headers["Content-Length"] = fmt.Sprintf("%d", size)headers["x-archive-size-hint"] = fmt.Sprintf("%d", size)}++ // This is IA's ITEM metadata, not file metadata+ headers, err = o.appendItemMetadataHeaders(headers, o.fs.opt)+ if err != nil {+ return err+ }+var mdata fs.Metadatamdata, err = fs.GetMetadataOptions(ctx, o.fs, src, options)if err == nil && mdata != nil {@@ -863,6 +884,51 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opreturn err}+func (o *Object) appendItemMetadataHeaders(headers map[string]string, options Options) (newHeaders map[string]string, err error) {+ metadataCounter := make(map[string]int)+ metadataValues := make(map[string][]string)++ // First pass: count occurrences and collect values+ for _, v := range options.ItemMetadata {+ parts := strings.SplitN(v, "=", 2)+ if len(parts) != 2 {+ return newHeaders, errors.New("item metadata key=value should be in the form key=value")+ }+ key, value := parts[0], parts[1]+ metadataCounter[key]+++ metadataValues[key] = append(metadataValues[key], value)+ }++ // Second pass: add headers with appropriate prefixes+ for key, count := range metadataCounter {+ if count == 1 {+ // Only one occurrence, use x-archive-meta-+ headers[fmt.Sprintf("x-archive-meta-%s", key)] = metadataValues[key][0]+ } else {+ // Multiple occurrences, use x-archive-meta01-, x-archive-meta02-, etc.+ for i, value := range metadataValues[key] {+ headers[fmt.Sprintf("x-archive-meta%02d-%s", i+1, key)] = value+ }+ }+ }++ if o.fs.opt.ItemDerive {+ headers["x-archive-queue-derive"] = "1"+ } else {+ headers["x-archive-queue-derive"] = "0"+ }++ fs.Debugf(o, "Setting IA item derive: %t", o.fs.opt.ItemDerive)++ for k, v := range headers {+ if strings.HasPrefix(k, "x-archive-meta") {+ fs.Debugf(o, "Setting IA item metadata: %s=%s", k, v)+ }+ }++ return headers, nil+}+// Remove an objectfunc (o *Object) Remove(ctx context.Context) (err error) {bucket, bucketPath := o.split()