"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fs: convert main options to new config system
Nick Craig-Wood4d2bc190cc5f
diff --git a/cmd/cmd.go b/cmd/cmd.goindex b540bc215..c8695b292 100644--- a/cmd/cmd.go+++ b/cmd/cmd.go@@ -14,7 +14,6 @@ import ("os""os/exec""path"- "regexp""runtime""runtime/pprof""strconv"@@ -49,7 +48,6 @@ var (cpuProfile = flags.StringP("cpuprofile", "", "", "Write cpu profile to file", "Debugging")memProfile = flags.StringP("memprofile", "", "", "Write memory profile to file", "Debugging")statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable)", "Logging")- dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes' per second", "Logging")version bool// ErrorserrorCommandNotFound = errors.New("command not found")@@ -472,13 +470,6 @@ func initConfig() {}})}-- if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); !m {- fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.")- ci.DataRateUnit = "bytes"- } else {- ci.DataRateUnit = *dataRateUnit- }}func resolveExitCode(err error) {diff --git a/cmdtest/environment_test.go b/cmdtest/environment_test.goindex f0688200d..5a56a692a 100644--- a/cmdtest/environment_test.go+++ b/cmdtest/environment_test.go@@ -66,7 +66,7 @@ func TestEnvironmentVariables(t *testing.T) {assert.NotContains(t, out, "fileAA1.txt") // depth 4}- // Test of debug logging while initialising flags from environment (tests #5241 Enhance1)+ // Test of debug logging while initialising flags from environment (tests #5341 Enhance1)env = "RCLONE_STATS=173ms"out, err = rcloneEnv(env, "version", "-vv")if assert.NoError(t, err) {diff --git a/fs/config.go b/fs/config.goindex 435fbf8e7..bd3d7aa89 100644--- a/fs/config.go+++ b/fs/config.go@@ -3,6 +3,7 @@ package fsimport ("context""errors"+ "fmt""net""os""strconv"@@ -13,7 +14,7 @@ import (// Globalvar (// globalConfig for rclone- globalConfig = NewConfig()+ globalConfig = new(ConfigInfo)// Read a value from the config file//@@ -49,184 +50,693 @@ var (ConfigEdit = "config_fs_edit")+// ConfigOptionsInfo describes the Options in use+var ConfigOptionsInfo = Options{{+ Name: "modify_window",+ Default: time.Nanosecond,+ Help: "Max time diff to be considered the same",+ Groups: "Copy",+}, {+ Name: "checkers",+ Default: 8,+ Help: "Number of checkers to run in parallel",+ Groups: "Performance",+}, {+ Name: "transfers",+ Default: 4,+ Help: "Number of file transfers to run in parallel",+ Groups: "Performance",+}, {+ Name: "checksum",+ ShortOpt: "c",+ Default: false,+ Help: "Check for changes with size & checksum (if available, or fallback to size only).",+ Groups: "Copy",+}, {+ Name: "size_only",+ Default: false,+ Help: "Skip based on size only, not modtime or checksum",+ Groups: "Copy",+}, {+ Name: "ignore_times",+ ShortOpt: "I",+ Default: false,+ Help: "Don't skip items that match size and time - transfer all unconditionally",+ Groups: "Copy",+}, {+ Name: "ignore_existing",+ Default: false,+ Help: "Skip all files that exist on destination",+ Groups: "Copy",+}, {+ Name: "ignore_errors",+ Default: false,+ Help: "Delete even if there are I/O errors",+ Groups: "Sync",+}, {+ Name: "dry_run",+ ShortOpt: "n",+ Default: false,+ Help: "Do a trial run with no permanent changes",+ Groups: "Config,Important",+}, {+ Name: "interactive",+ ShortOpt: "i",+ Default: false,+ Help: "Enable interactive mode",+ Groups: "Config,Important",+}, {+ Name: "contimeout",+ Default: 60 * time.Second,+ Help: "Connect timeout",+ Groups: "Networking",+}, {+ Name: "timeout",+ Default: 5 * 60 * time.Second,+ Help: "IO idle timeout",+ Groups: "Networking",+}, {+ Name: "expect_continue_timeout",+ Default: 1 * time.Second,+ Help: "Timeout when using expect / 100-continue in HTTP",+ Groups: "Networking",+}, {+ Name: "no_check_certificate",+ Default: false,+ Help: "Do not verify the server SSL certificate (insecure)",+ Groups: "Networking",+}, {+ Name: "ask_password",+ Default: true,+ Help: "Allow prompt for password for encrypted configuration",+ Groups: "Config",+}, {+ Name: "password_command",+ Default: SpaceSepList{},+ Help: "Command for supplying password for encrypted configuration",+ Groups: "Config",+}, {+ Name: "max_delete",+ Default: int64(-1),+ Help: "When synchronizing, limit the number of deletes",+ Groups: "Sync",+}, {+ Name: "max_delete_size",+ Default: SizeSuffix(-1),+ Help: "When synchronizing, limit the total size of deletes",+ Groups: "Sync",+}, {+ Name: "track_renames",+ Default: false,+ Help: "When synchronizing, track file renames and do a server-side move if possible",+ Groups: "Sync",+}, {+ Name: "track_renames_strategy",+ Default: "hash",+ Help: "Strategies to use when synchronizing using track-renames hash|modtime|leaf",+ Groups: "Sync",+}, {+ Name: "retries",+ Default: 3,+ Help: "Retry operations this many times if they fail",+ Groups: "Config",+}, {+ Name: "retries_sleep",+ Default: time.Duration(0),+ Help: "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)",+ Groups: "Config",+}, {+ Name: "low_level_retries",+ Default: 10,+ Help: "Number of low level retries to do",+ Groups: "Config",+}, {+ Name: "update",+ ShortOpt: "u",+ Default: false,+ Help: "Skip files that are newer on the destination",+ Groups: "Copy",+}, {+ Name: "use_server_modtime",+ Default: false,+ Help: "Use server modified time instead of object metadata",+ Groups: "Config",+}, {+ Name: "no_gzip_encoding",+ Default: false,+ Help: "Don't set Accept-Encoding: gzip",+ Groups: "Networking",+}, {+ Name: "max_depth",+ Default: -1,+ Help: "If set limits the recursion depth to this",+ Groups: "Filter",+}, {+ Name: "ignore_size",+ Default: false,+ Help: "Ignore size when skipping use modtime or checksum",+ Groups: "Copy",+}, {+ Name: "ignore_checksum",+ Default: false,+ Help: "Skip post copy check of checksums",+ Groups: "Copy",+}, {+ Name: "ignore_case_sync",+ Default: false,+ Help: "Ignore case when synchronizing",+ Groups: "Copy",+}, {+ Name: "fix_case",+ Default: false,+ Help: "Force rename of case insensitive dest to match source",+ Groups: "Sync",+}, {+ Name: "no_traverse",+ Default: false,+ Help: "Don't traverse destination file system on copy",+ Groups: "Copy",+}, {+ Name: "check_first",+ Default: false,+ Help: "Do all the checks before starting transfers",+ Groups: "Copy",+}, {+ Name: "no_check_dest",+ Default: false,+ Help: "Don't check the destination, copy regardless",+ Groups: "Copy",+}, {+ Name: "no_unicode_normalization",+ Default: false,+ Help: "Don't normalize unicode characters in filenames",+ Groups: "Config",+}, {+ Name: "no_update_modtime",+ Default: false,+ Help: "Don't update destination modtime if files identical",+ Groups: "Copy",+}, {+ Name: "no_update_dir_modtime",+ Default: false,+ Help: "Don't update directory modification times",+ Groups: "Copy",+}, {+ Name: "compare_dest",+ Default: []string{},+ Help: "Include additional server-side paths during comparison",+ Groups: "Copy",+}, {+ Name: "copy_dest",+ Default: []string{},+ Help: "Implies --compare-dest but also copies files from paths into destination",+ Groups: "Copy",+}, {+ Name: "backup_dir",+ Default: "",+ Help: "Make backups into hierarchy based in DIR",+ Groups: "Sync",+}, {+ Name: "suffix",+ Default: "",+ Help: "Suffix to add to changed files",+ Groups: "Sync",+}, {+ Name: "suffix_keep_extension",+ Default: false,+ Help: "Preserve the extension when using --suffix",+ Groups: "Sync",+}, {+ Name: "fast_list",+ Default: false,+ Help: "Use recursive list if available; uses more memory but fewer transactions",+ Groups: "Listing",+}, {+ Name: "tpslimit",+ Default: 0.0,+ Help: "Limit HTTP transactions per second to this",+ Groups: "Networking",+}, {+ Name: "tpslimit_burst",+ Default: 1,+ Help: "Max burst of transactions for --tpslimit",+ Groups: "Networking",+}, {+ Name: "user_agent",+ Default: "rclone/" + Version,+ Help: "Set the user-agent to a specified string",+ Groups: "Networking",+}, {+ Name: "immutable",+ Default: false,+ Help: "Do not modify files, fail if existing files have been modified",+ Groups: "Copy",+}, {+ Name: "auto_confirm",+ Default: false,+ Help: "If enabled, do not request console confirmation",+ Groups: "Config",+}, {+ Name: "stats_unit",+ Default: "bytes",+ Help: "Show data rate in stats as either 'bits' or 'bytes' per second",+ Groups: "Logging",+}, {+ Name: "stats_file_name_length",+ Default: 45,+ Help: "Max file name length in stats (0 for no limit)",+ Groups: "Logging",+}, {+ Name: "log_level",+ Default: LogLevelNotice,+ Help: "Log level DEBUG|INFO|NOTICE|ERROR",+ Groups: "Logging",+}, {+ Name: "stats_log_level",+ Default: LogLevelInfo,+ Help: "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR",+ Groups: "Logging",+}, {+ Name: "bwlimit",+ Default: BwTimetable{},+ Help: "Bandwidth limit in KiB/s, or use suffix B|K|M|G|T|P or a full timetable",+ Groups: "Networking",+}, {+ Name: "bwlimit_file",+ Default: BwTimetable{},+ Help: "Bandwidth limit per file in KiB/s, or use suffix B|K|M|G|T|P or a full timetable",+ Groups: "Networking",+}, {+ Name: "buffer_size",+ Default: SizeSuffix(16 << 20),+ Help: "In memory buffer size when reading files for each --transfer",+ Groups: "Performance",+}, {+ Name: "streaming_upload_cutoff",+ Default: SizeSuffix(100 * 1024),+ Help: "Cutoff for switching to chunked upload if file size is unknown, upload starts after reaching cutoff or when file ends",+ Groups: "Copy",+}, {+ Name: "dump",+ Default: DumpFlags(0),+ Help: "List of items to dump from: " + DumpFlagsList,+ Groups: "Debugging",+}, {+ Name: "max_transfer",+ Default: SizeSuffix(-1),+ Help: "Maximum size of data to transfer",+ Groups: "Copy",+}, {+ Name: "max_duration",+ Default: time.Duration(0),+ Help: "Maximum duration rclone will transfer data for",+ Groups: "Copy",+}, {+ Name: "cutoff_mode",+ Default: CutoffMode(0),+ Help: "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS",+ Groups: "Copy",+}, {+ Name: "max_backlog",+ Default: 10000,+ Help: "Maximum number of objects in sync or check backlog",+ Groups: "Copy,Check",+}, {+ Name: "max_stats_groups",+ Default: 1000,+ Help: "Maximum number of stats groups to keep in memory, on max oldest is discarded",+ Groups: "Logging",+}, {+ Name: "stats_one_line",+ Default: false,+ Help: "Make the stats fit on one line",+ Groups: "Logging",+}, {+ Name: "stats_one_line_date",+ Default: false,+ Help: "Enable --stats-one-line and add current date/time prefix",+ Groups: "Logging",+}, {+ Name: "stats_one_line_date_format",+ Default: "",+ Help: "Enable --stats-one-line-date and use custom formatted date: Enclose date string in double quotes (\"), see https://golang.org/pkg/time/#Time.Format",+ Groups: "Logging",+}, {+ Name: "error_on_no_transfer",+ Default: false,+ Help: "Sets exit code 9 if no files are transferred, useful in scripts",+ Groups: "Config",+}, {+ Name: "progress",+ ShortOpt: "P",+ Default: false,+ Help: "Show progress during transfer",+ Groups: "Logging",+}, {+ Name: "progress_terminal_title",+ Default: false,+ Help: "Show progress on the terminal title (requires -P/--progress)",+ Groups: "Logging",+}, {+ Name: "use_cookies",+ Default: false,+ Help: "Enable session cookiejar",+ Groups: "Networking",+}, {+ Name: "use_mmap",+ Default: false,+ Help: "Use mmap allocator (see docs)",+ Groups: "Config",+}, {+ Name: "ca_cert",+ Default: []string{},+ Help: "CA certificate used to verify servers",+ Groups: "Networking",+}, {+ Name: "client_cert",+ Default: "",+ Help: "Client SSL certificate (PEM) for mutual TLS auth",+ Groups: "Networking",+}, {+ Name: "client_key",+ Default: "",+ Help: "Client SSL private key (PEM) for mutual TLS auth",+ Groups: "Networking",+}, {+ Name: "multi_thread_cutoff",+ Default: SizeSuffix(256 * 1024 * 1024),+ Help: "Use multi-thread downloads for files above this size",+ Groups: "Copy",+}, {+ Name: "multi_thread_streams",+ Default: 4,+ Help: "Number of streams to use for multi-thread downloads",+ Groups: "Copy",+}, {+ Name: "multi_thread_write_buffer_size",+ Default: SizeSuffix(128 * 1024),+ Help: "In memory buffer size for writing when in multi-thread mode",+ Groups: "Copy",+}, {+ Name: "multi_thread_chunk_size",+ Default: SizeSuffix(64 * 1024 * 1024),+ Help: "Chunk size for multi-thread downloads / uploads, if not set by filesystem",+ Groups: "Copy",+}, {+ Name: "use_json_log",+ Default: false,+ Help: "Use json log format",+ Groups: "Logging",+}, {+ Name: "order_by",+ Default: "",+ Help: "Instructions on how to order the transfers, e.g. 'size,descending'",+ Groups: "Copy",+}, {+ Name: "refresh_times",+ Default: false,+ Help: "Refresh the modtime of remote files",+ Groups: "Copy",+}, {+ Name: "no_console",+ Default: false,+ Help: "Hide console window (supported on Windows only)",+ Groups: "Config",+}, {+ Name: "fs_cache_expire_duration",+ Default: 300 * time.Second,+ Help: "Cache remotes for this long (0 to disable caching)",+ Groups: "Config",+}, {+ Name: "fs_cache_expire_interval",+ Default: 60 * time.Second,+ Help: "Interval to check for expired remotes",+ Groups: "Config",+}, {+ Name: "disable_http2",+ Default: false,+ Help: "Disable HTTP/2 in the global transport",+ Groups: "Networking",+}, {+ Name: "human_readable",+ Default: false,+ Help: "Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi",+ Groups: "Config",+}, {+ Name: "kv_lock_time",+ Default: 1 * time.Second,+ Help: "Maximum time to keep key-value database locked by process",+ Groups: "Config",+}, {+ Name: "disable_http_keep_alives",+ Default: false,+ Help: "Disable HTTP keep-alives and use each connection once.",+ Groups: "Networking",+}, {+ Name: "metadata",+ ShortOpt: "M",+ Default: false,+ Help: "If set, preserve metadata when copying objects",+ Groups: "Metadata,Copy",+}, {+ Name: "server_side_across_configs",+ Default: false,+ Help: "Allow server-side operations (e.g. copy) to work across different configs",+ Groups: "Copy",+}, {+ Name: "color",+ Default: TerminalColorMode(0),+ Help: "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS",+ Groups: "Config",+}, {+ Name: "default_time",+ Default: Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)),+ Help: "Time to show if modtime is unknown for files and directories",+ Groups: "Config,Listing",+}, {+ Name: "inplace",+ Default: false,+ Help: "Download directly to destination file instead of atomic download to temp/rename",+ Groups: "Copy",+}, {+ Name: "metadata_mapper",+ Default: SpaceSepList{},+ Help: "Program to run to transforming metadata before upload",+ Groups: "Metadata",+}, {+ Name: "partial_suffix",+ Default: ".partial",+ Help: "Add partial-suffix to temporary file name when --inplace is not used",+ Groups: "Copy",+}}+// ConfigInfo is filesystem config optionstype ConfigInfo struct {- LogLevel LogLevel- StatsLogLevel LogLevel- UseJSONLog bool- DryRun bool- Interactive bool- CheckSum bool- SizeOnly bool- IgnoreTimes bool- IgnoreExisting bool- IgnoreErrors bool- ModifyWindow time.Duration- Checkers int- Transfers int- ConnectTimeout time.Duration // Connect timeout- Timeout time.Duration // Data channel timeout- ExpectContinueTimeout time.Duration- Dump DumpFlags- InsecureSkipVerify bool // Skip server certificate verification- DeleteMode DeleteMode- MaxDelete int64- MaxDeleteSize SizeSuffix- TrackRenames bool // Track file renames.- TrackRenamesStrategy string // Comma separated list of strategies used to track renames- Retries int // High-level retries- RetriesInterval time.Duration // --retries-sleep- LowLevelRetries int- UpdateOlder bool // Skip files that are newer on the destination- NoGzip bool // Disable compression- MaxDepth int- IgnoreSize bool- IgnoreChecksum bool- IgnoreCaseSync bool- FixCase bool- NoTraverse bool- CheckFirst bool- NoCheckDest bool- NoUnicodeNormalization bool- NoUpdateModTime bool- NoUpdateDirModTime bool- DataRateUnit string- CompareDest []string- CopyDest []string- BackupDir string- Suffix string- SuffixKeepExtension bool- UseListR bool- BufferSize SizeSuffix- BwLimit BwTimetable- BwLimitFile BwTimetable- TPSLimit float64- TPSLimitBurst int- BindAddr net.IP- DisableFeatures []string- UserAgent string- Immutable bool- AutoConfirm bool- StreamingUploadCutoff SizeSuffix- StatsFileNameLength int- AskPassword bool- PasswordCommand SpaceSepList- UseServerModTime bool- MaxTransfer SizeSuffix- MaxDuration time.Duration- CutoffMode CutoffMode- MaxBacklog int- MaxStatsGroups int- StatsOneLine bool- StatsOneLineDate bool // If we want a date prefix at all- StatsOneLineDateFormat string // If we want to customize the prefix- ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred- Progress bool- ProgressTerminalTitle bool- Cookie bool- UseMmap bool- CaCert []string // Client Side CA- ClientCert string // Client Side Cert- ClientKey string // Client Side Key- MultiThreadCutoff SizeSuffix- MultiThreadStreams int- MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags)- MultiThreadChunkSize SizeSuffix // Chunk size for multi-thread downloads / uploads, if not set by filesystem- MultiThreadWriteBufferSize SizeSuffix- OrderBy string // instructions on how to order the transfer- UploadHeaders []*HTTPOption- DownloadHeaders []*HTTPOption- Headers []*HTTPOption- MetadataSet Metadata // extra metadata to write when uploading- RefreshTimes bool- NoConsole bool- TrafficClass uint8- FsCacheExpireDuration time.Duration- FsCacheExpireInterval time.Duration- DisableHTTP2 bool- HumanReadable bool- KvLockTime time.Duration // maximum time to keep key-value database locked by process- DisableHTTPKeepAlives bool- Metadata bool- ServerSideAcrossConfigs bool- TerminalColorMode TerminalColorMode- DefaultTime Time // time that directories with no time should display- Inplace bool // Download directly to destination file instead of atomic download to temp/rename- PartialSuffix string- MetadataMapper SpaceSepList+ LogLevel LogLevel `config:"log_level"`+ StatsLogLevel LogLevel `config:"stats_log_level"`+ UseJSONLog bool `config:"use_json_log"`+ DryRun bool `config:"dry_run"`+ Interactive bool `config:"interactive"`+ CheckSum bool `config:"checksum"`+ SizeOnly bool `config:"size_only"`+ IgnoreTimes bool `config:"ignore_times"`+ IgnoreExisting bool `config:"ignore_existing"`+ IgnoreErrors bool `config:"ignore_errors"`+ ModifyWindow time.Duration `config:"modify_window"`+ Checkers int `config:"checkers"`+ Transfers int `config:"transfers"`+ ConnectTimeout time.Duration `config:"contimeout"` // Connect timeout+ Timeout time.Duration `config:"timeout"` // Data channel timeout+ ExpectContinueTimeout time.Duration `config:"expect_continue_timeout"`+ Dump DumpFlags `config:"dump"`+ InsecureSkipVerify bool `config:"no_check_certificate"` // Skip server certificate verification+ DeleteMode DeleteMode `config:"delete_mode"`+ MaxDelete int64 `config:"max_delete"`+ MaxDeleteSize SizeSuffix `config:"max_delete_size"`+ TrackRenames bool `config:"track_renames"` // Track file renames.+ TrackRenamesStrategy string `config:"track_renames_strategy"` // Comma separated list of strategies used to track renames+ Retries int `config:"retries"` // High-level retries+ RetriesInterval time.Duration `config:"retries_sleep"`+ LowLevelRetries int `config:"low_level_retries"`+ UpdateOlder bool `config:"update"` // Skip files that are newer on the destination+ NoGzip bool `config:"no_gzip_encoding"` // Disable compression+ MaxDepth int `config:"max_depth"`+ IgnoreSize bool `config:"ignore_size"`+ IgnoreChecksum bool `config:"ignore_checksum"`+ IgnoreCaseSync bool `config:"ignore_case_sync"`+ FixCase bool `config:"fix_case"`+ NoTraverse bool `config:"no_traverse"`+ CheckFirst bool `config:"check_first"`+ NoCheckDest bool `config:"no_check_dest"`+ NoUnicodeNormalization bool `config:"no_unicode_normalization"`+ NoUpdateModTime bool `config:"no_update_modtime"`+ NoUpdateDirModTime bool `config:"no_update_dir_modtime"`+ DataRateUnit string `config:"stats_unit"`+ CompareDest []string `config:"compare_dest"`+ CopyDest []string `config:"copy_dest"`+ BackupDir string `config:"backup_dir"`+ Suffix string `config:"suffix"`+ SuffixKeepExtension bool `config:"suffix_keep_extension"`+ UseListR bool `config:"fast_list"`+ BufferSize SizeSuffix `config:"buffer_size"`+ BwLimit BwTimetable `config:"bwlimit"`+ BwLimitFile BwTimetable `config:"bwlimit_file"`+ TPSLimit float64 `config:"tpslimit"`+ TPSLimitBurst int `config:"tpslimit_burst"`+ BindAddr net.IP `config:"bind_addr"`+ DisableFeatures []string `config:"disable"`+ UserAgent string `config:"user_agent"`+ Immutable bool `config:"immutable"`+ AutoConfirm bool `config:"auto_confirm"`+ StreamingUploadCutoff SizeSuffix `config:"streaming_upload_cutoff"`+ StatsFileNameLength int `config:"stats_file_name_length"`+ AskPassword bool `config:"ask_password"`+ PasswordCommand SpaceSepList `config:"password_command"`+ UseServerModTime bool `config:"use_server_modtime"`+ MaxTransfer SizeSuffix `config:"max_transfer"`+ MaxDuration time.Duration `config:"max_duration"`+ CutoffMode CutoffMode `config:"cutoff_mode"`+ MaxBacklog int `config:"max_backlog"`+ MaxStatsGroups int `config:"max_stats_groups"`+ StatsOneLine bool `config:"stats_one_line"`+ StatsOneLineDate bool `config:"stats_one_line_date"` // If we want a date prefix at all+ StatsOneLineDateFormat string `config:"stats_one_line_date_format"` // If we want to customize the prefix+ ErrorOnNoTransfer bool `config:"error_on_no_transfer"` // Set appropriate exit code if no files transferred+ Progress bool `config:"progress"`+ ProgressTerminalTitle bool `config:"progress_terminal_title"`+ Cookie bool `config:"use_cookies"`+ UseMmap bool `config:"use_mmap"`+ CaCert []string `config:"ca_cert"` // Client Side CA+ ClientCert string `config:"client_cert"` // Client Side Cert+ ClientKey string `config:"client_key"` // Client Side Key+ MultiThreadCutoff SizeSuffix `config:"multi_thread_cutoff"`+ MultiThreadStreams int `config:"multi_thread_streams"`+ MultiThreadSet bool `config:"multi_thread_set"` // whether MultiThreadStreams was set (set in fs/config/configflags)+ MultiThreadChunkSize SizeSuffix `config:"multi_thread_chunk_size"` // Chunk size for multi-thread downloads / uploads, if not set by filesystem+ MultiThreadWriteBufferSize SizeSuffix `config:"multi_thread_write_buffer_size"`+ OrderBy string `config:"order_by"` // instructions on how to order the transfer+ UploadHeaders []*HTTPOption `config:"upload_headers"`+ DownloadHeaders []*HTTPOption `config:"download_headers"`+ Headers []*HTTPOption `config:"headers"`+ MetadataSet Metadata `config:"metadata_set"` // extra metadata to write when uploading+ RefreshTimes bool `config:"refresh_times"`+ NoConsole bool `config:"no_console"`+ TrafficClass uint8 `config:"traffic_class"`+ FsCacheExpireDuration time.Duration `config:"fs_cache_expire_duration"`+ FsCacheExpireInterval time.Duration `config:"fs_cache_expire_interval"`+ DisableHTTP2 bool `config:"disable_http2"`+ HumanReadable bool `config:"human_readable"`+ KvLockTime time.Duration `config:"kv_lock_time"` // maximum time to keep key-value database locked by process+ DisableHTTPKeepAlives bool `config:"disable_http_keep_alives"`+ Metadata bool `config:"metadata"`+ ServerSideAcrossConfigs bool `config:"server_side_across_configs"`+ TerminalColorMode TerminalColorMode `config:"color"`+ DefaultTime Time `config:"default_time"` // time that directories with no time should display+ Inplace bool `config:"inplace"` // Download directly to destination file instead of atomic download to temp/rename+ PartialSuffix string `config:"partial_suffix"`+ MetadataMapper SpaceSepList `config:"metadata_mapper"`}-// NewConfig creates a new config with everything set to the default-// value. These are the ultimate defaults and are overridden by the-// config module.-func NewConfig() *ConfigInfo {- c := new(ConfigInfo)-+func init() {// Set any values which aren't the zero for the type- c.LogLevel = LogLevelNotice- c.StatsLogLevel = LogLevelInfo- c.ModifyWindow = time.Nanosecond- c.Checkers = 8- c.Transfers = 4- c.ConnectTimeout = 60 * time.Second- c.Timeout = 5 * 60 * time.Second- c.ExpectContinueTimeout = 1 * time.Second- c.DeleteMode = DeleteModeDefault- c.MaxDelete = -1- c.MaxDeleteSize = SizeSuffix(-1)- c.Retries = 3- c.LowLevelRetries = 10- c.MaxDepth = -1- c.DataRateUnit = "bytes"- c.BufferSize = SizeSuffix(16 << 20)- c.UserAgent = "rclone/" + Version- c.StreamingUploadCutoff = SizeSuffix(100 * 1024)- c.MaxStatsGroups = 1000- c.StatsFileNameLength = 45- c.AskPassword = true- c.TPSLimitBurst = 1- c.MaxTransfer = -1- c.MaxBacklog = 10000- // We do not want to set the default here. We use this variable being empty as part of the fall-through of options.- // c.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "- c.MultiThreadCutoff = SizeSuffix(256 * 1024 * 1024)- c.MultiThreadStreams = 4- c.MultiThreadChunkSize = SizeSuffix(64 * 1024 * 1024)- c.MultiThreadWriteBufferSize = SizeSuffix(128 * 1024)-- c.TrackRenamesStrategy = "hash"- c.FsCacheExpireDuration = 300 * time.Second- c.FsCacheExpireInterval = 60 * time.Second- c.KvLockTime = 1 * time.Second- c.DefaultTime = Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))- c.PartialSuffix = ".partial"-- // Perform a simple check for debug flags to enable debug logging during the flag initialization+ globalConfig.DeleteMode = DeleteModeDefault++ // Register the config and fill globalConfig with the defaults+ RegisterGlobalOptions(OptionsInfo{Name: "main", Opt: globalConfig, Options: ConfigOptionsInfo, Reload: globalConfig.Reload})++ // initial guess at log level from the flags+ globalConfig.LogLevel = initialLogLevel()+}++// Reload assumes the config has been edited and does what is necessary to make it live+func (ci *ConfigInfo) Reload(ctx context.Context) error {+ // Set -vv if --dump is in use+ if ci.Dump != 0 && ci.LogLevel != LogLevelDebug {+ Logf(nil, "Automatically setting -vv as --dump is enabled")+ ci.LogLevel = LogLevelDebug+ }++ // If --dry-run or -i then use NOTICE as minimum log level+ if (ci.DryRun || ci.Interactive) && ci.StatsLogLevel > LogLevelNotice {+ ci.StatsLogLevel = LogLevelNotice+ }++ // If --use-json-log then start the JSON logger+ if ci.UseJSONLog {+ InstallJSONLogger(ci.LogLevel)+ }++ // Check --compare-dest and --copy-dest+ if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 {+ return fmt.Errorf("can't use --compare-dest with --copy-dest")+ }++ // Check --stats-one-line and dependent flags+ switch {+ case len(ci.StatsOneLineDateFormat) > 0:+ ci.StatsOneLineDate = true+ ci.StatsOneLine = true+ case ci.StatsOneLineDate:+ ci.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "+ ci.StatsOneLine = true+ }++ // Check --partial-suffix+ if len(ci.PartialSuffix) > 16 {+ return fmt.Errorf("--partial-suffix: Expecting suffix length not greater than %d but got %d", 16, len(ci.PartialSuffix))+ }++ // Make sure some values are > 0+ nonZero := func(pi *int) {+ if *pi <= 0 {+ *pi = 1+ }+ }++ // Check --stats-unit+ if ci.DataRateUnit != "bits" && ci.DataRateUnit != "bytes" {+ Errorf(nil, "Unknown unit %q passed to --stats-unit. Defaulting to bytes.", ci.DataRateUnit)+ ci.DataRateUnit = "bytes"+ }++ // Check these are all > 0+ nonZero(&ci.Retries)+ nonZero(&ci.LowLevelRetries)+ nonZero(&ci.Transfers)+ nonZero(&ci.Checkers)++ return nil+}++// Initial logging level+//+// Perform a simple check for debug flags to enable debug logging during the flag initialization+func initialLogLevel() LogLevel {+ logLevel := LogLevelNoticefor argIndex, arg := range os.Args {if strings.HasPrefix(arg, "-vv") && strings.TrimRight(arg, "v") == "-" {- c.LogLevel = LogLevelDebug+ logLevel = LogLevelDebug}if arg == "--log-level=DEBUG" || (arg == "--log-level" && len(os.Args) > argIndex+1 && os.Args[argIndex+1] == "DEBUG") {- c.LogLevel = LogLevelDebug+ logLevel = LogLevelDebug}if strings.HasPrefix(arg, "--verbose=") {if level, err := strconv.Atoi(arg[10:]); err == nil && level >= 2 {- c.LogLevel = LogLevelDebug+ logLevel = LogLevelDebug}}}envValue, found := os.LookupEnv("RCLONE_LOG_LEVEL")if found && envValue == "DEBUG" {- c.LogLevel = LogLevelDebug+ logLevel = LogLevelDebug}-- return c+ return logLevel}// TimeoutOrInfinite returns ci.Timeout if > 0 or infinite otherwise-func (c *ConfigInfo) TimeoutOrInfinite() time.Duration {- if c.Timeout > 0 {- return c.Timeout+func (ci *ConfigInfo) TimeoutOrInfinite() time.Duration {+ if ci.Timeout > 0 {+ return ci.Timeout}return ModTimeNotSupported}diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.goindex ce7425343..056c0d9d2 100644--- a/fs/config/configflags/configflags.go+++ b/fs/config/configflags/configflags.go@@ -13,9 +13,6 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/config""github.com/rclone/rclone/fs/config/flags"- fsLog "github.com/rclone/rclone/fs/log"- "github.com/rclone/rclone/fs/rc"- "github.com/sirupsen/logrus""github.com/spf13/pflag")@@ -38,123 +35,30 @@ var (downloadHeaders []stringheaders []stringmetadataSet []string- partialSuffix string)// AddFlags adds the non filing system specific flags to the commandfunc AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {- rc.AddOption("main", ci)- // NB defaults which aren't the zero for the type should be set in fs/config.go NewConfig+ flags.AddFlagsFromOptions(flagSet, "", fs.ConfigOptionsInfo)++ // Add flags we haven't converted into options yetflags.CountVarP(flagSet, &verbose, "verbose", "v", "Print lots more stuff (repeat for more)", "Logging,Important")flags.BoolVarP(flagSet, &quiet, "quiet", "q", false, "Print as little stuff as possible", "Logging")- flags.DurationVarP(flagSet, &ci.ModifyWindow, "modify-window", "", ci.ModifyWindow, "Max time diff to be considered the same", "Copy")- flags.IntVarP(flagSet, &ci.Checkers, "checkers", "", ci.Checkers, "Number of checkers to run in parallel", "Performance")- flags.IntVarP(flagSet, &ci.Transfers, "transfers", "", ci.Transfers, "Number of file transfers to run in parallel", "Performance")flags.StringVarP(flagSet, &configPath, "config", "", config.GetConfigPath(), "Config file", "Config")flags.StringVarP(flagSet, &cacheDir, "cache-dir", "", config.GetCacheDir(), "Directory rclone will use for caching", "Config")flags.StringVarP(flagSet, &tempDir, "temp-dir", "", os.TempDir(), "Directory rclone will use for temporary files", "Config")- flags.BoolVarP(flagSet, &ci.CheckSum, "checksum", "c", ci.CheckSum, "Check for changes with size & checksum (if available, or fallback to size only).", "Copy")- flags.BoolVarP(flagSet, &ci.SizeOnly, "size-only", "", ci.SizeOnly, "Skip based on size only, not modtime or checksum", "Copy")- flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip items that match size and time - transfer all unconditionally", "Copy")- flags.BoolVarP(flagSet, &ci.IgnoreExisting, "ignore-existing", "", ci.IgnoreExisting, "Skip all files that exist on destination", "Copy")- flags.BoolVarP(flagSet, &ci.IgnoreErrors, "ignore-errors", "", ci.IgnoreErrors, "Delete even if there are I/O errors", "Sync")- flags.BoolVarP(flagSet, &ci.DryRun, "dry-run", "n", ci.DryRun, "Do a trial run with no permanent changes", "Config,Important")- flags.BoolVarP(flagSet, &ci.Interactive, "interactive", "i", ci.Interactive, "Enable interactive mode", "Config,Important")- flags.DurationVarP(flagSet, &ci.ConnectTimeout, "contimeout", "", ci.ConnectTimeout, "Connect timeout", "Networking")- flags.DurationVarP(flagSet, &ci.Timeout, "timeout", "", ci.Timeout, "IO idle timeout", "Networking")- flags.DurationVarP(flagSet, &ci.ExpectContinueTimeout, "expect-continue-timeout", "", ci.ExpectContinueTimeout, "Timeout when using expect / 100-continue in HTTP", "Networking")flags.BoolVarP(flagSet, &dumpHeaders, "dump-headers", "", false, "Dump HTTP headers - may contain sensitive info", "Debugging")flags.BoolVarP(flagSet, &dumpBodies, "dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info", "Debugging")- flags.BoolVarP(flagSet, &ci.InsecureSkipVerify, "no-check-certificate", "", ci.InsecureSkipVerify, "Do not verify the server SSL certificate (insecure)", "Networking")- flags.BoolVarP(flagSet, &ci.AskPassword, "ask-password", "", ci.AskPassword, "Allow prompt for password for encrypted configuration", "Config")- flags.FVarP(flagSet, &ci.PasswordCommand, "password-command", "", "Command for supplying password for encrypted configuration", "Config")flags.BoolVarP(flagSet, &deleteBefore, "delete-before", "", false, "When synchronizing, delete files on destination before transferring", "Sync")flags.BoolVarP(flagSet, &deleteDuring, "delete-during", "", false, "When synchronizing, delete files during transfer", "Sync")flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transferring (default)", "Sync")- flags.Int64VarP(flagSet, &ci.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes", "Sync")- flags.FVarP(flagSet, &ci.MaxDeleteSize, "max-delete-size", "", "When synchronizing, limit the total size of deletes", "Sync")- flags.BoolVarP(flagSet, &ci.TrackRenames, "track-renames", "", ci.TrackRenames, "When synchronizing, track file renames and do a server-side move if possible", "Sync")- flags.StringVarP(flagSet, &ci.TrackRenamesStrategy, "track-renames-strategy", "", ci.TrackRenamesStrategy, "Strategies to use when synchronizing using track-renames hash|modtime|leaf", "Sync")- flags.IntVarP(flagSet, &ci.Retries, "retries", "", 3, "Retry operations this many times if they fail", "Config")- flags.DurationVarP(flagSet, &ci.RetriesInterval, "retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)", "Config")- flags.IntVarP(flagSet, &ci.LowLevelRetries, "low-level-retries", "", ci.LowLevelRetries, "Number of low level retries to do", "Config")- flags.BoolVarP(flagSet, &ci.UpdateOlder, "update", "u", ci.UpdateOlder, "Skip files that are newer on the destination", "Copy")- flags.BoolVarP(flagSet, &ci.UseServerModTime, "use-server-modtime", "", ci.UseServerModTime, "Use server modified time instead of object metadata", "Config")- flags.BoolVarP(flagSet, &ci.NoGzip, "no-gzip-encoding", "", ci.NoGzip, "Don't set Accept-Encoding: gzip", "Networking")- flags.IntVarP(flagSet, &ci.MaxDepth, "max-depth", "", ci.MaxDepth, "If set limits the recursion depth to this", "Filter")- flags.BoolVarP(flagSet, &ci.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use modtime or checksum", "Copy")- flags.BoolVarP(flagSet, &ci.IgnoreChecksum, "ignore-checksum", "", ci.IgnoreChecksum, "Skip post copy check of checksums", "Copy")- flags.BoolVarP(flagSet, &ci.IgnoreCaseSync, "ignore-case-sync", "", ci.IgnoreCaseSync, "Ignore case when synchronizing", "Copy")- flags.BoolVarP(flagSet, &ci.FixCase, "fix-case", "", ci.FixCase, "Force rename of case insensitive dest to match source", "Sync")- flags.BoolVarP(flagSet, &ci.NoTraverse, "no-traverse", "", ci.NoTraverse, "Don't traverse destination file system on copy", "Copy")- flags.BoolVarP(flagSet, &ci.CheckFirst, "check-first", "", ci.CheckFirst, "Do all the checks before starting transfers", "Copy")- flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless", "Copy")- flags.BoolVarP(flagSet, &ci.NoUnicodeNormalization, "no-unicode-normalization", "", ci.NoUnicodeNormalization, "Don't normalize unicode characters in filenames", "Config")- flags.BoolVarP(flagSet, &ci.NoUpdateModTime, "no-update-modtime", "", ci.NoUpdateModTime, "Don't update destination modtime if files identical", "Copy")- flags.BoolVarP(flagSet, &ci.NoUpdateDirModTime, "no-update-dir-modtime", "", ci.NoUpdateModTime, "Don't update directory modification times", "Copy")- flags.StringArrayVarP(flagSet, &ci.CompareDest, "compare-dest", "", nil, "Include additional comma separated server-side paths during comparison", "Copy")- flags.StringArrayVarP(flagSet, &ci.CopyDest, "copy-dest", "", nil, "Implies --compare-dest but also copies files from paths into destination", "Copy")- flags.StringVarP(flagSet, &ci.BackupDir, "backup-dir", "", ci.BackupDir, "Make backups into hierarchy based in DIR", "Sync")- flags.StringVarP(flagSet, &ci.Suffix, "suffix", "", ci.Suffix, "Suffix to add to changed files", "Sync")- flags.BoolVarP(flagSet, &ci.SuffixKeepExtension, "suffix-keep-extension", "", ci.SuffixKeepExtension, "Preserve the extension when using --suffix", "Sync")- flags.BoolVarP(flagSet, &ci.UseListR, "fast-list", "", ci.UseListR, "Use recursive list if available; uses more memory but fewer transactions", "Listing")- flags.Float64VarP(flagSet, &ci.TPSLimit, "tpslimit", "", ci.TPSLimit, "Limit HTTP transactions per second to this", "Networking")- flags.IntVarP(flagSet, &ci.TPSLimitBurst, "tpslimit-burst", "", ci.TPSLimitBurst, "Max burst of transactions for --tpslimit", "Networking")flags.StringVarP(flagSet, &bindAddr, "bind", "", "", "Local address to bind to for outgoing connections, IPv4, IPv6 or name", "Networking")flags.StringVarP(flagSet, &disableFeatures, "disable", "", "", "Disable a comma separated list of features (use --disable help to see a list)", "Config")- flags.StringVarP(flagSet, &ci.UserAgent, "user-agent", "", ci.UserAgent, "Set the user-agent to a specified string", "Networking")- flags.BoolVarP(flagSet, &ci.Immutable, "immutable", "", ci.Immutable, "Do not modify files, fail if existing files have been modified", "Copy")- flags.BoolVarP(flagSet, &ci.AutoConfirm, "auto-confirm", "", ci.AutoConfirm, "If enabled, do not request console confirmation", "Config")- flags.IntVarP(flagSet, &ci.StatsFileNameLength, "stats-file-name-length", "", ci.StatsFileNameLength, "Max file name length in stats (0 for no limit)", "Logging")- flags.FVarP(flagSet, &ci.LogLevel, "log-level", "", "Log level DEBUG|INFO|NOTICE|ERROR", "Logging")- flags.FVarP(flagSet, &ci.StatsLogLevel, "stats-log-level", "", "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR", "Logging")- flags.FVarP(flagSet, &ci.BwLimit, "bwlimit", "", "Bandwidth limit in KiB/s, or use suffix B|K|M|G|T|P or a full timetable", "Networking")- flags.FVarP(flagSet, &ci.BwLimitFile, "bwlimit-file", "", "Bandwidth limit per file in KiB/s, or use suffix B|K|M|G|T|P or a full timetable", "Networking")- flags.FVarP(flagSet, &ci.BufferSize, "buffer-size", "", "In memory buffer size when reading files for each --transfer", "Performance")- flags.FVarP(flagSet, &ci.StreamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown, upload starts after reaching cutoff or when file ends", "Copy")- flags.FVarP(flagSet, &ci.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList, "Debugging")- flags.FVarP(flagSet, &ci.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer", "Copy")- flags.DurationVarP(flagSet, &ci.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for", "Copy")- flags.FVarP(flagSet, &ci.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS", "Copy")- flags.IntVarP(flagSet, &ci.MaxBacklog, "max-backlog", "", ci.MaxBacklog, "Maximum number of objects in sync or check backlog", "Copy,Check")- flags.IntVarP(flagSet, &ci.MaxStatsGroups, "max-stats-groups", "", ci.MaxStatsGroups, "Maximum number of stats groups to keep in memory, on max oldest is discarded", "Logging")- flags.BoolVarP(flagSet, &ci.StatsOneLine, "stats-one-line", "", ci.StatsOneLine, "Make the stats fit on one line", "Logging")- flags.BoolVarP(flagSet, &ci.StatsOneLineDate, "stats-one-line-date", "", ci.StatsOneLineDate, "Enable --stats-one-line and add current date/time prefix", "Logging")- flags.StringVarP(flagSet, &ci.StatsOneLineDateFormat, "stats-one-line-date-format", "", ci.StatsOneLineDateFormat, "Enable --stats-one-line-date and use custom formatted date: Enclose date string in double quotes (\"), see https://golang.org/pkg/time/#Time.Format", "Logging")- flags.BoolVarP(flagSet, &ci.ErrorOnNoTransfer, "error-on-no-transfer", "", ci.ErrorOnNoTransfer, "Sets exit code 9 if no files are transferred, useful in scripts", "Config")- flags.BoolVarP(flagSet, &ci.Progress, "progress", "P", ci.Progress, "Show progress during transfer", "Logging")- flags.BoolVarP(flagSet, &ci.ProgressTerminalTitle, "progress-terminal-title", "", ci.ProgressTerminalTitle, "Show progress on the terminal title (requires -P/--progress)", "Logging")- flags.BoolVarP(flagSet, &ci.Cookie, "use-cookies", "", ci.Cookie, "Enable session cookiejar", "Networking")- flags.BoolVarP(flagSet, &ci.UseMmap, "use-mmap", "", ci.UseMmap, "Use mmap allocator (see docs)", "Config")- flags.StringArrayVarP(flagSet, &ci.CaCert, "ca-cert", "", ci.CaCert, "CA certificate used to verify servers", "Networking")- flags.StringVarP(flagSet, &ci.ClientCert, "client-cert", "", ci.ClientCert, "Client SSL certificate (PEM) for mutual TLS auth", "Networking")- flags.StringVarP(flagSet, &ci.ClientKey, "client-key", "", ci.ClientKey, "Client SSL private key (PEM) for mutual TLS auth", "Networking")- flags.FVarP(flagSet, &ci.MultiThreadCutoff, "multi-thread-cutoff", "", "Use multi-thread downloads for files above this size", "Copy")- flags.IntVarP(flagSet, &ci.MultiThreadStreams, "multi-thread-streams", "", ci.MultiThreadStreams, "Number of streams to use for multi-thread downloads", "Copy")- flags.FVarP(flagSet, &ci.MultiThreadWriteBufferSize, "multi-thread-write-buffer-size", "", "In memory buffer size for writing when in multi-thread mode", "Copy")- flags.FVarP(flagSet, &ci.MultiThreadChunkSize, "multi-thread-chunk-size", "", "Chunk size for multi-thread downloads / uploads, if not set by filesystem", "Copy")- flags.BoolVarP(flagSet, &ci.UseJSONLog, "use-json-log", "", ci.UseJSONLog, "Use json log format", "Logging")- flags.StringVarP(flagSet, &ci.OrderBy, "order-by", "", ci.OrderBy, "Instructions on how to order the transfers, e.g. 'size,descending'", "Copy")flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions", "Networking")flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions", "Networking")flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions", "Networking")flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading", "Metadata")- flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files", "Copy")- flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)", "Config")flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21", "Networking")- flags.DurationVarP(flagSet, &ci.FsCacheExpireDuration, "fs-cache-expire-duration", "", ci.FsCacheExpireDuration, "Cache remotes for this long (0 to disable caching)", "Config")- flags.DurationVarP(flagSet, &ci.FsCacheExpireInterval, "fs-cache-expire-interval", "", ci.FsCacheExpireInterval, "Interval to check for expired remotes", "Config")- flags.BoolVarP(flagSet, &ci.DisableHTTP2, "disable-http2", "", ci.DisableHTTP2, "Disable HTTP/2 in the global transport", "Networking")- flags.BoolVarP(flagSet, &ci.HumanReadable, "human-readable", "", ci.HumanReadable, "Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi", "Config")- flags.DurationVarP(flagSet, &ci.KvLockTime, "kv-lock-time", "", ci.KvLockTime, "Maximum time to keep key-value database locked by process", "Config")- flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.", "Networking")- flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects", "Metadata,Copy")- flags.BoolVarP(flagSet, &ci.ServerSideAcrossConfigs, "server-side-across-configs", "", ci.ServerSideAcrossConfigs, "Allow server-side operations (e.g. copy) to work across different configs", "Copy")- flags.FVarP(flagSet, &ci.TerminalColorMode, "color", "", "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS", "Config")- flags.FVarP(flagSet, &ci.DefaultTime, "default-time", "", "Time to show if modtime is unknown for files and directories", "Config,Listing")- flags.BoolVarP(flagSet, &ci.Inplace, "inplace", "", ci.Inplace, "Download directly to destination file instead of atomic download to temp/rename", "Copy")- flags.StringVarP(flagSet, &partialSuffix, "partial-suffix", "", ci.PartialSuffix, "Add partial-suffix to temporary file name when --inplace is not used", "Copy")- flags.FVarP(flagSet, &ci.MetadataMapper, "metadata-mapper", "", "Program to run to transforming metadata before upload", "Metadata")}// ParseHeaders converts the strings passed in via the header flags into HTTPOptions@@ -174,8 +78,9 @@ func ParseHeaders(headers []string) []*fs.HTTPOption {return opts}-// SetFlags converts any flags into config which weren't straight forward+// SetFlags sets flags which aren't part of the config systemfunc SetFlags(ci *fs.ConfigInfo) {+ // Process obsolete --dump-headers and --dump-bodies flagsif dumpHeaders {ci.Dump |= fs.DumpHeadersfs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead")@@ -184,25 +89,23 @@ func SetFlags(ci *fs.ConfigInfo) {ci.Dump |= fs.DumpBodiesfs.Logf(nil, "--dump-bodies is obsolete - please use --dump bodies instead")}- if ci.Dump != 0 && verbose < 2 && ci.LogLevel != fs.LogLevelDebug {- fs.Logf(nil, "Automatically setting -vv as --dump is enabled")- verbose = 2- }+ // Process -v flagif verbose >= 2 {ci.LogLevel = fs.LogLevelDebug} else if verbose >= 1 {ci.LogLevel = fs.LogLevelInfo}- if (ci.DryRun || ci.Interactive) && ci.StatsLogLevel > fs.LogLevelNotice {- ci.StatsLogLevel = fs.LogLevelNotice- }++ // Process -q flagif quiet {if verbose > 0 {log.Fatalf("Can't set -v and -q")}ci.LogLevel = fs.LogLevelError}++ // Can't set log level, -v, -qlogLevelFlag := pflag.Lookup("log-level")if logLevelFlag != nil && logLevelFlag.Changed {if verbose > 0 {@@ -212,28 +115,8 @@ func SetFlags(ci *fs.ConfigInfo) {log.Fatalf("Can't set -q and --log-level")}}- if ci.UseJSONLog {- logrus.AddHook(fsLog.NewCallerHook())- logrus.SetFormatter(&logrus.JSONFormatter{- TimestampFormat: "2006-01-02T15:04:05.999999-07:00",- })- logrus.SetLevel(logrus.DebugLevel)- switch ci.LogLevel {- case fs.LogLevelEmergency, fs.LogLevelAlert:- logrus.SetLevel(logrus.PanicLevel)- case fs.LogLevelCritical:- logrus.SetLevel(logrus.FatalLevel)- case fs.LogLevelError:- logrus.SetLevel(logrus.ErrorLevel)- case fs.LogLevelWarning, fs.LogLevelNotice:- logrus.SetLevel(logrus.WarnLevel)- case fs.LogLevelInfo:- logrus.SetLevel(logrus.InfoLevel)- case fs.LogLevelDebug:- logrus.SetLevel(logrus.DebugLevel)- }- }+ // Process --delete-before, --delete-during and --delete-afterswitch {case deleteBefore && (deleteDuring || deleteAfter),deleteDuring && deleteAfter:@@ -248,19 +131,7 @@ func SetFlags(ci *fs.ConfigInfo) {ci.DeleteMode = fs.DeleteModeDefault}- if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 {- log.Fatalf(`Can't use --compare-dest with --copy-dest.`)- }-- switch {- case len(ci.StatsOneLineDateFormat) > 0:- ci.StatsOneLineDate = true- ci.StatsOneLine = true- case ci.StatsOneLineDate:- ci.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "- ci.StatsOneLine = true- }-+ // Process --bind into IP addressif bindAddr != "" {addrs, err := net.LookupIP(bindAddr)if err != nil {@@ -272,6 +143,7 @@ func SetFlags(ci *fs.ConfigInfo) {ci.BindAddr = addrs[0]}+ // Process --disableif disableFeatures != "" {if disableFeatures == "help" {log.Fatalf("Possible backend features are: %s\n", strings.Join(new(fs.Features).List(), ", "))@@ -279,6 +151,7 @@ func SetFlags(ci *fs.ConfigInfo) {ci.DisableFeatures = strings.Split(disableFeatures, ",")}+ // Process --headers-upload, --headers-download, --headersif len(uploadHeaders) != 0 {ci.UploadHeaders = ParseHeaders(uploadHeaders)}@@ -288,9 +161,8 @@ func SetFlags(ci *fs.ConfigInfo) {if len(headers) != 0 {ci.Headers = ParseHeaders(headers)}- if len(headers) != 0 {- ci.Headers = ParseHeaders(headers)- }++ // Process --metadata-setif len(metadataSet) != 0 {ci.MetadataSet = make(fs.Metadata, len(metadataSet))for _, kv := range metadataSet {@@ -302,6 +174,8 @@ func SetFlags(ci *fs.ConfigInfo) {}fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet)}++ // Process --dscpif len(dscp) != 0 {if value, ok := parseDSCP(dscp); ok {ci.TrafficClass = value << 2@@ -310,39 +184,24 @@ func SetFlags(ci *fs.ConfigInfo) {}}- // Set path to configuration file+ // Process --config pathif err := config.SetConfigPath(configPath); err != nil {log.Fatalf("--config: Failed to set %q as config path: %v", configPath, err)}- // Set path to cache dir+ // Process --cache-dir pathif err := config.SetCacheDir(cacheDir); err != nil {log.Fatalf("--cache-dir: Failed to set %q as cache dir: %v", cacheDir, err)}- // Set path to temp dir+ // Process --temp-dir pathif err := config.SetTempDir(tempDir); err != nil {log.Fatalf("--temp-dir: Failed to set %q as temp dir: %v", tempDir, err)}- // Set whether multi-thread-streams was set+ // Process --multi-thread-streams - set whether multi-thread-streams was setmultiThreadStreamsFlag := pflag.Lookup("multi-thread-streams")ci.MultiThreadSet = multiThreadStreamsFlag != nil && multiThreadStreamsFlag.Changed-- if len(partialSuffix) > 16 {- log.Fatalf("--partial-suffix: Expecting suffix length not greater than %d but got %d", 16, len(partialSuffix))- }- ci.PartialSuffix = partialSuffix-- // Make sure some values are > 0- nonZero := func(pi *int) {- if *pi <= 0 {- *pi = 1- }- }- nonZero(&ci.LowLevelRetries)- nonZero(&ci.Transfers)- nonZero(&ci.Checkers)}// parseHeaders converts DSCP names to valuediff --git a/fs/log.go b/fs/log.goindex d84458a37..40ccdbb27 100644--- a/fs/log.go+++ b/fs/log.go@@ -57,6 +57,9 @@ func (logLevelChoices) Type() string {// LogPrintPid enables process pid in logvar LogPrintPid = false+// InstallJSONLogger is a hook that --use-json-log calls+var InstallJSONLogger = func(logLevel LogLevel) {}+// LogPrint sends the text to the logger of levelvar LogPrint = func(level LogLevel, text string) {text = fmt.Sprintf("%-6s: %s", level, text)diff --git a/fs/log/caller_hook.go b/fs/log/caller_hook.goindex 13941bc02..4d760ae30 100644--- a/fs/log/caller_hook.go+++ b/fs/log/caller_hook.go@@ -5,9 +5,43 @@ import ("runtime""strings"+ "github.com/rclone/rclone/fs""github.com/sirupsen/logrus")+var loggerInstalled = false++// InstallJSONLogger installs the JSON logger at the specified log level+func InstallJSONLogger(logLevel fs.LogLevel) {+ if !loggerInstalled {+ logrus.AddHook(NewCallerHook())+ loggerInstalled = true+ }+ logrus.SetFormatter(&logrus.JSONFormatter{+ TimestampFormat: "2006-01-02T15:04:05.999999-07:00",+ })+ logrus.SetLevel(logrus.DebugLevel)+ switch logLevel {+ case fs.LogLevelEmergency, fs.LogLevelAlert:+ logrus.SetLevel(logrus.PanicLevel)+ case fs.LogLevelCritical:+ logrus.SetLevel(logrus.FatalLevel)+ case fs.LogLevelError:+ logrus.SetLevel(logrus.ErrorLevel)+ case fs.LogLevelWarning, fs.LogLevelNotice:+ logrus.SetLevel(logrus.WarnLevel)+ case fs.LogLevelInfo:+ logrus.SetLevel(logrus.InfoLevel)+ case fs.LogLevelDebug:+ logrus.SetLevel(logrus.DebugLevel)+ }+}++// install hook in fs to call to avoid circular dependency+func init() {+ fs.InstallJSONLogger = InstallJSONLogger+}+// CallerHook for log the calling file and line of the finetype CallerHook struct {Field string