"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fs: add context aware log functions so logs can be attributed
Nick Craig-Wood8c5a60f31d08
diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.goindex 5463658de..32054e67a 100644--- a/fs/accounting/stats_groups.go+++ b/fs/accounting/stats_groups.go@@ -17,6 +17,9 @@ var groups *statsGroupsfunc init() {// Init stats containergroups = newStatsGroups()++ // Attribute logs to stats groups+ fs.StatsGroupFromContext = StatsGroupFromContext}func rcListStats(ctx context.Context, in rc.Params) (rc.Params, error) {diff --git a/fs/accounting/stats_groups_test.go b/fs/accounting/stats_groups_test.goindex 4adfb04c4..ecaf08957 100644--- a/fs/accounting/stats_groups_test.go+++ b/fs/accounting/stats_groups_test.go@@ -310,3 +310,13 @@ func TestNewStatsGroupDoesNotStartAverageLoop(t *testing.T) {assert.False(t, startedAfterDone,"averageLoop should stop when the last transfer is done")}++// Check fs can read the stats group to attribute logs to it+func TestFsStatsGroupFromContext(t *testing.T) {+ ctx := context.Background()+ _, ok := fs.StatsGroupFromContext(ctx)+ assert.False(t, ok)+ group, ok := fs.StatsGroupFromContext(WithStatsGroup(ctx, "job/1"))+ assert.True(t, ok)+ assert.Equal(t, "job/1", group)+}diff --git a/fs/config.go b/fs/config.goindex 412bec35b..cafca05ad 100644--- a/fs/config.go+++ b/fs/config.go@@ -43,6 +43,22 @@ var (// implementation from the fsCountError = func(ctx context.Context, err error) error { return err }+ // StatsGroupFromContext returns the stats group from ctx if set.+ //+ // This is used to attribute logs to stats groups.+ //+ // This is a function pointer to decouple the accounting+ // implementation from the fs+ StatsGroupFromContext = func(ctx context.Context) (string, bool) { return "", false }++ // JobIDFromContext returns the ID of the rc job in ctx if set.+ //+ // This is used to attribute logs to the rc job which made them.+ //+ // This is a function pointer to decouple the rc jobs+ // implementation from the fs+ JobIDFromContext = func(ctx context.Context) (int64, bool) { return 0, false }+// ConfigProvider is the config key used for provider optionsConfigProvider = "provider"diff --git a/fs/log.go b/fs/log.goindex 8725b30fd..71a771b72 100644--- a/fs/log.go+++ b/fs/log.go@@ -140,11 +140,16 @@ func LogLevelToSlog(level LogLevel) slog.Level {return slogLevel}-func logSlog(level LogLevel, text string, attrs []any) {- logger.Log(context.Background(), LogLevelToSlog(level), text, attrs...)+func logSlog(ctx context.Context, level LogLevel, text string, attrs []any) {+ if ctx == nil {+ ctx = context.Background()+ }+ // ctx is passed to the log handler which uses it to attribute+ // the log to the rc job and stats group it was made by.+ logger.Log(ctx, LogLevelToSlog(level), text, attrs...)}-func logSlogWithObject(level LogLevel, o any, text string, attrs []any) {+func logSlogWithObject(ctx context.Context, level LogLevel, o any, text string, attrs []any) {if o != nil {var object stringswitch o.(type) {@@ -160,24 +165,46 @@ func logSlogWithObject(level LogLevel, o any, text string, attrs []any) {"objectType", fmt.Sprintf("%T", o),})}- logSlog(level, text, attrs)-}--// LogPrint produces a log string from the arguments passed in-func LogPrint(level LogLevel, o any, text string) {- logSlogWithObject(level, o, text, nil)+ logSlog(ctx, level, text, attrs)}-// LogPrintf produces a log string from the arguments passed in-func LogPrintf(level LogLevel, o any, text string, args ...any) {- text = fmt.Sprintf(text, args...)+func logPrintf(ctx context.Context, level LogLevel, o any, text string, args ...any) {+ // Note that text isn't assigned to here so go vet can see this+ // is a printf wrapper and check the format strings of the+ // callers.+ msg := fmt.Sprintf(text, args...)var fields []anyfor _, arg := range args {if item, ok := arg.(LogValueItem); ok {fields = append(fields, item.key, item.value)}}- logSlogWithObject(level, o, text, fields)+ logSlogWithObject(ctx, level, o, msg, fields)+}++func logLevelPrintf(ctx context.Context, level LogLevel, o any, text string, args ...any) {+ // This uses the global log level, not the one in ctx, as the+ // log handler filters on the global log level.+ if GetConfig(context.TODO()).LogLevel >= level {+ logPrintf(ctx, level, o, text, args...)+ }+}++// LogPrint produces a log string from the arguments passed in+func LogPrint(level LogLevel, o any, text string) {+ logSlogWithObject(context.Background(), level, o, text, nil)+}++// LogPrintf produces a log string from the arguments passed in+func LogPrintf(level LogLevel, o any, text string, args ...any) {+ logPrintf(context.Background(), level, o, text, args...)+}++// LogPrintfCtx produces a log string from the arguments passed in.+//+// The log is attributed to the stats group in ctx, if any.+func LogPrintfCtx(ctx context.Context, level LogLevel, o any, text string, args ...any) {+ logPrintf(ctx, level, o, text, args...)}// LogLevelPrint writes logs at the given level@@ -189,9 +216,7 @@ func LogLevelPrint(level LogLevel, o any, text string) {// LogLevelPrintf writes logs at the given levelfunc LogLevelPrintf(level LogLevel, o any, text string, args ...any) {- if GetConfig(context.TODO()).LogLevel >= level {- LogPrintf(level, o, text, args...)- }+ logLevelPrintf(context.Background(), level, o, text, args...)}// Panic writes alert log output for this Object or Fs and calls panic().@@ -273,6 +298,11 @@ func Errorf(o any, text string, args ...any) {LogLevelPrintf(LogLevelError, o, text, args...)}+// ErrorfCtx is Errorf with the log attributed to the stats group in ctx, if any.+func ErrorfCtx(ctx context.Context, o any, text string, args ...any) {+ logLevelPrintf(ctx, LogLevelError, o, text, args...)+}+// Print writes log output for this Object or Fs, same as Logf.func Print(o any, text string) {LogLevelPrint(LogLevelNotice, o, text)@@ -301,6 +331,11 @@ func Logf(o any, text string, args ...any) {LogLevelPrintf(LogLevelNotice, o, text, args...)}+// LogfCtx is Logf with the log attributed to the stats group in ctx, if any.+func LogfCtx(ctx context.Context, o any, text string, args ...any) {+ logLevelPrintf(ctx, LogLevelNotice, o, text, args...)+}+// Infoc writes info on transfers for this Object or Fs. Use this// level for logging transfers, deletions and things which should// appear with the -v flag.@@ -316,6 +351,11 @@ func Infof(o any, text string, args ...any) {LogLevelPrintf(LogLevelInfo, o, text, args...)}+// InfofCtx is Infof with the log attributed to the stats group in ctx, if any.+func InfofCtx(ctx context.Context, o any, text string, args ...any) {+ logLevelPrintf(ctx, LogLevelInfo, o, text, args...)+}+// Debug writes debugging output for this Object or Fs. Use this for// debug only. The user must have to specify -vv to see this.func Debug(o any, text string) {@@ -328,6 +368,11 @@ func Debugf(o any, text string, args ...any) {LogLevelPrintf(LogLevelDebug, o, text, args...)}+// DebugfCtx is Debugf with the log attributed to the stats group in ctx, if any.+func DebugfCtx(ctx context.Context, o any, text string, args ...any) {+ logLevelPrintf(ctx, LogLevelDebug, o, text, args...)+}+// LogDirName returns an object for the logger, logging a root// directory which would normally be "" as the Fsfunc LogDirName(f Fs, dir string) any {diff --git a/fs/log_test.go b/fs/log_test.goindex e7c790039..45c987f7d 100644--- a/fs/log_test.go+++ b/fs/log_test.go@@ -2,6 +2,7 @@ package fsimport ("bytes"+ "context""encoding/json""fmt""log/slog"@@ -41,19 +42,19 @@ func TestLogSlogWithObject(t *testing.T) {SetLogger(slog.NewTextHandler(&buf, nil))// Objects with a String method are rendered with it- logSlogWithObject(LogLevelError, withString{}, "message", nil)+ logSlogWithObject(context.Background(), LogLevelError, withString{}, "message", nil)assert.Contains(t, buf.String(), "object=hello")// Plain strings are rendered as themselvesbuf.Reset()- logSlogWithObject(LogLevelError, "potato", "message", nil)+ logSlogWithObject(context.Background(), LogLevelError, "potato", "message", nil)assert.Contains(t, buf.String(), "object=potato")// Anything else shows only its type as it may contain// sensitive data such as credentialsbuf.Reset()type secrets struct{ Password string }- logSlogWithObject(LogLevelError, &secrets{Password: "SECRET"}, "message", nil)+ logSlogWithObject(context.Background(), LogLevelError, &secrets{Password: "SECRET"}, "message", nil)assert.Contains(t, buf.String(), "object=*fs.secrets")assert.NotContains(t, buf.String(), "SECRET")}@@ -119,3 +120,79 @@ func TestLogLevelUnmarshalJSON(t *testing.T) {assert.Equal(t, test.want, logLevel, test.in)}}++type testGroupKey struct{}++// ctxHandler records the contexts it was called with+type ctxHandler struct {+ slog.Handler+ ctxs []context.Context+}++func (h *ctxHandler) Handle(ctx context.Context, r slog.Record) error {+ h.ctxs = append(h.ctxs, ctx)+ return h.Handler.Handle(ctx, r)+}++// The context aware log functions pass their context to the log+// handler which uses it to attribute the log.+func TestLogCtx(t *testing.T) {+ var buf bytes.Buffer+ oldLogger := logger+ defer func() { logger = oldLogger }()+ h := &ctxHandler{Handler: slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})}+ SetLogger(h)+ ci := GetConfig(context.Background())+ oldLogLevel := ci.LogLevel+ ci.LogLevel = LogLevelDebug+ defer func() { ci.LogLevel = oldLogLevel }()++ groupCtx := context.WithValue(context.Background(), testGroupKey{}, "job/1")+ check := func(want string) {+ t.Helper()+ var got struct {+ Level string `json:"level"`+ Msg string `json:"msg"`+ Object string `json:"object"`+ Size int `json:"size"`+ }+ require.NoError(t, json.Unmarshal(buf.Bytes(), &got), buf.String())+ assert.Equal(t, want, fmt.Sprintf("%s|%s|%s|%d", got.Level, got.Msg, got.Object, got.Size))+ buf.Reset()+ }++ ErrorfCtx(groupCtx, "obj", "error %d", 1)+ check("ERROR|error 1|obj|0")+ LogfCtx(groupCtx, nil, "notice")+ check("INFO+2|notice||0")+ InfofCtx(groupCtx, nil, "info")+ check("INFO|info||0")+ DebugfCtx(groupCtx, nil, "debug")+ check("DEBUG|debug||0")+ LogPrintfCtx(groupCtx, LogLevelInfo, nil, "size %v", LogValue("size", 3))+ check("INFO|size 3||3")++ // All of those were passed the context they were called with+ require.Len(t, h.ctxs, 5)+ for _, ctx := range h.ctxs {+ assert.Equal(t, "job/1", ctx.Value(testGroupKey{}))+ }++ // The non context functions and a nil context log with a+ // context which isn't attributed to anything+ h.ctxs = nil+ Errorf(nil, "plain")+ check("ERROR|plain||0")+ ErrorfCtx(nil, nil, "nil ctx") //nolint:staticcheck // deliberately testing a nil context+ check("ERROR|nil ctx||0")+ require.Len(t, h.ctxs, 2)+ for _, ctx := range h.ctxs {+ require.NotNil(t, ctx)+ assert.Nil(t, ctx.Value(testGroupKey{}))+ }++ // Logs below the log level are dropped+ ci.LogLevel = LogLevelNotice+ InfofCtx(groupCtx, nil, "dropped")+ assert.Equal(t, "", buf.String())+}