Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

fs: add context aware log functions so logs can be attributed

Nick Craig-Wood8c5a60f31d08
diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.go
index 5463658de..32054e67a 100644
--- a/fs/accounting/stats_groups.go
+++ b/fs/accounting/stats_groups.go
@@ -17,6 +17,9 @@ var groups *statsGroups
func init() {
// Init stats container
groups = newStatsGroups()
+
+ // Attribute logs to stats groups
+ fs.StatsGroupFromContext = StatsGroupFromContext
}
func rcListStats(ctx context.Context, in rc.Params) (rc.Params, error) {
diff --git a/fs/accounting/stats_groups_test.go b/fs/accounting/stats_groups_test.go
index 4adfb04c4..ecaf08957 100644
--- a/fs/accounting/stats_groups_test.go
+++ b/fs/accounting/stats_groups_test.go
@@ -310,3 +310,13 @@ func TestNewStatsGroupDoesNotStartAverageLoop(t *testing.T) {
assert.False(t, startedAfterDone,
"averageLoop should stop when the last transfer is done")
}
+
+// Check fs can read the stats group to attribute logs to it
+func TestFsStatsGroupFromContext(t *testing.T) {
+ ctx := context.Background()
+ _, ok := fs.StatsGroupFromContext(ctx)
+ assert.False(t, ok)
+ group, ok := fs.StatsGroupFromContext(WithStatsGroup(ctx, "job/1"))
+ assert.True(t, ok)
+ assert.Equal(t, "job/1", group)
+}
diff --git a/fs/config.go b/fs/config.go
index 412bec35b..cafca05ad 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -43,6 +43,22 @@ var (
// implementation from the fs
CountError = func(ctx context.Context, err error) error { return err }
+ // StatsGroupFromContext returns the stats group from ctx if set.
+ //
+ // This is used to attribute logs to stats groups.
+ //
+ // This is a function pointer to decouple the accounting
+ // implementation from the fs
+ StatsGroupFromContext = func(ctx context.Context) (string, bool) { return "", false }
+
+ // JobIDFromContext returns the ID of the rc job in ctx if set.
+ //
+ // This is used to attribute logs to the rc job which made them.
+ //
+ // This is a function pointer to decouple the rc jobs
+ // implementation from the fs
+ JobIDFromContext = func(ctx context.Context) (int64, bool) { return 0, false }
+
// ConfigProvider is the config key used for provider options
ConfigProvider = "provider"
diff --git a/fs/log.go b/fs/log.go
index 8725b30fd..71a771b72 100644
--- a/fs/log.go
+++ b/fs/log.go
@@ -140,11 +140,16 @@ func LogLevelToSlog(level LogLevel) slog.Level {
return slogLevel
}
-func logSlog(level LogLevel, text string, attrs []any) {
- logger.Log(context.Background(), LogLevelToSlog(level), text, attrs...)
+func logSlog(ctx context.Context, level LogLevel, text string, attrs []any) {
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ // ctx is passed to the log handler which uses it to attribute
+ // the log to the rc job and stats group it was made by.
+ logger.Log(ctx, LogLevelToSlog(level), text, attrs...)
}
-func logSlogWithObject(level LogLevel, o any, text string, attrs []any) {
+func logSlogWithObject(ctx context.Context, level LogLevel, o any, text string, attrs []any) {
if o != nil {
var object string
switch o.(type) {
@@ -160,24 +165,46 @@ func logSlogWithObject(level LogLevel, o any, text string, attrs []any) {
"objectType", fmt.Sprintf("%T", o),
})
}
- logSlog(level, text, attrs)
-}
-
-// LogPrint produces a log string from the arguments passed in
-func LogPrint(level LogLevel, o any, text string) {
- logSlogWithObject(level, o, text, nil)
+ logSlog(ctx, level, text, attrs)
}
-// LogPrintf produces a log string from the arguments passed in
-func LogPrintf(level LogLevel, o any, text string, args ...any) {
- text = fmt.Sprintf(text, args...)
+func logPrintf(ctx context.Context, level LogLevel, o any, text string, args ...any) {
+ // Note that text isn't assigned to here so go vet can see this
+ // is a printf wrapper and check the format strings of the
+ // callers.
+ msg := fmt.Sprintf(text, args...)
var fields []any
for _, arg := range args {
if item, ok := arg.(LogValueItem); ok {
fields = append(fields, item.key, item.value)
}
}
- logSlogWithObject(level, o, text, fields)
+ logSlogWithObject(ctx, level, o, msg, fields)
+}
+
+func logLevelPrintf(ctx context.Context, level LogLevel, o any, text string, args ...any) {
+ // This uses the global log level, not the one in ctx, as the
+ // log handler filters on the global log level.
+ if GetConfig(context.TODO()).LogLevel >= level {
+ logPrintf(ctx, level, o, text, args...)
+ }
+}
+
+// LogPrint produces a log string from the arguments passed in
+func LogPrint(level LogLevel, o any, text string) {
+ logSlogWithObject(context.Background(), level, o, text, nil)
+}
+
+// LogPrintf produces a log string from the arguments passed in
+func LogPrintf(level LogLevel, o any, text string, args ...any) {
+ logPrintf(context.Background(), level, o, text, args...)
+}
+
+// LogPrintfCtx produces a log string from the arguments passed in.
+//
+// The log is attributed to the stats group in ctx, if any.
+func LogPrintfCtx(ctx context.Context, level LogLevel, o any, text string, args ...any) {
+ logPrintf(ctx, level, o, text, args...)
}
// LogLevelPrint writes logs at the given level
@@ -189,9 +216,7 @@ func LogLevelPrint(level LogLevel, o any, text string) {
// LogLevelPrintf writes logs at the given level
func LogLevelPrintf(level LogLevel, o any, text string, args ...any) {
- if GetConfig(context.TODO()).LogLevel >= level {
- LogPrintf(level, o, text, args...)
- }
+ logLevelPrintf(context.Background(), level, o, text, args...)
}
// Panic writes alert log output for this Object or Fs and calls panic().
@@ -273,6 +298,11 @@ func Errorf(o any, text string, args ...any) {
LogLevelPrintf(LogLevelError, o, text, args...)
}
+// ErrorfCtx is Errorf with the log attributed to the stats group in ctx, if any.
+func ErrorfCtx(ctx context.Context, o any, text string, args ...any) {
+ logLevelPrintf(ctx, LogLevelError, o, text, args...)
+}
+
// Print writes log output for this Object or Fs, same as Logf.
func Print(o any, text string) {
LogLevelPrint(LogLevelNotice, o, text)
@@ -301,6 +331,11 @@ func Logf(o any, text string, args ...any) {
LogLevelPrintf(LogLevelNotice, o, text, args...)
}
+// LogfCtx is Logf with the log attributed to the stats group in ctx, if any.
+func LogfCtx(ctx context.Context, o any, text string, args ...any) {
+ logLevelPrintf(ctx, LogLevelNotice, o, text, args...)
+}
+
// Infoc writes info on transfers for this Object or Fs. Use this
// level for logging transfers, deletions and things which should
// appear with the -v flag.
@@ -316,6 +351,11 @@ func Infof(o any, text string, args ...any) {
LogLevelPrintf(LogLevelInfo, o, text, args...)
}
+// InfofCtx is Infof with the log attributed to the stats group in ctx, if any.
+func InfofCtx(ctx context.Context, o any, text string, args ...any) {
+ logLevelPrintf(ctx, LogLevelInfo, o, text, args...)
+}
+
// Debug writes debugging output for this Object or Fs. Use this for
// debug only. The user must have to specify -vv to see this.
func Debug(o any, text string) {
@@ -328,6 +368,11 @@ func Debugf(o any, text string, args ...any) {
LogLevelPrintf(LogLevelDebug, o, text, args...)
}
+// DebugfCtx is Debugf with the log attributed to the stats group in ctx, if any.
+func DebugfCtx(ctx context.Context, o any, text string, args ...any) {
+ logLevelPrintf(ctx, LogLevelDebug, o, text, args...)
+}
+
// LogDirName returns an object for the logger, logging a root
// directory which would normally be "" as the Fs
func LogDirName(f Fs, dir string) any {
diff --git a/fs/log_test.go b/fs/log_test.go
index e7c790039..45c987f7d 100644
--- a/fs/log_test.go
+++ b/fs/log_test.go
@@ -2,6 +2,7 @@ package fs
import (
"bytes"
+ "context"
"encoding/json"
"fmt"
"log/slog"
@@ -41,19 +42,19 @@ func TestLogSlogWithObject(t *testing.T) {
SetLogger(slog.NewTextHandler(&buf, nil))
// Objects with a String method are rendered with it
- logSlogWithObject(LogLevelError, withString{}, "message", nil)
+ logSlogWithObject(context.Background(), LogLevelError, withString{}, "message", nil)
assert.Contains(t, buf.String(), "object=hello")
// Plain strings are rendered as themselves
buf.Reset()
- logSlogWithObject(LogLevelError, "potato", "message", nil)
+ logSlogWithObject(context.Background(), LogLevelError, "potato", "message", nil)
assert.Contains(t, buf.String(), "object=potato")
// Anything else shows only its type as it may contain
// sensitive data such as credentials
buf.Reset()
type secrets struct{ Password string }
- logSlogWithObject(LogLevelError, &secrets{Password: "SECRET"}, "message", nil)
+ logSlogWithObject(context.Background(), LogLevelError, &secrets{Password: "SECRET"}, "message", nil)
assert.Contains(t, buf.String(), "object=*fs.secrets")
assert.NotContains(t, buf.String(), "SECRET")
}
@@ -119,3 +120,79 @@ func TestLogLevelUnmarshalJSON(t *testing.T) {
assert.Equal(t, test.want, logLevel, test.in)
}
}
+
+type testGroupKey struct{}
+
+// ctxHandler records the contexts it was called with
+type ctxHandler struct {
+ slog.Handler
+ ctxs []context.Context
+}
+
+func (h *ctxHandler) Handle(ctx context.Context, r slog.Record) error {
+ h.ctxs = append(h.ctxs, ctx)
+ return h.Handler.Handle(ctx, r)
+}
+
+// The context aware log functions pass their context to the log
+// handler which uses it to attribute the log.
+func TestLogCtx(t *testing.T) {
+ var buf bytes.Buffer
+ oldLogger := logger
+ defer func() { logger = oldLogger }()
+ h := &ctxHandler{Handler: slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})}
+ SetLogger(h)
+ ci := GetConfig(context.Background())
+ oldLogLevel := ci.LogLevel
+ ci.LogLevel = LogLevelDebug
+ defer func() { ci.LogLevel = oldLogLevel }()
+
+ groupCtx := context.WithValue(context.Background(), testGroupKey{}, "job/1")
+ check := func(want string) {
+ t.Helper()
+ var got struct {
+ Level string `json:"level"`
+ Msg string `json:"msg"`
+ Object string `json:"object"`
+ Size int `json:"size"`
+ }
+ require.NoError(t, json.Unmarshal(buf.Bytes(), &got), buf.String())
+ assert.Equal(t, want, fmt.Sprintf("%s|%s|%s|%d", got.Level, got.Msg, got.Object, got.Size))
+ buf.Reset()
+ }
+
+ ErrorfCtx(groupCtx, "obj", "error %d", 1)
+ check("ERROR|error 1|obj|0")
+ LogfCtx(groupCtx, nil, "notice")
+ check("INFO+2|notice||0")
+ InfofCtx(groupCtx, nil, "info")
+ check("INFO|info||0")
+ DebugfCtx(groupCtx, nil, "debug")
+ check("DEBUG|debug||0")
+ LogPrintfCtx(groupCtx, LogLevelInfo, nil, "size %v", LogValue("size", 3))
+ check("INFO|size 3||3")
+
+ // All of those were passed the context they were called with
+ require.Len(t, h.ctxs, 5)
+ for _, ctx := range h.ctxs {
+ assert.Equal(t, "job/1", ctx.Value(testGroupKey{}))
+ }
+
+ // The non context functions and a nil context log with a
+ // context which isn't attributed to anything
+ h.ctxs = nil
+ Errorf(nil, "plain")
+ check("ERROR|plain||0")
+ ErrorfCtx(nil, nil, "nil ctx") //nolint:staticcheck // deliberately testing a nil context
+ check("ERROR|nil ctx||0")
+ require.Len(t, h.ctxs, 2)
+ for _, ctx := range h.ctxs {
+ require.NotNil(t, ctx)
+ assert.Nil(t, ctx.Value(testGroupKey{}))
+ }
+
+ // Logs below the log level are dropped
+ ci.LogLevel = LogLevelNotice
+ InfofCtx(groupCtx, nil, "dropped")
+ assert.Equal(t, "", buf.String())
+}