Skip to content

Find, verify, and analyze leaked credentials

[chore] Remove logrus from circleci, filesystem, gitlab, and s3 sources (#1089)

Miccahd317ddb51a9d
diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.go
index 8d30c0e1f..44f00cac6 100644
--- a/pkg/engine/circleci.go
+++ b/pkg/engine/circleci.go
@@ -29,6 +29,10 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
}
circleSource := circleci.Source{}
+ ctx = context.WithValues(ctx,
+ "source_type", circleSource.Type().String(),
+ "source_name", "Circle CI",
+ )
err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
if err != nil {
return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.go
index 59ec8cf9d..61527bf8b 100644
--- a/pkg/engine/filesystem.go
+++ b/pkg/engine/filesystem.go
@@ -27,6 +27,11 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error {
}
fileSystemSource := filesystem.Source{}
+
+ ctx = context.WithValues(ctx,
+ "source_type", fileSystemSource.Type().String(),
+ "source_name", "filesystem",
+ )
err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())
if err != nil {
return errors.WrapPrefix(err, "could not init filesystem source", 0)
diff --git a/pkg/engine/git.go b/pkg/engine/git.go
index 59cb4f1a4..edc3b02d9 100644
--- a/pkg/engine/git.go
+++ b/pkg/engine/git.go
@@ -54,6 +54,10 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {
}
})
+ ctx = context.WithValues(ctx,
+ "source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
+ "source_name", "git",
+ )
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
diff --git a/pkg/engine/github.go b/pkg/engine/github.go
index ba3ec543f..927415df6 100644
--- a/pkg/engine/github.go
+++ b/pkg/engine/github.go
@@ -38,8 +38,8 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {
return err
}
ctx = context.WithValues(ctx,
- "source_type", source.Type(),
- "name", "trufflehog - github",
+ "source_type", source.Type().String(),
+ "source_name", "github",
)
err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
if err != nil {
diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go
index 3ae377085..c0257c8a5 100644
--- a/pkg/engine/gitlab.go
+++ b/pkg/engine/gitlab.go
@@ -53,6 +53,10 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
}
gitlabSource := gitlab.Source{}
+ ctx = context.WithValues(ctx,
+ "source_type", gitlabSource.Type().String(),
+ "source_name", "gitlab",
+ )
err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())
if err != nil {
return errors.WrapPrefix(err, "could not init GitLab source", 0)
diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go
index 88329a0a4..5ba708769 100644
--- a/pkg/engine/s3.go
+++ b/pkg/engine/s3.go
@@ -46,6 +46,10 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
}
s3Source := s3.Source{}
+ ctx = context.WithValues(ctx,
+ "source_type", s3Source.Type().String(),
+ "source_name", "s3",
+ )
err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
if err != nil {
return errors.WrapPrefix(err, "failed to init S3 source", 0)
diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.go
index 2868b5818..d12ba4705 100644
--- a/pkg/engine/syslog.go
+++ b/pkg/engine/syslog.go
@@ -42,6 +42,10 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error {
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
source := syslog.Source{}
+ ctx = context.WithValues(ctx,
+ "source_type", source.Type().String(),
+ "source_name", "syslog",
+ )
err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
source.InjectConnection(connection)
if err != nil {
diff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.go
index bdf22d4c1..f6d95e50c 100644
--- a/pkg/sources/circleci/circleci.go
+++ b/pkg/sources/circleci/circleci.go
@@ -9,7 +9,6 @@ import (
"sync/atomic"
"github.com/go-errors/errors"
- log "github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -111,14 +110,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
atomic.AddUint64(&scanned, 1)
- log.Debugf("scanned %d/%d projects", scanned, len(projects))
+ ctx.Logger().V(2).Info(fmt.Sprintf("scanned %d/%d projects", scanned, len(projects)))
return nil
})
}
_ = s.jobPool.Wait()
if scanErrs.Count() > 0 {
- log.Debugf("encountered %d errors while scanning; errors: %v", scanErrs.Count(), scanErrs)
+ ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)
}
return nil
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go
index 3dda10887..7562e59d8 100644
--- a/pkg/sources/filesystem/filesystem.go
+++ b/pkg/sources/filesystem/filesystem.go
@@ -9,7 +9,7 @@ import (
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
"github.com/go-errors/errors"
- log "github.com/sirupsen/logrus"
+ "github.com/go-logr/logr"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -35,7 +35,7 @@ type Source struct {
jobId int64
verify bool
paths []string
- log *log.Entry
+ log logr.Logger
filter *common.Filter
sources.Progress
}
@@ -59,7 +59,7 @@ func (s *Source) JobID() int64 {
// Init returns an initialized Filesystem source.
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
- s.log = log.WithField("source", s.Type()).WithField("name", name)
+ s.log = aCtx.Logger()
s.name = name
s.sourceId = sourceId
@@ -98,10 +98,11 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
path := filepath.Join(cleanPath, relativePath)
+ logger := ctx.Logger().WithValues("file", path)
fileStat, err := os.Stat(path)
if err != nil {
- log.WithError(err).Warnf("unable to stat file: %s", path)
+ logger.Error(err, "unable to stat file")
return nil
}
if !fileStat.Mode().IsRegular() {
@@ -114,15 +115,15 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
inputFile, err := os.Open(path)
if err != nil {
- log.Warn(err)
+ logger.V(2).Info("error opening file", "error", err)
return nil
}
defer inputFile.Close()
- log.WithField("file_path", path).Trace("scanning file")
+ logger.V(5).Info("scanning file")
reReader, err := diskbufferreader.New(inputFile)
if err != nil {
- log.WithError(err).Error("Could not create re-readable reader.")
+ logger.Error(err, "Could not create re-readable reader.")
}
defer reReader.Close()
diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.go
index a3e07525c..46fe85491 100644
--- a/pkg/sources/filesystem/filesystem_test.go
+++ b/pkg/sources/filesystem/filesystem_test.go
@@ -5,7 +5,6 @@ import (
"time"
"github.com/kylelemons/godebug/pretty"
- log "github.com/sirupsen/logrus"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -51,8 +50,6 @@ func TestSource_Scan(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
- log.SetLevel(log.DebugLevel)
- log.SetFormatter(&log.TextFormatter{ForceColors: true})
conn, err := anypb.New(tt.init.connection)
if err != nil {
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index 73a1283c0..73ddcf0e4 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -23,7 +23,6 @@ import (
"github.com/go-errors/errors"
gogit "github.com/go-git/go-git/v5"
"github.com/gobwas/glob"
- log "github.com/sirupsen/logrus"
"github.com/xanzy/go-gitlab"
"golang.org/x/exp/slices"
"google.golang.org/protobuf/proto"
@@ -69,7 +68,6 @@ func (s *Source) JobID() int64 {
// Init returns an initialized Gitlab source.
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
-
s.name = name
s.sourceId = sourceId
s.jobId = jobId
@@ -143,7 +141,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
// Get repo within target.
repos, errs := s.getRepos()
for _, repoErr := range errs {
- log.WithError(repoErr).Warn("error getting repo")
+ ctx.Logger().Info("error getting repo", "error", repoErr)
}
// End early if we had errors getting specified repos but none were validated.
@@ -153,13 +151,13 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
// Get all repos if not specified.
if len(repos) == 0 {
- projects, err := s.getAllProjects(apiClient)
+ projects, err := s.getAllProjects(ctx, apiClient)
if err != nil {
return fmt.Errorf("error getting all projects: %v", err)
}
// Turn projects into URLs for Git cloner.
for _, prj := range projects {
- if s.ignoreRepo(prj.PathWithNamespace) {
+ if s.ignoreRepo(ctx, prj.PathWithNamespace) {
continue
}
@@ -231,7 +229,7 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {
return false
}
-func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, error) {
+func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) ([]*gitlab.Project, error) {
// Projects without repo will get user projects, groups projects, and subgroup projects.
user, _, err := apiClient.Users.CurrentUser()
@@ -289,7 +287,10 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er
for {
grpPrjs, res, err := apiClient.Groups.ListGroupProjects(group.ID, listGroupProjectOptions)
if err != nil {
- log.WithError(err).WithField("group", group.FullPath).Warn("received error on listing group projects, you probably don't have permissions to do that")
+ ctx.Logger().Info("received error on listing group projects, you probably don't have permissions to do that",
+ "group", group.FullPath,
+ "error", err,
+ )
break
}
for _, prj := range grpPrjs {
@@ -305,7 +306,7 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er
for _, project := range projects {
projectNamesWithNamespace = append(projectNamesWithNamespace, project.NameWithNamespace)
}
- log.WithField("projects", strings.Join(projectNamesWithNamespace, ", ")).Debugf("Enumerated %d GitLab projects", len(projects))
+ ctx.Logger().V(2).Info("Enumerated GitLab projects", "count", len(projects), "projects", projectNamesWithNamespace)
var projectList []*gitlab.Project
for _, project := range projects {
@@ -334,7 +335,6 @@ func (s *Source) getRepos() ([]string, []error) {
}
func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
-
// If there is resume information available, limit this scan to only the repos that still need scanning.
reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
s.repos = reposToScan
@@ -343,15 +343,16 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
for i, repo := range s.repos {
i, repoURL := i, repo
s.jobPool.Go(func() error {
+ logger := ctx.Logger().WithValues("repo", repoURL)
if common.IsDone(ctx) {
// We are returning nil instead of the scanErrors slice here because
// we don't want to mark this scan as errored if we cancelled it.
- log.Debugf("Skipping repo %s because context was cancelled", repoURL)
+ logger.V(2).Info("Skipping repo because context was cancelled")
return nil
}
if len(repoURL) == 0 {
- log.Debugf("Skipping empty repo %s", repoURL)
+ logger.V(2).Info("Skipping empty repo")
return nil
}
@@ -383,35 +384,35 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
return nil
}
- log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repo)
+ logger.V(2).Info(fmt.Sprintf("Starting to scan repo %d/%d", i+1, len(s.repos)))
if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
scanErrs.Add(err)
return nil
}
- log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(s.repos), repo)
+ logger.V(2).Info(fmt.Sprintf("Completed scanning repo %d/%d", i+1, len(s.repos)))
return nil
})
}
_ = s.jobPool.Wait()
if scanErrs.Count() > 0 {
- log.Debugf("encountered %d errors while scanning; errors: %v", scanErrs.Count(), scanErrs)
+ ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)
}
s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Gitlab scan", "")
return nil
}
-func (s *Source) ignoreRepo(r string) bool {
+func (s *Source) ignoreRepo(ctx context.Context, r string) bool {
for _, ignore := range s.ignoreRepos {
g, err := glob.Compile(ignore)
if err != nil {
- log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)
+ ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", ignore)
continue
}
if g.Match(r) {
- log.Debugf("Ignoring repo %s", r)
+ ctx.Logger().V(2).Info("Ignoring repo", "repo", r)
return true
}
}
diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go
index 3b194a65a..68088c522 100644
--- a/pkg/sources/gitlab/gitlab_test.go
+++ b/pkg/sources/gitlab/gitlab_test.go
@@ -2,12 +2,10 @@ package gitlab
import (
"fmt"
- "io"
"reflect"
"testing"
"github.com/kylelemons/godebug/pretty"
- log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/types/known/anypb"
@@ -24,8 +22,6 @@ func TestSource_Scan(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- log.SetLevel(log.DebugLevel)
- log.SetFormatter(&log.TextFormatter{ForceColors: true})
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
@@ -142,7 +138,6 @@ func TestSource_Scan(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
- log.SetLevel(log.DebugLevel)
conn, err := anypb.New(tt.init.connection)
if err != nil {
@@ -200,7 +195,6 @@ func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
},
}
- log.SetOutput(io.Discard)
s := &Source{repos: []string{}}
for _, tt := range tests {
@@ -248,8 +242,6 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
},
}
- log.SetOutput(io.Discard)
-
for _, tt := range tests {
s := &Source{
repos: tt.repos,
diff --git a/pkg/sources/resume.go b/pkg/sources/resume.go
index d085b5d69..4969a5750 100644
--- a/pkg/sources/resume.go
+++ b/pkg/sources/resume.go
@@ -2,8 +2,6 @@ package sources
import (
"strings"
-
- "github.com/sirupsen/logrus"
)
// RemoveRepoFromResumeInfo removes the repoURL from the resume info.
@@ -18,8 +16,7 @@ func RemoveRepoFromResumeInfo(resumeRepos []string, repoURL string) []string {
if index == -1 {
// We should never be able to be here. But if we are, it means the resume info never had the repo added.
- // So log the error and do nothing.
- logrus.Errorf("repoURL (%q) not found in list of encode resume info: %v", repoURL, resumeRepos)
+ // So do nothing.
return resumeRepos
}
diff --git a/pkg/sources/resume_test.go b/pkg/sources/resume_test.go
index 56bda955d..6a89c05f7 100644
--- a/pkg/sources/resume_test.go
+++ b/pkg/sources/resume_test.go
@@ -1,11 +1,8 @@
package sources
import (
- "io"
"reflect"
"testing"
-
- "github.com/sirupsen/logrus"
)
func TestRemoveRepoFromResumeInfo(t *testing.T) {
@@ -31,9 +28,6 @@ func TestRemoveRepoFromResumeInfo(t *testing.T) {
},
}
- // Prevent the error message from showing up.
- logrus.SetOutput(io.Discard)
-
for _, tt := range tests {
gotResumeInfoSlice := RemoveRepoFromResumeInfo(tt.startingResumeInfoSlice, tt.repoURL)
if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) {
@@ -57,9 +51,6 @@ func TestEncodeResumeInfo(t *testing.T) {
},
}
- logger := logrus.New()
- logger.Out = io.Discard
-
for _, tt := range tests {
gotEncodedResumeInfo := EncodeResumeInfo(tt.startingResumeInfoSlice)
if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {
diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.go
index e1f962543..5d8ce03e2 100644
--- a/pkg/sources/s3/s3_test.go
+++ b/pkg/sources/s3/s3_test.go
@@ -9,7 +9,6 @@ import (
"time"
"github.com/kylelemons/godebug/pretty"
- log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/anypb"
@@ -62,14 +61,11 @@ func TestSource_Chunks(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- log.SetFormatter(&log.TextFormatter{ForceColors: true})
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
var cancelOnce sync.Once
defer cancelOnce.Do(cancel)
s := Source{}
- log.SetLevel(log.DebugLevel)
-
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)