Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[chore] Remove logrus from circleci, filesystem, gitlab, and s3 sources (#1089)
Miccahd317ddb51a9d
diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.goindex 8d30c0e1f..44f00cac6 100644--- a/pkg/engine/circleci.go+++ b/pkg/engine/circleci.go@@ -29,6 +29,10 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {}circleSource := circleci.Source{}+ ctx = context.WithValues(ctx,+ "source_type", circleSource.Type().String(),+ "source_name", "Circle CI",+ )err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())if err != nil {return errors.WrapPrefix(err, "failed to init Circle CI source", 0)diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.goindex 59ec8cf9d..61527bf8b 100644--- a/pkg/engine/filesystem.go+++ b/pkg/engine/filesystem.go@@ -27,6 +27,11 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error {}fileSystemSource := filesystem.Source{}++ ctx = context.WithValues(ctx,+ "source_type", fileSystemSource.Type().String(),+ "source_name", "filesystem",+ )err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())if err != nil {return errors.WrapPrefix(err, "could not init filesystem source", 0)diff --git a/pkg/engine/git.go b/pkg/engine/git.goindex 59cb4f1a4..edc3b02d9 100644--- a/pkg/engine/git.go+++ b/pkg/engine/git.go@@ -54,6 +54,10 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {}})+ ctx = context.WithValues(ctx,+ "source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),+ "source_name", "git",+ )e.sourcesWg.Add(1)go func() {defer common.RecoverWithExit(ctx)diff --git a/pkg/engine/github.go b/pkg/engine/github.goindex ba3ec543f..927415df6 100644--- a/pkg/engine/github.go+++ b/pkg/engine/github.go@@ -38,8 +38,8 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {return err}ctx = context.WithValues(ctx,- "source_type", source.Type(),- "name", "trufflehog - github",+ "source_type", source.Type().String(),+ "source_name", "github",)err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)if err != nil {diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.goindex 3ae377085..c0257c8a5 100644--- a/pkg/engine/gitlab.go+++ b/pkg/engine/gitlab.go@@ -53,6 +53,10 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {}gitlabSource := gitlab.Source{}+ ctx = context.WithValues(ctx,+ "source_type", gitlabSource.Type().String(),+ "source_name", "gitlab",+ )err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())if err != nil {return errors.WrapPrefix(err, "could not init GitLab source", 0)diff --git a/pkg/engine/s3.go b/pkg/engine/s3.goindex 88329a0a4..5ba708769 100644--- a/pkg/engine/s3.go+++ b/pkg/engine/s3.go@@ -46,6 +46,10 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {}s3Source := s3.Source{}+ ctx = context.WithValues(ctx,+ "source_type", s3Source.Type().String(),+ "source_name", "s3",+ )err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())if err != nil {return errors.WrapPrefix(err, "failed to init S3 source", 0)diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.goindex 2868b5818..d12ba4705 100644--- a/pkg/engine/syslog.go+++ b/pkg/engine/syslog.go@@ -42,6 +42,10 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error {return errors.WrapPrefix(err, "error unmarshalling connection", 0)}source := syslog.Source{}+ ctx = context.WithValues(ctx,+ "source_type", source.Type().String(),+ "source_name", "syslog",+ )err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)source.InjectConnection(connection)if err != nil {diff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.goindex bdf22d4c1..f6d95e50c 100644--- a/pkg/sources/circleci/circleci.go+++ b/pkg/sources/circleci/circleci.go@@ -9,7 +9,6 @@ import ("sync/atomic""github.com/go-errors/errors"- log "github.com/sirupsen/logrus""golang.org/x/sync/errgroup""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"@@ -111,14 +110,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err}atomic.AddUint64(&scanned, 1)- log.Debugf("scanned %d/%d projects", scanned, len(projects))+ ctx.Logger().V(2).Info(fmt.Sprintf("scanned %d/%d projects", scanned, len(projects)))return nil})}_ = s.jobPool.Wait()if scanErrs.Count() > 0 {- log.Debugf("encountered %d errors while scanning; errors: %v", scanErrs.Count(), scanErrs)+ ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)}return nildiff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.goindex 3dda10887..7562e59d8 100644--- a/pkg/sources/filesystem/filesystem.go+++ b/pkg/sources/filesystem/filesystem.go@@ -9,7 +9,7 @@ import (diskbufferreader "github.com/bill-rich/disk-buffer-reader""github.com/go-errors/errors"- log "github.com/sirupsen/logrus"+ "github.com/go-logr/logr""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"@@ -35,7 +35,7 @@ type Source struct {jobId int64verify boolpaths []string- log *log.Entry+ log logr.Loggerfilter *common.Filtersources.Progress}@@ -59,7 +59,7 @@ func (s *Source) JobID() int64 {// Init returns an initialized Filesystem source.func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {- s.log = log.WithField("source", s.Type()).WithField("name", name)+ s.log = aCtx.Logger()s.name = names.sourceId = sourceId@@ -98,10 +98,11 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err}path := filepath.Join(cleanPath, relativePath)+ logger := ctx.Logger().WithValues("file", path)fileStat, err := os.Stat(path)if err != nil {- log.WithError(err).Warnf("unable to stat file: %s", path)+ logger.Error(err, "unable to stat file")return nil}if !fileStat.Mode().IsRegular() {@@ -114,15 +115,15 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errinputFile, err := os.Open(path)if err != nil {- log.Warn(err)+ logger.V(2).Info("error opening file", "error", err)return nil}defer inputFile.Close()- log.WithField("file_path", path).Trace("scanning file")+ logger.V(5).Info("scanning file")reReader, err := diskbufferreader.New(inputFile)if err != nil {- log.WithError(err).Error("Could not create re-readable reader.")+ logger.Error(err, "Could not create re-readable reader.")}defer reReader.Close()diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.goindex a3e07525c..46fe85491 100644--- a/pkg/sources/filesystem/filesystem_test.go+++ b/pkg/sources/filesystem/filesystem_test.go@@ -5,7 +5,6 @@ import ("time""github.com/kylelemons/godebug/pretty"- log "github.com/sirupsen/logrus""google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/context"@@ -51,8 +50,6 @@ func TestSource_Scan(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {s := Source{}- log.SetLevel(log.DebugLevel)- log.SetFormatter(&log.TextFormatter{ForceColors: true})conn, err := anypb.New(tt.init.connection)if err != nil {diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex 73a1283c0..73ddcf0e4 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -23,7 +23,6 @@ import ("github.com/go-errors/errors"gogit "github.com/go-git/go-git/v5""github.com/gobwas/glob"- log "github.com/sirupsen/logrus""github.com/xanzy/go-gitlab""golang.org/x/exp/slices""google.golang.org/protobuf/proto"@@ -69,7 +68,6 @@ func (s *Source) JobID() int64 {// Init returns an initialized Gitlab source.func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {-s.name = names.sourceId = sourceIds.jobId = jobId@@ -143,7 +141,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err// Get repo within target.repos, errs := s.getRepos()for _, repoErr := range errs {- log.WithError(repoErr).Warn("error getting repo")+ ctx.Logger().Info("error getting repo", "error", repoErr)}// End early if we had errors getting specified repos but none were validated.@@ -153,13 +151,13 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err// Get all repos if not specified.if len(repos) == 0 {- projects, err := s.getAllProjects(apiClient)+ projects, err := s.getAllProjects(ctx, apiClient)if err != nil {return fmt.Errorf("error getting all projects: %v", err)}// Turn projects into URLs for Git cloner.for _, prj := range projects {- if s.ignoreRepo(prj.PathWithNamespace) {+ if s.ignoreRepo(ctx, prj.PathWithNamespace) {continue}@@ -231,7 +229,7 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {return false}-func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, error) {+func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) ([]*gitlab.Project, error) {// Projects without repo will get user projects, groups projects, and subgroup projects.user, _, err := apiClient.Users.CurrentUser()@@ -289,7 +287,10 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, erfor {grpPrjs, res, err := apiClient.Groups.ListGroupProjects(group.ID, listGroupProjectOptions)if err != nil {- log.WithError(err).WithField("group", group.FullPath).Warn("received error on listing group projects, you probably don't have permissions to do that")+ ctx.Logger().Info("received error on listing group projects, you probably don't have permissions to do that",+ "group", group.FullPath,+ "error", err,+ )break}for _, prj := range grpPrjs {@@ -305,7 +306,7 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, erfor _, project := range projects {projectNamesWithNamespace = append(projectNamesWithNamespace, project.NameWithNamespace)}- log.WithField("projects", strings.Join(projectNamesWithNamespace, ", ")).Debugf("Enumerated %d GitLab projects", len(projects))+ ctx.Logger().V(2).Info("Enumerated GitLab projects", "count", len(projects), "projects", projectNamesWithNamespace)var projectList []*gitlab.Projectfor _, project := range projects {@@ -334,7 +335,6 @@ func (s *Source) getRepos() ([]string, []error) {}func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {-// If there is resume information available, limit this scan to only the repos that still need scanning.reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)s.repos = reposToScan@@ -343,15 +343,16 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)for i, repo := range s.repos {i, repoURL := i, repos.jobPool.Go(func() error {+ logger := ctx.Logger().WithValues("repo", repoURL)if common.IsDone(ctx) {// We are returning nil instead of the scanErrors slice here because// we don't want to mark this scan as errored if we cancelled it.- log.Debugf("Skipping repo %s because context was cancelled", repoURL)+ logger.V(2).Info("Skipping repo because context was cancelled")return nil}if len(repoURL) == 0 {- log.Debugf("Skipping empty repo %s", repoURL)+ logger.V(2).Info("Skipping empty repo")return nil}@@ -383,35 +384,35 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)return nil}- log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repo)+ logger.V(2).Info(fmt.Sprintf("Starting to scan repo %d/%d", i+1, len(s.repos)))if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {scanErrs.Add(err)return nil}- log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(s.repos), repo)+ logger.V(2).Info(fmt.Sprintf("Completed scanning repo %d/%d", i+1, len(s.repos)))return nil})}_ = s.jobPool.Wait()if scanErrs.Count() > 0 {- log.Debugf("encountered %d errors while scanning; errors: %v", scanErrs.Count(), scanErrs)+ ctx.Logger().V(2).Info("encountered errors while scanning", "count", scanErrs.Count(), "errors", scanErrs)}s.SetProgressComplete(len(s.repos), len(s.repos), "Completed Gitlab scan", "")return nil}-func (s *Source) ignoreRepo(r string) bool {+func (s *Source) ignoreRepo(ctx context.Context, r string) bool {for _, ignore := range s.ignoreRepos {g, err := glob.Compile(ignore)if err != nil {- log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)+ ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", ignore)continue}if g.Match(r) {- log.Debugf("Ignoring repo %s", r)+ ctx.Logger().V(2).Info("Ignoring repo", "repo", r)return true}}diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.goindex 3b194a65a..68088c522 100644--- a/pkg/sources/gitlab/gitlab_test.go+++ b/pkg/sources/gitlab/gitlab_test.go@@ -2,12 +2,10 @@ package gitlabimport ("fmt"- "io""reflect""testing""github.com/kylelemons/godebug/pretty"- log "github.com/sirupsen/logrus""github.com/stretchr/testify/assert""golang.org/x/sync/errgroup""google.golang.org/protobuf/types/known/anypb"@@ -24,8 +22,6 @@ func TestSource_Scan(t *testing.T) {ctx, cancel := context.WithCancel(context.Background())defer cancel()- log.SetLevel(log.DebugLevel)- log.SetFormatter(&log.TextFormatter{ForceColors: true})secret, err := common.GetTestSecret(ctx)if err != nil {t.Fatal(fmt.Errorf("failed to access secret: %v", err))@@ -142,7 +138,6 @@ func TestSource_Scan(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {s := Source{}- log.SetLevel(log.DebugLevel)conn, err := anypb.New(tt.init.connection)if err != nil {@@ -200,7 +195,6 @@ func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {},}- log.SetOutput(io.Discard)s := &Source{repos: []string{}}for _, tt := range tests {@@ -248,8 +242,6 @@ func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {},}- log.SetOutput(io.Discard)-for _, tt := range tests {s := &Source{repos: tt.repos,diff --git a/pkg/sources/resume.go b/pkg/sources/resume.goindex d085b5d69..4969a5750 100644--- a/pkg/sources/resume.go+++ b/pkg/sources/resume.go@@ -2,8 +2,6 @@ package sourcesimport ("strings"-- "github.com/sirupsen/logrus")// RemoveRepoFromResumeInfo removes the repoURL from the resume info.@@ -18,8 +16,7 @@ func RemoveRepoFromResumeInfo(resumeRepos []string, repoURL string) []string {if index == -1 {// We should never be able to be here. But if we are, it means the resume info never had the repo added.- // So log the error and do nothing.- logrus.Errorf("repoURL (%q) not found in list of encode resume info: %v", repoURL, resumeRepos)+ // So do nothing.return resumeRepos}diff --git a/pkg/sources/resume_test.go b/pkg/sources/resume_test.goindex 56bda955d..6a89c05f7 100644--- a/pkg/sources/resume_test.go+++ b/pkg/sources/resume_test.go@@ -1,11 +1,8 @@package sourcesimport (- "io""reflect""testing"-- "github.com/sirupsen/logrus")func TestRemoveRepoFromResumeInfo(t *testing.T) {@@ -31,9 +28,6 @@ func TestRemoveRepoFromResumeInfo(t *testing.T) {},}- // Prevent the error message from showing up.- logrus.SetOutput(io.Discard)-for _, tt := range tests {gotResumeInfoSlice := RemoveRepoFromResumeInfo(tt.startingResumeInfoSlice, tt.repoURL)if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) {@@ -57,9 +51,6 @@ func TestEncodeResumeInfo(t *testing.T) {},}- logger := logrus.New()- logger.Out = io.Discard-for _, tt := range tests {gotEncodedResumeInfo := EncodeResumeInfo(tt.startingResumeInfoSlice)if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.goindex e1f962543..5d8ce03e2 100644--- a/pkg/sources/s3/s3_test.go+++ b/pkg/sources/s3/s3_test.go@@ -9,7 +9,6 @@ import ("time""github.com/kylelemons/godebug/pretty"- log "github.com/sirupsen/logrus""github.com/stretchr/testify/assert""google.golang.org/protobuf/types/known/anypb"@@ -62,14 +61,11 @@ func TestSource_Chunks(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- log.SetFormatter(&log.TextFormatter{ForceColors: true})ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)var cancelOnce sync.Oncedefer cancelOnce.Do(cancel)s := Source{}- log.SetLevel(log.DebugLevel)-conn, err := anypb.New(tt.init.connection)if err != nil {t.Fatal(err)