Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add filter as scan option to gitlab module's git scan (#919)
Jessica3d501975e407
diff --git a/main.go b/main.goindex c80c6bfdd..c5f325c11 100644--- a/main.go+++ b/main.go@@ -66,9 +66,11 @@ var (gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")// TODO: Add more GitLab options- gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()- gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()- gitlabScanToken = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()+ gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()+ gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()+ gitlabScanToken = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()+ gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()+ gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()@@ -223,10 +225,16 @@ func run(state overseer.State) {logrus.WithError(err).Fatal("Failed to scan Github.")}case gitlabScan.FullCommand():+ filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths)+ if err != nil {+ logrus.WithError(err).Fatal("could not create filter")+ }+gitlab := func(c *sources.Config) {c.Endpoint = *gitlabScanEndpointc.Token = *gitlabScanTokenc.Repos = *gitlabScanRepos+ c.Filter = filter}if err = e.ScanGitLab(ctx, sources.NewConfig(gitlab)); err != nil {diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.goindex b5586d2dd..f6274d583 100644--- a/pkg/engine/gitlab.go+++ b/pkg/engine/gitlab.go@@ -5,6 +5,7 @@ import ("runtime""github.com/go-errors/errors"+ gogit "github.com/go-git/go-git/v5""github.com/sirupsen/logrus""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"@@ -13,11 +14,19 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git""github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab")// ScanGitLab scans GitLab with the provided configuration.func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {+ logOptions := &gogit.LogOptions{}+ opts := []git.ScanOption{+ git.ScanOptionFilter(c.Filter),+ git.ScanOptionLogOptions(logOptions),+ }+ scanOptions := git.NewScanOptions(opts...)+connection := &sourcespb.GitLab{}switch {@@ -49,6 +58,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {if err != nil {return errors.WrapPrefix(err, "could not init GitLab source", 0)}+ gitlabSource.WithScanOptions(scanOptions)e.sourcesWg.Add(1)go func() {diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex 1004d9ba7..33fa410cd 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -41,6 +41,7 @@ type Source struct {repos []stringignoreRepos []stringgit *git.Git+ scanOptions *git.ScanOptionsaCtx context.ContextresumeInfoSlice []stringresumeInfoMutex sync.Mutex@@ -325,7 +326,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)return}log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repoURL)- err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)+ err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)if err != nil {errsMut.Lock()errs = append(errs, err)@@ -432,3 +433,7 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri// Add the offset to both the index and the repos to give the proper place and proper repo count.s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)}++func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {+ s.scanOptions = scanOptions+}