Skip to content

Find, verify, and analyze leaked credentials

Add filter as scan option to gitlab module's git scan (#919)

Jessica3d501975e407
diff --git a/main.go b/main.go
index c80c6bfdd..c5f325c11 100644
--- a/main.go
+++ b/main.go
@@ -66,9 +66,11 @@ var (
gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
// TODO: Add more GitLab options
- gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
- gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
- gitlabScanToken = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
+ gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
+ gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
+ gitlabScanToken = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
+ gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+ gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")
filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()
@@ -223,10 +225,16 @@ func run(state overseer.State) {
logrus.WithError(err).Fatal("Failed to scan Github.")
}
case gitlabScan.FullCommand():
+ filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths)
+ if err != nil {
+ logrus.WithError(err).Fatal("could not create filter")
+ }
+
gitlab := func(c *sources.Config) {
c.Endpoint = *gitlabScanEndpoint
c.Token = *gitlabScanToken
c.Repos = *gitlabScanRepos
+ c.Filter = filter
}
if err = e.ScanGitLab(ctx, sources.NewConfig(gitlab)); err != nil {
diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go
index b5586d2dd..f6274d583 100644
--- a/pkg/engine/gitlab.go
+++ b/pkg/engine/gitlab.go
@@ -5,6 +5,7 @@ import (
"runtime"
"github.com/go-errors/errors"
+ gogit "github.com/go-git/go-git/v5"
"github.com/sirupsen/logrus"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -13,11 +14,19 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
)
// ScanGitLab scans GitLab with the provided configuration.
func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
+ logOptions := &gogit.LogOptions{}
+ opts := []git.ScanOption{
+ git.ScanOptionFilter(c.Filter),
+ git.ScanOptionLogOptions(logOptions),
+ }
+ scanOptions := git.NewScanOptions(opts...)
+
connection := &sourcespb.GitLab{}
switch {
@@ -49,6 +58,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
if err != nil {
return errors.WrapPrefix(err, "could not init GitLab source", 0)
}
+ gitlabSource.WithScanOptions(scanOptions)
e.sourcesWg.Add(1)
go func() {
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index 1004d9ba7..33fa410cd 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -41,6 +41,7 @@ type Source struct {
repos []string
ignoreRepos []string
git *git.Git
+ scanOptions *git.ScanOptions
aCtx context.Context
resumeInfoSlice []string
resumeInfoMutex sync.Mutex
@@ -325,7 +326,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
return
}
log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repoURL)
- err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
+ err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
if err != nil {
errsMut.Lock()
errs = append(errs, err)
@@ -432,3 +433,7 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
// Add the offset to both the index and the repos to give the proper place and proper repo count.
s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
}
+
+func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
+ s.scanOptions = scanOptions
+}