Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Include and exclude paths args for gitscan (#6)
Bill Rich1422cc303a88
diff --git a/main.go b/main.goindex 6f4e9e762..489e831f7 100644--- a/main.go+++ b/main.go@@ -4,6 +4,7 @@ import ("context""encoding/json""fmt"+ "github.com/trufflesecurity/trufflehog/pkg/common""log""os""runtime"@@ -26,8 +27,8 @@ func main() {gitScan := cli.Command("git", "Find credentials in git repositories.")gitScanURI := gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()- // gitScanIncludePaths := gitScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()- // gitScanExcludePaths := gitScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()+ gitScanIncludePaths := gitScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()+ gitScanExcludePaths := gitScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()// gitScanSinceCommit := gitScan.Flag("since_commit", "Commit to start scan from.").String()gitScanBranch := gitScan.Flag("branch", "Branch to scan.").String()// gitScanMaxDepth := gitScan.Flag("max_depth", "Maximum depth of commits to scan.").Int()@@ -71,9 +72,14 @@ func main() {engine.WithDetectors(*verification, engine.DefaultDetectors()...),)+ filter, err := common.FilterFromFiles(*gitScanIncludePaths, *gitScanExcludePaths)+ if err != nil {+ logrus.WithError(err)+ }+switch cmd {case gitScan.FullCommand():- err := e.ScanGit(ctx, *gitScanURI, *gitScanBranch, "HEAD")+ err := e.ScanGit(ctx, *gitScanURI, *gitScanBranch, "HEAD", filter)if err != nil {logrus.WithError(err).Fatal("Failed to scan git.")}diff --git a/pkg/common/filter.go b/pkg/common/filter.gonew file mode 100644index 000000000..994a77062--- /dev/null+++ b/pkg/common/filter.go@@ -0,0 +1,100 @@+package common++import (+ "bufio"+ "fmt"+ log "github.com/sirupsen/logrus"+ "os"+ "regexp"+)++type Filter struct {+ include *FilterRuleSet+ exclude *FilterRuleSet+}++type FilterRuleSet []regexp.Regexp++// FilterEmpty returns a Filter that always passes.+func FilterEmpty() *Filter {+ filter, err := FilterFromFiles("", "")+ if err != nil {+ log.WithError(err).Fatalf("could not create empty filter")+ }+ return filter+}++// FilterFromFiles creates a Filter using from the rules in the provided include and exclude files.+func FilterFromFiles(includeFilterPath, excludeFilterPath string) (*Filter, error) {+ includeRules, err := FilterRulesFromFile(includeFilterPath)+ if err != nil {+ return nil, fmt.Errorf("could not create include rules: %s", err)+ }+ excludeRules, err := FilterRulesFromFile(excludeFilterPath)+ if err != nil {+ return nil, fmt.Errorf("could not create exclude rules: %s", err)+ }++ // If no includeFilterPath is provided, every pattern should pass the include rules.+ if includeFilterPath == "" {+ includeRules = &FilterRuleSet{*regexp.MustCompile("")}+ }++ filter := &Filter{+ include: includeRules,+ exclude: excludeRules,+ }++ return filter, nil+}++// FilterRulesFromFile loads the list of regular expression filter rules in `source` and creates a FilterRuleSet.+func FilterRulesFromFile(source string) (*FilterRuleSet, error) {+ rules := FilterRuleSet{}+ if source == "" {+ return &rules, nil+ }++ commentPattern := regexp.MustCompile(`^\s*#`)++ file, err := os.Open(source)+ if err != nil {+ log.WithError(err).Fatalf("unable to open filter file: %s", source)+ }+ defer file.Close()++ scanner := bufio.NewScanner(file)+ for scanner.Scan() {+ line := scanner.Text()+ if commentPattern.MatchString(line) {+ continue+ }+ pattern, err := regexp.Compile(line)+ if err != nil {+ return nil, fmt.Errorf("can not compile regular expression: %s", line)+ }+ rules = append(rules, *pattern)+ }+ return &rules, nil+}++// Pass returns true if the include FilterRuleSet matches the pattern and the exclude FilterRuleSet does not match.+func (filter *Filter) Pass(object string) bool {+ excluded := filter.exclude.Matches(object)+ included := filter.include.Matches(object)+ log.Debugf("test PathFilter: file: %s, included: %t, excluded: %t, pass: %t", object, included, excluded, !excluded && included)+ return !excluded && included+}++// Matches will return true if any of the regular expressions in the FilterRuleSet match the pattern.+func (rules *FilterRuleSet) Matches(object string) bool {+ if rules == nil {+ return false+ }+ for _, rule := range *rules {+ if rule.MatchString(object) {+ return true+ }+ }+ return false+}diff --git a/pkg/common/filter_test.go b/pkg/common/filter_test.gonew file mode 100644index 000000000..eac6fd621--- /dev/null+++ b/pkg/common/filter_test.go@@ -0,0 +1,168 @@+package common++import (+ "os"+ "regexp"+ "testing"+)++func TestFilterBasic(t *testing.T) {+ type filterTest struct {+ filter Filter+ pattern string+ pass bool+ }+ tests := map[string]filterTest{+ "IncludePassed": {+ filter: Filter{+ include: &FilterRuleSet{*regexp.MustCompile("test")},+ },+ pattern: "teststring",+ pass: true,+ },+ "IncludeFiltered": {+ filter: Filter{+ include: &FilterRuleSet{*regexp.MustCompile("nomatch")},+ },+ pattern: "teststring",+ pass: false,+ },+ "ExcludePassed": {+ filter: Filter{+ include: &FilterRuleSet{*regexp.MustCompile("")},+ exclude: &FilterRuleSet{*regexp.MustCompile("nomatch")},+ },+ pattern: "teststring",+ pass: true,+ },+ "ExcludeFiltered": {+ filter: Filter{+ include: &FilterRuleSet{*regexp.MustCompile("")},+ exclude: &FilterRuleSet{*regexp.MustCompile("test")},+ },+ pattern: "teststring",+ pass: false,+ },+ "IncludeExcludeDifferentPass": {+ filter: Filter{+ include: &FilterRuleSet{*regexp.MustCompile("test")},+ exclude: &FilterRuleSet{*regexp.MustCompile("nomatch")},+ },+ pattern: "teststring",+ pass: true,+ },+ "IncludeExcludeDifferentFiltered": {+ filter: Filter{+ include: &FilterRuleSet{*regexp.MustCompile("nomatch")},+ exclude: &FilterRuleSet{*regexp.MustCompile("test")},+ },+ pattern: "teststring",+ pass: false,+ },+ "IncludeExcludeSameFiltered": {+ filter: Filter{+ include: &FilterRuleSet{*regexp.MustCompile("test")},+ exclude: &FilterRuleSet{*regexp.MustCompile("test")},+ },+ pattern: "teststring",+ pass: false,+ },+ }++ for name, test := range tests {+ if test.filter.Pass(test.pattern) != test.pass {+ t.Errorf("%s: unexpected filter result. pattern: %q, pass: %t", name, test.pattern, !test.pass)+ }+ }+}++func TestFilterFromFile(t *testing.T) {+ type filterTest struct {+ includeFile bool+ excludeFile bool+ includeFileContents string+ excludeFileContents string+ pattern string+ pass bool+ }+ tests := map[string]filterTest{+ "includeFileOnlyPass": {+ includeFile: true,+ excludeFile: false,+ includeFileContents: "test",+ pattern: "test",+ pass: true,+ },+ "includeFileOnlyFiltered": {+ includeFile: true,+ excludeFile: false,+ includeFileContents: "nomatch",+ pattern: "test",+ pass: false,+ },+ "includeFileEmptyFiltered": {+ includeFile: true,+ excludeFile: false,+ includeFileContents: "",+ pattern: "test",+ pass: false,+ },+ "excludeFileOnlyPass": {+ includeFile: false,+ excludeFile: true,+ excludeFileContents: "nomatch",+ pattern: "test",+ pass: true,+ },+ "excludeFileOnlyFiltered": {+ includeFile: false,+ excludeFile: true,+ excludeFileContents: "test",+ pattern: "test",+ pass: false,+ },+ "BothFilesEmptyExcludeFiltered": {+ includeFile: true,+ excludeFile: true,+ excludeFileContents: "",+ includeFileContents: "",+ pattern: "test",+ pass: false,+ },+ }+ for name, test := range tests {+ var includeTestFile, excludeTestFile string+ if test.includeFile {+ includeTestFile = "/tmp/trufflehog_test_ifilter.txt"+ if err := testFilterWriteFile(includeTestFile, []byte(test.includeFileContents)); err != nil {+ t.Fatalf("failed to create include rules file: %s", err)+ }+ defer os.Remove(includeTestFile)+ }+ if test.excludeFile {+ excludeTestFile = "/tmp/trufflehog_test_xfilter.txt"+ if err := testFilterWriteFile(excludeTestFile, []byte(test.excludeFileContents)); err != nil {+ t.Fatalf("failed to create include rules file: %s", err)+ }+ defer os.Remove(excludeTestFile)+ }++ filter, err := FilterFromFiles(includeTestFile, excludeTestFile)+ if err != nil {+ t.Errorf("failed to create filter from files: %s", err)+ }++ if filter.Pass(test.pattern) != test.pass {+ t.Errorf("%s: unexpected filter result. pattern: %q, pass: %t", name, test.pattern, !test.pass)+ }+ }+}++func testFilterWriteFile(filename string, content []byte) error {+ f, err := os.Create(filename)+ if err != nil {+ return err+ }+ f.Write(content)+ return f.Close()+}diff --git a/pkg/engine/git.go b/pkg/engine/git.goindex a49d59123..7f290d1c3 100644--- a/pkg/engine/git.go+++ b/pkg/engine/git.go@@ -4,6 +4,7 @@ import ("context""errors""fmt"+ "github.com/trufflesecurity/trufflehog/pkg/common""runtime""strings"@@ -16,7 +17,7 @@ import ("github.com/trufflesecurity/trufflehog/pkg/sources/git")-func (e *Engine) ScanGit(ctx context.Context, gitScanURI, gitScanBranch, headRef string) error {+func (e *Engine) ScanGit(ctx context.Context, gitScanURI, gitScanBranch, headRef string, filter *common.Filter) error {var path stringswitch {case strings.HasPrefix(gitScanURI, "file://"):@@ -34,8 +35,9 @@ func (e *Engine) ScanGit(ctx context.Context, gitScanURI, gitScanBranch, headRef}scanOptions := &gogit.LogOptions{- All: true,- Order: gogit.LogOrderCommitterTime,+ All: true,+ Order: gogit.LogOrderCommitterTime,+ PathFilter: func(s string) bool { return filter.Pass(s) },}if gitScanBranch != "" {@@ -81,8 +83,9 @@ func (e *Engine) ScanGit(ctx context.Context, gitScanURI, gitScanBranch, headRef}).Debug("resolved common merge base between references")scanOptions = &gogit.LogOptions{- From: *headHash,- Order: gogit.LogOrderCommitterTime,+ From: *headHash,+ Order: gogit.LogOrderCommitterTime,+ PathFilter: func(s string) bool { return filter.Pass(s) },}}@@ -101,7 +104,7 @@ func (e *Engine) ScanGit(ctx context.Context, gitScanURI, gitScanBranch, headRef})go func() {- err := gitSource.ScanRepo(ctx, repo, scanOptions, &object.Commit{}, e.ChunksChan())+ err := gitSource.ScanRepo(ctx, repo, scanOptions, &object.Commit{}, filter, e.ChunksChan())if err != nil {logrus.WithError(err).Fatal("could not scan repo")}diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex d0ae6209e..9d529a6de 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -4,6 +4,7 @@ import ("bytes""context""fmt"+ "github.com/trufflesecurity/trufflehog/pkg/common""io""io/ioutil""net/url"@@ -136,7 +137,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errif err != nil {return err}- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, chunksChan)+ err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)if err != nil {return err}@@ -152,7 +153,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errif err != nil {return err}- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, chunksChan)+ err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)if err != nil {return err}@@ -177,7 +178,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errdefer os.RemoveAll(u)}- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, chunksChan)+ err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)if err != nil {return err@@ -247,7 +248,7 @@ func CloneRepoUsingUnauthenticated(url string) (clonePath string, repo *git.Reporeturn}-func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogOptions, untilCommit *object.Commit, chunksChan chan *sources.Chunk) error {+func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogOptions, untilCommit *object.Commit, filter *common.Filter, chunksChan chan *sources.Chunk) error {wg := &sync.WaitGroup{}remote, err := repo.Remote("origin")@@ -291,7 +292,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogO// go func(wg *sync.WaitGroup) {// defer wg.Done()// defer s.sem.Release(1)- err = s.scanCommitPatches(ctx, repo, commit, chunksChan)+ err = s.scanCommitPatches(ctx, repo, commit, chunksChan, filter)if err != nil {switch e := err.Error(); {case strings.Contains(e, "operation canceled"):@@ -335,6 +336,9 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogOreturn err}for fh := range status {+ if !filter.Pass(fh) {+ continue+ }metadata := s.sourceMetadataFunc(fh, "unstaged", "unstaged", safeRepo,)@@ -378,7 +382,7 @@ func GenerateLink(repo, commit, file string) string {return link}-func (s *Git) scanCommitPatches(ctx context.Context, repo *git.Repository, commit *object.Commit, chunksChan chan *sources.Chunk) error {+func (s *Git) scanCommitPatches(ctx context.Context, repo *git.Repository, commit *object.Commit, chunksChan chan *sources.Chunk, filter *common.Filter) error {defer func() {if err := recover(); err != nil {@@ -427,6 +431,9 @@ func (s *Git) scanCommitPatches(ctx context.Context, repo *git.Repository, commifor _, file := range patch.FilePatches() {bf, f := file.Files()filename := f.Path()+ if !filter.Pass(filename) {+ continue+ }if filename == "" {filename = bf.Path()}