Skip to content

Find, verify, and analyze leaked credentials

Include and exclude paths args for gitscan (#6)

Bill Rich1422cc303a88
diff --git a/main.go b/main.go
index 6f4e9e762..489e831f7 100644
--- a/main.go
+++ b/main.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
"log"
"os"
"runtime"
@@ -26,8 +27,8 @@ func main() {
gitScan := cli.Command("git", "Find credentials in git repositories.")
gitScanURI := gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()
- // gitScanIncludePaths := gitScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
- // gitScanExcludePaths := gitScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+ gitScanIncludePaths := gitScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+ gitScanExcludePaths := gitScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
// gitScanSinceCommit := gitScan.Flag("since_commit", "Commit to start scan from.").String()
gitScanBranch := gitScan.Flag("branch", "Branch to scan.").String()
// gitScanMaxDepth := gitScan.Flag("max_depth", "Maximum depth of commits to scan.").Int()
@@ -71,9 +72,14 @@ func main() {
engine.WithDetectors(*verification, engine.DefaultDetectors()...),
)
+ filter, err := common.FilterFromFiles(*gitScanIncludePaths, *gitScanExcludePaths)
+ if err != nil {
+ logrus.WithError(err)
+ }
+
switch cmd {
case gitScan.FullCommand():
- err := e.ScanGit(ctx, *gitScanURI, *gitScanBranch, "HEAD")
+ err := e.ScanGit(ctx, *gitScanURI, *gitScanBranch, "HEAD", filter)
if err != nil {
logrus.WithError(err).Fatal("Failed to scan git.")
}
diff --git a/pkg/common/filter.go b/pkg/common/filter.go
new file mode 100644
index 000000000..994a77062
--- /dev/null
+++ b/pkg/common/filter.go
@@ -0,0 +1,100 @@
+package common
+
+import (
+ "bufio"
+ "fmt"
+ log "github.com/sirupsen/logrus"
+ "os"
+ "regexp"
+)
+
+type Filter struct {
+ include *FilterRuleSet
+ exclude *FilterRuleSet
+}
+
+type FilterRuleSet []regexp.Regexp
+
+// FilterEmpty returns a Filter that always passes.
+func FilterEmpty() *Filter {
+ filter, err := FilterFromFiles("", "")
+ if err != nil {
+ log.WithError(err).Fatalf("could not create empty filter")
+ }
+ return filter
+}
+
+// FilterFromFiles creates a Filter using from the rules in the provided include and exclude files.
+func FilterFromFiles(includeFilterPath, excludeFilterPath string) (*Filter, error) {
+ includeRules, err := FilterRulesFromFile(includeFilterPath)
+ if err != nil {
+ return nil, fmt.Errorf("could not create include rules: %s", err)
+ }
+ excludeRules, err := FilterRulesFromFile(excludeFilterPath)
+ if err != nil {
+ return nil, fmt.Errorf("could not create exclude rules: %s", err)
+ }
+
+ // If no includeFilterPath is provided, every pattern should pass the include rules.
+ if includeFilterPath == "" {
+ includeRules = &FilterRuleSet{*regexp.MustCompile("")}
+ }
+
+ filter := &Filter{
+ include: includeRules,
+ exclude: excludeRules,
+ }
+
+ return filter, nil
+}
+
+// FilterRulesFromFile loads the list of regular expression filter rules in `source` and creates a FilterRuleSet.
+func FilterRulesFromFile(source string) (*FilterRuleSet, error) {
+ rules := FilterRuleSet{}
+ if source == "" {
+ return &rules, nil
+ }
+
+ commentPattern := regexp.MustCompile(`^\s*#`)
+
+ file, err := os.Open(source)
+ if err != nil {
+ log.WithError(err).Fatalf("unable to open filter file: %s", source)
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if commentPattern.MatchString(line) {
+ continue
+ }
+ pattern, err := regexp.Compile(line)
+ if err != nil {
+ return nil, fmt.Errorf("can not compile regular expression: %s", line)
+ }
+ rules = append(rules, *pattern)
+ }
+ return &rules, nil
+}
+
+// Pass returns true if the include FilterRuleSet matches the pattern and the exclude FilterRuleSet does not match.
+func (filter *Filter) Pass(object string) bool {
+ excluded := filter.exclude.Matches(object)
+ included := filter.include.Matches(object)
+ log.Debugf("test PathFilter: file: %s, included: %t, excluded: %t, pass: %t", object, included, excluded, !excluded && included)
+ return !excluded && included
+}
+
+// Matches will return true if any of the regular expressions in the FilterRuleSet match the pattern.
+func (rules *FilterRuleSet) Matches(object string) bool {
+ if rules == nil {
+ return false
+ }
+ for _, rule := range *rules {
+ if rule.MatchString(object) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/pkg/common/filter_test.go b/pkg/common/filter_test.go
new file mode 100644
index 000000000..eac6fd621
--- /dev/null
+++ b/pkg/common/filter_test.go
@@ -0,0 +1,168 @@
+package common
+
+import (
+ "os"
+ "regexp"
+ "testing"
+)
+
+func TestFilterBasic(t *testing.T) {
+ type filterTest struct {
+ filter Filter
+ pattern string
+ pass bool
+ }
+ tests := map[string]filterTest{
+ "IncludePassed": {
+ filter: Filter{
+ include: &FilterRuleSet{*regexp.MustCompile("test")},
+ },
+ pattern: "teststring",
+ pass: true,
+ },
+ "IncludeFiltered": {
+ filter: Filter{
+ include: &FilterRuleSet{*regexp.MustCompile("nomatch")},
+ },
+ pattern: "teststring",
+ pass: false,
+ },
+ "ExcludePassed": {
+ filter: Filter{
+ include: &FilterRuleSet{*regexp.MustCompile("")},
+ exclude: &FilterRuleSet{*regexp.MustCompile("nomatch")},
+ },
+ pattern: "teststring",
+ pass: true,
+ },
+ "ExcludeFiltered": {
+ filter: Filter{
+ include: &FilterRuleSet{*regexp.MustCompile("")},
+ exclude: &FilterRuleSet{*regexp.MustCompile("test")},
+ },
+ pattern: "teststring",
+ pass: false,
+ },
+ "IncludeExcludeDifferentPass": {
+ filter: Filter{
+ include: &FilterRuleSet{*regexp.MustCompile("test")},
+ exclude: &FilterRuleSet{*regexp.MustCompile("nomatch")},
+ },
+ pattern: "teststring",
+ pass: true,
+ },
+ "IncludeExcludeDifferentFiltered": {
+ filter: Filter{
+ include: &FilterRuleSet{*regexp.MustCompile("nomatch")},
+ exclude: &FilterRuleSet{*regexp.MustCompile("test")},
+ },
+ pattern: "teststring",
+ pass: false,
+ },
+ "IncludeExcludeSameFiltered": {
+ filter: Filter{
+ include: &FilterRuleSet{*regexp.MustCompile("test")},
+ exclude: &FilterRuleSet{*regexp.MustCompile("test")},
+ },
+ pattern: "teststring",
+ pass: false,
+ },
+ }
+
+ for name, test := range tests {
+ if test.filter.Pass(test.pattern) != test.pass {
+ t.Errorf("%s: unexpected filter result. pattern: %q, pass: %t", name, test.pattern, !test.pass)
+ }
+ }
+}
+
+func TestFilterFromFile(t *testing.T) {
+ type filterTest struct {
+ includeFile bool
+ excludeFile bool
+ includeFileContents string
+ excludeFileContents string
+ pattern string
+ pass bool
+ }
+ tests := map[string]filterTest{
+ "includeFileOnlyPass": {
+ includeFile: true,
+ excludeFile: false,
+ includeFileContents: "test",
+ pattern: "test",
+ pass: true,
+ },
+ "includeFileOnlyFiltered": {
+ includeFile: true,
+ excludeFile: false,
+ includeFileContents: "nomatch",
+ pattern: "test",
+ pass: false,
+ },
+ "includeFileEmptyFiltered": {
+ includeFile: true,
+ excludeFile: false,
+ includeFileContents: "",
+ pattern: "test",
+ pass: false,
+ },
+ "excludeFileOnlyPass": {
+ includeFile: false,
+ excludeFile: true,
+ excludeFileContents: "nomatch",
+ pattern: "test",
+ pass: true,
+ },
+ "excludeFileOnlyFiltered": {
+ includeFile: false,
+ excludeFile: true,
+ excludeFileContents: "test",
+ pattern: "test",
+ pass: false,
+ },
+ "BothFilesEmptyExcludeFiltered": {
+ includeFile: true,
+ excludeFile: true,
+ excludeFileContents: "",
+ includeFileContents: "",
+ pattern: "test",
+ pass: false,
+ },
+ }
+ for name, test := range tests {
+ var includeTestFile, excludeTestFile string
+ if test.includeFile {
+ includeTestFile = "/tmp/trufflehog_test_ifilter.txt"
+ if err := testFilterWriteFile(includeTestFile, []byte(test.includeFileContents)); err != nil {
+ t.Fatalf("failed to create include rules file: %s", err)
+ }
+ defer os.Remove(includeTestFile)
+ }
+ if test.excludeFile {
+ excludeTestFile = "/tmp/trufflehog_test_xfilter.txt"
+ if err := testFilterWriteFile(excludeTestFile, []byte(test.excludeFileContents)); err != nil {
+ t.Fatalf("failed to create include rules file: %s", err)
+ }
+ defer os.Remove(excludeTestFile)
+ }
+
+ filter, err := FilterFromFiles(includeTestFile, excludeTestFile)
+ if err != nil {
+ t.Errorf("failed to create filter from files: %s", err)
+ }
+
+ if filter.Pass(test.pattern) != test.pass {
+ t.Errorf("%s: unexpected filter result. pattern: %q, pass: %t", name, test.pattern, !test.pass)
+ }
+ }
+}
+
+func testFilterWriteFile(filename string, content []byte) error {
+ f, err := os.Create(filename)
+ if err != nil {
+ return err
+ }
+ f.Write(content)
+ return f.Close()
+}
diff --git a/pkg/engine/git.go b/pkg/engine/git.go
index a49d59123..7f290d1c3 100644
--- a/pkg/engine/git.go
+++ b/pkg/engine/git.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
"runtime"
"strings"
@@ -16,7 +17,7 @@ import (
"github.com/trufflesecurity/trufflehog/pkg/sources/git"
)
-func (e *Engine) ScanGit(ctx context.Context, gitScanURI, gitScanBranch, headRef string) error {
+func (e *Engine) ScanGit(ctx context.Context, gitScanURI, gitScanBranch, headRef string, filter *common.Filter) error {
var path string
switch {
case strings.HasPrefix(gitScanURI, "file://"):
@@ -34,8 +35,9 @@ func (e *Engine) ScanGit(ctx context.Context, gitScanURI, gitScanBranch, headRef
}
scanOptions := &gogit.LogOptions{
- All: true,
- Order: gogit.LogOrderCommitterTime,
+ All: true,
+ Order: gogit.LogOrderCommitterTime,
+ PathFilter: func(s string) bool { return filter.Pass(s) },
}
if gitScanBranch != "" {
@@ -81,8 +83,9 @@ func (e *Engine) ScanGit(ctx context.Context, gitScanURI, gitScanBranch, headRef
}).Debug("resolved common merge base between references")
scanOptions = &gogit.LogOptions{
- From: *headHash,
- Order: gogit.LogOrderCommitterTime,
+ From: *headHash,
+ Order: gogit.LogOrderCommitterTime,
+ PathFilter: func(s string) bool { return filter.Pass(s) },
}
}
@@ -101,7 +104,7 @@ func (e *Engine) ScanGit(ctx context.Context, gitScanURI, gitScanBranch, headRef
})
go func() {
- err := gitSource.ScanRepo(ctx, repo, scanOptions, &object.Commit{}, e.ChunksChan())
+ err := gitSource.ScanRepo(ctx, repo, scanOptions, &object.Commit{}, filter, e.ChunksChan())
if err != nil {
logrus.WithError(err).Fatal("could not scan repo")
}
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index d0ae6209e..9d529a6de 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"fmt"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
"io"
"io/ioutil"
"net/url"
@@ -136,7 +137,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
if err != nil {
return err
}
- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, chunksChan)
+ err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
if err != nil {
return err
}
@@ -152,7 +153,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
if err != nil {
return err
}
- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, chunksChan)
+ err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
if err != nil {
return err
}
@@ -177,7 +178,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
defer os.RemoveAll(u)
}
- err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, chunksChan)
+ err = s.git.ScanRepo(ctx, repo, &git.LogOptions{All: true}, nil, common.FilterEmpty(), chunksChan)
if err != nil {
return err
@@ -247,7 +248,7 @@ func CloneRepoUsingUnauthenticated(url string) (clonePath string, repo *git.Repo
return
}
-func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogOptions, untilCommit *object.Commit, chunksChan chan *sources.Chunk) error {
+func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogOptions, untilCommit *object.Commit, filter *common.Filter, chunksChan chan *sources.Chunk) error {
wg := &sync.WaitGroup{}
remote, err := repo.Remote("origin")
@@ -291,7 +292,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogO
// go func(wg *sync.WaitGroup) {
// defer wg.Done()
// defer s.sem.Release(1)
- err = s.scanCommitPatches(ctx, repo, commit, chunksChan)
+ err = s.scanCommitPatches(ctx, repo, commit, chunksChan, filter)
if err != nil {
switch e := err.Error(); {
case strings.Contains(e, "operation canceled"):
@@ -335,6 +336,9 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, opts *git.LogO
return err
}
for fh := range status {
+ if !filter.Pass(fh) {
+ continue
+ }
metadata := s.sourceMetadataFunc(
fh, "unstaged", "unstaged", safeRepo,
)
@@ -378,7 +382,7 @@ func GenerateLink(repo, commit, file string) string {
return link
}
-func (s *Git) scanCommitPatches(ctx context.Context, repo *git.Repository, commit *object.Commit, chunksChan chan *sources.Chunk) error {
+func (s *Git) scanCommitPatches(ctx context.Context, repo *git.Repository, commit *object.Commit, chunksChan chan *sources.Chunk, filter *common.Filter) error {
defer func() {
if err := recover(); err != nil {
@@ -427,6 +431,9 @@ func (s *Git) scanCommitPatches(ctx context.Context, repo *git.Repository, commi
for _, file := range patch.FilePatches() {
bf, f := file.Files()
filename := f.Path()
+ if !filter.Pass(filename) {
+ continue
+ }
if filename == "" {
filename = bf.Path()
}