Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Support globbing with ignore repos (#967)
Dustin Decker7de9bdd12daa
diff --git a/go.mod b/go.modindex 849b45e91..f5874867a 100644--- a/go.mod+++ b/go.mod@@ -30,6 +30,7 @@ require (github.com/go-logr/zapr v1.2.3github.com/go-redis/redis v6.15.9+incompatiblegithub.com/go-sql-driver/mysql v1.7.0+ github.com/gobwas/glob v0.2.3github.com/golang-jwt/jwt v3.2.2+incompatiblegithub.com/google/go-cmp v0.5.9github.com/google/go-github/v42 v42.0.0diff --git a/go.sum b/go.sumindex cac12f281..c083bb269 100644--- a/go.sum+++ b/go.sum@@ -176,6 +176,8 @@ github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wgithub.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 18cbea9ab..b3123996d 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -17,9 +17,9 @@ import ("github.com/bradleyfalzon/ghinstallation/v2""github.com/go-errors/errors"gogit "github.com/go-git/go-git/v5"+ "github.com/gobwas/glob""github.com/google/go-github/v42/github"log "github.com/sirupsen/logrus"- "golang.org/x/exp/slices""golang.org/x/oauth2""golang.org/x/sync/errgroup""google.golang.org/protobuf/proto"@@ -745,9 +745,16 @@ func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, err}func (s *Source) ignoreRepo(r string) bool {- if slices.Contains(s.ignoreRepos, r) {- s.log.Debugf("ignoring repo %s", r)- return true+ for _, ignore := range s.ignoreRepos {+ g, err := glob.Compile(ignore)+ if err != nil {+ s.log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)+ continue+ }+ if g.Match(r) {+ s.log.Debugf("ignoring repo %s", r)+ return true+ }}return false}diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex a337cb98b..5a9124ad0 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -71,7 +71,7 @@ func TestAddReposByOrg(t *testing.T) {})s := initTestSource(nil)- s.ignoreRepos = []string{"secret/super-secret-repo2"}+ s.ignoreRepos = []string{"secret/super-*-repo2"}// gock works here because github.NewClient is using the default HTTP Transporterr := s.addRepos(context.TODO(), "super-secret-org", s.getReposByOrg)assert.Nil(t, err)diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex 33fa410cd..74a83602a 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -20,6 +20,7 @@ import ("github.com/go-errors/errors"gogit "github.com/go-git/go-git/v5"+ "github.com/gobwas/glob"log "github.com/sirupsen/logrus""github.com/xanzy/go-gitlab""golang.org/x/exp/slices"@@ -344,6 +345,21 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)return errs}+func (s *Source) ignoreRepo(r string) bool {+ for _, ignore := range s.ignoreRepos {+ g, err := glob.Compile(ignore)+ if err != nil {+ log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)+ continue+ }+ if g.Match(r) {+ log.Debugf("Ignoring repo %s", r)+ return true+ }+ }+ return false+}+// Chunks emits chunks of bytes over a channel.func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {// Start client.@@ -370,10 +386,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err}// Turn projects into URLs for Git cloner.for _, prj := range projects {- if slices.Contains(s.ignoreRepos, prj.PathWithNamespace) {- log.Debugf("Ignoring repo %s", prj.PathWithNamespace)+ if s.ignoreRepo(prj.PathWithNamespace) {continue}+// Ensure the urls are valid before adding them to the repo list._, err := url.Parse(prj.HTTPURLToRepo)if err != nil {diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.goindex 265ebbab5..500fc4ac5 100644--- a/pkg/sources/gitlab/gitlab_test.go+++ b/pkg/sources/gitlab/gitlab_test.go@@ -46,7 +46,7 @@ func TestSource_Scan(t *testing.T) {wantErr bool}{{- name: "token auth, enumerate repo",+ name: "token auth, enumerate repo, with explicit ignore",init: init{name: "test source",connection: &sourcespb.GitLab{@@ -62,6 +62,23 @@ func TestSource_Scan(t *testing.T) {},wantReposScanned: 2,},+ {+ name: "token auth, enumerate repo, with glob ignore",+ init: init{+ name: "test source",+ connection: &sourcespb.GitLab{+ Credential: &sourcespb.GitLab_Token{+ Token: token,+ },+ IgnoreRepos: []string{"tes1188/*-gitlab"},+ },+ },+ wantChunk: &sources.Chunk{+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,+ SourceName: "test source",+ },+ wantReposScanned: 2,+ },{name: "token auth, scoped repo",init: init{