Skip to content

Find, verify, and analyze leaked credentials

Support globbing with ignore repos (#967)

Dustin Decker7de9bdd12daa
diff --git a/go.mod b/go.mod
index 849b45e91..f5874867a 100644
--- a/go.mod
+++ b/go.mod
@@ -30,6 +30,7 @@ require (
github.com/go-logr/zapr v1.2.3
github.com/go-redis/redis v6.15.9+incompatible
github.com/go-sql-driver/mysql v1.7.0
+ github.com/gobwas/glob v0.2.3
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-cmp v0.5.9
github.com/google/go-github/v42 v42.0.0
diff --git a/go.sum b/go.sum
index cac12f281..c083bb269 100644
--- a/go.sum
+++ b/go.sum
@@ -176,6 +176,8 @@ github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8w
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 18cbea9ab..b3123996d 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -17,9 +17,9 @@ import (
"github.com/bradleyfalzon/ghinstallation/v2"
"github.com/go-errors/errors"
gogit "github.com/go-git/go-git/v5"
+ "github.com/gobwas/glob"
"github.com/google/go-github/v42/github"
log "github.com/sirupsen/logrus"
- "golang.org/x/exp/slices"
"golang.org/x/oauth2"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
@@ -745,9 +745,16 @@ func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, err
}
func (s *Source) ignoreRepo(r string) bool {
- if slices.Contains(s.ignoreRepos, r) {
- s.log.Debugf("ignoring repo %s", r)
- return true
+ for _, ignore := range s.ignoreRepos {
+ g, err := glob.Compile(ignore)
+ if err != nil {
+ s.log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)
+ continue
+ }
+ if g.Match(r) {
+ s.log.Debugf("ignoring repo %s", r)
+ return true
+ }
}
return false
}
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index a337cb98b..5a9124ad0 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -71,7 +71,7 @@ func TestAddReposByOrg(t *testing.T) {
})
s := initTestSource(nil)
- s.ignoreRepos = []string{"secret/super-secret-repo2"}
+ s.ignoreRepos = []string{"secret/super-*-repo2"}
// gock works here because github.NewClient is using the default HTTP Transport
err := s.addRepos(context.TODO(), "super-secret-org", s.getReposByOrg)
assert.Nil(t, err)
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index 33fa410cd..74a83602a 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -20,6 +20,7 @@ import (
"github.com/go-errors/errors"
gogit "github.com/go-git/go-git/v5"
+ "github.com/gobwas/glob"
log "github.com/sirupsen/logrus"
"github.com/xanzy/go-gitlab"
"golang.org/x/exp/slices"
@@ -344,6 +345,21 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
return errs
}
+func (s *Source) ignoreRepo(r string) bool {
+ for _, ignore := range s.ignoreRepos {
+ g, err := glob.Compile(ignore)
+ if err != nil {
+ log.WithError(err).Errorf("could not compile ignore repo glob %s", ignore)
+ continue
+ }
+ if g.Match(r) {
+ log.Debugf("Ignoring repo %s", r)
+ return true
+ }
+ }
+ return false
+}
+
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
// Start client.
@@ -370,10 +386,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
// Turn projects into URLs for Git cloner.
for _, prj := range projects {
- if slices.Contains(s.ignoreRepos, prj.PathWithNamespace) {
- log.Debugf("Ignoring repo %s", prj.PathWithNamespace)
+ if s.ignoreRepo(prj.PathWithNamespace) {
continue
}
+
// Ensure the urls are valid before adding them to the repo list.
_, err := url.Parse(prj.HTTPURLToRepo)
if err != nil {
diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go
index 265ebbab5..500fc4ac5 100644
--- a/pkg/sources/gitlab/gitlab_test.go
+++ b/pkg/sources/gitlab/gitlab_test.go
@@ -46,7 +46,7 @@ func TestSource_Scan(t *testing.T) {
wantErr bool
}{
{
- name: "token auth, enumerate repo",
+ name: "token auth, enumerate repo, with explicit ignore",
init: init{
name: "test source",
connection: &sourcespb.GitLab{
@@ -62,6 +62,23 @@ func TestSource_Scan(t *testing.T) {
},
wantReposScanned: 2,
},
+ {
+ name: "token auth, enumerate repo, with glob ignore",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitLab{
+ Credential: &sourcespb.GitLab_Token{
+ Token: token,
+ },
+ IgnoreRepos: []string{"tes1188/*-gitlab"},
+ },
+ },
+ wantChunk: &sources.Chunk{
+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
+ SourceName: "test source",
+ },
+ wantReposScanned: 2,
+ },
{
name: "token auth, scoped repo",
init: init{