Skip to content

Find, verify, and analyze leaked credentials

[THOG-767] ignore gitlab repos (#853)

ahrav029519eb01a4
diff --git a/go.mod b/go.mod
index 26eef3659..e04547a18 100644
--- a/go.mod
+++ b/go.mod
@@ -53,6 +53,7 @@ require (
go.mongodb.org/mongo-driver v1.10.2
go.uber.org/zap v1.23.0
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
+ golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006
diff --git a/go.sum b/go.sum
index fb45af79b..700e5d69b 100644
--- a/go.sum
+++ b/go.sum
@@ -520,6 +520,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7 h1:lhh9BuEmjIBOdOhMIGSQzdW5LbTtFbUV1k9O/Rlq3SE=
+golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index f502beb6b..b12bdbf31 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -19,6 +19,7 @@ import (
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v42/github"
log "github.com/sirupsen/logrus"
+ "golang.org/x/exp/slices"
"golang.org/x/oauth2"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
@@ -698,22 +699,13 @@ func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, err
}
func (s *Source) ignoreRepo(r string) bool {
- if stringInSlice(r, s.ignoreRepos) {
+ if slices.Contains(s.ignoreRepos, r) {
s.log.Debugf("ignoring repo %s", r)
return true
}
return false
}
-func stringInSlice(s string, l []string) bool {
- for _, b := range l {
- if b == s {
- return true
- }
- }
- return false
-}
-
func (s *Source) getGistsByUser(ctx context.Context, user string) ([]string, error) {
var gistURLs []string
gistOpts := &github.GistListOptions{}
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index d932f69bf..334a82ab5 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -22,6 +22,7 @@ import (
gogit "github.com/go-git/go-git/v5"
log "github.com/sirupsen/logrus"
"github.com/xanzy/go-gitlab"
+ "golang.org/x/exp/slices"
"golang.org/x/sync/semaphore"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -38,6 +39,7 @@ type Source struct {
token string
url string
repos []string
+ ignoreRepos []string
git *git.Git
aCtx context.Context
resumeInfoSlice []string
@@ -80,7 +82,9 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
}
s.repos = conn.Repositories
+ s.ignoreRepos = conn.IgnoreRepos
s.url = conn.Endpoint
+
if conn.Endpoint != "" && !strings.HasSuffix(s.url, "/") {
s.url = s.url + "/"
}
@@ -365,6 +369,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
// Turn projects into URLs for Git cloner.
for _, prj := range projects {
+ if slices.Contains(s.ignoreRepos, prj.PathWithNamespace) {
+ log.Debugf("Ignoring repo %s", prj.PathWithNamespace)
+ continue
+ }
// Ensure the urls are valid before adding them to the repo list.
_, err := url.Parse(prj.HTTPURLToRepo)
if err != nil {
@@ -379,7 +387,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
s.repos = repos
// We must sort the repos so we can resume later if necessary.
- sort.Strings(s.repos)
+ slices.Sort(s.repos)
errs = s.scanRepos(ctx, chunksChan)
for _, err := range errs {
diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go
index 79c73ffea..265ebbab5 100644
--- a/pkg/sources/gitlab/gitlab_test.go
+++ b/pkg/sources/gitlab/gitlab_test.go
@@ -39,10 +39,11 @@ func TestSource_Scan(t *testing.T) {
connection *sourcespb.GitLab
}
tests := []struct {
- name string
- init init
- wantChunk *sources.Chunk
- wantErr bool
+ name string
+ init init
+ wantChunk *sources.Chunk
+ wantReposScanned int
+ wantErr bool
}{
{
name: "token auth, enumerate repo",
@@ -52,14 +53,14 @@ func TestSource_Scan(t *testing.T) {
Credential: &sourcespb.GitLab_Token{
Token: token,
},
+ IgnoreRepos: []string{"tes1188/learn-gitlab"},
},
},
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
SourceName: "test source",
- Verify: false,
},
- wantErr: false,
+ wantReposScanned: 2,
},
{
name: "token auth, scoped repo",
@@ -75,9 +76,8 @@ func TestSource_Scan(t *testing.T) {
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
SourceName: "test source scoped",
- Verify: false,
},
- wantErr: false,
+ wantReposScanned: 1,
},
{
name: "basic auth, scoped repo",
@@ -96,9 +96,8 @@ func TestSource_Scan(t *testing.T) {
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
SourceName: "test source basic auth scoped",
- Verify: false,
},
- wantErr: false,
+ wantReposScanned: 1,
},
{
name: "basic auth access token, scoped repo",
@@ -117,11 +116,11 @@ func TestSource_Scan(t *testing.T) {
wantChunk: &sources.Chunk{
SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,
SourceName: "test source basic auth access token scoped",
- Verify: false,
},
- wantErr: false,
+ wantReposScanned: 1,
},
}
+
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Source{}
@@ -156,6 +155,8 @@ func TestSource_Scan(t *testing.T) {
t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
}
}
+
+ assert.Equal(t, tt.wantReposScanned, len(s.repos))
if chunkCnt < 1 {
t.Errorf("0 chunks scanned.")
}