Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[THOG-767] ignore gitlab repos (#853)
ahrav029519eb01a4
diff --git a/go.mod b/go.modindex 26eef3659..e04547a18 100644--- a/go.mod+++ b/go.mod@@ -53,6 +53,7 @@ require (go.mongodb.org/mongo-driver v1.10.2go.uber.org/zap v1.23.0golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d+ golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006diff --git a/go.sum b/go.sumindex fb45af79b..700e5d69b 100644--- a/go.sum+++ b/go.sum@@ -520,6 +520,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=+golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7 h1:lhh9BuEmjIBOdOhMIGSQzdW5LbTtFbUV1k9O/Rlq3SE=+golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex f502beb6b..b12bdbf31 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -19,6 +19,7 @@ import (gogit "github.com/go-git/go-git/v5""github.com/google/go-github/v42/github"log "github.com/sirupsen/logrus"+ "golang.org/x/exp/slices""golang.org/x/oauth2""golang.org/x/sync/errgroup""google.golang.org/protobuf/proto"@@ -698,22 +699,13 @@ func (s *Source) getReposByUser(ctx context.Context, user string) ([]string, err}func (s *Source) ignoreRepo(r string) bool {- if stringInSlice(r, s.ignoreRepos) {+ if slices.Contains(s.ignoreRepos, r) {s.log.Debugf("ignoring repo %s", r)return true}return false}-func stringInSlice(s string, l []string) bool {- for _, b := range l {- if b == s {- return true- }- }- return false-}-func (s *Source) getGistsByUser(ctx context.Context, user string) ([]string, error) {var gistURLs []stringgistOpts := &github.GistListOptions{}diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex d932f69bf..334a82ab5 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -22,6 +22,7 @@ import (gogit "github.com/go-git/go-git/v5"log "github.com/sirupsen/logrus""github.com/xanzy/go-gitlab"+ "golang.org/x/exp/slices""golang.org/x/sync/semaphore""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"@@ -38,6 +39,7 @@ type Source struct {token stringurl stringrepos []string+ ignoreRepos []stringgit *git.GitaCtx context.ContextresumeInfoSlice []string@@ -80,7 +82,9 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,}s.repos = conn.Repositories+ s.ignoreRepos = conn.IgnoreReposs.url = conn.Endpoint+if conn.Endpoint != "" && !strings.HasSuffix(s.url, "/") {s.url = s.url + "/"}@@ -365,6 +369,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err}// Turn projects into URLs for Git cloner.for _, prj := range projects {+ if slices.Contains(s.ignoreRepos, prj.PathWithNamespace) {+ log.Debugf("Ignoring repo %s", prj.PathWithNamespace)+ continue+ }// Ensure the urls are valid before adding them to the repo list._, err := url.Parse(prj.HTTPURLToRepo)if err != nil {@@ -379,7 +387,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errs.repos = repos// We must sort the repos so we can resume later if necessary.- sort.Strings(s.repos)+ slices.Sort(s.repos)errs = s.scanRepos(ctx, chunksChan)for _, err := range errs {diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.goindex 79c73ffea..265ebbab5 100644--- a/pkg/sources/gitlab/gitlab_test.go+++ b/pkg/sources/gitlab/gitlab_test.go@@ -39,10 +39,11 @@ func TestSource_Scan(t *testing.T) {connection *sourcespb.GitLab}tests := []struct {- name string- init init- wantChunk *sources.Chunk- wantErr bool+ name string+ init init+ wantChunk *sources.Chunk+ wantReposScanned int+ wantErr bool}{{name: "token auth, enumerate repo",@@ -52,14 +53,14 @@ func TestSource_Scan(t *testing.T) {Credential: &sourcespb.GitLab_Token{Token: token,},+ IgnoreRepos: []string{"tes1188/learn-gitlab"},},},wantChunk: &sources.Chunk{SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,SourceName: "test source",- Verify: false,},- wantErr: false,+ wantReposScanned: 2,},{name: "token auth, scoped repo",@@ -75,9 +76,8 @@ func TestSource_Scan(t *testing.T) {wantChunk: &sources.Chunk{SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,SourceName: "test source scoped",- Verify: false,},- wantErr: false,+ wantReposScanned: 1,},{name: "basic auth, scoped repo",@@ -96,9 +96,8 @@ func TestSource_Scan(t *testing.T) {wantChunk: &sources.Chunk{SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,SourceName: "test source basic auth scoped",- Verify: false,},- wantErr: false,+ wantReposScanned: 1,},{name: "basic auth access token, scoped repo",@@ -117,11 +116,11 @@ func TestSource_Scan(t *testing.T) {wantChunk: &sources.Chunk{SourceType: sourcespb.SourceType_SOURCE_TYPE_GITLAB,SourceName: "test source basic auth access token scoped",- Verify: false,},- wantErr: false,+ wantReposScanned: 1,},}+for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {s := Source{}@@ -156,6 +155,8 @@ func TestSource_Scan(t *testing.T) {t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)}}++ assert.Equal(t, tt.wantReposScanned, len(s.repos))if chunkCnt < 1 {t.Errorf("0 chunks scanned.")}