Skip to content

Find, verify, and analyze leaked credentials

[THOG-248] fix broken gitlab tests (#437)

ahrav865bc5baf6e9
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index a27adf4ae..08bc49215 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -100,7 +100,11 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
s.conn = &conn
- s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, runtime.NumCPU(),
+ if concurrency == 0 {
+ concurrency = runtime.NumCPU()
+ }
+
+ s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, concurrency,
func(file, email, commit, repository, timestamp string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
@@ -216,11 +220,11 @@ func CloneRepo(userInfo *url.Userinfo, gitUrl string) (clonePath string, repo *g
cloneURL.User = userInfo
cloneCmd := exec.Command("git", "clone", cloneURL.String(), clonePath)
- //cloneCmd := exec.Command("date")
output, err := cloneCmd.CombinedOutput()
if err != nil {
err = errors.WrapPrefix(err, "error running 'git clone'", 0)
}
+
if cloneCmd.ProcessState == nil {
return "", nil, errors.New("clone command exited with no output")
}
@@ -399,7 +403,7 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk
return nil
}
-func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+func (s *Git) ScanRepo(_ context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
start := time.Now().UnixNano()
if err := s.ScanCommits(repo, repoPath, scanOptions, chunksChan); err != nil {
return err
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index bdfdc048a..f8ffe547d 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -10,6 +10,7 @@ import (
"sync"
"github.com/go-errors/errors"
+ gogit "github.com/go-git/go-git/v5"
log "github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
@@ -41,7 +42,7 @@ type Source struct {
jobSem *semaphore.Weighted
}
-// Ensure the Source satisfies the interface at compile time
+// Ensure the Source satisfies the interface at compile time.
var _ sources.Source = (*Source)(nil)
// Type returns the type of source.
@@ -98,7 +99,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
}
if len(s.url) == 0 {
- //assuming not custom gitlab url
+ // Assuming not custom gitlab url.
s.url = "https://gitlab.com/"
}
@@ -123,7 +124,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
}
func (s *Source) newClient() (*gitlab.Client, error) {
- // initialize a new api instance
+ // Initialize a new api instance.
switch s.authMethod {
case "OAUTH":
apiClient, err := gitlab.NewOAuthClient(s.token, gitlab.WithBaseURL(s.url))
@@ -158,7 +159,7 @@ func (s *Source) newClient() (*gitlab.Client, error) {
}
func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, error) {
- // projects without repo will get user projects, groups projects, and subgroup projects.
+ // Projects without repo will get user projects, groups projects, and subgroup projects.
user, _, err := apiClient.Users.CurrentUser()
if err != nil {
@@ -187,7 +188,7 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er
var groups []*gitlab.Group
listGroupsOptions := gitlab.ListGroupsOptions{
- AllAvailable: gitlab.Bool(false), // This actually grabs public groups on public GitLab if set to true
+ AllAvailable: gitlab.Bool(false), // This actually grabs public groups on public GitLab if set to true.
TopLevelOnly: gitlab.Bool(false),
Owned: gitlab.Bool(false),
}
@@ -262,95 +263,54 @@ func (s *Source) getRepos() ([]*url.URL, []error) {
func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk, repos []*url.URL) []error {
wg := sync.WaitGroup{}
- errs := []error{}
+ var errs []error
var errsMut sync.Mutex
- if s.authMethod == "UNAUTHENTICATED" {
- for i, u := range repos {
- if common.IsDone(ctx) {
- // We are returning nil instead of the scanErrors slice here because
- // we don't want to mark this scan as errored if we cancelled it.
- return nil
+
+ for i, u := range repos {
+ if common.IsDone(ctx) {
+ // We are returning nil instead of the scanErrors slice here because
+ // we don't want to mark this scan as errored if we cancelled it.
+ return nil
+ }
+ if err := s.jobSem.Acquire(ctx, 1); err != nil {
+ log.WithError(err).Debug("could not acquire semaphore")
+ continue
+ }
+ wg.Add(1)
+ go func(ctx context.Context, repoURL *url.URL, i int) {
+ defer s.jobSem.Release(1)
+ defer wg.Done()
+ if len(repoURL.String()) == 0 {
+ return
}
- if err := s.jobSem.Acquire(ctx, 1); err != nil {
- log.WithError(err).Debug("could not acquire semaphore")
- continue
+ s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")
+
+ var path string
+ var repo *gogit.Repository
+ var err error
+ if s.authMethod == "UNAUTHENTICATED" {
+ path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL.String())
+ } else {
+ path, repo, err = git.CloneRepoUsingToken(s.token, repoURL.String(), s.user)
}
- wg.Add(1)
- go func(ctx context.Context, repoURL *url.URL, i int) {
- defer s.jobSem.Release(1)
- defer wg.Done()
- if len(repoURL.String()) == 0 {
- return
- }
- s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")
-
- path, repo, err := git.CloneRepoUsingUnauthenticated(repoURL.String())
- defer os.RemoveAll(path)
- if err != nil {
- errsMut.Lock()
- errs = append(errs, err)
- errsMut.Unlock()
- return
- }
- log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(repos), repoURL.String())
- err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
- if err != nil {
- errsMut.Lock()
- errs = append(errs, err)
- errsMut.Unlock()
- return
- }
- log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(repos), repoURL.String())
- }(ctx, u, i)
- }
-
- } else {
- for i, u := range repos {
- if common.IsDone(ctx) {
- // We are returning nil instead of the scanErrors slice here because
- // we don't want to mark this scan as errored if we cancelled it.
- return nil
+ defer os.RemoveAll(path)
+ if err != nil {
+ errsMut.Lock()
+ errs = append(errs, err)
+ errsMut.Unlock()
+ return
}
- if err := s.jobSem.Acquire(ctx, 1); err != nil {
- log.WithError(err).Debug("could not acquire semaphore")
- continue
+ log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(repos), repoURL.String())
+ err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
+ if err != nil {
+ errsMut.Lock()
+ errs = append(errs, err)
+ errsMut.Unlock()
+ return
}
- wg.Add(1)
- go func(ctx context.Context, repoURL *url.URL, i int) {
- defer s.jobSem.Release(1)
- defer wg.Done()
- if len(repoURL.String()) == 0 {
- return
- }
- s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")
-
- // If a username is not provided we need to use a default one in order to clone a private repo.
- // Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user.
- user := s.user
- if user == "" {
- user = "placeholder"
- }
- path, repo, err := git.CloneRepoUsingToken(s.token, repoURL.String(), user)
- defer os.RemoveAll(path)
- if err != nil {
- errsMut.Lock()
- errs = append(errs, err)
- errsMut.Unlock()
- return
- }
- log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(repos), repoURL.String())
- err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
- if err != nil {
- errsMut.Lock()
- errs = append(errs, err)
- errsMut.Unlock()
- return
- }
- log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(repos), repoURL.String())
- }(ctx, u, i)
- }
+ log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(repos), repoURL.String())
+ }(ctx, u, i)
}
-
wg.Wait()
return errs
@@ -358,29 +318,29 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
- // start client
+ // Start client.
apiClient, err := s.newClient()
if err != nil {
return errors.New(err)
}
- // get repo within target
+ // Get repo within target.
repos, errs := s.getRepos()
for _, repoErr := range errs {
log.WithError(repoErr).Warn("error getting repo")
}
- // End early if we had errors getting specified repos but none were validated
+ // End early if we had errors getting specified repos but none were validated.
if len(errs) > 0 && len(repos) == 0 {
return errors.New("All specified repos had validation issues, ending scan")
}
- // get all repos if not specified
+ // Get all repos if not specified.
if repos == nil {
projects, err := s.getAllProjects(apiClient)
if err != nil {
return errors.New(err)
}
- // turn projects into URLs for Git cloner
+ // Turn projects into URLs for Git cloner.
for _, prj := range projects {
u, err := url.Parse(prj.HTTPURLToRepo)
if err != nil {
diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go
index 03e3993d8..82f7ba3bc 100644
--- a/pkg/sources/gitlab/gitlab_test.go
+++ b/pkg/sources/gitlab/gitlab_test.go
@@ -92,18 +92,25 @@ func TestSource_Scan(t *testing.T) {
}
chunksCh := make(chan *sources.Chunk, 1)
go func() {
+ defer close(chunksCh)
err = s.Chunks(ctx, chunksCh)
if (err != nil) != tt.wantErr {
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
return
}
}()
- gotChunk := <-chunksCh
+ var chunkCnt int
// Commits don't come in a deterministic order, so remove metadata comparison
- gotChunk.Data = nil
- gotChunk.SourceMetadata = nil
- if diff := pretty.Compare(gotChunk, tt.wantChunk); diff != "" {
- t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
+ for gotChunk := range chunksCh {
+ chunkCnt++
+ gotChunk.Data = nil
+ gotChunk.SourceMetadata = nil
+ if diff := pretty.Compare(gotChunk, tt.wantChunk); diff != "" {
+ t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ }
+ if chunkCnt < 1 {
+ t.Errorf("0 chunks scanned.")
}
})
}