Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[THOG-248] fix broken gitlab tests (#437)
ahrav865bc5baf6e9
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex a27adf4ae..08bc49215 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -100,7 +100,11 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,s.conn = &conn- s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, runtime.NumCPU(),+ if concurrency == 0 {+ concurrency = runtime.NumCPU()+ }++ s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, concurrency,func(file, email, commit, repository, timestamp string, line int64) *source_metadatapb.MetaData {return &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Git{@@ -216,11 +220,11 @@ func CloneRepo(userInfo *url.Userinfo, gitUrl string) (clonePath string, repo *gcloneURL.User = userInfocloneCmd := exec.Command("git", "clone", cloneURL.String(), clonePath)- //cloneCmd := exec.Command("date")output, err := cloneCmd.CombinedOutput()if err != nil {err = errors.WrapPrefix(err, "error running 'git clone'", 0)}+if cloneCmd.ProcessState == nil {return "", nil, errors.New("clone command exited with no output")}@@ -399,7 +403,7 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunkreturn nil}-func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {+func (s *Git) ScanRepo(_ context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {start := time.Now().UnixNano()if err := s.ScanCommits(repo, repoPath, scanOptions, chunksChan); err != nil {return errdiff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex bdfdc048a..f8ffe547d 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -10,6 +10,7 @@ import ("sync""github.com/go-errors/errors"+ gogit "github.com/go-git/go-git/v5"log "github.com/sirupsen/logrus""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/giturl"@@ -41,7 +42,7 @@ type Source struct {jobSem *semaphore.Weighted}-// Ensure the Source satisfies the interface at compile time+// Ensure the Source satisfies the interface at compile time.var _ sources.Source = (*Source)(nil)// Type returns the type of source.@@ -98,7 +99,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,}if len(s.url) == 0 {- //assuming not custom gitlab url+ // Assuming not custom gitlab url.s.url = "https://gitlab.com/"}@@ -123,7 +124,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,}func (s *Source) newClient() (*gitlab.Client, error) {- // initialize a new api instance+ // Initialize a new api instance.switch s.authMethod {case "OAUTH":apiClient, err := gitlab.NewOAuthClient(s.token, gitlab.WithBaseURL(s.url))@@ -158,7 +159,7 @@ func (s *Source) newClient() (*gitlab.Client, error) {}func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, error) {- // projects without repo will get user projects, groups projects, and subgroup projects.+ // Projects without repo will get user projects, groups projects, and subgroup projects.user, _, err := apiClient.Users.CurrentUser()if err != nil {@@ -187,7 +188,7 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, ervar groups []*gitlab.GrouplistGroupsOptions := gitlab.ListGroupsOptions{- AllAvailable: gitlab.Bool(false), // This actually grabs public groups on public GitLab if set to true+ AllAvailable: gitlab.Bool(false), // This actually grabs public groups on public GitLab if set to true.TopLevelOnly: gitlab.Bool(false),Owned: gitlab.Bool(false),}@@ -262,95 +263,54 @@ func (s *Source) getRepos() ([]*url.URL, []error) {func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk, repos []*url.URL) []error {wg := sync.WaitGroup{}- errs := []error{}+ var errs []errorvar errsMut sync.Mutex- if s.authMethod == "UNAUTHENTICATED" {- for i, u := range repos {- if common.IsDone(ctx) {- // We are returning nil instead of the scanErrors slice here because- // we don't want to mark this scan as errored if we cancelled it.- return nil++ for i, u := range repos {+ if common.IsDone(ctx) {+ // We are returning nil instead of the scanErrors slice here because+ // we don't want to mark this scan as errored if we cancelled it.+ return nil+ }+ if err := s.jobSem.Acquire(ctx, 1); err != nil {+ log.WithError(err).Debug("could not acquire semaphore")+ continue+ }+ wg.Add(1)+ go func(ctx context.Context, repoURL *url.URL, i int) {+ defer s.jobSem.Release(1)+ defer wg.Done()+ if len(repoURL.String()) == 0 {+ return}- if err := s.jobSem.Acquire(ctx, 1); err != nil {- log.WithError(err).Debug("could not acquire semaphore")- continue+ s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")++ var path string+ var repo *gogit.Repository+ var err error+ if s.authMethod == "UNAUTHENTICATED" {+ path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL.String())+ } else {+ path, repo, err = git.CloneRepoUsingToken(s.token, repoURL.String(), s.user)}- wg.Add(1)- go func(ctx context.Context, repoURL *url.URL, i int) {- defer s.jobSem.Release(1)- defer wg.Done()- if len(repoURL.String()) == 0 {- return- }- s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")-- path, repo, err := git.CloneRepoUsingUnauthenticated(repoURL.String())- defer os.RemoveAll(path)- if err != nil {- errsMut.Lock()- errs = append(errs, err)- errsMut.Unlock()- return- }- log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(repos), repoURL.String())- err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)- if err != nil {- errsMut.Lock()- errs = append(errs, err)- errsMut.Unlock()- return- }- log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(repos), repoURL.String())- }(ctx, u, i)- }-- } else {- for i, u := range repos {- if common.IsDone(ctx) {- // We are returning nil instead of the scanErrors slice here because- // we don't want to mark this scan as errored if we cancelled it.- return nil+ defer os.RemoveAll(path)+ if err != nil {+ errsMut.Lock()+ errs = append(errs, err)+ errsMut.Unlock()+ return}- if err := s.jobSem.Acquire(ctx, 1); err != nil {- log.WithError(err).Debug("could not acquire semaphore")- continue+ log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(repos), repoURL.String())+ err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)+ if err != nil {+ errsMut.Lock()+ errs = append(errs, err)+ errsMut.Unlock()+ return}- wg.Add(1)- go func(ctx context.Context, repoURL *url.URL, i int) {- defer s.jobSem.Release(1)- defer wg.Done()- if len(repoURL.String()) == 0 {- return- }- s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")-- // If a username is not provided we need to use a default one in order to clone a private repo.- // Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user.- user := s.user- if user == "" {- user = "placeholder"- }- path, repo, err := git.CloneRepoUsingToken(s.token, repoURL.String(), user)- defer os.RemoveAll(path)- if err != nil {- errsMut.Lock()- errs = append(errs, err)- errsMut.Unlock()- return- }- log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(repos), repoURL.String())- err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)- if err != nil {- errsMut.Lock()- errs = append(errs, err)- errsMut.Unlock()- return- }- log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(repos), repoURL.String())- }(ctx, u, i)- }+ log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(repos), repoURL.String())+ }(ctx, u, i)}-wg.Wait()return errs@@ -358,29 +318,29 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,// Chunks emits chunks of bytes over a channel.func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {- // start client+ // Start client.apiClient, err := s.newClient()if err != nil {return errors.New(err)}- // get repo within target+ // Get repo within target.repos, errs := s.getRepos()for _, repoErr := range errs {log.WithError(repoErr).Warn("error getting repo")}- // End early if we had errors getting specified repos but none were validated+ // End early if we had errors getting specified repos but none were validated.if len(errs) > 0 && len(repos) == 0 {return errors.New("All specified repos had validation issues, ending scan")}- // get all repos if not specified+ // Get all repos if not specified.if repos == nil {projects, err := s.getAllProjects(apiClient)if err != nil {return errors.New(err)}- // turn projects into URLs for Git cloner+ // Turn projects into URLs for Git cloner.for _, prj := range projects {u, err := url.Parse(prj.HTTPURLToRepo)if err != nil {diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.goindex 03e3993d8..82f7ba3bc 100644--- a/pkg/sources/gitlab/gitlab_test.go+++ b/pkg/sources/gitlab/gitlab_test.go@@ -92,18 +92,25 @@ func TestSource_Scan(t *testing.T) {}chunksCh := make(chan *sources.Chunk, 1)go func() {+ defer close(chunksCh)err = s.Chunks(ctx, chunksCh)if (err != nil) != tt.wantErr {t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)return}}()- gotChunk := <-chunksCh+ var chunkCnt int// Commits don't come in a deterministic order, so remove metadata comparison- gotChunk.Data = nil- gotChunk.SourceMetadata = nil- if diff := pretty.Compare(gotChunk, tt.wantChunk); diff != "" {- t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)+ for gotChunk := range chunksCh {+ chunkCnt+++ gotChunk.Data = nil+ gotChunk.SourceMetadata = nil+ if diff := pretty.Compare(gotChunk, tt.wantChunk); diff != "" {+ t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ }+ if chunkCnt < 1 {+ t.Errorf("0 chunks scanned.")}})}