Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Allow gitlab to resume from encoded resume info (#611)
trufflesteeeve10f4d02c318f
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 71d19da8f..e1b6c046f 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -24,12 +24,11 @@ import ("google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/giturl""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/giturl""github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer""github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"@@ -355,7 +354,8 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch}// If there is resume information available, limit this scan to only the repos that still need scanning.- progressIndexOffset := s.filterReposToResume(s.GetProgress().EncodedResumeInfo)+ reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)+ s.repos = reposToScanfor i, repoURL := range s.repos {if err := s.jobSem.Acquire(ctx, 1); err != nil {@@ -372,7 +372,11 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, chs.setProgressCompleteWithRepo(i+progressIndexOffset, repoURL)// Ensure the repo is removed from the resume info after being scanned.- defer s.removeRepoFromResumeInfo(repoURL)+ defer func(s *Source) {+ s.resumeInfoMutex.Lock()+ defer s.resumeInfoMutex.Unlock()+ s.resumeInfoSlice = sources.RemoveRepoFromResumeInfo(s.resumeInfoSlice, repoURL)+ }(s)if !strings.HasSuffix(repoURL, ".git") {return@@ -799,91 +803,7 @@ func (s *Source) setProgressCompleteWithRepo(index int, repoURL string) {sort.Strings(s.resumeInfoSlice)// Make the resume info string from the slice.- encodedResumeInfo := s.encodeResumeInfo()+ encodedResumeInfo := sources.EncodeResumeInfo(s.resumeInfoSlice)s.SetProgressComplete(index, len(s.repos), fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)}--// removeRepoFromResumeInfo removes the repoURL from the resume info.-func (s *Source) removeRepoFromResumeInfo(repoURL string) {- s.resumeInfoMutex.Lock()- defer s.resumeInfoMutex.Unlock()-- index := -1- for i, repo := range s.resumeInfoSlice {- if repoURL == repo {- index = i- }- }-- if index == -1 {- // We should never be able to be here. But if we are, it means the resume info never had the repo added.- // So log the error and do nothing.- s.log.Errorf("repoURL (%q) not found in list of encode resume info: %q", repoURL, s.EncodedResumeInfo)- return- }-- // This removes the element at the given index.- s.resumeInfoSlice = append(s.resumeInfoSlice[:index], s.resumeInfoSlice[index+1:]...)-}--func (s *Source) encodeResumeInfo() string {- return strings.Join(s.resumeInfoSlice, "\t")-}--func (s *Source) decodeResumeInfo(resumeInfo string) {- // strings.Split will, for an empty string, return []string{""},- // which is an element, where as when there is no resume info we want an empty slice.- if resumeInfo == "" {- return- }- s.resumeInfoSlice = strings.Split(resumeInfo, "\t")-}--// filterReposToResume filters the existing repos down to those that are included in the encoded resume info.-// It also returns the difference between the original length of the repos and the new length to use for progress reporting.-// It is required that both the resumeInfo repos and the existing repos in s.repos are sorted.-func (s *Source) filterReposToResume(resumeInfo string) int {- if resumeInfo == "" {- return 0- }-- s.resumeInfoMutex.Lock()- defer s.resumeInfoMutex.Unlock()-- s.decodeResumeInfo(resumeInfo)-- // Because this scanner is multithreaded, it is possible that we have scanned a range of repositories- // with some gaps of unlisted but completed repositories in between the ones in resumeInfo.- // So we know repositories that have not finished scanning are the ones included in the resumeInfo,- // and those that come after the last repository in the resumeInfo.- // However, it is possible that a resumed scan does not include all or even any of the repos within the resumeInfo.- // In this case, we must ensure we still scan all repos that come after the last found repo in the list.- reposToScan := []string{}- lastFoundRepoIndex := -1- resumeRepoIndex := 0- for i, repoURL := range s.repos {- // If the repoURL is bigger than what we're looking for, move to the next one.- if repoURL > s.resumeInfoSlice[resumeRepoIndex] {- resumeRepoIndex++- }-- // If we've found all of our repositories end the filter.- if resumeRepoIndex == len(s.resumeInfoSlice) {- break- }-- // If the repoURL is the one we're looking for, add it and update the lastFoundRepoIndex.- if repoURL == s.resumeInfoSlice[resumeRepoIndex] {- lastFoundRepoIndex = i- reposToScan = append(reposToScan, repoURL)- }- }-- // Append all repos after the last one we've found.- reposToScan = append(reposToScan, s.repos[lastFoundRepoIndex+1:]...)-- progressOffsetCount := len(s.repos) - len(reposToScan)- s.repos = reposToScan- return progressOffsetCount-}diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex 0db9a5277..f658e2be4 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -373,152 +373,3 @@ func Test_setProgressCompleteWithRepo(t *testing.T) {}}}--func Test_removeRepoFromResumeInfo(t *testing.T) {- tests := []struct {- startingResumeInfoSlice []string- repoURL string- wantResumeInfoSlice []string- }{- {- startingResumeInfoSlice: []string{"a", "b", "c"},- repoURL: "a",- wantResumeInfoSlice: []string{"b", "c"},- },- {- startingResumeInfoSlice: []string{"a", "b", "c"},- repoURL: "b",- wantResumeInfoSlice: []string{"a", "c"},- },- { // This is the probably can't happen case of a repo not in the list.- startingResumeInfoSlice: []string{"a", "b", "c"},- repoURL: "not in the list",- wantResumeInfoSlice: []string{"a", "b", "c"},- },- }-- logger := logrus.New()- logger.Out = io.Discard- s := &Source{- repos: []string{},- log: logger.WithField("no", "output"),- }-- for _, tt := range tests {- s.resumeInfoSlice = tt.startingResumeInfoSlice- s.removeRepoFromResumeInfo(tt.repoURL)- if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {- t.Errorf("s.removeRepoFromResumeInfo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)- }- }-}--func Test_encodeResumeInfo(t *testing.T) {- tests := []struct {- startingResumeInfoSlice []string- wantEncodedResumeInfo string- }{- {- startingResumeInfoSlice: []string{"a", "b", "c"},- wantEncodedResumeInfo: "a\tb\tc",- },- {- startingResumeInfoSlice: []string{},- wantEncodedResumeInfo: "",- },- }-- logger := logrus.New()- logger.Out = io.Discard- s := &Source{- repos: []string{},- log: logger.WithField("no", "output"),- }-- for _, tt := range tests {- s.resumeInfoSlice = tt.startingResumeInfoSlice- gotEncodedResumeInfo := s.encodeResumeInfo()- if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {- t.Errorf("s.encodeResumeInfo() got: %q, want: %q", gotEncodedResumeInfo, tt.wantEncodedResumeInfo)- }- }-}--func Test_decodeResumeInfo(t *testing.T) {- tests := []struct {- resumeInfo string- wantResumeInfoSlice []string- }{- {- resumeInfo: "a\tb\tc",- wantResumeInfoSlice: []string{"a", "b", "c"},- },- {- resumeInfo: "",- wantResumeInfoSlice: nil,- },- }-- for _, tt := range tests {- s := &Source{}- s.decodeResumeInfo(tt.resumeInfo)- if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {- t.Errorf("s.decodeResumeInfo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)- }- }-}--func Test_filterReposToResume(t *testing.T) {- startingRepos := []string{"a", "b", "c", "d", "e", "f", "g"}-- tests := map[string]struct {- resumeInfo string- wantProgressOffsetCount int- wantReposToScan []string- }{- "blank resume info": {- resumeInfo: "",- wantProgressOffsetCount: 0,- wantReposToScan: startingRepos,- },- "starting repos": {- resumeInfo: "a\tb",- wantProgressOffsetCount: 0,- wantReposToScan: startingRepos,- },- "early contiguous repos": {- resumeInfo: "b\tc",- wantProgressOffsetCount: 1,- wantReposToScan: []string{"b", "c", "d", "e", "f", "g"},- },- "non-contiguous repos": {- resumeInfo: "b\te",- wantProgressOffsetCount: 3,- wantReposToScan: []string{"b", "e", "f", "g"},- },- "no repos found in the repo list": {- resumeInfo: "not\tthere",- wantProgressOffsetCount: 0,- wantReposToScan: startingRepos,- },- "only some repos in the list": {- resumeInfo: "c\tnot\tthere",- wantProgressOffsetCount: 2,- wantReposToScan: []string{"c", "d", "e", "f", "g"},- },- }-- for name, tt := range tests {- s := &Source{- repos: startingRepos,- }-- gotProgressOffsetCount := s.filterReposToResume(tt.resumeInfo)- if gotProgressOffsetCount != tt.wantProgressOffsetCount {- t.Errorf("s.filterReposToResume() name: %q got: %d, want: %d", name, gotProgressOffsetCount, tt.wantProgressOffsetCount)- }- if !reflect.DeepEqual(s.repos, tt.wantReposToScan) {- t.Errorf("s.filterReposToResume() name: %q got: %v, want: %v", name, s.repos, tt.wantReposToScan)- }- }-}diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex c306bc407..ef8878fb0 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -6,12 +6,10 @@ import ("net/url""os""runtime"+ "sort""strings""sync"- "github.com/go-errors/errors"- gogit "github.com/go-git/go-git/v5"- log "github.com/sirupsen/logrus""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/giturl""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"@@ -19,6 +17,10 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer""github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"++ "github.com/go-errors/errors"+ gogit "github.com/go-git/go-git/v5"+ log "github.com/sirupsen/logrus""github.com/xanzy/go-gitlab""golang.org/x/sync/semaphore""google.golang.org/protobuf/proto"@@ -26,20 +28,22 @@ import ()type Source struct {- name string- sourceId int64- jobId int64- verify bool- authMethod string- user string- password string- token string- url string- repos []string- git *git.Git- aCtx context.Context+ name string+ sourceId int64+ jobId int64+ verify bool+ authMethod string+ user string+ password string+ token string+ url string+ repos []string+ git *git.Git+ aCtx context.Context+ resumeInfoSlice []string+ resumeInfoMutex sync.Mutex+ jobSem *semaphore.Weightedsources.Progress- jobSem *semaphore.Weighted}// Ensure the Source satisfies the interface at compile time.@@ -241,33 +245,35 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, erreturn projectList, nil}-func (s *Source) getRepos() ([]*url.URL, []error) {- var validRepos []*url.URL- var errs []error- if len(s.repos) > 0 {- for _, prj := range s.repos {- repo, err := giturl.NormalizeGitlabRepo(prj)- if err != nil {- errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))- continue- }+func (s *Source) getRepos() ([]string, []error) {+ if len(s.repos) == 0 {+ return nil, nil+ }- // The repo normalization has already successfully parsed the URL at this point, so we can ignore the error.- u, _ := url.ParseRequestURI(repo)- validRepos = append(validRepos, u)+ var validRepos []string+ var errs []error+ for _, prj := range s.repos {+ repo, err := giturl.NormalizeGitlabRepo(prj)+ if err != nil {+ errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))+ continue}- return validRepos, errs- }- return nil, nil+ validRepos = append(validRepos, repo)+ }+ return validRepos, errs}-func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk, repos []*url.URL) []error {+func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) []error {wg := sync.WaitGroup{}var errs []errorvar errsMut sync.Mutex- for i, u := range repos {+ // If there is resume information available, limit this scan to only the repos that still need scanning.+ reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)+ s.repos = reposToScan++ for i, repo := range s.repos {if common.IsDone(ctx) {// We are returning nil instead of the scanErrors slice here because// we don't want to mark this scan as errored if we cancelled it.@@ -278,19 +284,26 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,continue}wg.Add(1)- go func(ctx context.Context, repoURL *url.URL, i int) {+ go func(ctx context.Context, repoURL string, i int) {defer s.jobSem.Release(1)defer wg.Done()- if len(repoURL.String()) == 0 {+ if len(repoURL) == 0 {return}- s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")++ s.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)+ // Ensure the repo is removed from the resume info after being scanned.+ defer func(s *Source) {+ s.resumeInfoMutex.Lock()+ defer s.resumeInfoMutex.Unlock()+ s.resumeInfoSlice = sources.RemoveRepoFromResumeInfo(s.resumeInfoSlice, repoURL)+ }(s)var path stringvar repo *gogit.Repositoryvar err errorif s.authMethod == "UNAUTHENTICATED" {- path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL.String())+ path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL)} else {// If a username is not provided we need to use a default one in order to clone a private repo.// Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user.@@ -298,7 +311,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,if user == "" {user = "placeholder"}- path, repo, err = git.CloneRepoUsingToken(s.token, repoURL.String(), user)+ path, repo, err = git.CloneRepoUsingToken(s.token, repoURL, user)}defer os.RemoveAll(path)if err != nil {@@ -307,7 +320,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,errsMut.Unlock()return}- log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(repos), repoURL.String())+ log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repoURL)err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)if err != nil {errsMut.Lock()@@ -315,8 +328,8 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,errsMut.Unlock()return}- log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(repos), repoURL.String())- }(ctx, u, i)+ log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(s.repos), repoURL)+ }(ctx, repo, i)}wg.Wait()@@ -349,17 +362,23 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err}// Turn projects into URLs for Git cloner.for _, prj := range projects {- u, err := url.Parse(prj.HTTPURLToRepo)+ // Ensure the urls are valid before adding them to the repo list.+ _, err := url.Parse(prj.HTTPURLToRepo)if err != nil {fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)}- repos = append(repos, u)+ repos = append(repos, prj.HTTPURLToRepo)}if repos == nil {return errors.Errorf("unable to discover any repos")}}- errs = s.scanRepos(ctx, chunksChan, repos)++ s.repos = repos+ // We must sort the repos so we can resume later if necessary.+ sort.Strings(s.repos)++ errs = s.scanRepos(ctx, chunksChan)for _, err := range errs {log.WithError(err).WithFields(log.Fields{@@ -386,3 +405,19 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {}return false}++// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.+func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {+ s.resumeInfoMutex.Lock()+ defer s.resumeInfoMutex.Unlock()++ // Add the repoURL to the resume info slice.+ s.resumeInfoSlice = append(s.resumeInfoSlice, repoURL)+ sort.Strings(s.resumeInfoSlice)++ // Make the resume info string from the slice.+ encodedResumeInfo := sources.EncodeResumeInfo(s.resumeInfoSlice)++ // Add the offset to both the index and the repos to give the proper place and proper repo count.+ s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)+}diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.goindex c3183b146..c23087116 100644--- a/pkg/sources/gitlab/gitlab_test.go+++ b/pkg/sources/gitlab/gitlab_test.go@@ -3,6 +3,8 @@ package gitlabimport ("context""fmt"+ "io"+ "reflect""testing""github.com/kylelemons/godebug/pretty"@@ -160,3 +162,90 @@ func TestSource_Scan(t *testing.T) {})}}++func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {+ tests := []struct {+ startingResumeInfoSlice []string+ repoURL string+ wantResumeInfoSlice []string+ }{+ {+ startingResumeInfoSlice: []string{},+ repoURL: "a",+ wantResumeInfoSlice: []string{"a"},+ },+ {+ startingResumeInfoSlice: []string{"b"},+ repoURL: "a",+ wantResumeInfoSlice: []string{"a", "b"},+ },+ }++ log.SetOutput(io.Discard)+ s := &Source{repos: []string{}}++ for _, tt := range tests {+ s.resumeInfoSlice = tt.startingResumeInfoSlice+ s.setProgressCompleteWithRepo(0, 0, tt.repoURL)+ if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {+ t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)+ }+ }+}++func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {+ repos := []string{"a", "b", "c", "d", "e"}+ tests := map[string]struct {+ repos []string+ index int+ offset int+ wantPercentComplete int64+ wantSectionsCompleted int32+ wantSectionsRemaining int32+ }{+ "starting from the beginning, no offset": {+ repos: repos,+ index: 0,+ offset: 0,+ wantPercentComplete: 0,+ wantSectionsCompleted: 0,+ wantSectionsRemaining: 5,+ },+ "resume from the third, offset 2": {+ repos: repos[2:],+ index: 0,+ offset: 2,+ wantPercentComplete: 40,+ wantSectionsCompleted: 2,+ wantSectionsRemaining: 5,+ },+ "resume from the third, on last repo, offset 2": {+ repos: repos[2:],+ index: 2,+ offset: 2,+ wantPercentComplete: 80,+ wantSectionsCompleted: 4,+ wantSectionsRemaining: 5,+ },+ }++ log.SetOutput(io.Discard)++ for _, tt := range tests {+ s := &Source{+ repos: tt.repos,+ }++ s.setProgressCompleteWithRepo(tt.index, tt.offset, "")+ gotProgress := s.GetProgress()+ if gotProgress.PercentComplete != tt.wantPercentComplete {+ t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete)+ }+ if gotProgress.SectionsCompleted != tt.wantSectionsCompleted {+ t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted)+ }+ if gotProgress.SectionsRemaining != tt.wantSectionsRemaining {+ t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining)+ }+ }+}diff --git a/pkg/sources/resume.go b/pkg/sources/resume.gonew file mode 100644index 000000000..d085b5d69--- /dev/null+++ b/pkg/sources/resume.go@@ -0,0 +1,84 @@+package sources++import (+ "strings"++ "github.com/sirupsen/logrus"+)++// RemoveRepoFromResumeInfo removes the repoURL from the resume info.+func RemoveRepoFromResumeInfo(resumeRepos []string, repoURL string) []string {+ index := -1+ for i, repo := range resumeRepos {+ if repoURL == repo {+ index = i+ break+ }+ }++ if index == -1 {+ // We should never be able to be here. But if we are, it means the resume info never had the repo added.+ // So log the error and do nothing.+ logrus.Errorf("repoURL (%q) not found in list of encode resume info: %v", repoURL, resumeRepos)+ return resumeRepos+ }++ // This removes the element at the given index.+ return append(resumeRepos[:index], resumeRepos[index+1:]...)+}++// FilterReposToResume filters the existing repos down to those that are included in the encoded resume info.+// It returns the new slice of repos to be scanned.+// It also returns the difference between the original length of the repos and the new length to use for progress reporting.+// It is required that both the resumeInfo repos and the existing repos are sorted.+func FilterReposToResume(repos []string, resumeInfo string) (reposToScan []string, progressOffsetCount int) {+ if resumeInfo == "" {+ return repos, 0+ }++ resumeInfoSlice := DecodeResumeInfo(resumeInfo)++ // Because this scanner is multithreaded, it is possible that we have scanned a range of repositories+ // with some gaps of unlisted but completed repositories in between the ones in resumeInfo.+ // So we know repositories that have not finished scanning are the ones included in the resumeInfo,+ // and those that come after the last repository in the resumeInfo.+ // However, it is possible that a resumed scan does not include all or even any of the repos within the resumeInfo.+ // In this case, we must ensure we still scan all repos that come after the last found repo in the list.+ lastFoundRepoIndex := -1+ resumeRepoIndex := 0+ for i, repoURL := range repos {+ // If the repoURL is bigger than what we're looking for, move to the next one.+ if repoURL > resumeInfoSlice[resumeRepoIndex] {+ resumeRepoIndex+++ }++ // If we've found all of our repositories end the filter.+ if resumeRepoIndex == len(resumeInfoSlice) {+ break+ }++ // If the repoURL is the one we're looking for, add it and update the lastFoundRepoIndex.+ if repoURL == resumeInfoSlice[resumeRepoIndex] {+ lastFoundRepoIndex = i+ reposToScan = append(reposToScan, repoURL)+ }+ }++ // Append all repos after the last one we've found.+ reposToScan = append(reposToScan, repos[lastFoundRepoIndex+1:]...)+ progressOffsetCount = len(repos) - len(reposToScan)+ return+}++func EncodeResumeInfo(resumeInfoSlice []string) string {+ return strings.Join(resumeInfoSlice, "\t")+}++func DecodeResumeInfo(resumeInfo string) []string {+ // strings.Split will, for an empty string, return []string{""},+ // which is an element, where as when there is no resume info we want an empty slice.+ if resumeInfo == "" {+ return nil+ }+ return strings.Split(resumeInfo, "\t")+}diff --git a/pkg/sources/resume_test.go b/pkg/sources/resume_test.gonew file mode 100644index 000000000..56bda955d--- /dev/null+++ b/pkg/sources/resume_test.go@@ -0,0 +1,143 @@+package sources++import (+ "io"+ "reflect"+ "testing"++ "github.com/sirupsen/logrus"+)++func TestRemoveRepoFromResumeInfo(t *testing.T) {+ tests := []struct {+ startingResumeInfoSlice []string+ repoURL string+ wantResumeInfoSlice []string+ }{+ {+ startingResumeInfoSlice: []string{"a", "b", "c"},+ repoURL: "a",+ wantResumeInfoSlice: []string{"b", "c"},+ },+ {+ startingResumeInfoSlice: []string{"a", "b", "c"},+ repoURL: "b",+ wantResumeInfoSlice: []string{"a", "c"},+ },+ { // This is the probably can't happen case of a repo not in the list.+ startingResumeInfoSlice: []string{"a", "b", "c"},+ repoURL: "not in the list",+ wantResumeInfoSlice: []string{"a", "b", "c"},+ },+ }++ // Prevent the error message from showing up.+ logrus.SetOutput(io.Discard)++ for _, tt := range tests {+ gotResumeInfoSlice := RemoveRepoFromResumeInfo(tt.startingResumeInfoSlice, tt.repoURL)+ if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) {+ t.Errorf("RemoveRepoFromResumeInfo() got: %v, want: %v", gotResumeInfoSlice, tt.wantResumeInfoSlice)+ }+ }+}++func TestEncodeResumeInfo(t *testing.T) {+ tests := []struct {+ startingResumeInfoSlice []string+ wantEncodedResumeInfo string+ }{+ {+ startingResumeInfoSlice: []string{"a", "b", "c"},+ wantEncodedResumeInfo: "a\tb\tc",+ },+ {+ startingResumeInfoSlice: []string{},+ wantEncodedResumeInfo: "",+ },+ }++ logger := logrus.New()+ logger.Out = io.Discard++ for _, tt := range tests {+ gotEncodedResumeInfo := EncodeResumeInfo(tt.startingResumeInfoSlice)+ if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {+ t.Errorf("EncodeResumeInfo() got: %q, want: %q", gotEncodedResumeInfo, tt.wantEncodedResumeInfo)+ }+ }+}++func Test_decodeResumeInfo(t *testing.T) {+ tests := []struct {+ resumeInfo string+ wantResumeInfoSlice []string+ }{+ {+ resumeInfo: "a\tb\tc",+ wantResumeInfoSlice: []string{"a", "b", "c"},+ },+ {+ resumeInfo: "",+ wantResumeInfoSlice: nil,+ },+ }++ for _, tt := range tests {+ gotResumeInfoSlice := DecodeResumeInfo(tt.resumeInfo)+ if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) {+ t.Errorf("DecodeResumeInfo() got: %v, want: %v", gotResumeInfoSlice, tt.wantResumeInfoSlice)+ }+ }+}++func Test_filterReposToResume(t *testing.T) {+ startingRepos := []string{"a", "b", "c", "d", "e", "f", "g"}++ tests := map[string]struct {+ resumeInfo string+ wantProgressOffsetCount int+ wantReposToScan []string+ }{+ "blank resume info": {+ resumeInfo: "",+ wantProgressOffsetCount: 0,+ wantReposToScan: startingRepos,+ },+ "starting repos": {+ resumeInfo: "a\tb",+ wantProgressOffsetCount: 0,+ wantReposToScan: startingRepos,+ },+ "early contiguous repos": {+ resumeInfo: "b\tc",+ wantProgressOffsetCount: 1,+ wantReposToScan: []string{"b", "c", "d", "e", "f", "g"},+ },+ "non-contiguous repos": {+ resumeInfo: "b\te",+ wantProgressOffsetCount: 3,+ wantReposToScan: []string{"b", "e", "f", "g"},+ },+ "no repos found in the repo list": {+ resumeInfo: "not\tthere",+ wantProgressOffsetCount: 0,+ wantReposToScan: startingRepos,+ },+ "only some repos in the list": {+ resumeInfo: "c\tnot\tthere",+ wantProgressOffsetCount: 2,+ wantReposToScan: []string{"c", "d", "e", "f", "g"},+ },+ }++ for name, tt := range tests {+ gotReposToScan, gotProgressOffsetCount := FilterReposToResume(startingRepos, tt.resumeInfo)+ if !reflect.DeepEqual(gotReposToScan, tt.wantReposToScan) {+ t.Errorf("FilterReposToResume() name: %q got: %v, want: %v", name, gotReposToScan, tt.wantReposToScan)+ }+ if gotProgressOffsetCount != tt.wantProgressOffsetCount {+ t.Errorf("FilterReposToResume() name: %q got: %d, want: %d", name, gotProgressOffsetCount, tt.wantProgressOffsetCount)+ }+ }+}