Skip to content

Find, verify, and analyze leaked credentials

Allow gitlab to resume from encoded resume info (#611)

trufflesteeeve10f4d02c318f
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 71d19da8f..e1b6c046f 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -24,12 +24,11 @@ import (
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -355,7 +354,8 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
}
// If there is resume information available, limit this scan to only the repos that still need scanning.
- progressIndexOffset := s.filterReposToResume(s.GetProgress().EncodedResumeInfo)
+ reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
+ s.repos = reposToScan
for i, repoURL := range s.repos {
if err := s.jobSem.Acquire(ctx, 1); err != nil {
@@ -372,7 +372,11 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
s.setProgressCompleteWithRepo(i+progressIndexOffset, repoURL)
// Ensure the repo is removed from the resume info after being scanned.
- defer s.removeRepoFromResumeInfo(repoURL)
+ defer func(s *Source) {
+ s.resumeInfoMutex.Lock()
+ defer s.resumeInfoMutex.Unlock()
+ s.resumeInfoSlice = sources.RemoveRepoFromResumeInfo(s.resumeInfoSlice, repoURL)
+ }(s)
if !strings.HasSuffix(repoURL, ".git") {
return
@@ -799,91 +803,7 @@ func (s *Source) setProgressCompleteWithRepo(index int, repoURL string) {
sort.Strings(s.resumeInfoSlice)
// Make the resume info string from the slice.
- encodedResumeInfo := s.encodeResumeInfo()
+ encodedResumeInfo := sources.EncodeResumeInfo(s.resumeInfoSlice)
s.SetProgressComplete(index, len(s.repos), fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
}
-
-// removeRepoFromResumeInfo removes the repoURL from the resume info.
-func (s *Source) removeRepoFromResumeInfo(repoURL string) {
- s.resumeInfoMutex.Lock()
- defer s.resumeInfoMutex.Unlock()
-
- index := -1
- for i, repo := range s.resumeInfoSlice {
- if repoURL == repo {
- index = i
- }
- }
-
- if index == -1 {
- // We should never be able to be here. But if we are, it means the resume info never had the repo added.
- // So log the error and do nothing.
- s.log.Errorf("repoURL (%q) not found in list of encode resume info: %q", repoURL, s.EncodedResumeInfo)
- return
- }
-
- // This removes the element at the given index.
- s.resumeInfoSlice = append(s.resumeInfoSlice[:index], s.resumeInfoSlice[index+1:]...)
-}
-
-func (s *Source) encodeResumeInfo() string {
- return strings.Join(s.resumeInfoSlice, "\t")
-}
-
-func (s *Source) decodeResumeInfo(resumeInfo string) {
- // strings.Split will, for an empty string, return []string{""},
- // which is an element, where as when there is no resume info we want an empty slice.
- if resumeInfo == "" {
- return
- }
- s.resumeInfoSlice = strings.Split(resumeInfo, "\t")
-}
-
-// filterReposToResume filters the existing repos down to those that are included in the encoded resume info.
-// It also returns the difference between the original length of the repos and the new length to use for progress reporting.
-// It is required that both the resumeInfo repos and the existing repos in s.repos are sorted.
-func (s *Source) filterReposToResume(resumeInfo string) int {
- if resumeInfo == "" {
- return 0
- }
-
- s.resumeInfoMutex.Lock()
- defer s.resumeInfoMutex.Unlock()
-
- s.decodeResumeInfo(resumeInfo)
-
- // Because this scanner is multithreaded, it is possible that we have scanned a range of repositories
- // with some gaps of unlisted but completed repositories in between the ones in resumeInfo.
- // So we know repositories that have not finished scanning are the ones included in the resumeInfo,
- // and those that come after the last repository in the resumeInfo.
- // However, it is possible that a resumed scan does not include all or even any of the repos within the resumeInfo.
- // In this case, we must ensure we still scan all repos that come after the last found repo in the list.
- reposToScan := []string{}
- lastFoundRepoIndex := -1
- resumeRepoIndex := 0
- for i, repoURL := range s.repos {
- // If the repoURL is bigger than what we're looking for, move to the next one.
- if repoURL > s.resumeInfoSlice[resumeRepoIndex] {
- resumeRepoIndex++
- }
-
- // If we've found all of our repositories end the filter.
- if resumeRepoIndex == len(s.resumeInfoSlice) {
- break
- }
-
- // If the repoURL is the one we're looking for, add it and update the lastFoundRepoIndex.
- if repoURL == s.resumeInfoSlice[resumeRepoIndex] {
- lastFoundRepoIndex = i
- reposToScan = append(reposToScan, repoURL)
- }
- }
-
- // Append all repos after the last one we've found.
- reposToScan = append(reposToScan, s.repos[lastFoundRepoIndex+1:]...)
-
- progressOffsetCount := len(s.repos) - len(reposToScan)
- s.repos = reposToScan
- return progressOffsetCount
-}
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 0db9a5277..f658e2be4 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -373,152 +373,3 @@ func Test_setProgressCompleteWithRepo(t *testing.T) {
}
}
}
-
-func Test_removeRepoFromResumeInfo(t *testing.T) {
- tests := []struct {
- startingResumeInfoSlice []string
- repoURL string
- wantResumeInfoSlice []string
- }{
- {
- startingResumeInfoSlice: []string{"a", "b", "c"},
- repoURL: "a",
- wantResumeInfoSlice: []string{"b", "c"},
- },
- {
- startingResumeInfoSlice: []string{"a", "b", "c"},
- repoURL: "b",
- wantResumeInfoSlice: []string{"a", "c"},
- },
- { // This is the probably can't happen case of a repo not in the list.
- startingResumeInfoSlice: []string{"a", "b", "c"},
- repoURL: "not in the list",
- wantResumeInfoSlice: []string{"a", "b", "c"},
- },
- }
-
- logger := logrus.New()
- logger.Out = io.Discard
- s := &Source{
- repos: []string{},
- log: logger.WithField("no", "output"),
- }
-
- for _, tt := range tests {
- s.resumeInfoSlice = tt.startingResumeInfoSlice
- s.removeRepoFromResumeInfo(tt.repoURL)
- if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
- t.Errorf("s.removeRepoFromResumeInfo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
- }
- }
-}
-
-func Test_encodeResumeInfo(t *testing.T) {
- tests := []struct {
- startingResumeInfoSlice []string
- wantEncodedResumeInfo string
- }{
- {
- startingResumeInfoSlice: []string{"a", "b", "c"},
- wantEncodedResumeInfo: "a\tb\tc",
- },
- {
- startingResumeInfoSlice: []string{},
- wantEncodedResumeInfo: "",
- },
- }
-
- logger := logrus.New()
- logger.Out = io.Discard
- s := &Source{
- repos: []string{},
- log: logger.WithField("no", "output"),
- }
-
- for _, tt := range tests {
- s.resumeInfoSlice = tt.startingResumeInfoSlice
- gotEncodedResumeInfo := s.encodeResumeInfo()
- if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {
- t.Errorf("s.encodeResumeInfo() got: %q, want: %q", gotEncodedResumeInfo, tt.wantEncodedResumeInfo)
- }
- }
-}
-
-func Test_decodeResumeInfo(t *testing.T) {
- tests := []struct {
- resumeInfo string
- wantResumeInfoSlice []string
- }{
- {
- resumeInfo: "a\tb\tc",
- wantResumeInfoSlice: []string{"a", "b", "c"},
- },
- {
- resumeInfo: "",
- wantResumeInfoSlice: nil,
- },
- }
-
- for _, tt := range tests {
- s := &Source{}
- s.decodeResumeInfo(tt.resumeInfo)
- if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
- t.Errorf("s.decodeResumeInfo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
- }
- }
-}
-
-func Test_filterReposToResume(t *testing.T) {
- startingRepos := []string{"a", "b", "c", "d", "e", "f", "g"}
-
- tests := map[string]struct {
- resumeInfo string
- wantProgressOffsetCount int
- wantReposToScan []string
- }{
- "blank resume info": {
- resumeInfo: "",
- wantProgressOffsetCount: 0,
- wantReposToScan: startingRepos,
- },
- "starting repos": {
- resumeInfo: "a\tb",
- wantProgressOffsetCount: 0,
- wantReposToScan: startingRepos,
- },
- "early contiguous repos": {
- resumeInfo: "b\tc",
- wantProgressOffsetCount: 1,
- wantReposToScan: []string{"b", "c", "d", "e", "f", "g"},
- },
- "non-contiguous repos": {
- resumeInfo: "b\te",
- wantProgressOffsetCount: 3,
- wantReposToScan: []string{"b", "e", "f", "g"},
- },
- "no repos found in the repo list": {
- resumeInfo: "not\tthere",
- wantProgressOffsetCount: 0,
- wantReposToScan: startingRepos,
- },
- "only some repos in the list": {
- resumeInfo: "c\tnot\tthere",
- wantProgressOffsetCount: 2,
- wantReposToScan: []string{"c", "d", "e", "f", "g"},
- },
- }
-
- for name, tt := range tests {
- s := &Source{
- repos: startingRepos,
- }
-
- gotProgressOffsetCount := s.filterReposToResume(tt.resumeInfo)
- if gotProgressOffsetCount != tt.wantProgressOffsetCount {
- t.Errorf("s.filterReposToResume() name: %q got: %d, want: %d", name, gotProgressOffsetCount, tt.wantProgressOffsetCount)
- }
- if !reflect.DeepEqual(s.repos, tt.wantReposToScan) {
- t.Errorf("s.filterReposToResume() name: %q got: %v, want: %v", name, s.repos, tt.wantReposToScan)
- }
- }
-}
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index c306bc407..ef8878fb0 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -6,12 +6,10 @@ import (
"net/url"
"os"
"runtime"
+ "sort"
"strings"
"sync"
- "github.com/go-errors/errors"
- gogit "github.com/go-git/go-git/v5"
- log "github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -19,6 +17,10 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
+
+ "github.com/go-errors/errors"
+ gogit "github.com/go-git/go-git/v5"
+ log "github.com/sirupsen/logrus"
"github.com/xanzy/go-gitlab"
"golang.org/x/sync/semaphore"
"google.golang.org/protobuf/proto"
@@ -26,20 +28,22 @@ import (
)
type Source struct {
- name string
- sourceId int64
- jobId int64
- verify bool
- authMethod string
- user string
- password string
- token string
- url string
- repos []string
- git *git.Git
- aCtx context.Context
+ name string
+ sourceId int64
+ jobId int64
+ verify bool
+ authMethod string
+ user string
+ password string
+ token string
+ url string
+ repos []string
+ git *git.Git
+ aCtx context.Context
+ resumeInfoSlice []string
+ resumeInfoMutex sync.Mutex
+ jobSem *semaphore.Weighted
sources.Progress
- jobSem *semaphore.Weighted
}
// Ensure the Source satisfies the interface at compile time.
@@ -241,33 +245,35 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er
return projectList, nil
}
-func (s *Source) getRepos() ([]*url.URL, []error) {
- var validRepos []*url.URL
- var errs []error
- if len(s.repos) > 0 {
- for _, prj := range s.repos {
- repo, err := giturl.NormalizeGitlabRepo(prj)
- if err != nil {
- errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
- continue
- }
+func (s *Source) getRepos() ([]string, []error) {
+ if len(s.repos) == 0 {
+ return nil, nil
+ }
- // The repo normalization has already successfully parsed the URL at this point, so we can ignore the error.
- u, _ := url.ParseRequestURI(repo)
- validRepos = append(validRepos, u)
+ var validRepos []string
+ var errs []error
+ for _, prj := range s.repos {
+ repo, err := giturl.NormalizeGitlabRepo(prj)
+ if err != nil {
+ errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
+ continue
}
- return validRepos, errs
- }
- return nil, nil
+ validRepos = append(validRepos, repo)
+ }
+ return validRepos, errs
}
-func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk, repos []*url.URL) []error {
+func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) []error {
wg := sync.WaitGroup{}
var errs []error
var errsMut sync.Mutex
- for i, u := range repos {
+ // If there is resume information available, limit this scan to only the repos that still need scanning.
+ reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
+ s.repos = reposToScan
+
+ for i, repo := range s.repos {
if common.IsDone(ctx) {
// We are returning nil instead of the scanErrors slice here because
// we don't want to mark this scan as errored if we cancelled it.
@@ -278,19 +284,26 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
continue
}
wg.Add(1)
- go func(ctx context.Context, repoURL *url.URL, i int) {
+ go func(ctx context.Context, repoURL string, i int) {
defer s.jobSem.Release(1)
defer wg.Done()
- if len(repoURL.String()) == 0 {
+ if len(repoURL) == 0 {
return
}
- s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")
+
+ s.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)
+ // Ensure the repo is removed from the resume info after being scanned.
+ defer func(s *Source) {
+ s.resumeInfoMutex.Lock()
+ defer s.resumeInfoMutex.Unlock()
+ s.resumeInfoSlice = sources.RemoveRepoFromResumeInfo(s.resumeInfoSlice, repoURL)
+ }(s)
var path string
var repo *gogit.Repository
var err error
if s.authMethod == "UNAUTHENTICATED" {
- path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL.String())
+ path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL)
} else {
// If a username is not provided we need to use a default one in order to clone a private repo.
// Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user.
@@ -298,7 +311,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
if user == "" {
user = "placeholder"
}
- path, repo, err = git.CloneRepoUsingToken(s.token, repoURL.String(), user)
+ path, repo, err = git.CloneRepoUsingToken(s.token, repoURL, user)
}
defer os.RemoveAll(path)
if err != nil {
@@ -307,7 +320,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
errsMut.Unlock()
return
}
- log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(repos), repoURL.String())
+ log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repoURL)
err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
if err != nil {
errsMut.Lock()
@@ -315,8 +328,8 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
errsMut.Unlock()
return
}
- log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(repos), repoURL.String())
- }(ctx, u, i)
+ log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(s.repos), repoURL)
+ }(ctx, repo, i)
}
wg.Wait()
@@ -349,17 +362,23 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
// Turn projects into URLs for Git cloner.
for _, prj := range projects {
- u, err := url.Parse(prj.HTTPURLToRepo)
+ // Ensure the urls are valid before adding them to the repo list.
+ _, err := url.Parse(prj.HTTPURLToRepo)
if err != nil {
fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
}
- repos = append(repos, u)
+ repos = append(repos, prj.HTTPURLToRepo)
}
if repos == nil {
return errors.Errorf("unable to discover any repos")
}
}
- errs = s.scanRepos(ctx, chunksChan, repos)
+
+ s.repos = repos
+ // We must sort the repos so we can resume later if necessary.
+ sort.Strings(s.repos)
+
+ errs = s.scanRepos(ctx, chunksChan)
for _, err := range errs {
log.WithError(err).WithFields(
log.Fields{
@@ -386,3 +405,19 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {
}
return false
}
+
+// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
+func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
+ s.resumeInfoMutex.Lock()
+ defer s.resumeInfoMutex.Unlock()
+
+ // Add the repoURL to the resume info slice.
+ s.resumeInfoSlice = append(s.resumeInfoSlice, repoURL)
+ sort.Strings(s.resumeInfoSlice)
+
+ // Make the resume info string from the slice.
+ encodedResumeInfo := sources.EncodeResumeInfo(s.resumeInfoSlice)
+
+ // Add the offset to both the index and the repos to give the proper place and proper repo count.
+ s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
+}
diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.go
index c3183b146..c23087116 100644
--- a/pkg/sources/gitlab/gitlab_test.go
+++ b/pkg/sources/gitlab/gitlab_test.go
@@ -3,6 +3,8 @@ package gitlab
import (
"context"
"fmt"
+ "io"
+ "reflect"
"testing"
"github.com/kylelemons/godebug/pretty"
@@ -160,3 +162,90 @@ func TestSource_Scan(t *testing.T) {
})
}
}
+
+func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
+ tests := []struct {
+ startingResumeInfoSlice []string
+ repoURL string
+ wantResumeInfoSlice []string
+ }{
+ {
+ startingResumeInfoSlice: []string{},
+ repoURL: "a",
+ wantResumeInfoSlice: []string{"a"},
+ },
+ {
+ startingResumeInfoSlice: []string{"b"},
+ repoURL: "a",
+ wantResumeInfoSlice: []string{"a", "b"},
+ },
+ }
+
+ log.SetOutput(io.Discard)
+ s := &Source{repos: []string{}}
+
+ for _, tt := range tests {
+ s.resumeInfoSlice = tt.startingResumeInfoSlice
+ s.setProgressCompleteWithRepo(0, 0, tt.repoURL)
+ if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
+ t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
+ }
+ }
+}
+
+func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
+ repos := []string{"a", "b", "c", "d", "e"}
+ tests := map[string]struct {
+ repos []string
+ index int
+ offset int
+ wantPercentComplete int64
+ wantSectionsCompleted int32
+ wantSectionsRemaining int32
+ }{
+ "starting from the beginning, no offset": {
+ repos: repos,
+ index: 0,
+ offset: 0,
+ wantPercentComplete: 0,
+ wantSectionsCompleted: 0,
+ wantSectionsRemaining: 5,
+ },
+ "resume from the third, offset 2": {
+ repos: repos[2:],
+ index: 0,
+ offset: 2,
+ wantPercentComplete: 40,
+ wantSectionsCompleted: 2,
+ wantSectionsRemaining: 5,
+ },
+ "resume from the third, on last repo, offset 2": {
+ repos: repos[2:],
+ index: 2,
+ offset: 2,
+ wantPercentComplete: 80,
+ wantSectionsCompleted: 4,
+ wantSectionsRemaining: 5,
+ },
+ }
+
+ log.SetOutput(io.Discard)
+
+ for _, tt := range tests {
+ s := &Source{
+ repos: tt.repos,
+ }
+
+ s.setProgressCompleteWithRepo(tt.index, tt.offset, "")
+ gotProgress := s.GetProgress()
+ if gotProgress.PercentComplete != tt.wantPercentComplete {
+ t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete)
+ }
+ if gotProgress.SectionsCompleted != tt.wantSectionsCompleted {
+ t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted)
+ }
+ if gotProgress.SectionsRemaining != tt.wantSectionsRemaining {
+ t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining)
+ }
+ }
+}
diff --git a/pkg/sources/resume.go b/pkg/sources/resume.go
new file mode 100644
index 000000000..d085b5d69
--- /dev/null
+++ b/pkg/sources/resume.go
@@ -0,0 +1,84 @@
+package sources
+
+import (
+ "strings"
+
+ "github.com/sirupsen/logrus"
+)
+
+// RemoveRepoFromResumeInfo removes the repoURL from the resume info.
+func RemoveRepoFromResumeInfo(resumeRepos []string, repoURL string) []string {
+ index := -1
+ for i, repo := range resumeRepos {
+ if repoURL == repo {
+ index = i
+ break
+ }
+ }
+
+ if index == -1 {
+ // We should never be able to be here. But if we are, it means the resume info never had the repo added.
+ // So log the error and do nothing.
+ logrus.Errorf("repoURL (%q) not found in list of encode resume info: %v", repoURL, resumeRepos)
+ return resumeRepos
+ }
+
+ // This removes the element at the given index.
+ return append(resumeRepos[:index], resumeRepos[index+1:]...)
+}
+
+// FilterReposToResume filters the existing repos down to those that are included in the encoded resume info.
+// It returns the new slice of repos to be scanned.
+// It also returns the difference between the original length of the repos and the new length to use for progress reporting.
+// It is required that both the resumeInfo repos and the existing repos are sorted.
+func FilterReposToResume(repos []string, resumeInfo string) (reposToScan []string, progressOffsetCount int) {
+ if resumeInfo == "" {
+ return repos, 0
+ }
+
+ resumeInfoSlice := DecodeResumeInfo(resumeInfo)
+
+ // Because this scanner is multithreaded, it is possible that we have scanned a range of repositories
+ // with some gaps of unlisted but completed repositories in between the ones in resumeInfo.
+ // So we know repositories that have not finished scanning are the ones included in the resumeInfo,
+ // and those that come after the last repository in the resumeInfo.
+ // However, it is possible that a resumed scan does not include all or even any of the repos within the resumeInfo.
+ // In this case, we must ensure we still scan all repos that come after the last found repo in the list.
+ lastFoundRepoIndex := -1
+ resumeRepoIndex := 0
+ for i, repoURL := range repos {
+ // If the repoURL is bigger than what we're looking for, move to the next one.
+ if repoURL > resumeInfoSlice[resumeRepoIndex] {
+ resumeRepoIndex++
+ }
+
+ // If we've found all of our repositories end the filter.
+ if resumeRepoIndex == len(resumeInfoSlice) {
+ break
+ }
+
+ // If the repoURL is the one we're looking for, add it and update the lastFoundRepoIndex.
+ if repoURL == resumeInfoSlice[resumeRepoIndex] {
+ lastFoundRepoIndex = i
+ reposToScan = append(reposToScan, repoURL)
+ }
+ }
+
+ // Append all repos after the last one we've found.
+ reposToScan = append(reposToScan, repos[lastFoundRepoIndex+1:]...)
+ progressOffsetCount = len(repos) - len(reposToScan)
+ return
+}
+
+func EncodeResumeInfo(resumeInfoSlice []string) string {
+ return strings.Join(resumeInfoSlice, "\t")
+}
+
+func DecodeResumeInfo(resumeInfo string) []string {
+ // strings.Split will, for an empty string, return []string{""},
+ // which is an element, where as when there is no resume info we want an empty slice.
+ if resumeInfo == "" {
+ return nil
+ }
+ return strings.Split(resumeInfo, "\t")
+}
diff --git a/pkg/sources/resume_test.go b/pkg/sources/resume_test.go
new file mode 100644
index 000000000..56bda955d
--- /dev/null
+++ b/pkg/sources/resume_test.go
@@ -0,0 +1,143 @@
+package sources
+
+import (
+ "io"
+ "reflect"
+ "testing"
+
+ "github.com/sirupsen/logrus"
+)
+
+func TestRemoveRepoFromResumeInfo(t *testing.T) {
+ tests := []struct {
+ startingResumeInfoSlice []string
+ repoURL string
+ wantResumeInfoSlice []string
+ }{
+ {
+ startingResumeInfoSlice: []string{"a", "b", "c"},
+ repoURL: "a",
+ wantResumeInfoSlice: []string{"b", "c"},
+ },
+ {
+ startingResumeInfoSlice: []string{"a", "b", "c"},
+ repoURL: "b",
+ wantResumeInfoSlice: []string{"a", "c"},
+ },
+ { // This is the probably can't happen case of a repo not in the list.
+ startingResumeInfoSlice: []string{"a", "b", "c"},
+ repoURL: "not in the list",
+ wantResumeInfoSlice: []string{"a", "b", "c"},
+ },
+ }
+
+ // Prevent the error message from showing up.
+ logrus.SetOutput(io.Discard)
+
+ for _, tt := range tests {
+ gotResumeInfoSlice := RemoveRepoFromResumeInfo(tt.startingResumeInfoSlice, tt.repoURL)
+ if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) {
+ t.Errorf("RemoveRepoFromResumeInfo() got: %v, want: %v", gotResumeInfoSlice, tt.wantResumeInfoSlice)
+ }
+ }
+}
+
+func TestEncodeResumeInfo(t *testing.T) {
+ tests := []struct {
+ startingResumeInfoSlice []string
+ wantEncodedResumeInfo string
+ }{
+ {
+ startingResumeInfoSlice: []string{"a", "b", "c"},
+ wantEncodedResumeInfo: "a\tb\tc",
+ },
+ {
+ startingResumeInfoSlice: []string{},
+ wantEncodedResumeInfo: "",
+ },
+ }
+
+ logger := logrus.New()
+ logger.Out = io.Discard
+
+ for _, tt := range tests {
+ gotEncodedResumeInfo := EncodeResumeInfo(tt.startingResumeInfoSlice)
+ if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {
+ t.Errorf("EncodeResumeInfo() got: %q, want: %q", gotEncodedResumeInfo, tt.wantEncodedResumeInfo)
+ }
+ }
+}
+
+func Test_decodeResumeInfo(t *testing.T) {
+ tests := []struct {
+ resumeInfo string
+ wantResumeInfoSlice []string
+ }{
+ {
+ resumeInfo: "a\tb\tc",
+ wantResumeInfoSlice: []string{"a", "b", "c"},
+ },
+ {
+ resumeInfo: "",
+ wantResumeInfoSlice: nil,
+ },
+ }
+
+ for _, tt := range tests {
+ gotResumeInfoSlice := DecodeResumeInfo(tt.resumeInfo)
+ if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) {
+ t.Errorf("DecodeResumeInfo() got: %v, want: %v", gotResumeInfoSlice, tt.wantResumeInfoSlice)
+ }
+ }
+}
+
+func Test_filterReposToResume(t *testing.T) {
+ startingRepos := []string{"a", "b", "c", "d", "e", "f", "g"}
+
+ tests := map[string]struct {
+ resumeInfo string
+ wantProgressOffsetCount int
+ wantReposToScan []string
+ }{
+ "blank resume info": {
+ resumeInfo: "",
+ wantProgressOffsetCount: 0,
+ wantReposToScan: startingRepos,
+ },
+ "starting repos": {
+ resumeInfo: "a\tb",
+ wantProgressOffsetCount: 0,
+ wantReposToScan: startingRepos,
+ },
+ "early contiguous repos": {
+ resumeInfo: "b\tc",
+ wantProgressOffsetCount: 1,
+ wantReposToScan: []string{"b", "c", "d", "e", "f", "g"},
+ },
+ "non-contiguous repos": {
+ resumeInfo: "b\te",
+ wantProgressOffsetCount: 3,
+ wantReposToScan: []string{"b", "e", "f", "g"},
+ },
+ "no repos found in the repo list": {
+ resumeInfo: "not\tthere",
+ wantProgressOffsetCount: 0,
+ wantReposToScan: startingRepos,
+ },
+ "only some repos in the list": {
+ resumeInfo: "c\tnot\tthere",
+ wantProgressOffsetCount: 2,
+ wantReposToScan: []string{"c", "d", "e", "f", "g"},
+ },
+ }
+
+ for name, tt := range tests {
+ gotReposToScan, gotProgressOffsetCount := FilterReposToResume(startingRepos, tt.resumeInfo)
+ if !reflect.DeepEqual(gotReposToScan, tt.wantReposToScan) {
+ t.Errorf("FilterReposToResume() name: %q got: %v, want: %v", name, gotReposToScan, tt.wantReposToScan)
+ }
+ if gotProgressOffsetCount != tt.wantProgressOffsetCount {
+ t.Errorf("FilterReposToResume() name: %q got: %d, want: %d", name, gotProgressOffsetCount, tt.wantProgressOffsetCount)
+ }
+ }
+}