Skip to content

Find, verify, and analyze leaked credentials

Add placeholder for encoded resume info in SetProgressComplete

steeevea770f643df5b
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go
index bb5ff064e..9349a1aa7 100644
--- a/pkg/sources/filesystem/filesystem.go
+++ b/pkg/sources/filesystem/filesystem.go
@@ -80,7 +80,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
for i, path := range s.paths {
- s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", path))
+ s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", path), "")
cleanPath := filepath.Clean(path)
done := false
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 1911c7f41..a078b71bf 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -127,7 +127,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
token := cred.BasicAuth.Password
for i, repoURI := range s.conn.Repositories {
- s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI))
+ s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
if len(repoURI) == 0 {
continue
}
@@ -143,7 +143,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
case *sourcespb.Git_Unauthenticated:
for i, repoURI := range s.conn.Repositories {
- s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI))
+ s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", repoURI), "")
if len(repoURI) == 0 {
continue
}
@@ -162,7 +162,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
for i, u := range s.conn.Directories {
- s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", u))
+ s.SetProgressComplete(i, len(s.conn.Repositories), fmt.Sprintf("Repo: %s", u), "")
if len(u) == 0 {
continue
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 231171c3d..1e812972d 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -304,7 +304,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
defer s.jobSem.Release(1)
defer wg.Done()
- s.SetProgressComplete(i, len(s.repos), fmt.Sprintf("Repo: %s", repoURL))
+ s.SetProgressComplete(i, len(s.repos), fmt.Sprintf("Repo: %s", repoURL), "")
if !strings.HasSuffix(repoURL, ".git") {
return
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index f3643621e..9f7cdcb3a 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -279,7 +279,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
if len(repoURL.String()) == 0 {
return
}
- s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL))
+ s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")
path, repo, err := git.CloneRepoUsingUnauthenticated(repoURL.String())
defer os.RemoveAll(path)
@@ -319,7 +319,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,
if len(repoURL.String()) == 0 {
return
}
- s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL))
+ s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")
path, repo, err := git.CloneRepoUsingToken(s.token, repoURL.String(), s.user)
defer os.RemoveAll(path)
diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go
index 8e5b78bdb..9e64b1b12 100644
--- a/pkg/sources/s3/s3.go
+++ b/pkg/sources/s3/s3.go
@@ -138,7 +138,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
return nil
}
- s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket))
+ s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
s.log.Debugf("Scanning bucket: %s", bucket)
region, err := s3manager.GetBucketRegionWithClient(context.Background(), client, bucket)
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index 234d9a5a7..eaecae20c 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -47,16 +47,22 @@ type Progress struct {
mut sync.Mutex
PercentComplete int64
Message string
+ EncodedResumeInfo string
SectionsCompleted int32
SectionsRemaining int32
}
-// SetProgressComplete sets job complete percentage based on passed in scope array of highest level objects. i is the current iteration in the loop of target scope, scope should be len(scoped_items)
-func (p *Progress) SetProgressComplete(i, scope int, message string) {
+// SetProgressComplete sets job progress information for a running job based on the highest level objects in the source.
+// i is the current iteration in the loop of target scope
+// scope should be the len(scopedItems)
+// message is the public facing user information about the current progress
+// encodedResumeInfo is an optional string representing any information necessary to resume the job if interrupted
+func (p *Progress) SetProgressComplete(i, scope int, message, encodedResumeInfo string) {
p.mut.Lock()
defer p.mut.Unlock()
p.Message = message
+ p.EncodedResumeInfo = encodedResumeInfo
p.SectionsCompleted = int32(i)
p.SectionsRemaining = int32(scope)
p.PercentComplete = int64((float64(i) / float64(scope)) * 100)