Skip to content

Find, verify, and analyze leaked credentials

Allow github to resume from encoded resume info (#601)

trufflesteeevefd79a367f100
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 6e7055947..6feae4f48 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -3,11 +3,11 @@ package github
import (
"context"
"fmt"
- "math/rand"
"net/http"
"os"
"regexp"
"runtime"
+ "sort"
"strconv"
"strings"
"sync"
@@ -36,21 +36,23 @@ import (
)
type Source struct {
- name string
- sourceID int64
- jobID int64
- verify bool
- repos []string
- orgs []string
- members []string
- git *git.Git
- httpClient *http.Client
- aCtx context.Context
+ name string
+ sourceID int64
+ jobID int64
+ verify bool
+ repos []string
+ orgs []string
+ members []string
+ git *git.Git
+ httpClient *http.Client
+ aCtx context.Context
+ log *log.Entry
+ token string
+ conn *sourcespb.GitHub
+ jobSem *semaphore.Weighted
+ resumeInfoSlice []string
+ resumeInfoMutex *sync.Mutex
sources.Progress
- log *log.Entry
- token string
- conn *sourcespb.GitHub
- jobSem *semaphore.Weighted
}
// Ensure the Source satisfies the interface at compile time
@@ -323,11 +325,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
s.normalizeRepos(ctx, apiClient)
- if _, ok := os.LookupEnv("DO_NOT_RANDOMIZE"); !ok {
- // Randomize channel scan order on each scan
- rand.Seed(time.Now().UnixNano())
- rand.Shuffle(len(s.repos), func(i, j int) { s.repos[i], s.repos[j] = s.repos[j], s.repos[i] })
- }
+ // We must sort the repos so we can resume later if necessary.
+ sort.Strings(s.repos)
return s.scan(ctx, installationClient, chunksChan)
}
@@ -347,6 +346,9 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
}
}
+ // If there is resume information available, limit this scan to only the repos that still need scanning.
+ progressIndexOffset := s.filterReposToResume(s.GetProgress().EncodedResumeInfo)
+
for i, repoURL := range s.repos {
if err := s.jobSem.Acquire(ctx, 1); err != nil {
// Acquire blocks until it can acquire the semaphore or returns an
@@ -360,7 +362,9 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
defer s.jobSem.Release(1)
defer wg.Done()
- s.SetProgressComplete(i, len(s.repos), fmt.Sprintf("Repo: %s", repoURL), "")
+ s.setProgressCompleteWithRepo(i+progressIndexOffset, repoURL)
+ // Ensure the repo is removed from the resume info after being scanned.
+ defer s.removeRepoFromResumeInfo(repoURL)
if !strings.HasSuffix(repoURL, ".git") {
return
@@ -728,3 +732,102 @@ func (s *Source) normalizeRepos(ctx context.Context, apiClient *github.Client) {
s.repos = append(s.repos, key)
}
}
+
+// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
+func (s *Source) setProgressCompleteWithRepo(index int, repoURL string) {
+ s.resumeInfoMutex.Lock()
+ defer s.resumeInfoMutex.Unlock()
+
+ // Add the repoURL to the resume info slice.
+ s.resumeInfoSlice = append(s.resumeInfoSlice, repoURL)
+ sort.Strings(s.resumeInfoSlice)
+
+ // Make the resume info string from the slice.
+ encodedResumeInfo := s.encodeResumeInfo()
+
+ s.SetProgressComplete(index, len(s.repos), fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
+}
+
+// removeRepoFromResumeInfo removes the repoURL from the resume info.
+func (s *Source) removeRepoFromResumeInfo(repoURL string) {
+ s.resumeInfoMutex.Lock()
+ defer s.resumeInfoMutex.Unlock()
+
+ index := -1
+ for i, repo := range s.resumeInfoSlice {
+ if repoURL == repo {
+ index = i
+ }
+ }
+
+ if index == -1 {
+ // We should never be able to be here. But if we are, it means the resume info never had the repo added.
+ // So log the error and do nothing.
+ s.log.Errorf("repoURL (%q) not found in list of encode resume info: %q", repoURL, s.EncodedResumeInfo)
+ return
+ }
+
+ // This removes the element at the given index.
+ s.resumeInfoSlice = append(s.resumeInfoSlice[:index], s.resumeInfoSlice[index+1:]...)
+}
+
+func (s *Source) encodeResumeInfo() string {
+ return strings.Join(s.resumeInfoSlice, "\t")
+}
+
+func (s *Source) decodeResumeInfo(resumeInfo string) {
+ // strings.Split will, for an empty string, return []string{""},
+ // which is an element, where as when there is no resume info we want an empty slice.
+ if resumeInfo == "" {
+ return
+ }
+ s.resumeInfoSlice = strings.Split(resumeInfo, "\t")
+}
+
+// filterReposToResume filters the existing repos down to those that are included in the encoded resume info.
+// It also returns the difference between the original length of the repos and the new length to use for progress reporting.
+// It is required that both the resumeInfo repos and the existing repos in s.repos are sorted.
+func (s *Source) filterReposToResume(resumeInfo string) int {
+ if resumeInfo == "" {
+ return 0
+ }
+
+ s.resumeInfoMutex.Lock()
+ defer s.resumeInfoMutex.Unlock()
+
+ s.decodeResumeInfo(resumeInfo)
+
+ // Because this scanner is multithreaded, it is possible that we have scanned a range of repositories
+ // with some gaps of unlisted but completed repositories in between the ones in resumeInfo.
+ // So we know repositories that have not finished scanning are the ones included in the resumeInfo,
+ // and those that come after the last repository in the resumeInfo.
+ // However, it is possible that a resumed scan does not include all or even any of the repos within the resumeInfo.
+ // In this case, we must ensure we still scan all repos that come after the last found repo in the list.
+ reposToScan := []string{}
+ lastFoundRepoIndex := -1
+ resumeRepoIndex := 0
+ for i, repoURL := range s.repos {
+ // If the repoURL is bigger than what we're looking for, move to the next one.
+ if repoURL > s.resumeInfoSlice[resumeRepoIndex] {
+ resumeRepoIndex++
+ }
+
+ // If we've found all of our repositories end the filter.
+ if resumeRepoIndex == len(s.resumeInfoSlice) {
+ break
+ }
+
+ // If the repoURL is the one we're looking for, add it and update the lastFoundRepoIndex.
+ if repoURL == s.resumeInfoSlice[resumeRepoIndex] {
+ lastFoundRepoIndex = i
+ reposToScan = append(reposToScan, repoURL)
+ }
+ }
+
+ // Append all repos after the last one we've found.
+ reposToScan = append(reposToScan, s.repos[lastFoundRepoIndex+1:]...)
+
+ progressOffsetCount := len(s.repos) - len(reposToScan)
+ s.repos = reposToScan
+ return progressOffsetCount
+}
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 35988a785..15d6bb99e 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -8,13 +8,17 @@ import (
"crypto/x509"
"encoding/pem"
"fmt"
+ "io"
"net/http"
+ "reflect"
"sort"
"strconv"
+ "sync"
"testing"
"time"
"github.com/google/go-github/v42/github"
+ "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -336,3 +340,191 @@ func TestEnumerateWithApp(t *testing.T) {
assert.True(t, gock.IsDone())
}
+
+// This only tests the resume info slice portion of setProgressCompleteWithRepo.
+func Test_setProgressCompleteWithRepo(t *testing.T) {
+ tests := []struct {
+ startingResumeInfoSlice []string
+ repoURL string
+ wantResumeInfoSlice []string
+ }{
+ {
+ startingResumeInfoSlice: []string{},
+ repoURL: "a",
+ wantResumeInfoSlice: []string{"a"},
+ },
+ {
+ startingResumeInfoSlice: []string{"b"},
+ repoURL: "a",
+ wantResumeInfoSlice: []string{"a", "b"},
+ },
+ }
+
+ logger := logrus.New()
+ logger.Out = io.Discard
+ s := &Source{
+ repos: []string{},
+ log: logger.WithField("no", "output"),
+ resumeInfoMutex: &sync.Mutex{},
+ }
+
+ for _, tt := range tests {
+ s.resumeInfoSlice = tt.startingResumeInfoSlice
+ s.setProgressCompleteWithRepo(0, tt.repoURL)
+ if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
+ t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
+ }
+ }
+}
+
+func Test_removeRepoFromResumeInfo(t *testing.T) {
+ tests := []struct {
+ startingResumeInfoSlice []string
+ repoURL string
+ wantResumeInfoSlice []string
+ }{
+ {
+ startingResumeInfoSlice: []string{"a", "b", "c"},
+ repoURL: "a",
+ wantResumeInfoSlice: []string{"b", "c"},
+ },
+ {
+ startingResumeInfoSlice: []string{"a", "b", "c"},
+ repoURL: "b",
+ wantResumeInfoSlice: []string{"a", "c"},
+ },
+ { // This is the probably can't happen case of a repo not in the list.
+ startingResumeInfoSlice: []string{"a", "b", "c"},
+ repoURL: "not in the list",
+ wantResumeInfoSlice: []string{"a", "b", "c"},
+ },
+ }
+
+ logger := logrus.New()
+ logger.Out = io.Discard
+ s := &Source{
+ repos: []string{},
+ log: logger.WithField("no", "output"),
+ resumeInfoMutex: &sync.Mutex{},
+ }
+
+ for _, tt := range tests {
+ s.resumeInfoSlice = tt.startingResumeInfoSlice
+ s.removeRepoFromResumeInfo(tt.repoURL)
+ if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
+ t.Errorf("s.removeRepoFromResumeInfo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
+ }
+ }
+}
+
+func Test_encodeResumeInfo(t *testing.T) {
+ tests := []struct {
+ startingResumeInfoSlice []string
+ wantEncodedResumeInfo string
+ }{
+ {
+ startingResumeInfoSlice: []string{"a", "b", "c"},
+ wantEncodedResumeInfo: "a\tb\tc",
+ },
+ {
+ startingResumeInfoSlice: []string{},
+ wantEncodedResumeInfo: "",
+ },
+ }
+
+ logger := logrus.New()
+ logger.Out = io.Discard
+ s := &Source{
+ repos: []string{},
+ log: logger.WithField("no", "output"),
+ resumeInfoMutex: &sync.Mutex{},
+ }
+
+ for _, tt := range tests {
+ s.resumeInfoSlice = tt.startingResumeInfoSlice
+ gotEncodedResumeInfo := s.encodeResumeInfo()
+ if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {
+ t.Errorf("s.encodeResumeInfo() got: %q, want: %q", gotEncodedResumeInfo, tt.wantEncodedResumeInfo)
+ }
+ }
+}
+
+func Test_decodeResumeInfo(t *testing.T) {
+ tests := []struct {
+ resumeInfo string
+ wantResumeInfoSlice []string
+ }{
+ {
+ resumeInfo: "a\tb\tc",
+ wantResumeInfoSlice: []string{"a", "b", "c"},
+ },
+ {
+ resumeInfo: "",
+ wantResumeInfoSlice: nil,
+ },
+ }
+
+ for _, tt := range tests {
+ s := &Source{}
+ s.decodeResumeInfo(tt.resumeInfo)
+ if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
+ t.Errorf("s.decodeResumeInfo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
+ }
+ }
+}
+
+func Test_filterReposToResume(t *testing.T) {
+ startingRepos := []string{"a", "b", "c", "d", "e", "f", "g"}
+
+ tests := map[string]struct {
+ resumeInfo string
+ wantProgressOffsetCount int
+ wantReposToScan []string
+ }{
+ "blank resume info": {
+ resumeInfo: "",
+ wantProgressOffsetCount: 0,
+ wantReposToScan: startingRepos,
+ },
+ "starting repos": {
+ resumeInfo: "a\tb",
+ wantProgressOffsetCount: 0,
+ wantReposToScan: startingRepos,
+ },
+ "early contiguous repos": {
+ resumeInfo: "b\tc",
+ wantProgressOffsetCount: 1,
+ wantReposToScan: []string{"b", "c", "d", "e", "f", "g"},
+ },
+ "non-contiguous repos": {
+ resumeInfo: "b\te",
+ wantProgressOffsetCount: 3,
+ wantReposToScan: []string{"b", "e", "f", "g"},
+ },
+ "no repos found in the repo list": {
+ resumeInfo: "not\tthere",
+ wantProgressOffsetCount: 0,
+ wantReposToScan: startingRepos,
+ },
+ "only some repos in the list": {
+ resumeInfo: "c\tnot\tthere",
+ wantProgressOffsetCount: 2,
+ wantReposToScan: []string{"c", "d", "e", "f", "g"},
+ },
+ }
+
+ for name, tt := range tests {
+ s := &Source{
+ repos: startingRepos,
+ resumeInfoMutex: &sync.Mutex{},
+ }
+
+ gotProgressOffsetCount := s.filterReposToResume(tt.resumeInfo)
+ if gotProgressOffsetCount != tt.wantProgressOffsetCount {
+ t.Errorf("s.filterReposToResume() name: %q got: %d, want: %d", name, gotProgressOffsetCount, tt.wantProgressOffsetCount)
+ }
+ if !reflect.DeepEqual(s.repos, tt.wantReposToScan) {
+ t.Errorf("s.filterReposToResume() name: %q got: %v, want: %v", name, s.repos, tt.wantReposToScan)
+ }
+ }
+}