Skip to content

Find, verify, and analyze leaked credentials

Delete progress tracking from GCS source (#1190)

ahravac19de75bfa3
diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.go
index 9735479c3..aa3a5bfd1 100644
--- a/pkg/sources/gcs/gcs.go
+++ b/pkg/sources/gcs/gcs.go
@@ -1,11 +1,9 @@
package gcs
import (
- "encoding/json"
"fmt"
"io"
"os"
- "sort"
"strconv"
"sync"
@@ -22,23 +20,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
-// Source represents a GCS source.
-type Source struct {
- name string
- jobId int64
- sourceId int64
- concurrency int
- verify bool
-
- gcsManager objectManager
- stats *attributes
- log logr.Logger
- chunksCh chan *sources.Chunk
-
- mu sync.Mutex
- sources.Progress // not safe for concurrent access.
-}
-
// Ensure the Source satisfies the interface at compile time.
var _ sources.Source = (*Source)(nil)
@@ -58,6 +39,22 @@ func (s *Source) JobID() int64 {
return s.jobId
}
+// Source represents a GCS source.
+type Source struct {
+ name string
+ jobId int64
+ sourceId int64
+ concurrency int
+ verify bool
+
+ gcsManager objectManager
+ stats *attributes
+ log logr.Logger
+ chunksCh chan *sources.Chunk
+
+ sources.Progress
+}
+
// Init returns an initialized GCS source.
func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
s.log = aCtx.Logger()
@@ -74,6 +71,25 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
+ gcsManager, err := configureGCSManager(aCtx, &conn, concurrency)
+ if err != nil {
+ return err
+ }
+ s.gcsManager = gcsManager
+
+ s.log.V(2).Info("enumerating buckets and objects")
+ if err := s.enumerate(aCtx); err != nil {
+ return fmt.Errorf("error enumerating buckets and objects: %w", err)
+ }
+
+ return nil
+}
+
+func configureGCSManager(aCtx context.Context, conn *sourcespb.GCS, concurrency int) (*gcsManager, error) {
+ if conn == nil {
+ return nil, fmt.Errorf("GCS connection is nil, cannot configure GCS manager")
+ }
+
var gcsManagerAuthOption gcsManagerOption
switch conn.Credential.(type) {
@@ -82,7 +98,7 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6
case *sourcespb.GCS_ServiceAccountFile:
b, err := os.ReadFile(conn.GetServiceAccountFile())
if err != nil {
- return fmt.Errorf("error reading GCS JSON Service Account file: %w", err)
+ return nil, fmt.Errorf("error reading GCS JSON Service Account file: %w", err)
}
gcsManagerAuthOption = withJSONServiceAccount(aCtx, b)
case *sourcespb.GCS_JsonServiceAccount:
@@ -92,39 +108,28 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6
case *sourcespb.GCS_Unauthenticated:
gcsManagerAuthOption = withoutAuthentication()
default:
- return fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)
+ return nil, fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)
}
- resume, err := setResumeBucketOffset(s.Progress.EncodedResumeInfo)
- if err != nil {
- return fmt.Errorf("error setting resume info: %w", err)
- }
-
gcsManagerOpts := []gcsManagerOption{
withConcurrency(concurrency),
- withBucketOffsets(resume),
withMaxObjectSize(conn.MaxObjectSize),
gcsManagerAuthOption,
}
- if setGCSManagerBucketOptions(&conn) != nil {
- gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(&conn))
+ if setGCSManagerBucketOptions(conn) != nil {
+ gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(conn))
}
- if setGCSManagerObjectOptions(&conn) != nil {
- gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(&conn))
+ if setGCSManagerObjectOptions(conn) != nil {
+ gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(conn))
}
- if s.gcsManager, err = newGCSManager(conn.ProjectId, gcsManagerOpts...); err != nil {
- return fmt.Errorf("error creating GCS manager: %w", err)
- }
-
- if len(resume) > 0 {
- s.log.V(2).Info("resuming from previous run")
- return nil
+ gcsManager, err := newGCSManager(conn.ProjectId, gcsManagerOpts...)
+ if err != nil {
+ return nil, fmt.Errorf("error creating GCS manager: %w", err)
}
- s.log.V(2).Info("new run, enumerating buckets and objects")
- return s.enumerate(aCtx)
+ return gcsManager, nil
}
func setGCSManagerBucketOptions(conn *sourcespb.GCS) gcsManagerOption {
@@ -148,145 +153,7 @@ func setGCSManagerOptions(include, exclude []string, includeFn, excludeFn func([
return nil
}
-type progressInfo struct {
- processedBucketCnt,
- totalBucketsCnt,
- processedObjectsCnt,
- totalObjectsCnt int
-
- // bucketObjects tracks processing and processed objects for each bucket.
- bucketObjects map[string]*objectsProgress
-}
-
-// objectsProgress keeps track of the progress of processing objects in a bucket.
-// It is marshalled/unmarshalled to/from a string as part of the Source's Progress.
-type objectsProgress struct {
- // IsBuckedProcessed is true if all objects in the bucket have been processed.
- IsBucketProcessed bool
- // ProcessedCnt is the number of objects that have been processed.
- ProcessedCnt,
- // TotalCnt is the total number of objects in the bucket.
- TotalCnt int
- // Processed is the last object that was fully processed.
- Processed string
- // Processing are the objects that were not fully processed.
- Processing map[string]struct{}
-}
-
-func newObjectsProgress(cnt int) *objectsProgress {
- return &objectsProgress{TotalCnt: cnt, Processing: map[string]struct{}{}}
-}
-
-// newProgressInfo constructs a progressInfo from the Source's Progress information.
-func newProgressInfo(ctx context.Context, s *sources.Progress) (*progressInfo, error) {
- totalObjCnt := s.SectionsRemaining
- processedObjCnt := s.SectionsCompleted
- info := &progressInfo{
- totalObjectsCnt: int(totalObjCnt),
- processedObjectsCnt: int(processedObjCnt),
- }
-
- encodeResume := s.EncodedResumeInfo
- if encodeResume == "" {
- info.bucketObjects = make(map[string]*objectsProgress)
- return info, nil
- }
-
- var m map[string]*objectsProgress
- if err := json.Unmarshal([]byte(encodeResume), &m); err != nil {
- ctx.Logger().V(1).Info("error unmarshalling progress info", "error", err)
- return info, nil
- }
- info.bucketObjects = m
-
- return info, nil
-}
-
-type progressStateFn func(string, string, *progressInfo)
-
-func (p *progressInfo) setProcessStatus(obj object, fn progressStateFn) {
- fn(obj.bucket, obj.name, p)
-}
-
-func setProcessingBucketObject(bucket, obj string, progress *progressInfo) {
- progress.processing(bucket, obj)
-}
-
-func (p *progressInfo) processing(bkt, obj string) {
- p.bucketObjects[bkt].Processing[obj] = struct{}{}
-}
-
-func setProcessedBucketObject(bucket, obj string, progress *progressInfo) {
- progress.processed(bucket, obj)
-}
-
-func (p *progressInfo) processed(bkt, obj string) {
- // Remove the object from the processing list and increment the processed count.
- delete(p.bucketObjects[bkt].Processing, obj)
-
- p.bucketObjects[bkt].ProcessedCnt++
- // If all objects in the bucket have been processed, mark the bucket as processed.
- if p.bucketObjects[bkt].ProcessedCnt == p.bucketObjects[bkt].TotalCnt {
- p.bucketObjects[bkt].IsBucketProcessed = true
- p.processedBucketCnt++
- }
- p.processedObjectsCnt++
-
- // If the object is not greater (lexicographically) than the last processed object, we can skip it.
- // This ensures we keep the greatest object name as the last processed object.
- if obj < p.bucketObjects[bkt].Processed {
- return
- }
- p.bucketObjects[bkt].Processed = obj
-}
-
-func setResumeBucketOffset(s string) (map[string]offsetInfo, error) {
- resumeInfo := progressInfo{}
- if s != "" {
- if err := json.Unmarshal([]byte(s), &resumeInfo.bucketObjects); err != nil {
- return nil, fmt.Errorf("error unmarshalling resume info: %w", err)
- }
- }
-
- return calcBktOffset(resumeInfo.bucketObjects)
-}
-
-// In order to calculate the bucket offset, we need to know the last object
-// that was processed for each bucket, as well as any objects that were processing.
-// If the bucket was fully processed, we don't need to set an offset.
-// If there were no objects processing, we can just use the last object that was processed.
-// If there were objects processing, we need to find the first object (lexicographically)
-// that was processing, as that is the next object that needs to be processed.
-func calcBktOffset(resumeInfo map[string]*objectsProgress) (map[string]offsetInfo, error) {
- bucketOffset := make(map[string]offsetInfo, len(resumeInfo))
- for objName, progress := range resumeInfo {
- info := offsetInfo{}
- if progress.IsBucketProcessed {
- info.isBucketProcessed = true
- bucketOffset[objName] = info
- continue
- }
- if len(progress.Processing) == 0 {
- info.lastProcessedObject = progress.Processed
- bucketOffset[objName] = info
- continue
- }
-
- processing := make([]string, 0, len(progress.Processing))
- for k := range progress.Processing {
- processing = append(processing, k)
- }
- sort.Strings(processing)
- info.lastProcessedObject = processing[0]
- bucketOffset[objName] = info
- }
-
- return bucketOffset, nil
-}
-
-// enumerate all the objects and buckets in the source and use the results to
-// set the progress information. This will be used track progression of the scan,
-// and to resume the scan if it is interrupted.
+// enumerate all the objects and buckets in the source.
func (s *Source) enumerate(ctx context.Context) error {
stats, err := s.gcsManager.attributes(ctx)
if err != nil {
@@ -294,18 +161,7 @@ func (s *Source) enumerate(ctx context.Context) error {
}
s.stats = stats
- info := progressInfo{
- totalBucketsCnt: int(s.stats.numBuckets),
- totalObjectsCnt: int(s.stats.numObjects),
- }
-
- progress := make(map[string]*objectsProgress, len(s.stats.bucketObjects))
- for k, v := range s.stats.bucketObjects {
- progress[k] = newObjectsProgress(int(v))
- }
- info.bucketObjects = progress
-
- return s.setProgress(ctx, &info, fmt.Sprintf("enumerated %d buckets, %d objects", s.stats.numBuckets, s.stats.numObjects))
+ return nil
}
// Chunks emits chunks of bytes over a channel.
@@ -316,11 +172,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
s.chunksCh = chunksChan
- progress, err := newProgressInfo(ctx, &s.Progress)
- if err != nil {
- return fmt.Errorf("error constructing progress info: %w", err)
- }
-
var wg sync.WaitGroup
for obj := range objectCh {
obj := obj
@@ -330,54 +181,18 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
continue
}
- if err := s.startProcessing(ctx, progress, o); err != nil {
- ctx.Logger().V(1).Info("GCS source error starting to process objects", "error", err)
- continue
- }
wg.Add(1)
- go func(obj object, resume *progressInfo) {
+ go func(obj object) {
defer wg.Done()
if err := s.processObject(ctx, o); err != nil {
ctx.Logger().V(1).Info("error setting start progress progress", "name", o.name, "error", err)
return
}
- if err := s.endProcessing(ctx, resume, o); err != nil {
- ctx.Logger().V(1).Info("error setting end progress progress", "name", o.name, "error", err)
- }
- }(o, progress)
+ }(o)
}
wg.Wait()
- ctx.Logger().Info("GCS source finished processing", "name", s.name)
- return nil
-}
-
-func (s *Source) startProcessing(ctx context.Context, progress *progressInfo, o object) error {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- progress.setProcessStatus(o, setProcessingBucketObject)
- return s.setProgress(ctx, progress, fmt.Sprintf("GCS source beginning to process object %s in bucket %s", o.name, o.bucket))
-}
-
-func (s *Source) endProcessing(ctx context.Context, progress *progressInfo, o object) error {
- s.mu.Lock()
- defer s.mu.Unlock()
-
- progress.setProcessStatus(o, setProcessedBucketObject)
- return s.setProgress(ctx, progress, fmt.Sprintf("GCS source finished processing object %s in bucket %s", o.name, o.bucket))
-}
-
-func (s *Source) setProgress(ctx context.Context, progress *progressInfo, msg string) error {
- ctx.Logger().V(5).Info(msg)
-
- b, err := json.Marshal(progress.bucketObjects)
- if err != nil {
- return fmt.Errorf("error marshalling resume info: %w", err)
- }
- s.SetProgressComplete(progress.processedObjectsCnt, progress.totalObjectsCnt, msg, string(b))
-
return nil
}
diff --git a/pkg/sources/gcs/gcs_test.go b/pkg/sources/gcs/gcs_test.go
index 7a893ee1b..871ab9eca 100644
--- a/pkg/sources/gcs/gcs_test.go
+++ b/pkg/sources/gcs/gcs_test.go
@@ -1,7 +1,6 @@
package gcs
import (
- "encoding/json"
"fmt"
"io"
"sort"
@@ -32,7 +31,7 @@ func TestSourceInit(t *testing.T) {
source, conn := createTestSource(&sourcespb.GCS{
ProjectId: testProjectID,
IncludeBuckets: []string{
- "bucket1",
+ publicBucket,
},
ExcludeBuckets: []string{
perfTestBucketGlob,
@@ -40,22 +39,15 @@ func TestSourceInit(t *testing.T) {
ExcludeObjects: []string{
"object1",
},
- Credential: &sourcespb.GCS_ApiKey{
- ApiKey: testAPIKey,
- },
+ Credential: &sourcespb.GCS_Unauthenticated{},
})
- // Set the EncodeResumeInfo to prevent enumeration.
- b, err := json.Marshal(map[string]objectsProgress{"other": {}})
- assert.Nil(t, err)
- source.Progress.EncodedResumeInfo = string(b)
-
- err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+ err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)
assert.Nil(t, err)
assert.NotNil(t, source.gcsManager)
}
-func TestSourceInit_Conn(t *testing.T) {
+func TestConfigureGCSManager(t *testing.T) {
testCases := []struct {
name string
conn *sourcespb.GCS
@@ -78,7 +70,6 @@ func TestSourceInit_Conn(t *testing.T) {
want: &gcsManager{
projectID: testProjectID,
excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
- hasEnumerated: true,
},
},
{
@@ -96,7 +87,6 @@ func TestSourceInit_Conn(t *testing.T) {
want: &gcsManager{
projectID: testProjectID,
includeBuckets: map[string]struct{}{"bucket1": {}},
- hasEnumerated: true,
},
},
{
@@ -118,27 +108,20 @@ func TestSourceInit_Conn(t *testing.T) {
projectID: testProjectID,
includeObjects: map[string]struct{}{"object1": {}},
excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
- hasEnumerated: true,
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
- source, conn := createTestSource(tc.conn)
- // Set the EncodeResumeInfo to prevent enumeration.
- b, err := json.Marshal(map[string]objectsProgress{"other": {}})
- assert.Nil(t, err)
- source.Progress.EncodedResumeInfo = string(b)
-
- err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+ ctx := context.Background()
+ got, err := configureGCSManager(ctx, tc.conn, 8)
if err != nil && !tc.wantErr {
- t.Errorf("source.Init() got: %v, want: %v", err, nil)
- return
+ t.Errorf("source.configureGCSManager() error = %v", err)
}
if !tc.wantErr {
- if diff := cmp.Diff(tc.want, source.gcsManager,
+ if diff := cmp.Diff(tc.want, got,
cmp.AllowUnexported(gcsManager{}),
cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "concurrency", "buckets", "maxObjectSize", "attr"),
); diff != "" {
@@ -149,65 +132,6 @@ func TestSourceInit_Conn(t *testing.T) {
}
}
-func TestSourceInit_Progress(t *testing.T) {
- complete := map[string]objectsProgress{testBucket: {
- Processed: object2,
- IsBucketProcessed: true,
- ProcessedCnt: 1,
- TotalCnt: 1,
- }}
- completeEncodeResumeInfo, err := json.Marshal(complete)
- assert.Nil(t, err)
-
- processing := map[string]objectsProgress{
- testBucket: {
- IsBucketProcessed: false,
- ProcessedCnt: 1,
- Processed: object2,
- Processing: map[string]struct{}{object1: {}},
- TotalCnt: 2,
- },
- }
- processingEncodeResumeInfo, err := json.Marshal(processing)
- assert.Nil(t, err)
-
- testCases := []struct {
- name string
- conn *sourcespb.GCS
- encodeResumeInfo string
- gcsManagerBuckets map[string]bucket
- }{
- {
- name: "no progress",
- conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}, ExcludeBuckets: []string{perfTestBucketGlob}},
- gcsManagerBuckets: map[string]bucket{},
- },
- {
- name: "progress set, no processing objects in any buckets",
- conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
- encodeResumeInfo: string(completeEncodeResumeInfo),
- gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: "", shouldInclude: false}},
- },
- {
- name: "progress set, processing objects in some buckets",
- conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
- encodeResumeInfo: string(processingEncodeResumeInfo),
- gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: object1, shouldInclude: true}},
- },
- }
-
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- source, conn := createTestSource(tc.conn)
- source.Progress.EncodedResumeInfo = tc.encodeResumeInfo
-
- err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)
- assert.Nil(t, err)
- assert.Equal(t, tc.gcsManagerBuckets, source.gcsManager.(*gcsManager).buckets)
- })
- }
-}
-
type mockObjectManager struct {
wantErr bool
}
@@ -353,11 +277,6 @@ func TestSourceInit_Enumerate(t *testing.T) {
assert.Equal(t, uint64(5), source.stats.numObjects)
assert.Equal(t, uint32(1), source.stats.numBuckets)
assert.Equal(t, uint64(5), source.stats.bucketObjects[testBucket])
- // Ensure progress is initialized correctly.
- p := map[string]*objectsProgress{testBucket: {TotalCnt: 5, Processing: map[string]struct{}{}}}
- b, err := json.Marshal(p)
- assert.Nil(t, err)
- assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
}
func TestSourceChunks_ListObjects_Error(t *testing.T) {
@@ -370,57 +289,3 @@ func TestSourceChunks_ListObjects_Error(t *testing.T) {
err := source.Chunks(ctx, chunksCh)
assert.True(t, err != nil)
}
-
-func TestSourceChunks_ProgressSet(t *testing.T) {
- ctx := context.Background()
- chunksCh := make(chan *sources.Chunk, 1)
-
- source := &Source{
- gcsManager: &mockObjectManager{},
- chunksCh: chunksCh,
- }
- err := source.enumerate(ctx)
- assert.Nil(t, err)
-
- go func() {
- defer close(chunksCh)
- err := source.Chunks(ctx, chunksCh)
- assert.Nil(t, err)
- }()
-
- want := make([]*sources.Chunk, 0, 5)
- for i := 0; i < 5; i++ {
- want = append(want, createTestSourceChunk(i))
- }
-
- got := make([]*sources.Chunk, 0, 5)
- for ch := range chunksCh {
- got = append(got, ch)
- }
-
- // Ensure we get 5 objects back.
- assert.Equal(t, len(want), len(got))
-
- processing := map[string]struct{}{"object0": {}, "object1": {}, "object2": {}, "object3": {}, "object4": {}}
- for i := 0; i < 5; i++ {
- delete(processing, fmt.Sprintf("object%d", i))
- }
-
- // Test that the resume progress is set.
- // The processed values should be the greatest object name lexicographically.
- resume := map[string]objectsProgress{
- testBucket: {
- Processed: "object4",
- Processing: processing,
- IsBucketProcessed: true,
- ProcessedCnt: 5,
- TotalCnt: 5,
- },
- }
- b, err := json.Marshal(resume)
- assert.Nil(t, err)
-
- assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
- assert.Equal(t, int32(5), source.Progress.SectionsCompleted)
- assert.Equal(t, int64(100), source.Progress.PercentComplete)
-}