Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Delete progress tracking from GCS source (#1190)
ahravac19de75bfa3
diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.goindex 9735479c3..aa3a5bfd1 100644--- a/pkg/sources/gcs/gcs.go+++ b/pkg/sources/gcs/gcs.go@@ -1,11 +1,9 @@package gcsimport (- "encoding/json""fmt""io""os"- "sort""strconv""sync"@@ -22,23 +20,6 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/sources")-// Source represents a GCS source.-type Source struct {- name string- jobId int64- sourceId int64- concurrency int- verify bool-- gcsManager objectManager- stats *attributes- log logr.Logger- chunksCh chan *sources.Chunk-- mu sync.Mutex- sources.Progress // not safe for concurrent access.-}-// Ensure the Source satisfies the interface at compile time.var _ sources.Source = (*Source)(nil)@@ -58,6 +39,22 @@ func (s *Source) JobID() int64 {return s.jobId}+// Source represents a GCS source.+type Source struct {+ name string+ jobId int64+ sourceId int64+ concurrency int+ verify bool++ gcsManager objectManager+ stats *attributes+ log logr.Logger+ chunksCh chan *sources.Chunk++ sources.Progress+}+// Init returns an initialized GCS source.func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {s.log = aCtx.Logger()@@ -74,6 +71,25 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6return errors.WrapPrefix(err, "error unmarshalling connection", 0)}+ gcsManager, err := configureGCSManager(aCtx, &conn, concurrency)+ if err != nil {+ return err+ }+ s.gcsManager = gcsManager++ s.log.V(2).Info("enumerating buckets and objects")+ if err := s.enumerate(aCtx); err != nil {+ return fmt.Errorf("error enumerating buckets and objects: %w", err)+ }++ return nil+}++func configureGCSManager(aCtx context.Context, conn *sourcespb.GCS, concurrency int) (*gcsManager, error) {+ if conn == nil {+ return nil, fmt.Errorf("GCS connection is nil, cannot configure GCS manager")+ }+var gcsManagerAuthOption gcsManagerOptionswitch conn.Credential.(type) {@@ -82,7 +98,7 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6case *sourcespb.GCS_ServiceAccountFile:b, err := os.ReadFile(conn.GetServiceAccountFile())if err != nil {- return fmt.Errorf("error reading GCS JSON Service Account file: %w", err)+ return nil, fmt.Errorf("error reading GCS JSON Service Account file: %w", err)}gcsManagerAuthOption = withJSONServiceAccount(aCtx, b)case *sourcespb.GCS_JsonServiceAccount:@@ -92,39 +108,28 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6case *sourcespb.GCS_Unauthenticated:gcsManagerAuthOption = withoutAuthentication()default:- return fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)+ return nil, fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)}- resume, err := setResumeBucketOffset(s.Progress.EncodedResumeInfo)- if err != nil {- return fmt.Errorf("error setting resume info: %w", err)- }-gcsManagerOpts := []gcsManagerOption{withConcurrency(concurrency),- withBucketOffsets(resume),withMaxObjectSize(conn.MaxObjectSize),gcsManagerAuthOption,}- if setGCSManagerBucketOptions(&conn) != nil {- gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(&conn))+ if setGCSManagerBucketOptions(conn) != nil {+ gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(conn))}- if setGCSManagerObjectOptions(&conn) != nil {- gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(&conn))+ if setGCSManagerObjectOptions(conn) != nil {+ gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(conn))}- if s.gcsManager, err = newGCSManager(conn.ProjectId, gcsManagerOpts...); err != nil {- return fmt.Errorf("error creating GCS manager: %w", err)- }-- if len(resume) > 0 {- s.log.V(2).Info("resuming from previous run")- return nil+ gcsManager, err := newGCSManager(conn.ProjectId, gcsManagerOpts...)+ if err != nil {+ return nil, fmt.Errorf("error creating GCS manager: %w", err)}- s.log.V(2).Info("new run, enumerating buckets and objects")- return s.enumerate(aCtx)+ return gcsManager, nil}func setGCSManagerBucketOptions(conn *sourcespb.GCS) gcsManagerOption {@@ -148,145 +153,7 @@ func setGCSManagerOptions(include, exclude []string, includeFn, excludeFn func([return nil}-type progressInfo struct {- processedBucketCnt,- totalBucketsCnt,- processedObjectsCnt,- totalObjectsCnt int-- // bucketObjects tracks processing and processed objects for each bucket.- bucketObjects map[string]*objectsProgress-}--// objectsProgress keeps track of the progress of processing objects in a bucket.-// It is marshalled/unmarshalled to/from a string as part of the Source's Progress.-type objectsProgress struct {- // IsBuckedProcessed is true if all objects in the bucket have been processed.- IsBucketProcessed bool- // ProcessedCnt is the number of objects that have been processed.- ProcessedCnt,- // TotalCnt is the total number of objects in the bucket.- TotalCnt int- // Processed is the last object that was fully processed.- Processed string- // Processing are the objects that were not fully processed.- Processing map[string]struct{}-}--func newObjectsProgress(cnt int) *objectsProgress {- return &objectsProgress{TotalCnt: cnt, Processing: map[string]struct{}{}}-}--// newProgressInfo constructs a progressInfo from the Source's Progress information.-func newProgressInfo(ctx context.Context, s *sources.Progress) (*progressInfo, error) {- totalObjCnt := s.SectionsRemaining- processedObjCnt := s.SectionsCompleted- info := &progressInfo{- totalObjectsCnt: int(totalObjCnt),- processedObjectsCnt: int(processedObjCnt),- }-- encodeResume := s.EncodedResumeInfo- if encodeResume == "" {- info.bucketObjects = make(map[string]*objectsProgress)- return info, nil- }-- var m map[string]*objectsProgress- if err := json.Unmarshal([]byte(encodeResume), &m); err != nil {- ctx.Logger().V(1).Info("error unmarshalling progress info", "error", err)- return info, nil- }- info.bucketObjects = m-- return info, nil-}--type progressStateFn func(string, string, *progressInfo)--func (p *progressInfo) setProcessStatus(obj object, fn progressStateFn) {- fn(obj.bucket, obj.name, p)-}--func setProcessingBucketObject(bucket, obj string, progress *progressInfo) {- progress.processing(bucket, obj)-}--func (p *progressInfo) processing(bkt, obj string) {- p.bucketObjects[bkt].Processing[obj] = struct{}{}-}--func setProcessedBucketObject(bucket, obj string, progress *progressInfo) {- progress.processed(bucket, obj)-}--func (p *progressInfo) processed(bkt, obj string) {- // Remove the object from the processing list and increment the processed count.- delete(p.bucketObjects[bkt].Processing, obj)-- p.bucketObjects[bkt].ProcessedCnt++- // If all objects in the bucket have been processed, mark the bucket as processed.- if p.bucketObjects[bkt].ProcessedCnt == p.bucketObjects[bkt].TotalCnt {- p.bucketObjects[bkt].IsBucketProcessed = true- p.processedBucketCnt++- }- p.processedObjectsCnt++-- // If the object is not greater (lexicographically) than the last processed object, we can skip it.- // This ensures we keep the greatest object name as the last processed object.- if obj < p.bucketObjects[bkt].Processed {- return- }- p.bucketObjects[bkt].Processed = obj-}--func setResumeBucketOffset(s string) (map[string]offsetInfo, error) {- resumeInfo := progressInfo{}- if s != "" {- if err := json.Unmarshal([]byte(s), &resumeInfo.bucketObjects); err != nil {- return nil, fmt.Errorf("error unmarshalling resume info: %w", err)- }- }-- return calcBktOffset(resumeInfo.bucketObjects)-}--// In order to calculate the bucket offset, we need to know the last object-// that was processed for each bucket, as well as any objects that were processing.-// If the bucket was fully processed, we don't need to set an offset.-// If there were no objects processing, we can just use the last object that was processed.-// If there were objects processing, we need to find the first object (lexicographically)-// that was processing, as that is the next object that needs to be processed.-func calcBktOffset(resumeInfo map[string]*objectsProgress) (map[string]offsetInfo, error) {- bucketOffset := make(map[string]offsetInfo, len(resumeInfo))- for objName, progress := range resumeInfo {- info := offsetInfo{}- if progress.IsBucketProcessed {- info.isBucketProcessed = true- bucketOffset[objName] = info- continue- }- if len(progress.Processing) == 0 {- info.lastProcessedObject = progress.Processed- bucketOffset[objName] = info- continue- }-- processing := make([]string, 0, len(progress.Processing))- for k := range progress.Processing {- processing = append(processing, k)- }- sort.Strings(processing)- info.lastProcessedObject = processing[0]- bucketOffset[objName] = info- }-- return bucketOffset, nil-}--// enumerate all the objects and buckets in the source and use the results to-// set the progress information. This will be used track progression of the scan,-// and to resume the scan if it is interrupted.+// enumerate all the objects and buckets in the source.func (s *Source) enumerate(ctx context.Context) error {stats, err := s.gcsManager.attributes(ctx)if err != nil {@@ -294,18 +161,7 @@ func (s *Source) enumerate(ctx context.Context) error {}s.stats = stats- info := progressInfo{- totalBucketsCnt: int(s.stats.numBuckets),- totalObjectsCnt: int(s.stats.numObjects),- }-- progress := make(map[string]*objectsProgress, len(s.stats.bucketObjects))- for k, v := range s.stats.bucketObjects {- progress[k] = newObjectsProgress(int(v))- }- info.bucketObjects = progress-- return s.setProgress(ctx, &info, fmt.Sprintf("enumerated %d buckets, %d objects", s.stats.numBuckets, s.stats.numObjects))+ return nil}// Chunks emits chunks of bytes over a channel.@@ -316,11 +172,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err}s.chunksCh = chunksChan- progress, err := newProgressInfo(ctx, &s.Progress)- if err != nil {- return fmt.Errorf("error constructing progress info: %w", err)- }-var wg sync.WaitGroupfor obj := range objectCh {obj := obj@@ -330,54 +181,18 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errcontinue}- if err := s.startProcessing(ctx, progress, o); err != nil {- ctx.Logger().V(1).Info("GCS source error starting to process objects", "error", err)- continue- }wg.Add(1)- go func(obj object, resume *progressInfo) {+ go func(obj object) {defer wg.Done()if err := s.processObject(ctx, o); err != nil {ctx.Logger().V(1).Info("error setting start progress progress", "name", o.name, "error", err)return}- if err := s.endProcessing(ctx, resume, o); err != nil {- ctx.Logger().V(1).Info("error setting end progress progress", "name", o.name, "error", err)- }- }(o, progress)+ }(o)}wg.Wait()- ctx.Logger().Info("GCS source finished processing", "name", s.name)- return nil-}--func (s *Source) startProcessing(ctx context.Context, progress *progressInfo, o object) error {- s.mu.Lock()- defer s.mu.Unlock()-- progress.setProcessStatus(o, setProcessingBucketObject)- return s.setProgress(ctx, progress, fmt.Sprintf("GCS source beginning to process object %s in bucket %s", o.name, o.bucket))-}--func (s *Source) endProcessing(ctx context.Context, progress *progressInfo, o object) error {- s.mu.Lock()- defer s.mu.Unlock()-- progress.setProcessStatus(o, setProcessedBucketObject)- return s.setProgress(ctx, progress, fmt.Sprintf("GCS source finished processing object %s in bucket %s", o.name, o.bucket))-}--func (s *Source) setProgress(ctx context.Context, progress *progressInfo, msg string) error {- ctx.Logger().V(5).Info(msg)-- b, err := json.Marshal(progress.bucketObjects)- if err != nil {- return fmt.Errorf("error marshalling resume info: %w", err)- }- s.SetProgressComplete(progress.processedObjectsCnt, progress.totalObjectsCnt, msg, string(b))-return nil}diff --git a/pkg/sources/gcs/gcs_test.go b/pkg/sources/gcs/gcs_test.goindex 7a893ee1b..871ab9eca 100644--- a/pkg/sources/gcs/gcs_test.go+++ b/pkg/sources/gcs/gcs_test.go@@ -1,7 +1,6 @@package gcsimport (- "encoding/json""fmt""io""sort"@@ -32,7 +31,7 @@ func TestSourceInit(t *testing.T) {source, conn := createTestSource(&sourcespb.GCS{ProjectId: testProjectID,IncludeBuckets: []string{- "bucket1",+ publicBucket,},ExcludeBuckets: []string{perfTestBucketGlob,@@ -40,22 +39,15 @@ func TestSourceInit(t *testing.T) {ExcludeObjects: []string{"object1",},- Credential: &sourcespb.GCS_ApiKey{- ApiKey: testAPIKey,- },+ Credential: &sourcespb.GCS_Unauthenticated{},})- // Set the EncodeResumeInfo to prevent enumeration.- b, err := json.Marshal(map[string]objectsProgress{"other": {}})- assert.Nil(t, err)- source.Progress.EncodedResumeInfo = string(b)-- err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)+ err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)assert.Nil(t, err)assert.NotNil(t, source.gcsManager)}-func TestSourceInit_Conn(t *testing.T) {+func TestConfigureGCSManager(t *testing.T) {testCases := []struct {name stringconn *sourcespb.GCS@@ -78,7 +70,6 @@ func TestSourceInit_Conn(t *testing.T) {want: &gcsManager{projectID: testProjectID,excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},- hasEnumerated: true,},},{@@ -96,7 +87,6 @@ func TestSourceInit_Conn(t *testing.T) {want: &gcsManager{projectID: testProjectID,includeBuckets: map[string]struct{}{"bucket1": {}},- hasEnumerated: true,},},{@@ -118,27 +108,20 @@ func TestSourceInit_Conn(t *testing.T) {projectID: testProjectID,includeObjects: map[string]struct{}{"object1": {}},excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},- hasEnumerated: true,},},}for _, tc := range testCases {t.Run(tc.name, func(t *testing.T) {- source, conn := createTestSource(tc.conn)- // Set the EncodeResumeInfo to prevent enumeration.- b, err := json.Marshal(map[string]objectsProgress{"other": {}})- assert.Nil(t, err)- source.Progress.EncodedResumeInfo = string(b)-- err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)+ ctx := context.Background()+ got, err := configureGCSManager(ctx, tc.conn, 8)if err != nil && !tc.wantErr {- t.Errorf("source.Init() got: %v, want: %v", err, nil)- return+ t.Errorf("source.configureGCSManager() error = %v", err)}if !tc.wantErr {- if diff := cmp.Diff(tc.want, source.gcsManager,+ if diff := cmp.Diff(tc.want, got,cmp.AllowUnexported(gcsManager{}),cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "concurrency", "buckets", "maxObjectSize", "attr"),); diff != "" {@@ -149,65 +132,6 @@ func TestSourceInit_Conn(t *testing.T) {}}-func TestSourceInit_Progress(t *testing.T) {- complete := map[string]objectsProgress{testBucket: {- Processed: object2,- IsBucketProcessed: true,- ProcessedCnt: 1,- TotalCnt: 1,- }}- completeEncodeResumeInfo, err := json.Marshal(complete)- assert.Nil(t, err)-- processing := map[string]objectsProgress{- testBucket: {- IsBucketProcessed: false,- ProcessedCnt: 1,- Processed: object2,- Processing: map[string]struct{}{object1: {}},- TotalCnt: 2,- },- }- processingEncodeResumeInfo, err := json.Marshal(processing)- assert.Nil(t, err)-- testCases := []struct {- name string- conn *sourcespb.GCS- encodeResumeInfo string- gcsManagerBuckets map[string]bucket- }{- {- name: "no progress",- conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}, ExcludeBuckets: []string{perfTestBucketGlob}},- gcsManagerBuckets: map[string]bucket{},- },- {- name: "progress set, no processing objects in any buckets",- conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},- encodeResumeInfo: string(completeEncodeResumeInfo),- gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: "", shouldInclude: false}},- },- {- name: "progress set, processing objects in some buckets",- conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},- encodeResumeInfo: string(processingEncodeResumeInfo),- gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: object1, shouldInclude: true}},- },- }-- for _, tc := range testCases {- t.Run(tc.name, func(t *testing.T) {- source, conn := createTestSource(tc.conn)- source.Progress.EncodedResumeInfo = tc.encodeResumeInfo-- err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)- assert.Nil(t, err)- assert.Equal(t, tc.gcsManagerBuckets, source.gcsManager.(*gcsManager).buckets)- })- }-}-type mockObjectManager struct {wantErr bool}@@ -353,11 +277,6 @@ func TestSourceInit_Enumerate(t *testing.T) {assert.Equal(t, uint64(5), source.stats.numObjects)assert.Equal(t, uint32(1), source.stats.numBuckets)assert.Equal(t, uint64(5), source.stats.bucketObjects[testBucket])- // Ensure progress is initialized correctly.- p := map[string]*objectsProgress{testBucket: {TotalCnt: 5, Processing: map[string]struct{}{}}}- b, err := json.Marshal(p)- assert.Nil(t, err)- assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)}func TestSourceChunks_ListObjects_Error(t *testing.T) {@@ -370,57 +289,3 @@ func TestSourceChunks_ListObjects_Error(t *testing.T) {err := source.Chunks(ctx, chunksCh)assert.True(t, err != nil)}--func TestSourceChunks_ProgressSet(t *testing.T) {- ctx := context.Background()- chunksCh := make(chan *sources.Chunk, 1)-- source := &Source{- gcsManager: &mockObjectManager{},- chunksCh: chunksCh,- }- err := source.enumerate(ctx)- assert.Nil(t, err)-- go func() {- defer close(chunksCh)- err := source.Chunks(ctx, chunksCh)- assert.Nil(t, err)- }()-- want := make([]*sources.Chunk, 0, 5)- for i := 0; i < 5; i++ {- want = append(want, createTestSourceChunk(i))- }-- got := make([]*sources.Chunk, 0, 5)- for ch := range chunksCh {- got = append(got, ch)- }-- // Ensure we get 5 objects back.- assert.Equal(t, len(want), len(got))-- processing := map[string]struct{}{"object0": {}, "object1": {}, "object2": {}, "object3": {}, "object4": {}}- for i := 0; i < 5; i++ {- delete(processing, fmt.Sprintf("object%d", i))- }-- // Test that the resume progress is set.- // The processed values should be the greatest object name lexicographically.- resume := map[string]objectsProgress{- testBucket: {- Processed: "object4",- Processing: processing,- IsBucketProcessed: true,- ProcessedCnt: 5,- TotalCnt: 5,- },- }- b, err := json.Marshal(resume)- assert.Nil(t, err)-- assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)- assert.Equal(t, int32(5), source.Progress.SectionsCompleted)- assert.Equal(t, int64(100), source.Progress.PercentComplete)-}