Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add resuming capability to GCS source (#1161)
ahravc617bd7a4e6b
diff --git a/main.go b/main.goindex d303206b6..a24f6fae7 100644--- a/main.go+++ b/main.go@@ -103,15 +103,16 @@ var (s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()gcsScan = cli.Command("gcs", "Find credentials in GCS buckets.")- gcsProjectID = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String()+ gcsProjectID = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can NOT be used with unauth scan. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String()gcsCloudEnv = gcsScan.Flag("cloud-environment", "Use Application Default Credentials, IAM credentials to authenticate.").Bool()gcsServiceAccount = gcsScan.Flag("service-account", "Path to GCS service account JSON file.").ExistingFile()gcsWithoutAuth = gcsScan.Flag("without-auth", "Scan GCS buckets without authentication. This will only work for public buckets").Bool()gcsAPIKey = gcsScan.Flag("api-key", "GCS API key used to authenticate. Can be provided with environment variable GOOGLE_API_KEY.").Envar("GOOGLE_API_KEY").String()- gcsIncludeBuckets = gcsScan.Flag("include-buckets", "Buckets to scan. You can repeat this flag. Globs are supported").Short('i').Strings()- gcsExcludeBuckets = gcsScan.Flag("exclude-buckets", "Buckets to exclude from scan. You can repeat this flag. Globs are supported").Short('x').Strings()- gcsIncludeObjects = gcsScan.Flag("include-objects", "Objects to scan. You can repeat this flag. Globs are supported").Short('I').Strings()- gcsExcludeObjects = gcsScan.Flag("exclude-objects", "Objects to exclude from scan. You can repeat this flag. Globs are supported").Short('X').Strings()+ gcsIncludeBuckets = gcsScan.Flag("include-buckets", "Buckets to scan. Comma seperated list of buckets. You can repeat this flag. Globs are supported").Short('I').Strings()+ gcsExcludeBuckets = gcsScan.Flag("exclude-buckets", "Buckets to exclude from scan. Comma separated list of buckets. Globs are supported").Short('X').Strings()+ gcsIncludeObjects = gcsScan.Flag("include-objects", "Objects to scan. Comma separated list of objects. you can repeat this flag. Globs are supported").Short('i').Strings()+ gcsExcludeObjects = gcsScan.Flag("exclude-objects", "Objects to exclude from scan. Comma separated list of objects. You can repeat this flag. Globs are supported").Short('x').Strings()+ gcsMaxObjectSize = gcsScan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("10MB").Bytes()syslogScan = cli.Command("syslog", "Scan syslog")syslogAddress = syslogScan.Flag("address", "Address and port to listen on for syslog. Example: 127.0.0.1:514").String()@@ -401,11 +402,12 @@ func run(state overseer.State) {ServiceAccount: *gcsServiceAccount,WithoutAuth: *gcsWithoutAuth,ApiKey: *gcsAPIKey,- IncludeBuckets: *gcsIncludeBuckets,- ExcludeBuckets: *gcsExcludeBuckets,- IncludeObjects: *gcsIncludeObjects,- ExcludeObjects: *gcsExcludeObjects,+ IncludeBuckets: commaSeperatedToSlice(*gcsIncludeBuckets),+ ExcludeBuckets: commaSeperatedToSlice(*gcsExcludeBuckets),+ IncludeObjects: commaSeperatedToSlice(*gcsIncludeObjects),+ ExcludeObjects: commaSeperatedToSlice(*gcsExcludeObjects),Concurrency: *concurrency,+ MaxObjectSize: int64(*gcsMaxObjectSize),}if err := e.ScanGCS(ctx, cfg); err != nil {logFatal(err, "Failed to scan GCS.")@@ -455,6 +457,20 @@ func run(state overseer.State) {}}+func commaSeperatedToSlice(s []string) []string {+ var result []string+ for _, items := range s {+ for _, item := range strings.Split(items, ",") {+ item = strings.TrimSpace(item)+ if item == "" {+ continue+ }+ result = append(result, item)+ }+ }+ return result+}+func printAverageDetectorTime(e *engine.Engine) {fmt.Fprintln(os.Stderr, "Average detector time is the measurement of average time spent on each detector when results are returned.")for detectorName, durations := range e.DetectorAvgTime() {diff --git a/pkg/engine/gcs.go b/pkg/engine/gcs.goindex 26fdffd51..75f3a4453 100644--- a/pkg/engine/gcs.go+++ b/pkg/engine/gcs.go@@ -15,10 +15,17 @@ import (// ScanGCS with the provided options.func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {- if c.ProjectID == "" {+ // Project ID is required if using any authenticated access.+ if c.ProjectID == "" && !c.WithoutAuth {return fmt.Errorf("project ID is required")}+ // If using unauthenticated access, the project ID is not used.+ if c.ProjectID != "" && c.WithoutAuth {+ c.ProjectID = ""+ ctx.Logger().Info("project ID is not used when using unauthenticated access, ignoring provided project ID")+ }+connection := &sourcespb.GCS{ProjectId: c.ProjectID,IncludeBuckets: c.IncludeBuckets,@@ -43,7 +50,7 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {"source_type", source.Type().String(),"source_name", "gcs",)- if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, c.Concurrency); err != nil {+ if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, int(c.Concurrency)); err != nil {return fmt.Errorf("failed to initialize GCS source: %w", err)}diff --git a/pkg/engine/gcs_test.go b/pkg/engine/gcs_test.goindex 1621a9236..e66e7bd92 100644--- a/pkg/engine/gcs_test.go+++ b/pkg/engine/gcs_test.go@@ -1,6 +1,7 @@package engineimport (+ "strings""testing""github.com/trufflesecurity/trufflehog/v3/pkg/context"@@ -24,11 +25,13 @@ func TestScanGCS(t *testing.T) {},},{- name: "missing project ID",- gcsConfig: sources.GCSConfig{- ApiKey: "abc123",- },- wantErr: true,+ name: "missing project ID, with auth",+ gcsConfig: sources.GCSConfig{ApiKey: "abc123"},+ wantErr: true,+ },+ {+ name: "missing project ID, without auth, public scan",+ gcsConfig: sources.GCSConfig{WithoutAuth: true},},{name: "multiple selected auth methods",@@ -44,7 +47,8 @@ func TestScanGCS(t *testing.T) {{name: "no auth method selected",gcsConfig: sources.GCSConfig{- ProjectID: "test-project",+ ProjectID: "test-project",+ MaxObjectSize: 10 * 1024 * 1024,},},}@@ -53,7 +57,7 @@ func TestScanGCS(t *testing.T) {t.Run(test.name, func(t *testing.T) {e := &Engine{}err := e.ScanGCS(context.Background(), test.gcsConfig)- if err != nil && !test.wantErr {+ if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {t.Errorf("ScanGCS() got: %v, want: %v", err, nil)return}diff --git a/pkg/pb/sourcespb/sources.pb.go b/pkg/pb/sourcespb/sources.pb.goindex ccb2f0ae8..9d93e6f86 100644--- a/pkg/pb/sourcespb/sources.pb.go+++ b/pkg/pb/sourcespb/sources.pb.go@@ -963,6 +963,7 @@ type GCS struct {ExcludeBuckets []string `protobuf:"bytes,7,rep,name=exclude_buckets,json=excludeBuckets,proto3" json:"exclude_buckets,omitempty"`IncludeObjects []string `protobuf:"bytes,8,rep,name=include_objects,json=includeObjects,proto3" json:"include_objects,omitempty"`ExcludeObjects []string `protobuf:"bytes,9,rep,name=exclude_objects,json=excludeObjects,proto3" json:"exclude_objects,omitempty"`+ MaxObjectSize int64 `protobuf:"varint,10,opt,name=max_object_size,json=maxObjectSize,proto3" json:"max_object_size,omitempty"`}func (x *GCS) Reset() {@@ -1074,6 +1075,13 @@ func (x *GCS) GetExcludeObjects() []string {return nil}+func (x *GCS) GetMaxObjectSize() int64 {+ if x != nil {+ return x.MaxObjectSize+ }+ return 0+}+type isGCS_Credential interface {isGCS_Credential()}diff --git a/pkg/pb/sourcespb/sources.pb.validate.go b/pkg/pb/sourcespb/sources.pb.validate.goindex d675096d4..565b6dbc0 100644--- a/pkg/pb/sourcespb/sources.pb.validate.go+++ b/pkg/pb/sourcespb/sources.pb.validate.go@@ -1244,6 +1244,8 @@ func (m *GCS) validate(all bool) error {// no validation rules for ProjectId+ // no validation rules for MaxObjectSize+switch m.Credential.(type) {case *GCS_JsonServiceAccount:diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.goindex 64a5f3486..1bd141ec9 100644--- a/pkg/sources/gcs/gcs.go+++ b/pkg/sources/gcs/gcs.go@@ -1,9 +1,11 @@package gcsimport (+ "encoding/json""fmt""io""os"+ "sort""sync"diskbufferreader "github.com/bill-rich/disk-buffer-reader"@@ -28,9 +30,12 @@ type Source struct {verify boolgcsManager objectManager+ stats *attributeslog logr.LoggerchunksCh chan *sources.Chunk- sources.Progress++ mu sync.Mutex+ sources.Progress // not safe for concurrent access.}// Ensure the Source satisfies the interface at compile time.@@ -90,7 +95,17 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6}- gcsManagerOpts := []gcsManagerOption{withConcurrency(concurrency), gcsManagerAuthOption}+ resume, err := setResumeBucketOffset(s.Progress.EncodedResumeInfo)+ if err != nil {+ return fmt.Errorf("error setting resume info: %w", err)+ }++ gcsManagerOpts := []gcsManagerOption{+ withConcurrency(concurrency),+ withBucketOffsets(resume),+ withMaxObjectSize(conn.MaxObjectSize),+ gcsManagerAuthOption,+ }if setGCSManagerBucketOptions(&conn) != nil {gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(&conn))}@@ -102,7 +117,13 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6return fmt.Errorf("error creating GCS manager: %w", err)}- return nil+ if len(resume) > 0 {+ s.log.V(2).Info("resuming from previous run")+ return nil+ }++ s.log.V(2).Info("new run, enumerating buckets and objects")+ return s.enumerate(aCtx)}func setGCSManagerBucketOptions(conn *sourcespb.GCS) gcsManagerOption {@@ -126,6 +147,166 @@ func setGCSManagerOptions(include, exclude []string, includeFn, excludeFn func([return nil}+type progressInfo struct {+ processedBucketCnt,+ totalBucketsCnt,+ processedObjectsCnt,+ totalObjectsCnt int++ // bucketObjects tracks processing and processed objects for each bucket.+ bucketObjects map[string]*objectsProgress+}++// objectsProgress keeps track of the progress of processing objects in a bucket.+// It is marshalled/unmarshalled to/from a string as part of the Source's Progress.+type objectsProgress struct {+ // IsBuckedProcessed is true if all objects in the bucket have been processed.+ IsBucketProcessed bool+ // ProcessedCnt is the number of objects that have been processed.+ ProcessedCnt,+ // TotalCnt is the total number of objects in the bucket.+ TotalCnt int+ // Processed is the last object that was fully processed.+ Processed string+ // Processing are the objects that were not fully processed.+ Processing map[string]struct{}+}++func newObjectsProgress(cnt int) *objectsProgress {+ return &objectsProgress{TotalCnt: cnt, Processing: map[string]struct{}{}}+}++// newProgressInfo constructs a progressInfo from the Source's Progress information.+func newProgressInfo(ctx context.Context, s *sources.Progress) (*progressInfo, error) {+ totalObjCnt := s.SectionsRemaining+ processedObjCnt := s.SectionsCompleted+ info := &progressInfo{+ totalObjectsCnt: int(totalObjCnt),+ processedObjectsCnt: int(processedObjCnt),+ }++ encodeResume := s.EncodedResumeInfo+ if encodeResume == "" {+ info.bucketObjects = make(map[string]*objectsProgress)+ return info, nil+ }++ var m map[string]*objectsProgress+ if err := json.Unmarshal([]byte(encodeResume), &m); err != nil {+ ctx.Logger().V(1).Info("error unmarshalling progress info", "error", err)+ return info, nil+ }+ info.bucketObjects = m++ return info, nil+}++type progressStateFn func(string, string, *progressInfo)++func (p *progressInfo) setProcessStatus(obj object, fn progressStateFn) {+ fn(obj.bucket, obj.name, p)+}++func setProcessingBucketObject(bucket, obj string, progress *progressInfo) {+ progress.processing(bucket, obj)+}++func (p *progressInfo) processing(bkt, obj string) {+ p.bucketObjects[bkt].Processing[obj] = struct{}{}+}++func setProcessedBucketObject(bucket, obj string, progress *progressInfo) {+ progress.processed(bucket, obj)+}++func (p *progressInfo) processed(bkt, obj string) {+ // Remove the object from the processing list and increment the processed count.+ delete(p.bucketObjects[bkt].Processing, obj)++ p.bucketObjects[bkt].ProcessedCnt+++ // If all objects in the bucket have been processed, mark the bucket as processed.+ if p.bucketObjects[bkt].ProcessedCnt == p.bucketObjects[bkt].TotalCnt {+ p.bucketObjects[bkt].IsBucketProcessed = true+ p.processedBucketCnt+++ }+ p.processedObjectsCnt++++ // If the object is not greater (lexicographically) than the last processed object, we can skip it.+ // This ensures we keep the greatest object name as the last processed object.+ if obj < p.bucketObjects[bkt].Processed {+ return+ }+ p.bucketObjects[bkt].Processed = obj+}++func setResumeBucketOffset(s string) (map[string]offsetInfo, error) {+ resumeInfo := progressInfo{}+ if s != "" {+ if err := json.Unmarshal([]byte(s), &resumeInfo.bucketObjects); err != nil {+ return nil, fmt.Errorf("error unmarshalling resume info: %w", err)+ }+ }++ return calcBktOffset(resumeInfo.bucketObjects)+}++// In order to calculate the bucket offset, we need to know the last object+// that was processed for each bucket, as well as any objects that were processing.+// If the bucket was fully processed, we don't need to set an offset.+// If there were no objects processing, we can just use the last object that was processed.+// If there were objects processing, we need to find the first object (lexicographically)+// that was processing, as that is the next object that needs to be processed.+func calcBktOffset(resumeInfo map[string]*objectsProgress) (map[string]offsetInfo, error) {+ bucketOffset := make(map[string]offsetInfo, len(resumeInfo))+ for objName, progress := range resumeInfo {+ info := offsetInfo{}+ if progress.IsBucketProcessed {+ info.isBucketProcessed = true+ bucketOffset[objName] = info+ continue+ }+ if len(progress.Processing) == 0 {+ info.lastProcessedObject = progress.Processed+ bucketOffset[objName] = info+ continue+ }++ processing := make([]string, 0, len(progress.Processing))+ for k := range progress.Processing {+ processing = append(processing, k)+ }+ sort.Strings(processing)+ info.lastProcessedObject = processing[0]+ bucketOffset[objName] = info+ }++ return bucketOffset, nil+}++// enumerate all the objects and buckets in the source and use the results to+// set the progress information. This will be used track progression of the scan,+// and to resume the scan if it is interrupted.+func (s *Source) enumerate(ctx context.Context) error {+ stats, err := s.gcsManager.attributes(ctx)+ if err != nil {+ return fmt.Errorf("error getting attributes during enumeration: %w", err)+ }+ s.stats = stats++ info := progressInfo{+ totalBucketsCnt: int(s.stats.numBuckets),+ totalObjectsCnt: int(s.stats.numObjects),+ }++ progress := make(map[string]*objectsProgress, len(s.stats.bucketObjects))+ for k, v := range s.stats.bucketObjects {+ progress[k] = newObjectsProgress(int(v))+ }+ info.bucketObjects = progress++ return s.setProgress(ctx, &info, fmt.Sprintf("enumerated %d buckets, %d objects", s.stats.numBuckets, s.stats.numObjects))+}+// Chunks emits chunks of bytes over a channel.func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {objectCh, err := s.gcsManager.listObjects(ctx)@@ -134,6 +315,11 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err}s.chunksCh = chunksChan+ progress, err := newProgressInfo(ctx, &s.Progress)+ if err != nil {+ return fmt.Errorf("error constructing progress info: %w", err)+ }+var wg sync.WaitGroupfor obj := range objectCh {obj := obj@@ -143,14 +329,22 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errcontinue}+ if err := s.startProcessing(ctx, progress, o); err != nil {+ ctx.Logger().V(1).Info("GCS source error starting to process objects", "error", err)+ continue+ }wg.Add(1)- go func(obj object) {+ go func(obj object, resume *progressInfo) {defer wg.Done()if err := s.processObject(ctx, o); err != nil {- ctx.Logger().V(1).Info("GCS source error processing objects", "name", s.name, "error", err)+ ctx.Logger().V(1).Info("error setting start progress progress", "name", o.name, "error", err)+ return+ }+ if err := s.endProcessing(ctx, resume, o); err != nil {+ ctx.Logger().V(1).Info("error setting end progress progress", "name", o.name, "error", err)}- }(o)+ }(o, progress)}wg.Wait()@@ -158,6 +352,34 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errreturn nil}+func (s *Source) startProcessing(ctx context.Context, progress *progressInfo, o object) error {+ s.mu.Lock()+ defer s.mu.Unlock()++ progress.setProcessStatus(o, setProcessingBucketObject)+ return s.setProgress(ctx, progress, fmt.Sprintf("GCS source beginning to process object %s in bucket %s", o.name, o.bucket))+}++func (s *Source) endProcessing(ctx context.Context, progress *progressInfo, o object) error {+ s.mu.Lock()+ defer s.mu.Unlock()++ progress.setProcessStatus(o, setProcessedBucketObject)+ return s.setProgress(ctx, progress, fmt.Sprintf("GCS source finished processing object %s in bucket %s", o.name, o.bucket))+}++func (s *Source) setProgress(ctx context.Context, progress *progressInfo, msg string) error {+ ctx.Logger().V(5).Info(msg)++ b, err := json.Marshal(progress.bucketObjects)+ if err != nil {+ return fmt.Errorf("error marshalling resume info: %w", err)+ }+ s.SetProgressComplete(progress.processedObjectsCnt, progress.totalObjectsCnt, msg, string(b))++ return nil+}+func (s *Source) processObject(ctx context.Context, o object) error {chunkSkel := &sources.Chunk{SourceName: s.name,diff --git a/pkg/sources/gcs/gcs_integration_test.go b/pkg/sources/gcs/gcs_integration_test.goindex 357844791..6c8fe2a09 100644--- a/pkg/sources/gcs/gcs_integration_test.go+++ b/pkg/sources/gcs/gcs_integration_test.go@@ -23,7 +23,7 @@ func TestChunks(t *testing.T) {source, conn := createTestSource(&sourcespb.GCS{ProjectId: testProjectID,Credential: &sourcespb.GCS_Adc{},- ExcludeBuckets: []string{perfTestBucketGlob, "public-trufflehog-test-bucket"},+ ExcludeBuckets: []string{perfTestBucketGlob, publicBucket},})err := source.Init(ctx, "test", 1, 1, true, conn, 8)@@ -58,6 +58,65 @@ func TestChunks(t *testing.T) {}}+func TestChunks_PublicBucket(t *testing.T) {+ ctx := context.Background()++ source, conn := createTestSource(&sourcespb.GCS{+ Credential: &sourcespb.GCS_Unauthenticated{},+ IncludeBuckets: []string{publicBucket},+ })++ err := source.Init(ctx, "test", 1, 1, true, conn, 8)+ assert.Nil(t, err)++ chunksCh := make(chan *sources.Chunk, 1)++ go func() {+ defer close(chunksCh)+ err := source.Chunks(ctx, chunksCh)+ assert.Nil(t, err)+ }()++ want := []*sources.Chunk{+ {+ SourceName: "test",+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,+ SourceID: 0,+ Verify: true,+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Gcs{+ Gcs: &source_metadatapb.GCS{+ Filename: "aws1.txt",+ Bucket: publicBucket,+ ContentType: "text/plain",+ Email: "",+ Link: "https://storage.googleapis.com/download/storage/v1/b/public-trufflehog-test-bucket/o/aws1.txt?generation=1678334408999764&alt=media",+ Acls: []string{},+ },+ },+ },+ },+ }++ got := make([]*sources.Chunk, 0, len(want))+ for chunk := range chunksCh {+ got = append(got, chunk)+ }+ sort.Slice(got, func(i, j int) bool {+ return got[i].SourceMetadata.GetGcs().Filename < got[j].SourceMetadata.GetGcs().Filename+ })++ assert.Equal(t, len(want), len(got))++ for i, chunk := range got {+ if diff := cmp.Diff(want[i].SourceMetadata.GetGcs(), chunk.SourceMetadata.GetGcs(),+ cmpopts.IgnoreFields(source_metadatapb.GCS{}, "state", "sizeCache", "unknownFields", "CreatedAt", "UpdatedAt"),+ ); diff != "" {+ t.Errorf("chunk mismatch (-want +got):\n%s", diff)+ }+ }+}+func createTestChunks() []*sources.Chunk {objects := []object{{diff --git a/pkg/sources/gcs/gcs_manager.go b/pkg/sources/gcs/gcs_manager.goindex f272074a4..3fb58a8e0 100644--- a/pkg/sources/gcs/gcs_manager.go+++ b/pkg/sources/gcs/gcs_manager.go@@ -5,6 +5,7 @@ import ("fmt""io""runtime"+ "sync""sync/atomic""time"@@ -22,8 +23,8 @@ import ()const (- defaultMaxObjectSize = 500 * 1024 * 1024 // 500MB- maxObjectSizeLimit = 1 * 1024 * 1024 * 1024 // 1GB+ defaultMaxObjectSize = 10 * 1024 * 1024 // 10MB+ maxObjectSizeLimit = 50 * 1024 * 1024 // 50MB)var (@@ -32,6 +33,7 @@ var (type objectManager interface {listObjects(context.Context) (chan io.Reader, error)+ attributes(ctx context.Context) (*attributes, error)}// bucketManager is a simplified *storage.Client wrapper.@@ -47,6 +49,8 @@ type bucketManager interface {// It's main purpose is to retrieve objects from GCS.type gcsManager struct {projectID string+ withoutAuth,+ hasEnumerated boolconcurrency intworkerPool *errgroup.Group@@ -59,16 +63,19 @@ type gcsManager struct {excludeBuckets,includeObjects,excludeObjects map[string]struct{}+buckets map[string]bucket- client bucketManager+ attr *attributes++ client bucketManager}// bucket is a simplified *storage.BucketHandle wrapper.// It also provides a mechanism for resuming an interrupted iteration.// This reduces the number of objects that need to be re-processed.type bucket struct {- name string- numObjects uint64+ shouldInclude bool+ name string// startOffset is the name of the object to resume from. (inclusive)// This works because GCS objects are sorted lexicographically.// If this is empty, the iteration will start from the beginning.@@ -76,6 +83,40 @@ type bucket struct {*storage.BucketHandle}+// offsetInfo is used to resume an interrupted iteration.+type offsetInfo struct {+ isBucketProcessed bool+ lastProcessedObject string+}++// attributes contains metadata about the GCS source.+// This will be collected during the initial scan.+type attributes struct {+ numBuckets uint32+ numObjects uint64+ mu sync.RWMutex+ bucketObjects map[string]uint64+}++func newStats(numBkts int) *attributes {+ return &attributes{+ numBuckets: uint32(numBkts),+ bucketObjects: make(map[string]uint64, numBkts),+ }+}++func (s *attributes) incObjects() {+ s.mu.Lock()+ s.numObjects+++ s.mu.Unlock()+}++func (s *attributes) setBucketCnt(bkt string, cnt uint64) {+ s.mu.Lock()+ s.bucketObjects[bkt] = cnt+ s.mu.Unlock()+}+type gcsManagerOption func(*gcsManager) error// withAPIKey uses the provided API key when creating a new GCS client.@@ -107,8 +148,8 @@ func withJSONServiceAccount(ctx context.Context, jsonServiceAccount []byte) gcsM// withDefaultADC uses the default application credentials when creating a new GCS client.func withDefaultADC(ctx context.Context) gcsManagerOption {+ client, err := defaultADC(ctx)return func(m *gcsManager) error {- client, err := defaultADC(ctx)if err != nil {return err}@@ -127,6 +168,7 @@ func withoutAuthentication() gcsManagerOption {return err}m.client = client+ m.withoutAuth = truereturn nil}}@@ -254,25 +296,23 @@ func withMaxObjectSize(maxObjectSize int64) gcsManagerOption {// withBucketOffsets sets the offset for each bucket.// This is used to resume listing objects for a bucket.-func withBucketOffsets(offsets map[string]string) gcsManagerOption {+func withBucketOffsets(offsets map[string]offsetInfo) gcsManagerOption {bkts := make(map[string]bucket, len(offsets))- for bkt, val := range offsets {+ for bkt, offst := range offsets {bkts[bkt] = bucket{- startOffset: val,- name: bkt,+ shouldInclude: !offst.isBucketProcessed,+ name: bkt,+ startOffset: offst.lastProcessedObject,}}return func(m *gcsManager) error {+ m.hasEnumerated = len(offsets) > 0 // offset means we are resuming.m.buckets = bktsreturn nil}}func newGCSManager(projectID string, opts ...gcsManagerOption) (*gcsManager, error) {- if projectID == "" {- return nil, fmt.Errorf("project ID is required")- }-// Default values for the manager.gcs := &gcsManager{projectID: projectID,@@ -287,6 +327,10 @@ func newGCSManager(projectID string, opts ...gcsManagerOption) (*gcsManager, err}}+ if projectID == "" && !gcs.withoutAuth {+ return nil, fmt.Errorf("project ID is required, when using authentication")+ }+// If no client was provided, use the default application credentials.// A client is required to perform any operations.if gcs.client == nil {@@ -323,6 +367,70 @@ type object struct {io.Reader}+func (g *gcsManager) attributes(ctx context.Context) (*attributes, error) {+ // Get all the buckets in the project.+ buckets, err := g.listBuckets(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to list buckets: %w", err)+ }++ return g.enumerate(ctx, buckets)+}++func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes, error) {+ logger := ctx.Logger().WithValues("phase", "enumeration")++ logger.V(5).Info("enumerating buckets", "numBuckets", len(bkts))+ stats := newStats(len(bkts))++ for _, bkt := range bkts {+ bkt := bkt+ g.workerPool.Go(func() error {+ // List all the objects in the bucket and calculate a attributes.+ g.setupBktHandle(&bkt)++ q, err := setObjectQuery(&bkt)+ if err != nil {+ logger.Error(err, "failed to set object query", "bucket", bkt.name)+ return nil+ }++ var count uint64+ objs := bkt.Objects(ctx, q)+ for {+ obj, err := objs.Next()+ if errors.Is(err, iterator.Done) {+ logger.V(5).Info("finished listing objects in bucket")+ break+ }+ if err != nil {+ logger.V(1).Info("failed to list objects", "bucket", bkt.name, "error", err)+ return nil+ }+ if obj == nil {+ logger.V(5).Info("object is nil")+ continue+ }++ if !g.shouldIncludeObject(ctx, obj.Name) || g.shouldExcludeObject(ctx, obj.Name) {+ continue+ }+ count+++ stats.incObjects()+ }++ stats.setBucketCnt(bkt.name, count)+ return nil+ })+ }++ _ = g.workerPool.Wait()+ g.attr = stats+ logger.V(5).Info("finished enumerating buckets", "num-objects", stats.numObjects, "num-buckets", stats.numBuckets)++ return stats, nil+}+func (g *gcsManager) listObjects(ctx context.Context) (chan io.Reader, error) {ch := make(chan io.Reader, 100) // TODO (ahrav): Determine optimal buffer size.@@ -376,6 +484,12 @@ func (g *gcsManager) listObjects(ctx context.Context) (chan io.Reader, error) {func (g *gcsManager) listBuckets(ctx context.Context) ([]bucket, error) {var buckets []bucket+ if g.withoutAuth {+ for name := range g.includeBuckets {+ buckets = append(buckets, bucket{name: name})+ }+ return buckets, nil+ }bkts := g.client.Buckets(ctx, g.projectID)for {@@ -412,15 +526,7 @@ func (g *gcsManager) listBucketObjects(ctx context.Context, bkt *bucket) (chan ilogger := ctx.Logger().WithValues("bucket", bkt.name)logger.V(5).Info("listing object(s) in bucket")- b := g.client.Bucket(bkt.name).Retryer(- storage.WithBackoff(gax.Backoff{- Initial: 2 * time.Second,- Max: 30 * time.Second,- Multiplier: 1.5,- }),- storage.WithPolicy(storage.RetryAlways),- )- bkt.BucketHandle = b+ g.setupBktHandle(bkt)// TODO (ahrav): Look to extend gcsManager to allow for exact buckets/objects// include filters. This will increase performance substantially@@ -432,6 +538,18 @@ func (g *gcsManager) listBucketObjects(ctx context.Context, bkt *bucket) (chan ireturn ch, errCh}+func (g *gcsManager) setupBktHandle(bkt *bucket) {+ b := g.client.Bucket(bkt.name).Retryer(+ storage.WithBackoff(gax.Backoff{+ Initial: 2 * time.Second,+ Max: 30 * time.Second,+ Multiplier: 1.5,+ }),+ storage.WithPolicy(storage.RetryAlways),+ )+ bkt.BucketHandle = b+}+func (g *gcsManager) bucketObjects(ctx context.Context, bkt *bucket, ch chan<- io.Reader) error {q, err := setObjectQuery(bkt)if err != nil {@@ -445,12 +563,16 @@ func (g *gcsManager) bucketObjects(ctx context.Context, bkt *bucket, ch chan<- ictx.Logger().V(5).Info("finished listing objects in bucket")break}- if !g.shouldIncludeObject(ctx, obj.Name) || g.shouldExcludeObject(ctx, obj.Name) {+ if err != nil {+ return fmt.Errorf("failed to retrieve object iterator: %w", err)+ }+ if obj == nil {+ ctx.Logger().V(5).Info("object is nil")continue}- if err != nil {- return fmt.Errorf("failed to retrieve iterator: %w", err)+ if !g.shouldIncludeObject(ctx, obj.Name) || g.shouldExcludeObject(ctx, obj.Name) {+ continue}o, err := g.constructObject(ctx, bkt.Object(obj.Name))@@ -458,7 +580,6 @@ func (g *gcsManager) bucketObjects(ctx context.Context, bkt *bucket, ch chan<- ictx.Logger().V(1).Info("failed to create object", "object-name", obj.Name, "error", err)continue}- bkt.numObjects++ // atomic add NOT needed, object listing is sequential.ch <- o}return nildiff --git a/pkg/sources/gcs/gcs_manager_test.go b/pkg/sources/gcs/gcs_manager_test.goindex 2fe316f82..95bab761f 100644--- a/pkg/sources/gcs/gcs_manager_test.go+++ b/pkg/sources/gcs/gcs_manager_test.go@@ -1,8 +1,10 @@package gcsimport (+ "fmt""sort""testing"+ "time""github.com/google/go-cmp/cmp""github.com/google/go-cmp/cmp/cmpopts"@@ -13,6 +15,7 @@ import ()const (+ publicBucket = "public-trufflehog-test-bucket"testProjectID = "trufflehog-testing"testAPIKey = "somekeys"testBucket = "test-bkt-th"@@ -48,9 +51,13 @@ func TestNewGcsManager(t *testing.T) {},{name: "new gcs manager, without auth",- projID: testProjectID,+ projID: "",opts: []gcsManagerOption{withoutAuthentication()},- want: &gcsManager{projectID: testProjectID, concurrency: defaultConcurrency, maxObjectSize: defaultMaxObjectSize},+ want: &gcsManager{+ concurrency: defaultConcurrency,+ maxObjectSize: defaultMaxObjectSize,+ withoutAuth: true,+ },},{name: "new gcs manager, with api key",@@ -78,15 +85,18 @@ func TestNewGcsManager(t *testing.T) {projID: testProjectID,opts: []gcsManagerOption{withDefaultADC(ctx),- withBucketOffsets(map[string]string{testBucket: "moar.txt", testBucket2: "moar2.txt"}),+ withBucketOffsets(map[string]offsetInfo{+ testBucket: {lastProcessedObject: "moar1.txt", isBucketProcessed: false},+ testBucket2: {lastProcessedObject: "", isBucketProcessed: true}}),},want: &gcsManager{projectID: testProjectID,+ hasEnumerated: true,concurrency: defaultConcurrency,maxObjectSize: defaultMaxObjectSize,buckets: map[string]bucket{- testBucket: {name: testBucket, startOffset: "moar.txt"},- testBucket2: {name: testBucket2, startOffset: "moar2.txt"},+ testBucket: {name: testBucket, startOffset: "moar.txt", shouldInclude: true},+ testBucket2: {name: testBucket2, startOffset: "", shouldInclude: false},},},},@@ -331,7 +341,6 @@ func TestNewGcsManager(t *testing.T) {if err != nil {return}-// The client should never be nil.if got.client == nil {t.Errorf("newGCSManager() client should not be nil")@@ -348,6 +357,91 @@ func TestNewGcsManager(t *testing.T) {}}+func TestGCSManagerStats(t *testing.T) {+ ctx := context.Background()++ testCases := []struct {+ name string+ projectID string+ opts []gcsManagerOption+ wantStats *attributes+ }{+ {+ name: "enumerate, all buckets, with objects",+ projectID: testProjectID,+ opts: []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{perfTestBucketGlob, publicBucket})},+ wantStats: &attributes{+ numBuckets: 4,+ numObjects: 5,+ bucketObjects: map[string]uint64{testBucket: 2, testBucket2: 1, testBucket3: 1, testBucket4: 1},+ },+ },+ {+ name: "list objects, with exclude objects",+ projectID: testProjectID,+ opts: []gcsManagerOption{+ withDefaultADC(ctx),+ withExcludeObjects([]string{"aws1.txt", "moar2.txt"}),+ withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),+ },+ wantStats: &attributes{+ numBuckets: 4,+ numObjects: 3,+ bucketObjects: map[string]uint64{testBucket: 0, testBucket2: 1, testBucket3: 1, testBucket4: 1},+ },+ },+ {+ name: "list objects, with include objects",+ projectID: testProjectID,+ opts: []gcsManagerOption{+ withDefaultADC(ctx),+ withIncludeObjects([]string{"aws1.txt"}),+ withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),+ },+ wantStats: &attributes{+ numBuckets: 4,+ numObjects: 1,+ bucketObjects: map[string]uint64{testBucket: 1, testBucket2: 0, testBucket3: 0, testBucket4: 0},+ },+ },+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ gm, err := newGCSManager(tc.projectID, tc.opts...)+ if err != nil {+ t.Fatalf("newGCSManager() error = %v", err)+ }++ got, err := gm.attributes(ctx)+ if err != nil {+ t.Errorf("attributes() error = %v", err)+ }++ if diff := cmp.Diff(got, tc.wantStats, cmp.AllowUnexported(attributes{}), cmpopts.IgnoreFields(attributes{}, "mu")); diff != "" {+ t.Errorf("attributes() got: %v, want: %v, diff: %v", got, tc.wantStats, diff)+ }+ })+ }+}++func TestGCSManagerStats_Time(t *testing.T) {+ ctx := context.Background()++ opts := []gcsManagerOption{withDefaultADC(ctx)}+ gm, err := newGCSManager(testProjectID, opts...)+ if err != nil {+ t.Fatalf("newGCSManager() error = %v", err)+ }++ start := time.Now()+ var stats *attributes+ stats, _ = gm.attributes(ctx)+ end := time.Since(start).Seconds()++ fmt.Printf("Time taken to get %d objects: %f seconds\n", stats.numObjects, end)+}+func TestGCSManagerListObjects(t *testing.T) {ctx := context.Background()@@ -370,7 +464,7 @@ func TestGCSManagerListObjects(t *testing.T) {{name: "list objects, all buckets, with objects",projectID: testProjectID,- opts: []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{perfTestBucketGlob, "public-trufflehog-test-bucket"})},+ opts: []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{perfTestBucketGlob, publicBucket})},want: []object{{name: "aws1.txt",@@ -444,8 +538,7 @@ func TestGCSManagerListObjects(t *testing.T) {{name: "list objects, exclude buckets, with objects",projectID: testProjectID,- opts: []gcsManagerOption{withDefaultADC(ctx),- withExcludeBuckets([]string{testBucket, testBucket2, perfTestBucketGlob, "public-trufflehog-test-bucket"})},+ opts: []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{testBucket, testBucket2, perfTestBucketGlob, publicBucket})},want: []object{{name: "moar.txt",@@ -473,7 +566,7 @@ func TestGCSManagerListObjects(t *testing.T) {opts: []gcsManagerOption{withDefaultADC(ctx),withExcludeObjects([]string{"aws1.txt", "moar2.txt"}),- withExcludeBuckets([]string{perfTestBucketGlob, "public-trufflehog-test-bucket"}),+ withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),},want: []object{{@@ -531,7 +624,7 @@ func TestGCSManagerListObjects(t *testing.T) {opts: []gcsManagerOption{withDefaultADC(ctx),withIncludeObjects([]string{"aws1.txt"}),- withExcludeBuckets([]string{perfTestBucketGlob, "public-trufflehog-test-bucket"}),+ withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),},want: []object{{@@ -553,7 +646,7 @@ func TestGCSManagerListObjects(t *testing.T) {withDefaultADC(ctx),withIncludeBuckets([]string{testBucket}),withExcludeObjects([]string{"aws1.txt"}),- withExcludeBuckets([]string{perfTestBucketGlob, "public-trufflehog-test-bucket"}),+ withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),},want: []object{{@@ -640,7 +733,7 @@ func TestGCSManagerListObjects_Resuming(t *testing.T) {opts: []gcsManagerOption{withDefaultADC(ctx),withIncludeBuckets([]string{testBucket}),- withBucketOffsets(map[string]string{testBucket: "moar2.txt"}),+ withBucketOffsets(map[string]offsetInfo{testBucket: {lastProcessedObject: "moar2.txt"}}),},want: []object{{@@ -758,14 +851,14 @@ func Test_isObjectSizeValid(t *testing.T) {want: true,},{- name: "valid object size, 100 MB",- objSize: int64(100 * common.MB),+ name: "valid object size, 49 MB",+ objSize: int64(49 * common.MB),want: true,},{- name: "valid object size, 1 GB",+ name: "invalid object size, 1 GB",objSize: int64(1 * common.GB),- want: true,+ want: false,},{name: "invalid object size, 2 GB",diff --git a/pkg/sources/gcs/gcs_test.go b/pkg/sources/gcs/gcs_test.goindex 600c2683b..973a5d9c6 100644--- a/pkg/sources/gcs/gcs_test.go+++ b/pkg/sources/gcs/gcs_test.go@@ -1,6 +1,7 @@package gcsimport (+ "encoding/json""fmt""io""sort"@@ -43,7 +44,12 @@ func TestSourceInit(t *testing.T) {},})- err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)+ // Set the EncodeResumeInfo to prevent enumeration.+ b, err := json.Marshal(map[string]objectsProgress{"other": {}})+ assert.Nil(t, err)+ source.Progress.EncodedResumeInfo = string(b)++ err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)assert.Nil(t, err)assert.NotNil(t, source.gcsManager)}@@ -64,9 +70,14 @@ func TestSourceInit_Conn(t *testing.T) {conn: &sourcespb.GCS{ProjectId: testProjectID,Credential: &sourcespb.GCS_Adc{},+ ExcludeBuckets: []string{+ perfTestBucketGlob,+ },},want: &gcsManager{- projectID: testProjectID,+ projectID: testProjectID,+ excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},+ hasEnumerated: true,},},{@@ -84,6 +95,7 @@ func TestSourceInit_Conn(t *testing.T) {want: &gcsManager{projectID: testProjectID,includeBuckets: map[string]struct{}{"bucket1": {}},+ hasEnumerated: true,},},{@@ -97,10 +109,15 @@ func TestSourceInit_Conn(t *testing.T) {ExcludeObjects: []string{"object2",},+ ExcludeBuckets: []string{+ perfTestBucketGlob,+ },},want: &gcsManager{projectID: testProjectID,includeObjects: map[string]struct{}{"object1": {}},+ excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},+ hasEnumerated: true,},},}@@ -108,8 +125,12 @@ func TestSourceInit_Conn(t *testing.T) {for _, tc := range testCases {t.Run(tc.name, func(t *testing.T) {source, conn := createTestSource(tc.conn)+ // Set the EncodeResumeInfo to prevent enumeration.+ b, err := json.Marshal(map[string]objectsProgress{"other": {}})+ assert.Nil(t, err)+ source.Progress.EncodedResumeInfo = string(b)- err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)+ err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)if err != nil && !tc.wantErr {t.Errorf("source.Init() got: %v, want: %v", err, nil)return@@ -118,7 +139,7 @@ func TestSourceInit_Conn(t *testing.T) {if !tc.wantErr {if diff := cmp.Diff(tc.want, source.gcsManager,cmp.AllowUnexported(gcsManager{}),- cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "concurrency", "buckets", "maxObjectSize"),+ cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "concurrency", "buckets", "maxObjectSize", "attr"),); diff != "" {t.Errorf("source.Init() diff: (-want +got)\n%s", diff)}@@ -127,10 +148,81 @@ func TestSourceInit_Conn(t *testing.T) {}}+func TestSourceInit_Progress(t *testing.T) {+ complete := map[string]objectsProgress{testBucket: {+ Processed: object2,+ IsBucketProcessed: true,+ ProcessedCnt: 1,+ TotalCnt: 1,+ }}+ completeEncodeResumeInfo, err := json.Marshal(complete)+ assert.Nil(t, err)++ processing := map[string]objectsProgress{+ testBucket: {+ IsBucketProcessed: false,+ ProcessedCnt: 1,+ Processed: object2,+ Processing: map[string]struct{}{object1: {}},+ TotalCnt: 2,+ },+ }+ processingEncodeResumeInfo, err := json.Marshal(processing)+ assert.Nil(t, err)++ testCases := []struct {+ name string+ conn *sourcespb.GCS+ encodeResumeInfo string+ gcsManagerBuckets map[string]bucket+ }{+ {+ name: "no progress",+ conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}, ExcludeBuckets: []string{perfTestBucketGlob}},+ gcsManagerBuckets: map[string]bucket{},+ },+ {+ name: "progress set, no processing objects in any buckets",+ conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},+ encodeResumeInfo: string(completeEncodeResumeInfo),+ gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: "", shouldInclude: false}},+ },+ {+ name: "progress set, processing objects in some buckets",+ conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},+ encodeResumeInfo: string(processingEncodeResumeInfo),+ gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: object1, shouldInclude: true}},+ },+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ source, conn := createTestSource(tc.conn)+ source.Progress.EncodedResumeInfo = tc.encodeResumeInfo++ err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)+ assert.Nil(t, err)+ assert.Equal(t, tc.gcsManagerBuckets, source.gcsManager.(*gcsManager).buckets)+ })+ }+}+type mockObjectManager struct {wantErr bool}+func (m *mockObjectManager) attributes(_ context.Context) (*attributes, error) {+ if m.wantErr {+ return nil, fmt.Errorf("some error")+ }++ return &attributes{+ numObjects: 5,+ numBuckets: 1,+ bucketObjects: map[string]uint64{testBucket: 5},+ }, nil+}+type mockReader struct {offset intdata []byte@@ -207,6 +299,9 @@ func TestSourceChunks_ListObjects(t *testing.T) {chunksCh: chunksCh,}+ err := source.enumerate(ctx)+ assert.Nil(t, err)+go func() {defer close(chunksCh)err := source.Chunks(ctx, chunksCh)@@ -243,6 +338,24 @@ func TestSourceChunks_ListObjects(t *testing.T) {}+func TestSourceInit_Enumerate(t *testing.T) {+ ctx := context.Background()+ source := &Source{gcsManager: &mockObjectManager{}}++ err := source.enumerate(ctx)+ assert.Nil(t, err)++ // Ensure the attributes are set.+ assert.Equal(t, uint64(5), source.stats.numObjects)+ assert.Equal(t, uint32(1), source.stats.numBuckets)+ assert.Equal(t, uint64(5), source.stats.bucketObjects[testBucket])+ // Ensure progress is initialized correctly.+ p := map[string]*objectsProgress{testBucket: {TotalCnt: 5, Processing: map[string]struct{}{}}}+ b, err := json.Marshal(p)+ assert.Nil(t, err)+ assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)+}+func TestSourceChunks_ListObjects_Error(t *testing.T) {ctx := context.Background()source := &Source{gcsManager: &mockObjectManager{wantErr: true}}@@ -253,3 +366,57 @@ func TestSourceChunks_ListObjects_Error(t *testing.T) {err := source.Chunks(ctx, chunksCh)assert.True(t, err != nil)}++func TestSourceChunks_ProgressSet(t *testing.T) {+ ctx := context.Background()+ chunksCh := make(chan *sources.Chunk, 1)++ source := &Source{+ gcsManager: &mockObjectManager{},+ chunksCh: chunksCh,+ }+ err := source.enumerate(ctx)+ assert.Nil(t, err)++ go func() {+ defer close(chunksCh)+ err := source.Chunks(ctx, chunksCh)+ assert.Nil(t, err)+ }()++ want := make([]*sources.Chunk, 0, 5)+ for i := 0; i < 5; i++ {+ want = append(want, createTestSourceChunk(i))+ }++ got := make([]*sources.Chunk, 0, 5)+ for ch := range chunksCh {+ got = append(got, ch)+ }++ // Ensure we get 5 objects back.+ assert.Equal(t, len(want), len(got))++ processing := map[string]struct{}{"object0": {}, "object1": {}, "object2": {}, "object3": {}, "object4": {}}+ for i := 0; i < 5; i++ {+ delete(processing, fmt.Sprintf("object%d", i))+ }++ // Test that the resume progress is set.+ // The processed values should be the greatest object name lexicographically.+ resume := map[string]objectsProgress{+ testBucket: {+ Processed: "object4",+ Processing: processing,+ IsBucketProcessed: true,+ ProcessedCnt: 5,+ TotalCnt: 5,+ },+ }+ b, err := json.Marshal(resume)+ assert.Nil(t, err)++ assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)+ assert.Equal(t, int32(5), source.Progress.SectionsCompleted)+ assert.Equal(t, int64(100), source.Progress.PercentComplete)+}diff --git a/pkg/sources/sources.go b/pkg/sources/sources.goindex 261596025..aa41abaff 100644--- a/pkg/sources/sources.go+++ b/pkg/sources/sources.go@@ -57,6 +57,8 @@ type GCSConfig struct {ProjectID,// ServiceAccount is the service account to use to authenticate with the source.ServiceAccount string+ // MaxObjectSize is the maximum object size to scan.+ MaxObjectSize int64// Concurrency is the number of concurrent workers to use to scan the source.Concurrency int// IncludeBuckets is a list of buckets to include in the scan.diff --git a/proto/sources.proto b/proto/sources.protoindex 8d96e2483..86adadcf5 100644--- a/proto/sources.proto+++ b/proto/sources.proto@@ -137,6 +137,7 @@ message GCS {repeated string exclude_buckets = 7;repeated string include_objects = 8;repeated string exclude_objects = 9;+ int64 max_object_size = 10;}message Git {