Skip to content

Find, verify, and analyze leaked credentials

Add resuming capability to GCS source (#1161)

ahravc617bd7a4e6b
diff --git a/main.go b/main.go
index d303206b6..a24f6fae7 100644
--- a/main.go
+++ b/main.go
@@ -103,15 +103,16 @@ var (
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
gcsScan = cli.Command("gcs", "Find credentials in GCS buckets.")
- gcsProjectID = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String()
+ gcsProjectID = gcsScan.Flag("project-id", "GCS project ID used to authenticate. Can NOT be used with unauth scan. Can be provided with environment variable GOOGLE_CLOUD_PROJECT.").Envar("GOOGLE_CLOUD_PROJECT").String()
gcsCloudEnv = gcsScan.Flag("cloud-environment", "Use Application Default Credentials, IAM credentials to authenticate.").Bool()
gcsServiceAccount = gcsScan.Flag("service-account", "Path to GCS service account JSON file.").ExistingFile()
gcsWithoutAuth = gcsScan.Flag("without-auth", "Scan GCS buckets without authentication. This will only work for public buckets").Bool()
gcsAPIKey = gcsScan.Flag("api-key", "GCS API key used to authenticate. Can be provided with environment variable GOOGLE_API_KEY.").Envar("GOOGLE_API_KEY").String()
- gcsIncludeBuckets = gcsScan.Flag("include-buckets", "Buckets to scan. You can repeat this flag. Globs are supported").Short('i').Strings()
- gcsExcludeBuckets = gcsScan.Flag("exclude-buckets", "Buckets to exclude from scan. You can repeat this flag. Globs are supported").Short('x').Strings()
- gcsIncludeObjects = gcsScan.Flag("include-objects", "Objects to scan. You can repeat this flag. Globs are supported").Short('I').Strings()
- gcsExcludeObjects = gcsScan.Flag("exclude-objects", "Objects to exclude from scan. You can repeat this flag. Globs are supported").Short('X').Strings()
+ gcsIncludeBuckets = gcsScan.Flag("include-buckets", "Buckets to scan. Comma seperated list of buckets. You can repeat this flag. Globs are supported").Short('I').Strings()
+ gcsExcludeBuckets = gcsScan.Flag("exclude-buckets", "Buckets to exclude from scan. Comma separated list of buckets. Globs are supported").Short('X').Strings()
+ gcsIncludeObjects = gcsScan.Flag("include-objects", "Objects to scan. Comma separated list of objects. you can repeat this flag. Globs are supported").Short('i').Strings()
+ gcsExcludeObjects = gcsScan.Flag("exclude-objects", "Objects to exclude from scan. Comma separated list of objects. You can repeat this flag. Globs are supported").Short('x').Strings()
+ gcsMaxObjectSize = gcsScan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("10MB").Bytes()
syslogScan = cli.Command("syslog", "Scan syslog")
syslogAddress = syslogScan.Flag("address", "Address and port to listen on for syslog. Example: 127.0.0.1:514").String()
@@ -401,11 +402,12 @@ func run(state overseer.State) {
ServiceAccount: *gcsServiceAccount,
WithoutAuth: *gcsWithoutAuth,
ApiKey: *gcsAPIKey,
- IncludeBuckets: *gcsIncludeBuckets,
- ExcludeBuckets: *gcsExcludeBuckets,
- IncludeObjects: *gcsIncludeObjects,
- ExcludeObjects: *gcsExcludeObjects,
+ IncludeBuckets: commaSeperatedToSlice(*gcsIncludeBuckets),
+ ExcludeBuckets: commaSeperatedToSlice(*gcsExcludeBuckets),
+ IncludeObjects: commaSeperatedToSlice(*gcsIncludeObjects),
+ ExcludeObjects: commaSeperatedToSlice(*gcsExcludeObjects),
Concurrency: *concurrency,
+ MaxObjectSize: int64(*gcsMaxObjectSize),
}
if err := e.ScanGCS(ctx, cfg); err != nil {
logFatal(err, "Failed to scan GCS.")
@@ -455,6 +457,20 @@ func run(state overseer.State) {
}
}
+func commaSeperatedToSlice(s []string) []string {
+ var result []string
+ for _, items := range s {
+ for _, item := range strings.Split(items, ",") {
+ item = strings.TrimSpace(item)
+ if item == "" {
+ continue
+ }
+ result = append(result, item)
+ }
+ }
+ return result
+}
+
func printAverageDetectorTime(e *engine.Engine) {
fmt.Fprintln(os.Stderr, "Average detector time is the measurement of average time spent on each detector when results are returned.")
for detectorName, durations := range e.DetectorAvgTime() {
diff --git a/pkg/engine/gcs.go b/pkg/engine/gcs.go
index 26fdffd51..75f3a4453 100644
--- a/pkg/engine/gcs.go
+++ b/pkg/engine/gcs.go
@@ -15,10 +15,17 @@ import (
// ScanGCS with the provided options.
func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
- if c.ProjectID == "" {
+ // Project ID is required if using any authenticated access.
+ if c.ProjectID == "" && !c.WithoutAuth {
return fmt.Errorf("project ID is required")
}
+ // If using unauthenticated access, the project ID is not used.
+ if c.ProjectID != "" && c.WithoutAuth {
+ c.ProjectID = ""
+ ctx.Logger().Info("project ID is not used when using unauthenticated access, ignoring provided project ID")
+ }
+
connection := &sourcespb.GCS{
ProjectId: c.ProjectID,
IncludeBuckets: c.IncludeBuckets,
@@ -43,7 +50,7 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
"source_type", source.Type().String(),
"source_name", "gcs",
)
- if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, c.Concurrency); err != nil {
+ if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, int(c.Concurrency)); err != nil {
return fmt.Errorf("failed to initialize GCS source: %w", err)
}
diff --git a/pkg/engine/gcs_test.go b/pkg/engine/gcs_test.go
index 1621a9236..e66e7bd92 100644
--- a/pkg/engine/gcs_test.go
+++ b/pkg/engine/gcs_test.go
@@ -1,6 +1,7 @@
package engine
import (
+ "strings"
"testing"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -24,11 +25,13 @@ func TestScanGCS(t *testing.T) {
},
},
{
- name: "missing project ID",
- gcsConfig: sources.GCSConfig{
- ApiKey: "abc123",
- },
- wantErr: true,
+ name: "missing project ID, with auth",
+ gcsConfig: sources.GCSConfig{ApiKey: "abc123"},
+ wantErr: true,
+ },
+ {
+ name: "missing project ID, without auth, public scan",
+ gcsConfig: sources.GCSConfig{WithoutAuth: true},
},
{
name: "multiple selected auth methods",
@@ -44,7 +47,8 @@ func TestScanGCS(t *testing.T) {
{
name: "no auth method selected",
gcsConfig: sources.GCSConfig{
- ProjectID: "test-project",
+ ProjectID: "test-project",
+ MaxObjectSize: 10 * 1024 * 1024,
},
},
}
@@ -53,7 +57,7 @@ func TestScanGCS(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
e := &Engine{}
err := e.ScanGCS(context.Background(), test.gcsConfig)
- if err != nil && !test.wantErr {
+ if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
return
}
diff --git a/pkg/pb/sourcespb/sources.pb.go b/pkg/pb/sourcespb/sources.pb.go
index ccb2f0ae8..9d93e6f86 100644
--- a/pkg/pb/sourcespb/sources.pb.go
+++ b/pkg/pb/sourcespb/sources.pb.go
@@ -963,6 +963,7 @@ type GCS struct {
ExcludeBuckets []string `protobuf:"bytes,7,rep,name=exclude_buckets,json=excludeBuckets,proto3" json:"exclude_buckets,omitempty"`
IncludeObjects []string `protobuf:"bytes,8,rep,name=include_objects,json=includeObjects,proto3" json:"include_objects,omitempty"`
ExcludeObjects []string `protobuf:"bytes,9,rep,name=exclude_objects,json=excludeObjects,proto3" json:"exclude_objects,omitempty"`
+ MaxObjectSize int64 `protobuf:"varint,10,opt,name=max_object_size,json=maxObjectSize,proto3" json:"max_object_size,omitempty"`
}
func (x *GCS) Reset() {
@@ -1074,6 +1075,13 @@ func (x *GCS) GetExcludeObjects() []string {
return nil
}
+func (x *GCS) GetMaxObjectSize() int64 {
+ if x != nil {
+ return x.MaxObjectSize
+ }
+ return 0
+}
+
type isGCS_Credential interface {
isGCS_Credential()
}
diff --git a/pkg/pb/sourcespb/sources.pb.validate.go b/pkg/pb/sourcespb/sources.pb.validate.go
index d675096d4..565b6dbc0 100644
--- a/pkg/pb/sourcespb/sources.pb.validate.go
+++ b/pkg/pb/sourcespb/sources.pb.validate.go
@@ -1244,6 +1244,8 @@ func (m *GCS) validate(all bool) error {
// no validation rules for ProjectId
+ // no validation rules for MaxObjectSize
+
switch m.Credential.(type) {
case *GCS_JsonServiceAccount:
diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.go
index 64a5f3486..1bd141ec9 100644
--- a/pkg/sources/gcs/gcs.go
+++ b/pkg/sources/gcs/gcs.go
@@ -1,9 +1,11 @@
package gcs
import (
+ "encoding/json"
"fmt"
"io"
"os"
+ "sort"
"sync"
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
@@ -28,9 +30,12 @@ type Source struct {
verify bool
gcsManager objectManager
+ stats *attributes
log logr.Logger
chunksCh chan *sources.Chunk
- sources.Progress
+
+ mu sync.Mutex
+ sources.Progress // not safe for concurrent access.
}
// Ensure the Source satisfies the interface at compile time.
@@ -90,7 +95,17 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6
}
- gcsManagerOpts := []gcsManagerOption{withConcurrency(concurrency), gcsManagerAuthOption}
+ resume, err := setResumeBucketOffset(s.Progress.EncodedResumeInfo)
+ if err != nil {
+ return fmt.Errorf("error setting resume info: %w", err)
+ }
+
+ gcsManagerOpts := []gcsManagerOption{
+ withConcurrency(concurrency),
+ withBucketOffsets(resume),
+ withMaxObjectSize(conn.MaxObjectSize),
+ gcsManagerAuthOption,
+ }
if setGCSManagerBucketOptions(&conn) != nil {
gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(&conn))
}
@@ -102,7 +117,13 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6
return fmt.Errorf("error creating GCS manager: %w", err)
}
- return nil
+ if len(resume) > 0 {
+ s.log.V(2).Info("resuming from previous run")
+ return nil
+ }
+
+ s.log.V(2).Info("new run, enumerating buckets and objects")
+ return s.enumerate(aCtx)
}
func setGCSManagerBucketOptions(conn *sourcespb.GCS) gcsManagerOption {
@@ -126,6 +147,166 @@ func setGCSManagerOptions(include, exclude []string, includeFn, excludeFn func([
return nil
}
+type progressInfo struct {
+ processedBucketCnt,
+ totalBucketsCnt,
+ processedObjectsCnt,
+ totalObjectsCnt int
+
+ // bucketObjects tracks processing and processed objects for each bucket.
+ bucketObjects map[string]*objectsProgress
+}
+
+// objectsProgress keeps track of the progress of processing objects in a bucket.
+// It is marshalled/unmarshalled to/from a string as part of the Source's Progress.
+type objectsProgress struct {
+ // IsBuckedProcessed is true if all objects in the bucket have been processed.
+ IsBucketProcessed bool
+ // ProcessedCnt is the number of objects that have been processed.
+ ProcessedCnt,
+ // TotalCnt is the total number of objects in the bucket.
+ TotalCnt int
+ // Processed is the last object that was fully processed.
+ Processed string
+ // Processing are the objects that were not fully processed.
+ Processing map[string]struct{}
+}
+
+func newObjectsProgress(cnt int) *objectsProgress {
+ return &objectsProgress{TotalCnt: cnt, Processing: map[string]struct{}{}}
+}
+
+// newProgressInfo constructs a progressInfo from the Source's Progress information.
+func newProgressInfo(ctx context.Context, s *sources.Progress) (*progressInfo, error) {
+ totalObjCnt := s.SectionsRemaining
+ processedObjCnt := s.SectionsCompleted
+ info := &progressInfo{
+ totalObjectsCnt: int(totalObjCnt),
+ processedObjectsCnt: int(processedObjCnt),
+ }
+
+ encodeResume := s.EncodedResumeInfo
+ if encodeResume == "" {
+ info.bucketObjects = make(map[string]*objectsProgress)
+ return info, nil
+ }
+
+ var m map[string]*objectsProgress
+ if err := json.Unmarshal([]byte(encodeResume), &m); err != nil {
+ ctx.Logger().V(1).Info("error unmarshalling progress info", "error", err)
+ return info, nil
+ }
+ info.bucketObjects = m
+
+ return info, nil
+}
+
+type progressStateFn func(string, string, *progressInfo)
+
+func (p *progressInfo) setProcessStatus(obj object, fn progressStateFn) {
+ fn(obj.bucket, obj.name, p)
+}
+
+func setProcessingBucketObject(bucket, obj string, progress *progressInfo) {
+ progress.processing(bucket, obj)
+}
+
+func (p *progressInfo) processing(bkt, obj string) {
+ p.bucketObjects[bkt].Processing[obj] = struct{}{}
+}
+
+func setProcessedBucketObject(bucket, obj string, progress *progressInfo) {
+ progress.processed(bucket, obj)
+}
+
+func (p *progressInfo) processed(bkt, obj string) {
+ // Remove the object from the processing list and increment the processed count.
+ delete(p.bucketObjects[bkt].Processing, obj)
+
+ p.bucketObjects[bkt].ProcessedCnt++
+ // If all objects in the bucket have been processed, mark the bucket as processed.
+ if p.bucketObjects[bkt].ProcessedCnt == p.bucketObjects[bkt].TotalCnt {
+ p.bucketObjects[bkt].IsBucketProcessed = true
+ p.processedBucketCnt++
+ }
+ p.processedObjectsCnt++
+
+ // If the object is not greater (lexicographically) than the last processed object, we can skip it.
+ // This ensures we keep the greatest object name as the last processed object.
+ if obj < p.bucketObjects[bkt].Processed {
+ return
+ }
+ p.bucketObjects[bkt].Processed = obj
+}
+
+func setResumeBucketOffset(s string) (map[string]offsetInfo, error) {
+ resumeInfo := progressInfo{}
+ if s != "" {
+ if err := json.Unmarshal([]byte(s), &resumeInfo.bucketObjects); err != nil {
+ return nil, fmt.Errorf("error unmarshalling resume info: %w", err)
+ }
+ }
+
+ return calcBktOffset(resumeInfo.bucketObjects)
+}
+
+// In order to calculate the bucket offset, we need to know the last object
+// that was processed for each bucket, as well as any objects that were processing.
+// If the bucket was fully processed, we don't need to set an offset.
+// If there were no objects processing, we can just use the last object that was processed.
+// If there were objects processing, we need to find the first object (lexicographically)
+// that was processing, as that is the next object that needs to be processed.
+func calcBktOffset(resumeInfo map[string]*objectsProgress) (map[string]offsetInfo, error) {
+ bucketOffset := make(map[string]offsetInfo, len(resumeInfo))
+ for objName, progress := range resumeInfo {
+ info := offsetInfo{}
+ if progress.IsBucketProcessed {
+ info.isBucketProcessed = true
+ bucketOffset[objName] = info
+ continue
+ }
+ if len(progress.Processing) == 0 {
+ info.lastProcessedObject = progress.Processed
+ bucketOffset[objName] = info
+ continue
+ }
+
+ processing := make([]string, 0, len(progress.Processing))
+ for k := range progress.Processing {
+ processing = append(processing, k)
+ }
+ sort.Strings(processing)
+ info.lastProcessedObject = processing[0]
+ bucketOffset[objName] = info
+ }
+
+ return bucketOffset, nil
+}
+
+// enumerate all the objects and buckets in the source and use the results to
+// set the progress information. This will be used track progression of the scan,
+// and to resume the scan if it is interrupted.
+func (s *Source) enumerate(ctx context.Context) error {
+ stats, err := s.gcsManager.attributes(ctx)
+ if err != nil {
+ return fmt.Errorf("error getting attributes during enumeration: %w", err)
+ }
+ s.stats = stats
+
+ info := progressInfo{
+ totalBucketsCnt: int(s.stats.numBuckets),
+ totalObjectsCnt: int(s.stats.numObjects),
+ }
+
+ progress := make(map[string]*objectsProgress, len(s.stats.bucketObjects))
+ for k, v := range s.stats.bucketObjects {
+ progress[k] = newObjectsProgress(int(v))
+ }
+ info.bucketObjects = progress
+
+ return s.setProgress(ctx, &info, fmt.Sprintf("enumerated %d buckets, %d objects", s.stats.numBuckets, s.stats.numObjects))
+}
+
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
objectCh, err := s.gcsManager.listObjects(ctx)
@@ -134,6 +315,11 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
}
s.chunksCh = chunksChan
+ progress, err := newProgressInfo(ctx, &s.Progress)
+ if err != nil {
+ return fmt.Errorf("error constructing progress info: %w", err)
+ }
+
var wg sync.WaitGroup
for obj := range objectCh {
obj := obj
@@ -143,14 +329,22 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
continue
}
+ if err := s.startProcessing(ctx, progress, o); err != nil {
+ ctx.Logger().V(1).Info("GCS source error starting to process objects", "error", err)
+ continue
+ }
wg.Add(1)
- go func(obj object) {
+ go func(obj object, resume *progressInfo) {
defer wg.Done()
if err := s.processObject(ctx, o); err != nil {
- ctx.Logger().V(1).Info("GCS source error processing objects", "name", s.name, "error", err)
+ ctx.Logger().V(1).Info("error setting start progress progress", "name", o.name, "error", err)
+ return
+ }
+ if err := s.endProcessing(ctx, resume, o); err != nil {
+ ctx.Logger().V(1).Info("error setting end progress progress", "name", o.name, "error", err)
}
- }(o)
+ }(o, progress)
}
wg.Wait()
@@ -158,6 +352,34 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
return nil
}
+func (s *Source) startProcessing(ctx context.Context, progress *progressInfo, o object) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ progress.setProcessStatus(o, setProcessingBucketObject)
+ return s.setProgress(ctx, progress, fmt.Sprintf("GCS source beginning to process object %s in bucket %s", o.name, o.bucket))
+}
+
+func (s *Source) endProcessing(ctx context.Context, progress *progressInfo, o object) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ progress.setProcessStatus(o, setProcessedBucketObject)
+ return s.setProgress(ctx, progress, fmt.Sprintf("GCS source finished processing object %s in bucket %s", o.name, o.bucket))
+}
+
+func (s *Source) setProgress(ctx context.Context, progress *progressInfo, msg string) error {
+ ctx.Logger().V(5).Info(msg)
+
+ b, err := json.Marshal(progress.bucketObjects)
+ if err != nil {
+ return fmt.Errorf("error marshalling resume info: %w", err)
+ }
+ s.SetProgressComplete(progress.processedObjectsCnt, progress.totalObjectsCnt, msg, string(b))
+
+ return nil
+}
+
func (s *Source) processObject(ctx context.Context, o object) error {
chunkSkel := &sources.Chunk{
SourceName: s.name,
diff --git a/pkg/sources/gcs/gcs_integration_test.go b/pkg/sources/gcs/gcs_integration_test.go
index 357844791..6c8fe2a09 100644
--- a/pkg/sources/gcs/gcs_integration_test.go
+++ b/pkg/sources/gcs/gcs_integration_test.go
@@ -23,7 +23,7 @@ func TestChunks(t *testing.T) {
source, conn := createTestSource(&sourcespb.GCS{
ProjectId: testProjectID,
Credential: &sourcespb.GCS_Adc{},
- ExcludeBuckets: []string{perfTestBucketGlob, "public-trufflehog-test-bucket"},
+ ExcludeBuckets: []string{perfTestBucketGlob, publicBucket},
})
err := source.Init(ctx, "test", 1, 1, true, conn, 8)
@@ -58,6 +58,65 @@ func TestChunks(t *testing.T) {
}
}
+func TestChunks_PublicBucket(t *testing.T) {
+ ctx := context.Background()
+
+ source, conn := createTestSource(&sourcespb.GCS{
+ Credential: &sourcespb.GCS_Unauthenticated{},
+ IncludeBuckets: []string{publicBucket},
+ })
+
+ err := source.Init(ctx, "test", 1, 1, true, conn, 8)
+ assert.Nil(t, err)
+
+ chunksCh := make(chan *sources.Chunk, 1)
+
+ go func() {
+ defer close(chunksCh)
+ err := source.Chunks(ctx, chunksCh)
+ assert.Nil(t, err)
+ }()
+
+ want := []*sources.Chunk{
+ {
+ SourceName: "test",
+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,
+ SourceID: 0,
+ Verify: true,
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Gcs{
+ Gcs: &source_metadatapb.GCS{
+ Filename: "aws1.txt",
+ Bucket: publicBucket,
+ ContentType: "text/plain",
+ Email: "",
+ Link: "https://storage.googleapis.com/download/storage/v1/b/public-trufflehog-test-bucket/o/aws1.txt?generation=1678334408999764&alt=media",
+ Acls: []string{},
+ },
+ },
+ },
+ },
+ }
+
+ got := make([]*sources.Chunk, 0, len(want))
+ for chunk := range chunksCh {
+ got = append(got, chunk)
+ }
+ sort.Slice(got, func(i, j int) bool {
+ return got[i].SourceMetadata.GetGcs().Filename < got[j].SourceMetadata.GetGcs().Filename
+ })
+
+ assert.Equal(t, len(want), len(got))
+
+ for i, chunk := range got {
+ if diff := cmp.Diff(want[i].SourceMetadata.GetGcs(), chunk.SourceMetadata.GetGcs(),
+ cmpopts.IgnoreFields(source_metadatapb.GCS{}, "state", "sizeCache", "unknownFields", "CreatedAt", "UpdatedAt"),
+ ); diff != "" {
+ t.Errorf("chunk mismatch (-want +got):\n%s", diff)
+ }
+ }
+}
+
func createTestChunks() []*sources.Chunk {
objects := []object{
{
diff --git a/pkg/sources/gcs/gcs_manager.go b/pkg/sources/gcs/gcs_manager.go
index f272074a4..3fb58a8e0 100644
--- a/pkg/sources/gcs/gcs_manager.go
+++ b/pkg/sources/gcs/gcs_manager.go
@@ -5,6 +5,7 @@ import (
"fmt"
"io"
"runtime"
+ "sync"
"sync/atomic"
"time"
@@ -22,8 +23,8 @@ import (
)
const (
- defaultMaxObjectSize = 500 * 1024 * 1024 // 500MB
- maxObjectSizeLimit = 1 * 1024 * 1024 * 1024 // 1GB
+ defaultMaxObjectSize = 10 * 1024 * 1024 // 10MB
+ maxObjectSizeLimit = 50 * 1024 * 1024 // 50MB
)
var (
@@ -32,6 +33,7 @@ var (
type objectManager interface {
listObjects(context.Context) (chan io.Reader, error)
+ attributes(ctx context.Context) (*attributes, error)
}
// bucketManager is a simplified *storage.Client wrapper.
@@ -47,6 +49,8 @@ type bucketManager interface {
// It's main purpose is to retrieve objects from GCS.
type gcsManager struct {
projectID string
+ withoutAuth,
+ hasEnumerated bool
concurrency int
workerPool *errgroup.Group
@@ -59,16 +63,19 @@ type gcsManager struct {
excludeBuckets,
includeObjects,
excludeObjects map[string]struct{}
+
buckets map[string]bucket
- client bucketManager
+ attr *attributes
+
+ client bucketManager
}
// bucket is a simplified *storage.BucketHandle wrapper.
// It also provides a mechanism for resuming an interrupted iteration.
// This reduces the number of objects that need to be re-processed.
type bucket struct {
- name string
- numObjects uint64
+ shouldInclude bool
+ name string
// startOffset is the name of the object to resume from. (inclusive)
// This works because GCS objects are sorted lexicographically.
// If this is empty, the iteration will start from the beginning.
@@ -76,6 +83,40 @@ type bucket struct {
*storage.BucketHandle
}
+// offsetInfo is used to resume an interrupted iteration.
+type offsetInfo struct {
+ isBucketProcessed bool
+ lastProcessedObject string
+}
+
+// attributes contains metadata about the GCS source.
+// This will be collected during the initial scan.
+type attributes struct {
+ numBuckets uint32
+ numObjects uint64
+ mu sync.RWMutex
+ bucketObjects map[string]uint64
+}
+
+func newStats(numBkts int) *attributes {
+ return &attributes{
+ numBuckets: uint32(numBkts),
+ bucketObjects: make(map[string]uint64, numBkts),
+ }
+}
+
+func (s *attributes) incObjects() {
+ s.mu.Lock()
+ s.numObjects++
+ s.mu.Unlock()
+}
+
+func (s *attributes) setBucketCnt(bkt string, cnt uint64) {
+ s.mu.Lock()
+ s.bucketObjects[bkt] = cnt
+ s.mu.Unlock()
+}
+
type gcsManagerOption func(*gcsManager) error
// withAPIKey uses the provided API key when creating a new GCS client.
@@ -107,8 +148,8 @@ func withJSONServiceAccount(ctx context.Context, jsonServiceAccount []byte) gcsM
// withDefaultADC uses the default application credentials when creating a new GCS client.
func withDefaultADC(ctx context.Context) gcsManagerOption {
+ client, err := defaultADC(ctx)
return func(m *gcsManager) error {
- client, err := defaultADC(ctx)
if err != nil {
return err
}
@@ -127,6 +168,7 @@ func withoutAuthentication() gcsManagerOption {
return err
}
m.client = client
+ m.withoutAuth = true
return nil
}
}
@@ -254,25 +296,23 @@ func withMaxObjectSize(maxObjectSize int64) gcsManagerOption {
// withBucketOffsets sets the offset for each bucket.
// This is used to resume listing objects for a bucket.
-func withBucketOffsets(offsets map[string]string) gcsManagerOption {
+func withBucketOffsets(offsets map[string]offsetInfo) gcsManagerOption {
bkts := make(map[string]bucket, len(offsets))
- for bkt, val := range offsets {
+ for bkt, offst := range offsets {
bkts[bkt] = bucket{
- startOffset: val,
- name: bkt,
+ shouldInclude: !offst.isBucketProcessed,
+ name: bkt,
+ startOffset: offst.lastProcessedObject,
}
}
return func(m *gcsManager) error {
+ m.hasEnumerated = len(offsets) > 0 // offset means we are resuming.
m.buckets = bkts
return nil
}
}
func newGCSManager(projectID string, opts ...gcsManagerOption) (*gcsManager, error) {
- if projectID == "" {
- return nil, fmt.Errorf("project ID is required")
- }
-
// Default values for the manager.
gcs := &gcsManager{
projectID: projectID,
@@ -287,6 +327,10 @@ func newGCSManager(projectID string, opts ...gcsManagerOption) (*gcsManager, err
}
}
+ if projectID == "" && !gcs.withoutAuth {
+ return nil, fmt.Errorf("project ID is required, when using authentication")
+ }
+
// If no client was provided, use the default application credentials.
// A client is required to perform any operations.
if gcs.client == nil {
@@ -323,6 +367,70 @@ type object struct {
io.Reader
}
+func (g *gcsManager) attributes(ctx context.Context) (*attributes, error) {
+ // Get all the buckets in the project.
+ buckets, err := g.listBuckets(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list buckets: %w", err)
+ }
+
+ return g.enumerate(ctx, buckets)
+}
+
+func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes, error) {
+ logger := ctx.Logger().WithValues("phase", "enumeration")
+
+ logger.V(5).Info("enumerating buckets", "numBuckets", len(bkts))
+ stats := newStats(len(bkts))
+
+ for _, bkt := range bkts {
+ bkt := bkt
+ g.workerPool.Go(func() error {
+ // List all the objects in the bucket and calculate a attributes.
+ g.setupBktHandle(&bkt)
+
+ q, err := setObjectQuery(&bkt)
+ if err != nil {
+ logger.Error(err, "failed to set object query", "bucket", bkt.name)
+ return nil
+ }
+
+ var count uint64
+ objs := bkt.Objects(ctx, q)
+ for {
+ obj, err := objs.Next()
+ if errors.Is(err, iterator.Done) {
+ logger.V(5).Info("finished listing objects in bucket")
+ break
+ }
+ if err != nil {
+ logger.V(1).Info("failed to list objects", "bucket", bkt.name, "error", err)
+ return nil
+ }
+ if obj == nil {
+ logger.V(5).Info("object is nil")
+ continue
+ }
+
+ if !g.shouldIncludeObject(ctx, obj.Name) || g.shouldExcludeObject(ctx, obj.Name) {
+ continue
+ }
+ count++
+ stats.incObjects()
+ }
+
+ stats.setBucketCnt(bkt.name, count)
+ return nil
+ })
+ }
+
+ _ = g.workerPool.Wait()
+ g.attr = stats
+ logger.V(5).Info("finished enumerating buckets", "num-objects", stats.numObjects, "num-buckets", stats.numBuckets)
+
+ return stats, nil
+}
+
func (g *gcsManager) listObjects(ctx context.Context) (chan io.Reader, error) {
ch := make(chan io.Reader, 100) // TODO (ahrav): Determine optimal buffer size.
@@ -376,6 +484,12 @@ func (g *gcsManager) listObjects(ctx context.Context) (chan io.Reader, error) {
func (g *gcsManager) listBuckets(ctx context.Context) ([]bucket, error) {
var buckets []bucket
+ if g.withoutAuth {
+ for name := range g.includeBuckets {
+ buckets = append(buckets, bucket{name: name})
+ }
+ return buckets, nil
+ }
bkts := g.client.Buckets(ctx, g.projectID)
for {
@@ -412,15 +526,7 @@ func (g *gcsManager) listBucketObjects(ctx context.Context, bkt *bucket) (chan i
logger := ctx.Logger().WithValues("bucket", bkt.name)
logger.V(5).Info("listing object(s) in bucket")
- b := g.client.Bucket(bkt.name).Retryer(
- storage.WithBackoff(gax.Backoff{
- Initial: 2 * time.Second,
- Max: 30 * time.Second,
- Multiplier: 1.5,
- }),
- storage.WithPolicy(storage.RetryAlways),
- )
- bkt.BucketHandle = b
+ g.setupBktHandle(bkt)
// TODO (ahrav): Look to extend gcsManager to allow for exact buckets/objects
// include filters. This will increase performance substantially
@@ -432,6 +538,18 @@ func (g *gcsManager) listBucketObjects(ctx context.Context, bkt *bucket) (chan i
return ch, errCh
}
+func (g *gcsManager) setupBktHandle(bkt *bucket) {
+ b := g.client.Bucket(bkt.name).Retryer(
+ storage.WithBackoff(gax.Backoff{
+ Initial: 2 * time.Second,
+ Max: 30 * time.Second,
+ Multiplier: 1.5,
+ }),
+ storage.WithPolicy(storage.RetryAlways),
+ )
+ bkt.BucketHandle = b
+}
+
func (g *gcsManager) bucketObjects(ctx context.Context, bkt *bucket, ch chan<- io.Reader) error {
q, err := setObjectQuery(bkt)
if err != nil {
@@ -445,12 +563,16 @@ func (g *gcsManager) bucketObjects(ctx context.Context, bkt *bucket, ch chan<- i
ctx.Logger().V(5).Info("finished listing objects in bucket")
break
}
- if !g.shouldIncludeObject(ctx, obj.Name) || g.shouldExcludeObject(ctx, obj.Name) {
+ if err != nil {
+ return fmt.Errorf("failed to retrieve object iterator: %w", err)
+ }
+ if obj == nil {
+ ctx.Logger().V(5).Info("object is nil")
continue
}
- if err != nil {
- return fmt.Errorf("failed to retrieve iterator: %w", err)
+ if !g.shouldIncludeObject(ctx, obj.Name) || g.shouldExcludeObject(ctx, obj.Name) {
+ continue
}
o, err := g.constructObject(ctx, bkt.Object(obj.Name))
@@ -458,7 +580,6 @@ func (g *gcsManager) bucketObjects(ctx context.Context, bkt *bucket, ch chan<- i
ctx.Logger().V(1).Info("failed to create object", "object-name", obj.Name, "error", err)
continue
}
- bkt.numObjects++ // atomic add NOT needed, object listing is sequential.
ch <- o
}
return nil
diff --git a/pkg/sources/gcs/gcs_manager_test.go b/pkg/sources/gcs/gcs_manager_test.go
index 2fe316f82..95bab761f 100644
--- a/pkg/sources/gcs/gcs_manager_test.go
+++ b/pkg/sources/gcs/gcs_manager_test.go
@@ -1,8 +1,10 @@
package gcs
import (
+ "fmt"
"sort"
"testing"
+ "time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
@@ -13,6 +15,7 @@ import (
)
const (
+ publicBucket = "public-trufflehog-test-bucket"
testProjectID = "trufflehog-testing"
testAPIKey = "somekeys"
testBucket = "test-bkt-th"
@@ -48,9 +51,13 @@ func TestNewGcsManager(t *testing.T) {
},
{
name: "new gcs manager, without auth",
- projID: testProjectID,
+ projID: "",
opts: []gcsManagerOption{withoutAuthentication()},
- want: &gcsManager{projectID: testProjectID, concurrency: defaultConcurrency, maxObjectSize: defaultMaxObjectSize},
+ want: &gcsManager{
+ concurrency: defaultConcurrency,
+ maxObjectSize: defaultMaxObjectSize,
+ withoutAuth: true,
+ },
},
{
name: "new gcs manager, with api key",
@@ -78,15 +85,18 @@ func TestNewGcsManager(t *testing.T) {
projID: testProjectID,
opts: []gcsManagerOption{
withDefaultADC(ctx),
- withBucketOffsets(map[string]string{testBucket: "moar.txt", testBucket2: "moar2.txt"}),
+ withBucketOffsets(map[string]offsetInfo{
+ testBucket: {lastProcessedObject: "moar1.txt", isBucketProcessed: false},
+ testBucket2: {lastProcessedObject: "", isBucketProcessed: true}}),
},
want: &gcsManager{
projectID: testProjectID,
+ hasEnumerated: true,
concurrency: defaultConcurrency,
maxObjectSize: defaultMaxObjectSize,
buckets: map[string]bucket{
- testBucket: {name: testBucket, startOffset: "moar.txt"},
- testBucket2: {name: testBucket2, startOffset: "moar2.txt"},
+ testBucket: {name: testBucket, startOffset: "moar.txt", shouldInclude: true},
+ testBucket2: {name: testBucket2, startOffset: "", shouldInclude: false},
},
},
},
@@ -331,7 +341,6 @@ func TestNewGcsManager(t *testing.T) {
if err != nil {
return
}
-
// The client should never be nil.
if got.client == nil {
t.Errorf("newGCSManager() client should not be nil")
@@ -348,6 +357,91 @@ func TestNewGcsManager(t *testing.T) {
}
}
+func TestGCSManagerStats(t *testing.T) {
+ ctx := context.Background()
+
+ testCases := []struct {
+ name string
+ projectID string
+ opts []gcsManagerOption
+ wantStats *attributes
+ }{
+ {
+ name: "enumerate, all buckets, with objects",
+ projectID: testProjectID,
+ opts: []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{perfTestBucketGlob, publicBucket})},
+ wantStats: &attributes{
+ numBuckets: 4,
+ numObjects: 5,
+ bucketObjects: map[string]uint64{testBucket: 2, testBucket2: 1, testBucket3: 1, testBucket4: 1},
+ },
+ },
+ {
+ name: "list objects, with exclude objects",
+ projectID: testProjectID,
+ opts: []gcsManagerOption{
+ withDefaultADC(ctx),
+ withExcludeObjects([]string{"aws1.txt", "moar2.txt"}),
+ withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),
+ },
+ wantStats: &attributes{
+ numBuckets: 4,
+ numObjects: 3,
+ bucketObjects: map[string]uint64{testBucket: 0, testBucket2: 1, testBucket3: 1, testBucket4: 1},
+ },
+ },
+ {
+ name: "list objects, with include objects",
+ projectID: testProjectID,
+ opts: []gcsManagerOption{
+ withDefaultADC(ctx),
+ withIncludeObjects([]string{"aws1.txt"}),
+ withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),
+ },
+ wantStats: &attributes{
+ numBuckets: 4,
+ numObjects: 1,
+ bucketObjects: map[string]uint64{testBucket: 1, testBucket2: 0, testBucket3: 0, testBucket4: 0},
+ },
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ gm, err := newGCSManager(tc.projectID, tc.opts...)
+ if err != nil {
+ t.Fatalf("newGCSManager() error = %v", err)
+ }
+
+ got, err := gm.attributes(ctx)
+ if err != nil {
+ t.Errorf("attributes() error = %v", err)
+ }
+
+ if diff := cmp.Diff(got, tc.wantStats, cmp.AllowUnexported(attributes{}), cmpopts.IgnoreFields(attributes{}, "mu")); diff != "" {
+ t.Errorf("attributes() got: %v, want: %v, diff: %v", got, tc.wantStats, diff)
+ }
+ })
+ }
+}
+
+func TestGCSManagerStats_Time(t *testing.T) {
+ ctx := context.Background()
+
+ opts := []gcsManagerOption{withDefaultADC(ctx)}
+ gm, err := newGCSManager(testProjectID, opts...)
+ if err != nil {
+ t.Fatalf("newGCSManager() error = %v", err)
+ }
+
+ start := time.Now()
+ var stats *attributes
+ stats, _ = gm.attributes(ctx)
+ end := time.Since(start).Seconds()
+
+ fmt.Printf("Time taken to get %d objects: %f seconds\n", stats.numObjects, end)
+}
+
func TestGCSManagerListObjects(t *testing.T) {
ctx := context.Background()
@@ -370,7 +464,7 @@ func TestGCSManagerListObjects(t *testing.T) {
{
name: "list objects, all buckets, with objects",
projectID: testProjectID,
- opts: []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{perfTestBucketGlob, "public-trufflehog-test-bucket"})},
+ opts: []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{perfTestBucketGlob, publicBucket})},
want: []object{
{
name: "aws1.txt",
@@ -444,8 +538,7 @@ func TestGCSManagerListObjects(t *testing.T) {
{
name: "list objects, exclude buckets, with objects",
projectID: testProjectID,
- opts: []gcsManagerOption{withDefaultADC(ctx),
- withExcludeBuckets([]string{testBucket, testBucket2, perfTestBucketGlob, "public-trufflehog-test-bucket"})},
+ opts: []gcsManagerOption{withDefaultADC(ctx), withExcludeBuckets([]string{testBucket, testBucket2, perfTestBucketGlob, publicBucket})},
want: []object{
{
name: "moar.txt",
@@ -473,7 +566,7 @@ func TestGCSManagerListObjects(t *testing.T) {
opts: []gcsManagerOption{
withDefaultADC(ctx),
withExcludeObjects([]string{"aws1.txt", "moar2.txt"}),
- withExcludeBuckets([]string{perfTestBucketGlob, "public-trufflehog-test-bucket"}),
+ withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),
},
want: []object{
{
@@ -531,7 +624,7 @@ func TestGCSManagerListObjects(t *testing.T) {
opts: []gcsManagerOption{
withDefaultADC(ctx),
withIncludeObjects([]string{"aws1.txt"}),
- withExcludeBuckets([]string{perfTestBucketGlob, "public-trufflehog-test-bucket"}),
+ withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),
},
want: []object{
{
@@ -553,7 +646,7 @@ func TestGCSManagerListObjects(t *testing.T) {
withDefaultADC(ctx),
withIncludeBuckets([]string{testBucket}),
withExcludeObjects([]string{"aws1.txt"}),
- withExcludeBuckets([]string{perfTestBucketGlob, "public-trufflehog-test-bucket"}),
+ withExcludeBuckets([]string{perfTestBucketGlob, publicBucket}),
},
want: []object{
{
@@ -640,7 +733,7 @@ func TestGCSManagerListObjects_Resuming(t *testing.T) {
opts: []gcsManagerOption{
withDefaultADC(ctx),
withIncludeBuckets([]string{testBucket}),
- withBucketOffsets(map[string]string{testBucket: "moar2.txt"}),
+ withBucketOffsets(map[string]offsetInfo{testBucket: {lastProcessedObject: "moar2.txt"}}),
},
want: []object{
{
@@ -758,14 +851,14 @@ func Test_isObjectSizeValid(t *testing.T) {
want: true,
},
{
- name: "valid object size, 100 MB",
- objSize: int64(100 * common.MB),
+ name: "valid object size, 49 MB",
+ objSize: int64(49 * common.MB),
want: true,
},
{
- name: "valid object size, 1 GB",
+ name: "invalid object size, 1 GB",
objSize: int64(1 * common.GB),
- want: true,
+ want: false,
},
{
name: "invalid object size, 2 GB",
diff --git a/pkg/sources/gcs/gcs_test.go b/pkg/sources/gcs/gcs_test.go
index 600c2683b..973a5d9c6 100644
--- a/pkg/sources/gcs/gcs_test.go
+++ b/pkg/sources/gcs/gcs_test.go
@@ -1,6 +1,7 @@
package gcs
import (
+ "encoding/json"
"fmt"
"io"
"sort"
@@ -43,7 +44,12 @@ func TestSourceInit(t *testing.T) {
},
})
- err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+ // Set the EncodeResumeInfo to prevent enumeration.
+ b, err := json.Marshal(map[string]objectsProgress{"other": {}})
+ assert.Nil(t, err)
+ source.Progress.EncodedResumeInfo = string(b)
+
+ err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
assert.Nil(t, err)
assert.NotNil(t, source.gcsManager)
}
@@ -64,9 +70,14 @@ func TestSourceInit_Conn(t *testing.T) {
conn: &sourcespb.GCS{
ProjectId: testProjectID,
Credential: &sourcespb.GCS_Adc{},
+ ExcludeBuckets: []string{
+ perfTestBucketGlob,
+ },
},
want: &gcsManager{
- projectID: testProjectID,
+ projectID: testProjectID,
+ excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
+ hasEnumerated: true,
},
},
{
@@ -84,6 +95,7 @@ func TestSourceInit_Conn(t *testing.T) {
want: &gcsManager{
projectID: testProjectID,
includeBuckets: map[string]struct{}{"bucket1": {}},
+ hasEnumerated: true,
},
},
{
@@ -97,10 +109,15 @@ func TestSourceInit_Conn(t *testing.T) {
ExcludeObjects: []string{
"object2",
},
+ ExcludeBuckets: []string{
+ perfTestBucketGlob,
+ },
},
want: &gcsManager{
projectID: testProjectID,
includeObjects: map[string]struct{}{"object1": {}},
+ excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
+ hasEnumerated: true,
},
},
}
@@ -108,8 +125,12 @@ func TestSourceInit_Conn(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
source, conn := createTestSource(tc.conn)
+ // Set the EncodeResumeInfo to prevent enumeration.
+ b, err := json.Marshal(map[string]objectsProgress{"other": {}})
+ assert.Nil(t, err)
+ source.Progress.EncodedResumeInfo = string(b)
- err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+ err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
if err != nil && !tc.wantErr {
t.Errorf("source.Init() got: %v, want: %v", err, nil)
return
@@ -118,7 +139,7 @@ func TestSourceInit_Conn(t *testing.T) {
if !tc.wantErr {
if diff := cmp.Diff(tc.want, source.gcsManager,
cmp.AllowUnexported(gcsManager{}),
- cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "concurrency", "buckets", "maxObjectSize"),
+ cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "concurrency", "buckets", "maxObjectSize", "attr"),
); diff != "" {
t.Errorf("source.Init() diff: (-want +got)\n%s", diff)
}
@@ -127,10 +148,81 @@ func TestSourceInit_Conn(t *testing.T) {
}
}
+func TestSourceInit_Progress(t *testing.T) {
+ complete := map[string]objectsProgress{testBucket: {
+ Processed: object2,
+ IsBucketProcessed: true,
+ ProcessedCnt: 1,
+ TotalCnt: 1,
+ }}
+ completeEncodeResumeInfo, err := json.Marshal(complete)
+ assert.Nil(t, err)
+
+ processing := map[string]objectsProgress{
+ testBucket: {
+ IsBucketProcessed: false,
+ ProcessedCnt: 1,
+ Processed: object2,
+ Processing: map[string]struct{}{object1: {}},
+ TotalCnt: 2,
+ },
+ }
+ processingEncodeResumeInfo, err := json.Marshal(processing)
+ assert.Nil(t, err)
+
+ testCases := []struct {
+ name string
+ conn *sourcespb.GCS
+ encodeResumeInfo string
+ gcsManagerBuckets map[string]bucket
+ }{
+ {
+ name: "no progress",
+ conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}, ExcludeBuckets: []string{perfTestBucketGlob}},
+ gcsManagerBuckets: map[string]bucket{},
+ },
+ {
+ name: "progress set, no processing objects in any buckets",
+ conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
+ encodeResumeInfo: string(completeEncodeResumeInfo),
+ gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: "", shouldInclude: false}},
+ },
+ {
+ name: "progress set, processing objects in some buckets",
+ conn: &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
+ encodeResumeInfo: string(processingEncodeResumeInfo),
+ gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: object1, shouldInclude: true}},
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ source, conn := createTestSource(tc.conn)
+ source.Progress.EncodedResumeInfo = tc.encodeResumeInfo
+
+ err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+ assert.Nil(t, err)
+ assert.Equal(t, tc.gcsManagerBuckets, source.gcsManager.(*gcsManager).buckets)
+ })
+ }
+}
+
type mockObjectManager struct {
wantErr bool
}
+func (m *mockObjectManager) attributes(_ context.Context) (*attributes, error) {
+ if m.wantErr {
+ return nil, fmt.Errorf("some error")
+ }
+
+ return &attributes{
+ numObjects: 5,
+ numBuckets: 1,
+ bucketObjects: map[string]uint64{testBucket: 5},
+ }, nil
+}
+
type mockReader struct {
offset int
data []byte
@@ -207,6 +299,9 @@ func TestSourceChunks_ListObjects(t *testing.T) {
chunksCh: chunksCh,
}
+ err := source.enumerate(ctx)
+ assert.Nil(t, err)
+
go func() {
defer close(chunksCh)
err := source.Chunks(ctx, chunksCh)
@@ -243,6 +338,24 @@ func TestSourceChunks_ListObjects(t *testing.T) {
}
+func TestSourceInit_Enumerate(t *testing.T) {
+ ctx := context.Background()
+ source := &Source{gcsManager: &mockObjectManager{}}
+
+ err := source.enumerate(ctx)
+ assert.Nil(t, err)
+
+ // Ensure the attributes are set.
+ assert.Equal(t, uint64(5), source.stats.numObjects)
+ assert.Equal(t, uint32(1), source.stats.numBuckets)
+ assert.Equal(t, uint64(5), source.stats.bucketObjects[testBucket])
+ // Ensure progress is initialized correctly.
+ p := map[string]*objectsProgress{testBucket: {TotalCnt: 5, Processing: map[string]struct{}{}}}
+ b, err := json.Marshal(p)
+ assert.Nil(t, err)
+ assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
+}
+
func TestSourceChunks_ListObjects_Error(t *testing.T) {
ctx := context.Background()
source := &Source{gcsManager: &mockObjectManager{wantErr: true}}
@@ -253,3 +366,57 @@ func TestSourceChunks_ListObjects_Error(t *testing.T) {
err := source.Chunks(ctx, chunksCh)
assert.True(t, err != nil)
}
+
+func TestSourceChunks_ProgressSet(t *testing.T) {
+ ctx := context.Background()
+ chunksCh := make(chan *sources.Chunk, 1)
+
+ source := &Source{
+ gcsManager: &mockObjectManager{},
+ chunksCh: chunksCh,
+ }
+ err := source.enumerate(ctx)
+ assert.Nil(t, err)
+
+ go func() {
+ defer close(chunksCh)
+ err := source.Chunks(ctx, chunksCh)
+ assert.Nil(t, err)
+ }()
+
+ want := make([]*sources.Chunk, 0, 5)
+ for i := 0; i < 5; i++ {
+ want = append(want, createTestSourceChunk(i))
+ }
+
+ got := make([]*sources.Chunk, 0, 5)
+ for ch := range chunksCh {
+ got = append(got, ch)
+ }
+
+ // Ensure we get 5 objects back.
+ assert.Equal(t, len(want), len(got))
+
+ processing := map[string]struct{}{"object0": {}, "object1": {}, "object2": {}, "object3": {}, "object4": {}}
+ for i := 0; i < 5; i++ {
+ delete(processing, fmt.Sprintf("object%d", i))
+ }
+
+ // Test that the resume progress is set.
+ // The processed values should be the greatest object name lexicographically.
+ resume := map[string]objectsProgress{
+ testBucket: {
+ Processed: "object4",
+ Processing: processing,
+ IsBucketProcessed: true,
+ ProcessedCnt: 5,
+ TotalCnt: 5,
+ },
+ }
+ b, err := json.Marshal(resume)
+ assert.Nil(t, err)
+
+ assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
+ assert.Equal(t, int32(5), source.Progress.SectionsCompleted)
+ assert.Equal(t, int64(100), source.Progress.PercentComplete)
+}
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index 261596025..aa41abaff 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -57,6 +57,8 @@ type GCSConfig struct {
ProjectID,
// ServiceAccount is the service account to use to authenticate with the source.
ServiceAccount string
+ // MaxObjectSize is the maximum object size to scan.
+ MaxObjectSize int64
// Concurrency is the number of concurrent workers to use to scan the source.
Concurrency int
// IncludeBuckets is a list of buckets to include in the scan.
diff --git a/proto/sources.proto b/proto/sources.proto
index 8d96e2483..86adadcf5 100644
--- a/proto/sources.proto
+++ b/proto/sources.proto
@@ -137,6 +137,7 @@ message GCS {
repeated string exclude_buckets = 7;
repeated string include_objects = 8;
repeated string exclude_objects = 9;
+ int64 max_object_size = 10;
}
message Git {