Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Use md5 hash for resuming key (#1203)
ahrav2fbf86a6abcf
diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.goindex 9d866fbe4..f05c1ea95 100644--- a/pkg/sources/gcs/gcs.go+++ b/pkg/sources/gcs/gcs.go@@ -273,7 +273,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errctx.Logger().V(1).Info("error setting start progress progress", "name", o.name, "error", err)return}- s.setProgress(ctx, o.name, persistableCache)+ s.setProgress(ctx, o.md5, o.name, persistableCache)}(o)}wg.Wait()@@ -295,14 +295,14 @@ func (s *Source) setupCache(ctx context.Context) *persistableCache {return persistCache}-func (s *Source) setProgress(ctx context.Context, objName string, cache cache.Cache) {+func (s *Source) setProgress(ctx context.Context, md5, objName string, cache cache.Cache) {s.mu.Lock()defer s.mu.Unlock()ctx.Logger().V(5).Info("setting progress for object", "object-name", objName)s.SectionsCompleted++- cache.Set(objName, objName)+ cache.Set(md5, md5)s.Progress.SectionsRemaining = int32(s.stats.numObjects)s.Progress.PercentComplete = int64(float64(s.SectionsCompleted) / float64(s.stats.numObjects) * 100)}diff --git a/pkg/sources/gcs/gcs_manager.go b/pkg/sources/gcs/gcs_manager.goindex 79e69b930..1dcd15cee 100644--- a/pkg/sources/gcs/gcs_manager.go+++ b/pkg/sources/gcs/gcs_manager.go@@ -2,6 +2,7 @@ package gcsimport (aCtx "context"+ "encoding/base64""fmt""io""net/http"@@ -371,6 +372,7 @@ type object struct {contentType stringowner stringlink string+ md5 string// acl represents an ACLEntities.// https://pkg.go.dev/cloud.google.com/go/storage#ACLEntityacl []string@@ -648,6 +650,7 @@ func (g *gcsManager) constructObject(ctx context.Context, obj *storage.ObjectHano.contentType = attrs.ContentTypeo.owner = attrs.Ownero.link = attrs.MediaLink+ o.md5 = base64.StdEncoding.EncodeToString(attrs.MD5)o.createdAt = attrs.Createdo.updatedAt = attrs.Updatedo.acl = objectACLs(attrs.ACL)diff --git a/pkg/sources/gcs/gcs_manager_test.go b/pkg/sources/gcs/gcs_manager_test.goindex a373839ea..2df844372 100644--- a/pkg/sources/gcs/gcs_manager_test.go+++ b/pkg/sources/gcs/gcs_manager_test.go@@ -480,6 +480,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 150,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",acl: []string{},+ md5: "ek30X4Oc1ZSsI2sZUkzskg==",},{name: "moar2.txt",@@ -488,6 +489,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 12,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",acl: []string{},+ md5: "5Z/5eUEET4XfUpfhwwLSYA==",},{name: "aws3.txt",@@ -496,6 +498,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 150,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",acl: []string{},+ md5: "JusDN4oMjpnuDOz5FVrOgQ==",},{name: "moar.txt",@@ -504,6 +507,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 6,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",acl: []string{},+ md5: "CffgLxKQviEdpweiZvFTsw==",},{name: "AMAZON_FASHION_5.json",@@ -512,6 +516,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 1413469,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",acl: []string{},+ md5: "Wh2PSDqjRpUlY3kFDUKitw==",},},wantNumBkt: 4,@@ -529,6 +534,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 150,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",acl: []string{},+ md5: "ek30X4Oc1ZSsI2sZUkzskg==",},{name: "moar2.txt",@@ -537,6 +543,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 12,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",acl: []string{},+ md5: "5Z/5eUEET4XfUpfhwwLSYA==",},},wantNumBkt: 1,@@ -554,6 +561,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 6,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",acl: []string{},+ md5: "CffgLxKQviEdpweiZvFTsw==",},{name: "AMAZON_FASHION_5.json",@@ -562,6 +570,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 1413469,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",acl: []string{},+ md5: "Wh2PSDqjRpUlY3kFDUKitw==",},},wantNumBkt: 2,@@ -583,6 +592,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 150,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",acl: []string{},+ md5: "JusDN4oMjpnuDOz5FVrOgQ==",},{name: "moar.txt",@@ -591,6 +601,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 6,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",acl: []string{},+ md5: "CffgLxKQviEdpweiZvFTsw==",},{name: "AMAZON_FASHION_5.json",@@ -599,6 +610,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 1413469,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",acl: []string{},+ md5: "Wh2PSDqjRpUlY3kFDUKitw==",},},wantNumBkt: 4, // We still list objects in all buckets.@@ -620,6 +632,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 150,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",acl: []string{},+ md5: "ek30X4Oc1ZSsI2sZUkzskg==",},},wantNumBkt: 1,@@ -641,6 +654,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 150,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",acl: []string{},+ md5: "ek30X4Oc1ZSsI2sZUkzskg==",},},wantNumBkt: 4,@@ -663,6 +677,7 @@ func TestGCSManagerListObjects(t *testing.T) {size: 12,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",acl: []string{},+ md5: "5Z/5eUEET4XfUpfhwwLSYA==",},},wantNumBkt: 1,@@ -750,6 +765,7 @@ func TestGCSManagerListObjects_Resuming(t *testing.T) {size: 12,link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",acl: []string{},+ md5: "5Z/5eUEET4XfUpfhwwLSYA==",},},wantNumBkt: 1,diff --git a/pkg/sources/gcs/gcs_test.go b/pkg/sources/gcs/gcs_test.goindex a41a45644..a9e7acb81 100644--- a/pkg/sources/gcs/gcs_test.go+++ b/pkg/sources/gcs/gcs_test.go@@ -263,6 +263,7 @@ func createTestObject(id int) object {link: fmt.Sprintf("https://storage.googleapis.com/%s/%s", testBucket, fmt.Sprintf("object%d", id)),acl: []string{"authenticatedUsers"},size: 42,+ md5: fmt.Sprintf("md5hash%d", id),Reader: &mockReader{data: []byte(fmt.Sprintf("hello world %d", id))},createdAt: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),}@@ -399,7 +400,7 @@ func TestSourceChunks_ProgressSet(t *testing.T) {// Test that the resume progress is set.var progress strings.Builderfor i := range got {- progress.WriteString(fmt.Sprintf("object%d", i))+ progress.WriteString(fmt.Sprintf("md5hash%d", i))// Add a comma if not the last element.if i != len(got)-1 {progress.WriteString(",")@@ -408,8 +409,8 @@ func TestSourceChunks_ProgressSet(t *testing.T) {encodeResume := strings.Split(source.Progress.EncodedResumeInfo, ",")sort.Slice(encodeResume, func(i, j int) bool {- numI, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[i], "object"))- numJ, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[j], "object"))+ numI, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[i], "md5hash"))+ numJ, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[j], "md5hash"))return numI < numJ})