Skip to content

Find, verify, and analyze leaked credentials

Use md5 hash for resuming key (#1203)

ahrav2fbf86a6abcf
diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.go
index 9d866fbe4..f05c1ea95 100644
--- a/pkg/sources/gcs/gcs.go
+++ b/pkg/sources/gcs/gcs.go
@@ -273,7 +273,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
ctx.Logger().V(1).Info("error setting start progress progress", "name", o.name, "error", err)
return
}
- s.setProgress(ctx, o.name, persistableCache)
+ s.setProgress(ctx, o.md5, o.name, persistableCache)
}(o)
}
wg.Wait()
@@ -295,14 +295,14 @@ func (s *Source) setupCache(ctx context.Context) *persistableCache {
return persistCache
}
-func (s *Source) setProgress(ctx context.Context, objName string, cache cache.Cache) {
+func (s *Source) setProgress(ctx context.Context, md5, objName string, cache cache.Cache) {
s.mu.Lock()
defer s.mu.Unlock()
ctx.Logger().V(5).Info("setting progress for object", "object-name", objName)
s.SectionsCompleted++
- cache.Set(objName, objName)
+ cache.Set(md5, md5)
s.Progress.SectionsRemaining = int32(s.stats.numObjects)
s.Progress.PercentComplete = int64(float64(s.SectionsCompleted) / float64(s.stats.numObjects) * 100)
}
diff --git a/pkg/sources/gcs/gcs_manager.go b/pkg/sources/gcs/gcs_manager.go
index 79e69b930..1dcd15cee 100644
--- a/pkg/sources/gcs/gcs_manager.go
+++ b/pkg/sources/gcs/gcs_manager.go
@@ -2,6 +2,7 @@ package gcs
import (
aCtx "context"
+ "encoding/base64"
"fmt"
"io"
"net/http"
@@ -371,6 +372,7 @@ type object struct {
contentType string
owner string
link string
+ md5 string
// acl represents an ACLEntities.
// https://pkg.go.dev/cloud.google.com/go/storage#ACLEntity
acl []string
@@ -648,6 +650,7 @@ func (g *gcsManager) constructObject(ctx context.Context, obj *storage.ObjectHan
o.contentType = attrs.ContentType
o.owner = attrs.Owner
o.link = attrs.MediaLink
+ o.md5 = base64.StdEncoding.EncodeToString(attrs.MD5)
o.createdAt = attrs.Created
o.updatedAt = attrs.Updated
o.acl = objectACLs(attrs.ACL)
diff --git a/pkg/sources/gcs/gcs_manager_test.go b/pkg/sources/gcs/gcs_manager_test.go
index a373839ea..2df844372 100644
--- a/pkg/sources/gcs/gcs_manager_test.go
+++ b/pkg/sources/gcs/gcs_manager_test.go
@@ -480,6 +480,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
acl: []string{},
+ md5: "ek30X4Oc1ZSsI2sZUkzskg==",
},
{
name: "moar2.txt",
@@ -488,6 +489,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 12,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
acl: []string{},
+ md5: "5Z/5eUEET4XfUpfhwwLSYA==",
},
{
name: "aws3.txt",
@@ -496,6 +498,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
acl: []string{},
+ md5: "JusDN4oMjpnuDOz5FVrOgQ==",
},
{
name: "moar.txt",
@@ -504,6 +507,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 6,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
acl: []string{},
+ md5: "CffgLxKQviEdpweiZvFTsw==",
},
{
name: "AMAZON_FASHION_5.json",
@@ -512,6 +516,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 1413469,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
acl: []string{},
+ md5: "Wh2PSDqjRpUlY3kFDUKitw==",
},
},
wantNumBkt: 4,
@@ -529,6 +534,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
acl: []string{},
+ md5: "ek30X4Oc1ZSsI2sZUkzskg==",
},
{
name: "moar2.txt",
@@ -537,6 +543,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 12,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
acl: []string{},
+ md5: "5Z/5eUEET4XfUpfhwwLSYA==",
},
},
wantNumBkt: 1,
@@ -554,6 +561,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 6,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
acl: []string{},
+ md5: "CffgLxKQviEdpweiZvFTsw==",
},
{
name: "AMAZON_FASHION_5.json",
@@ -562,6 +570,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 1413469,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
acl: []string{},
+ md5: "Wh2PSDqjRpUlY3kFDUKitw==",
},
},
wantNumBkt: 2,
@@ -583,6 +592,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
acl: []string{},
+ md5: "JusDN4oMjpnuDOz5FVrOgQ==",
},
{
name: "moar.txt",
@@ -591,6 +601,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 6,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
acl: []string{},
+ md5: "CffgLxKQviEdpweiZvFTsw==",
},
{
name: "AMAZON_FASHION_5.json",
@@ -599,6 +610,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 1413469,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
acl: []string{},
+ md5: "Wh2PSDqjRpUlY3kFDUKitw==",
},
},
wantNumBkt: 4, // We still list objects in all buckets.
@@ -620,6 +632,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
acl: []string{},
+ md5: "ek30X4Oc1ZSsI2sZUkzskg==",
},
},
wantNumBkt: 1,
@@ -641,6 +654,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 150,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
acl: []string{},
+ md5: "ek30X4Oc1ZSsI2sZUkzskg==",
},
},
wantNumBkt: 4,
@@ -663,6 +677,7 @@ func TestGCSManagerListObjects(t *testing.T) {
size: 12,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
acl: []string{},
+ md5: "5Z/5eUEET4XfUpfhwwLSYA==",
},
},
wantNumBkt: 1,
@@ -750,6 +765,7 @@ func TestGCSManagerListObjects_Resuming(t *testing.T) {
size: 12,
link: "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
acl: []string{},
+ md5: "5Z/5eUEET4XfUpfhwwLSYA==",
},
},
wantNumBkt: 1,
diff --git a/pkg/sources/gcs/gcs_test.go b/pkg/sources/gcs/gcs_test.go
index a41a45644..a9e7acb81 100644
--- a/pkg/sources/gcs/gcs_test.go
+++ b/pkg/sources/gcs/gcs_test.go
@@ -263,6 +263,7 @@ func createTestObject(id int) object {
link: fmt.Sprintf("https://storage.googleapis.com/%s/%s", testBucket, fmt.Sprintf("object%d", id)),
acl: []string{"authenticatedUsers"},
size: 42,
+ md5: fmt.Sprintf("md5hash%d", id),
Reader: &mockReader{data: []byte(fmt.Sprintf("hello world %d", id))},
createdAt: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
}
@@ -399,7 +400,7 @@ func TestSourceChunks_ProgressSet(t *testing.T) {
// Test that the resume progress is set.
var progress strings.Builder
for i := range got {
- progress.WriteString(fmt.Sprintf("object%d", i))
+ progress.WriteString(fmt.Sprintf("md5hash%d", i))
// Add a comma if not the last element.
if i != len(got)-1 {
progress.WriteString(",")
@@ -408,8 +409,8 @@ func TestSourceChunks_ProgressSet(t *testing.T) {
encodeResume := strings.Split(source.Progress.EncodedResumeInfo, ",")
sort.Slice(encodeResume, func(i, j int) bool {
- numI, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[i], "object"))
- numJ, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[j], "object"))
+ numI, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[i], "md5hash"))
+ numJ, _ := strconv.Atoi(strings.TrimPrefix(encodeResume[j], "md5hash"))
return numI < numJ
})