Skip to content

Find, verify, and analyze leaked credentials

[chore] - move objectManager interface (#1332)

ahrav948828ba8cea
diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.go
index f05c1ea95..6658c5288 100644
--- a/pkg/sources/gcs/gcs.go
+++ b/pkg/sources/gcs/gcs.go
@@ -49,6 +49,11 @@ func (s *Source) JobID() int64 {
return s.jobId
}
+type objectManager interface {
+ ListObjects(context.Context) (chan io.Reader, error)
+ Attributes(ctx context.Context) (*attributes, error)
+}
+
// Source represents a GCS source.
type Source struct {
name string
@@ -231,7 +236,7 @@ func setGCSManagerOptions(include, exclude []string, includeFn, excludeFn func([
// enumerate all the objects and buckets in the source.
// This will be used to calculate progress.
func (s *Source) enumerate(ctx context.Context) error {
- stats, err := s.gcsManager.attributes(ctx)
+ stats, err := s.gcsManager.Attributes(ctx)
if err != nil {
return fmt.Errorf("error getting attributes during enumeration: %w", err)
}
@@ -244,7 +249,7 @@ func (s *Source) enumerate(ctx context.Context) error {
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
persistableCache := s.setupCache(ctx)
- objectCh, err := s.gcsManager.listObjects(ctx)
+ objectCh, err := s.gcsManager.ListObjects(ctx)
if err != nil {
return fmt.Errorf("error listing objects: %w", err)
}
diff --git a/pkg/sources/gcs/gcs_manager.go b/pkg/sources/gcs/gcs_manager.go
index 65d805a52..fcbf5a709 100644
--- a/pkg/sources/gcs/gcs_manager.go
+++ b/pkg/sources/gcs/gcs_manager.go
@@ -31,11 +31,6 @@ const (
var defaultConcurrency = runtime.NumCPU()
-type objectManager interface {
- listObjects(context.Context) (chan io.Reader, error)
- attributes(ctx context.Context) (*attributes, error)
-}
-
// bucketManager is a simplified *storage.Client wrapper.
// It provides only a subset of methods that are needed by the gcsManager.
type bucketManager interface {
@@ -381,7 +376,7 @@ type object struct {
io.Reader
}
-func (g *gcsManager) attributes(ctx context.Context) (*attributes, error) {
+func (g *gcsManager) Attributes(ctx context.Context) (*attributes, error) {
// Get all the buckets in the project.
buckets, err := g.listBuckets(ctx)
if err != nil {
@@ -404,7 +399,7 @@ func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes,
for _, bkt := range bkts {
bkt := bkt
g.workerPool.Go(func() error {
- // List all the objects in the bucket and calculate a attributes.
+ // List all the objects in the bucket and calculate attributes.
g.setupBktHandle(&bkt)
q, err := setObjectQuery(&bkt)
@@ -448,7 +443,7 @@ func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes,
return stats, nil
}
-func (g *gcsManager) listObjects(ctx context.Context) (chan io.Reader, error) {
+func (g *gcsManager) ListObjects(ctx context.Context) (chan io.Reader, error) {
ch := make(chan io.Reader, 100) // TODO (ahrav): Determine optimal buffer size.
// Get all the buckets in the project.
diff --git a/pkg/sources/gcs/gcs_manager_test.go b/pkg/sources/gcs/gcs_manager_test.go
index 8c215bf26..2cbd2b39c 100644
--- a/pkg/sources/gcs/gcs_manager_test.go
+++ b/pkg/sources/gcs/gcs_manager_test.go
@@ -420,13 +420,13 @@ func TestGCSManagerStats(t *testing.T) {
t.Fatalf("newGCSManager() error = %v", err)
}
- got, err := gm.attributes(ctx)
+ got, err := gm.Attributes(ctx)
if err != nil {
- t.Errorf("attributes() error = %v", err)
+ t.Errorf("Attributes() error = %v", err)
}
if diff := cmp.Diff(got, tc.wantStats, cmp.AllowUnexported(attributes{}), cmpopts.IgnoreFields(attributes{}, "mu")); diff != "" {
- t.Errorf("attributes() got: %v, want: %v, diff: %v", got, tc.wantStats, diff)
+ t.Errorf("Attributes() got: %v, want: %v, diff: %v", got, tc.wantStats, diff)
}
})
}
@@ -443,7 +443,7 @@ func TestGCSManagerStats_Time(t *testing.T) {
start := time.Now()
var stats *attributes
- stats, _ = gm.attributes(ctx)
+ stats, _ = gm.Attributes(ctx)
end := time.Since(start).Seconds()
fmt.Printf("Time taken to get %d objects: %f seconds\n", stats.numObjects, end)
@@ -690,7 +690,7 @@ func TestGCSManagerListObjects(t *testing.T) {
mgr, err := newGCSManager(tc.projectID, tc.opts...)
assert.Nil(t, err)
- got, err := mgr.listObjects(ctx)
+ got, err := mgr.ListObjects(ctx)
if (err != nil) != tc.wantErr {
t.Errorf("GCSManager.ListObjects() error = %v, wantErr %v", err, tc.wantErr)
return
@@ -714,7 +714,7 @@ func TestGCSManagerListObjects(t *testing.T) {
}
if len(res) != len(tc.want) {
- t.Errorf("gcsManager.listObjects() got: %v, want: %v", res, tc.want)
+ t.Errorf("gcsManager.ListObjects() got: %v, want: %v", res, tc.want)
}
// Test the bucket and object counts.
@@ -728,7 +728,7 @@ func TestGCSManagerListObjects(t *testing.T) {
// Test the objects are equal.
if diff := cmp.Diff(res, tc.want, cmp.AllowUnexported(object{}), cmpopts.IgnoreFields(object{}, "Reader", "createdAt", "updatedAt")); diff != "" {
- t.Errorf("gcsManager.listObjects() mismatch (-want +got):\n%s", diff)
+ t.Errorf("gcsManager.ListObjects() mismatch (-want +got):\n%s", diff)
}
}()
@@ -778,7 +778,7 @@ func TestGCSManagerListObjects_Resuming(t *testing.T) {
mgr, err := newGCSManager(tc.projectID, tc.opts...)
assert.Nil(t, err)
- got, err := mgr.listObjects(ctx)
+ got, err := mgr.ListObjects(ctx)
if (err != nil) != tc.wantErr {
t.Errorf("GCSManager.ListObjects() error = %v, wantErr %v", err, tc.wantErr)
return
@@ -802,7 +802,7 @@ func TestGCSManagerListObjects_Resuming(t *testing.T) {
}
if len(res) != len(tc.want) {
- t.Errorf("gcsManager.listObjects() got: %v, want: %v", res, tc.want)
+ t.Errorf("gcsManager.ListObjects() got: %v, want: %v", res, tc.want)
}
// Test the bucket and object counts.
@@ -816,7 +816,7 @@ func TestGCSManagerListObjects_Resuming(t *testing.T) {
// Test the objects are equal.
if diff := cmp.Diff(res, tc.want, cmp.AllowUnexported(object{}), cmpopts.IgnoreFields(object{}, "Reader", "createdAt", "updatedAt")); diff != "" {
- t.Errorf("gcsManager.listObjects() mismatch (-want +got):\n%s", diff)
+ t.Errorf("gcsManager.ListObjects() mismatch (-want +got):\n%s", diff)
}
}()
diff --git a/pkg/sources/gcs/gcs_test.go b/pkg/sources/gcs/gcs_test.go
index a9e7acb81..126facfe4 100644
--- a/pkg/sources/gcs/gcs_test.go
+++ b/pkg/sources/gcs/gcs_test.go
@@ -210,7 +210,7 @@ type mockObjectManager struct {
wantErr bool
}
-func (m *mockObjectManager) attributes(_ context.Context) (*attributes, error) {
+func (m *mockObjectManager) Attributes(_ context.Context) (*attributes, error) {
if m.wantErr {
return nil, fmt.Errorf("some error")
}
@@ -237,7 +237,7 @@ func (m *mockReader) Read(p []byte) (n int, err error) {
return
}
-func (m *mockObjectManager) listObjects(context.Context) (chan io.Reader, error) {
+func (m *mockObjectManager) ListObjects(context.Context) (chan io.Reader, error) {
if m.wantErr {
return nil, fmt.Errorf("some error")
}