Skip to content

Find, verify, and analyze leaked credentials

test(sources/s3): fix infinite blocking and timeout issue in TestSource_Chunks (#4048)

Eng Zer Juna4838d4ebd4c
diff --git a/pkg/sources/s3/s3_integration_test.go b/pkg/sources/s3/s3_integration_test.go
index 3276e40fd..02eea5611 100644
--- a/pkg/sources/s3/s3_integration_test.go
+++ b/pkg/sources/s3/s3_integration_test.go
@@ -82,37 +82,6 @@ func TestSource_ChunksLarge(t *testing.T) {
assert.Equal(t, got, wantChunkCount)
}
-func TestSourceChunksNoResumption(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
- defer cancel()
-
- s := Source{}
- connection := &sourcespb.S3{
- Credential: &sourcespb.S3_Unauthenticated{},
- Buckets: []string{"trufflesec-ahrav-test-2"},
- }
- conn, err := anypb.New(connection)
- if err != nil {
- t.Fatal(err)
- }
-
- err = s.Init(ctx, "test name", 0, 0, false, conn, 1)
- chunksCh := make(chan *sources.Chunk)
- go func() {
- defer close(chunksCh)
- err = s.Chunks(ctx, chunksCh)
- assert.Nil(t, err)
- }()
-
- wantChunkCount := 19787
- got := 0
-
- for range chunksCh {
- got++
- }
- assert.Equal(t, got, wantChunkCount)
-}
-
func TestSource_Validate(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
defer cancel()
@@ -251,34 +220,50 @@ func TestSource_Validate(t *testing.T) {
func TestSourceChunksNoResumption(t *testing.T) {
t.Parallel()
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
- defer cancel()
-
- s := Source{}
- connection := &sourcespb.S3{
- Credential: &sourcespb.S3_Unauthenticated{},
- Buckets: []string{"integration-resumption-tests"},
- }
- conn, err := anypb.New(connection)
- if err != nil {
- t.Fatal(err)
+ tests := []struct {
+ bucket string
+ wantChunkCount int
+ }{
+ {
+ bucket: "trufflesec-ahrav-test-2",
+ wantChunkCount: 19787,
+ },
+ {
+ bucket: "integration-resumption-tests",
+ wantChunkCount: 19787,
+ },
}
- err = s.Init(ctx, "test name", 0, 0, false, conn, 1)
- chunksCh := make(chan *sources.Chunk)
- go func() {
- defer close(chunksCh)
- err = s.Chunks(ctx, chunksCh)
- assert.Nil(t, err)
- }()
-
- wantChunkCount := 19787
- got := 0
+ for _, tt := range tests {
+ t.Run(tt.bucket, func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
+ defer cancel()
+
+ s := Source{}
+ connection := &sourcespb.S3{
+ Credential: &sourcespb.S3_Unauthenticated{},
+ Buckets: []string{tt.bucket},
+ }
+ conn, err := anypb.New(connection)
+ if err != nil {
+ t.Fatal(err)
+ }
- for range chunksCh {
- got++
+ err = s.Init(ctx, "test name", 0, 0, false, conn, 1)
+ chunksCh := make(chan *sources.Chunk)
+ go func() {
+ defer close(chunksCh)
+ err = s.Chunks(ctx, chunksCh)
+ assert.Nil(t, err)
+ }()
+
+ got := 0
+ for range chunksCh {
+ got++
+ }
+ assert.Equal(t, tt.wantChunkCount, got)
+ })
}
- assert.Equal(t, wantChunkCount, got)
}
func TestSourceChunksResumption(t *testing.T) {
diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.go
index 5f2f4aed7..4f0892ed3 100644
--- a/pkg/sources/s3/s3_test.go
+++ b/pkg/sources/s3/s3_test.go
@@ -4,7 +4,6 @@ import (
"encoding/base64"
"fmt"
"os"
- "sync"
"testing"
"time"
@@ -99,8 +98,7 @@ func TestSource_Chunks(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
- var cancelOnce sync.Once
- defer cancelOnce.Do(cancel)
+ defer cancel()
for k, v := range tt.init.setEnv {
t.Setenv(k, v)
@@ -117,26 +115,47 @@ func TestSource_Chunks(t *testing.T) {
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
return
}
- chunksCh := make(chan *sources.Chunk)
- var wg sync.WaitGroup
- wg.Add(1)
+ chunksCh := make(chan *sources.Chunk, 1)
go func() {
- defer wg.Done()
+ defer close(chunksCh)
err = s.Chunks(ctx, chunksCh)
if (err != nil) != tt.wantErr {
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
os.Exit(1)
}
}()
- gotChunk := <-chunksCh
- wantData, _ := base64.StdEncoding.DecodeString(tt.wantChunkData)
- if diff := pretty.Compare(gotChunk.Data, wantData); diff != "" {
- t.Errorf("%s: Source.Chunks() diff: (-got +want)\n%s", tt.name, diff)
+ waitFn := func() {
+ receivedFirstChunk := false
+ for {
+ select {
+ case <-ctx.Done():
+ t.Errorf("TestSource_Chunks timed out: %v", ctx.Err())
+ return
+ case gotChunk, ok := <-chunksCh:
+ if !ok {
+ t.Logf("Source.Chunks() finished, channel closed")
+ assert.Equal(t, "", s.GetProgress().EncodedResumeInfo)
+ assert.Equal(t, int64(100), s.GetProgress().PercentComplete)
+ return
+ }
+ if receivedFirstChunk {
+ // wantChunkData is the first chunk data. After the first chunk has
+ // been received and matched below, we want to drain chunksCh
+ // so Source.Chunks() can finish completely.
+ continue
+ }
+
+ receivedFirstChunk = true
+ wantData, _ := base64.StdEncoding.DecodeString(tt.wantChunkData)
+
+ if diff := pretty.Compare(gotChunk.Data, wantData); diff != "" {
+ t.Logf("%s: Source.Chunks() diff: (-got +want)\n%s", tt.name, diff)
+ }
+ }
+ }
}
- wg.Wait()
- assert.Equal(t, "", s.GetProgress().EncodedResumeInfo)
- assert.Equal(t, int64(100), s.GetProgress().PercentComplete)
+ waitFn()
})
}
}