Skip to content

Find, verify, and analyze leaked credentials

List only the include prefixes in S3 scans (#5377)

Shahzad Haidere28360875917
diff --git a/README.md b/README.md
index 58f70bb75..0a001e615 100644
--- a/README.md
+++ b/README.md
@@ -684,7 +684,7 @@ Two things to watch when using `--include-extension`. A dotfile counts as all ex
These flags also apply to object keys only, not to files inside an archive. `--exclude-extension=mp4` skips `clip.mp4` sitting in the bucket, but not a `clip.mp4` packed inside `media.zip`. Excluding `zip` skips the archive entirely.
-Filtering happens after TruffleHog lists a bucket, so it cuts the cost of downloading objects but not the cost of listing them. A bucket is still paginated in full even when a prefix covers a small part of it.
+TruffleHog asks S3 to list only the keys under the include prefixes, so a prefix that covers a small part of a large bucket is listed quickly. Exclude prefixes and extensions are applied after listing, so they cut the cost of downloading objects but not the cost of listing them.
Prefixes and extensions are applied together. An object has to pass both to be scanned, so the command below scans `src/main.tf` but skips both `src/bundle.zip` and `docs/guide.tf`:
diff --git a/pkg/sources/s3/README.md b/pkg/sources/s3/README.md
index d10724104..bf0e6531a 100644
--- a/pkg/sources/s3/README.md
+++ b/pkg/sources/s3/README.md
@@ -41,7 +41,9 @@ Prefixes are literal, so `--include-prefix=log` matches `logs/app.txt` and `logs
The two kinds are combined with AND: an object has to pass both to be scanned. Entries are trimmed, and blank ones are ignored.
-Filtering happens after listing, so it saves `GetObject` requests but not `ListObjectsV2` requests. A bucket is still paginated in full even when a prefix covers a small part of it. Pushing prefixes into `ListObjectsV2Input.Prefix` would cut that too, and has not been done.
+Include prefixes are also sent to S3 as `ListObjectsV2Input.Prefix`. A bucket is listed once per include prefix instead of in full, so the listing covers only the keys they select, and each prefix costs at least one `ListObjectsV2` request per bucket even when nothing is under it. A prefix under another, like `src/vendor/` under `src/`, is dropped, since the shorter one already lists it. Directory buckets are listed in full, because they only accept a prefix that ends in `/`.
+
+Exclude prefixes and extensions have no server-side form, so they are matched after listing. They save `GetObject` requests but not `ListObjectsV2` requests.
Three edge cases worth knowing:
@@ -55,7 +57,7 @@ Prefixes are matched with `strings.HasPrefix`, so an exclude prefix naming one k
The same prefixes and extensions apply to every bucket in a scan. There is no way to scope a prefix to one bucket.
-If a filter excludes every object in a bucket, the scan logs `Scanned no objects in bucket` with the count, so a mistyped prefix does not look like a clean scan of an empty bucket.
+If a filter excludes every object in a bucket, the scan logs `Scanned no objects in bucket` with the count. If nothing is listed under the include prefixes at all, it logs `Found no objects under the include prefixes`. Either way, a mistyped prefix does not look like a clean scan of an empty bucket.
## Objects skipped regardless of configuration
@@ -83,6 +85,8 @@ The endpoint is parsed in `Init`, so a malformed one fails at startup rather tha
Only consecutive completions are checkpointed. If objects 0 to 5 and 7 to 8 are done but 6 is not, the checkpoint stops at 5, so resuming may rescan a few objects but never misses one.
+With include prefixes the scan lists each in turn. They are listed in key order and none is under another, so keys still arrive in ascending order and the checkpoint stays a single key. Resuming passes it as `StartAfter` to every prefix, and S3 returns nothing for the prefixes already finished. A checkpoint from a scan that listed the whole bucket resumes the same way.
+
This matters when adding a new skip to `pageChunker`: a skipped object still has to call `UpdateObjectCompletion`. Skipping without it stalls the checkpoint at the first skipped object, and a resumed scan then redoes everything after it.
The OSS CLI does not persist progress between runs, so resumption only takes effect where something stores and returns `EncodedResumeInfo`.
diff --git a/pkg/sources/s3/filter.go b/pkg/sources/s3/filter.go
index 9f343084c..cd56d2406 100644
--- a/pkg/sources/s3/filter.go
+++ b/pkg/sources/s3/filter.go
@@ -18,6 +18,8 @@ import (
//
// The zero value and a nil *objectFilter both include everything.
type objectFilter struct {
+ // Sorted, and none starts with another, so a scan can list each in turn and
+ // still see keys in order.
includePrefixes []string
excludePrefixes []string
@@ -40,7 +42,7 @@ func newObjectFilter(includePrefixes, excludePrefixes, includeExtensions, exclud
}
return &objectFilter{
- includePrefixes: cleanEntries(includePrefixes, strings.TrimSpace),
+ includePrefixes: outermostPrefixes(cleanEntries(includePrefixes, strings.TrimSpace)),
excludePrefixes: cleanEntries(excludePrefixes, strings.TrimSpace),
includeExtensions: include,
excludeExtensions: exclude,
@@ -55,6 +57,15 @@ func (f *objectFilter) shouldInclude(key string) bool {
return f.passesPrefixes(key) && f.passesExtensions(key)
}
+// listPrefixes returns the include prefixes for S3 to list in place of the whole
+// bucket, or nil when the whole bucket is listed.
+func (f *objectFilter) listPrefixes() []string {
+ if f == nil {
+ return nil
+ }
+ return f.includePrefixes
+}
+
func (f *objectFilter) passesPrefixes(key string) bool {
for _, prefix := range f.excludePrefixes {
if strings.HasPrefix(key, prefix) {
@@ -113,6 +124,20 @@ func cleanExtensions(values []string) []string {
return slices.Compact(exts)
}
+// outermostPrefixes sorts prefixes and drops each one that starts with another,
+// since the shorter one already covers every key under it. Once sorted, a prefix
+// that starts with any kept prefix starts with the last one kept.
+func outermostPrefixes(prefixes []string) []string {
+ slices.Sort(prefixes)
+ var kept []string
+ for _, prefix := range prefixes {
+ if len(kept) == 0 || !strings.HasPrefix(prefix, kept[len(kept)-1]) {
+ kept = append(kept, prefix)
+ }
+ }
+ return kept
+}
+
// cleanEntries normalizes each entry and drops the ones that come out empty,
// returning nil when nothing is left.
func cleanEntries(values []string, normalize func(string) string) []string {
diff --git a/pkg/sources/s3/filter_test.go b/pkg/sources/s3/filter_test.go
index e4ed5353c..5ccaaeee8 100644
--- a/pkg/sources/s3/filter_test.go
+++ b/pkg/sources/s3/filter_test.go
@@ -21,6 +21,7 @@ func TestObjectFilter_Unconfigured(t *testing.T) {
assert.True(t, filter.shouldInclude("any/key.zip"))
assert.True(t, filter.shouldInclude("Makefile"))
assert.True(t, (*objectFilter)(nil).shouldInclude("any/key.zip"))
+ assert.Nil(t, (*objectFilter)(nil).listPrefixes())
}
func TestObjectFilter_ExcludePrefix(t *testing.T) {
@@ -48,6 +49,17 @@ func TestObjectFilter_IncludeAndExcludePrefixes(t *testing.T) {
assert.False(t, filter.shouldInclude("docs/readme.md"), "matches no include prefix")
}
+// A scan lists the include prefixes in turn, so they must come out sorted with
+// none under another, or keys would be listed twice or out of order.
+func TestObjectFilter_IncludePrefixesAreOutermost(t *testing.T) {
+ include := []string{"src/vendor/", "src/", "docs/", "src/", "a", "ab", "ac"}
+ filter := newTestObjectFilter(t, include, nil, nil, nil)
+
+ assert.Equal(t, []string{"a", "docs/", "src/"}, filter.listPrefixes())
+ assert.True(t, filter.shouldInclude("src/vendor/dep.go"), "a dropped prefix is still covered")
+ assert.True(t, filter.shouldInclude("ac/key.txt"))
+}
+
func TestObjectFilter_ExcludeExtension(t *testing.T) {
filter := newTestObjectFilter(t, nil, nil, nil, []string{"zip", ".MP4"})
diff --git a/pkg/sources/s3/paginator.go b/pkg/sources/s3/paginator.go
new file mode 100644
index 000000000..05870e4d7
--- /dev/null
+++ b/pkg/sources/s3/paginator.go
@@ -0,0 +1,71 @@
+package s3
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+// directoryBucketSuffix ends every S3 Express directory bucket name, and AWS reserves it for them.
+const directoryBucketSuffix = "--x-s3"
+
+// listInputs returns the listings that cover what a scan of the bucket can include: one per include
+// prefix, so S3 skips the keys outside them, or one of the whole bucket.
+//
+// The include prefixes are sorted and none starts with another, so listing them in turn returns keys
+// in ascending order, just as one listing of the bucket would. That keeps the checkpoint a single
+// start-after key, and passing it to every listing resumes exactly where the scan stopped: S3
+// returns nothing for the prefixes the scan already finished.
+func (s *Source) listInputs(bucket string, startAfter *string) []*s3.ListObjectsV2Input {
+ prefixes := s.objectFilter.listPrefixes()
+ // Directory buckets only accept a prefix that ends in "/", so they are listed whole and filtered after.
+ if len(prefixes) == 0 || strings.HasSuffix(bucket, directoryBucketSuffix) {
+ return []*s3.ListObjectsV2Input{{Bucket: &bucket, StartAfter: startAfter}}
+ }
+
+ inputs := make([]*s3.ListObjectsV2Input, len(prefixes))
+ for i, prefix := range prefixes {
+ inputs[i] = &s3.ListObjectsV2Input{Bucket: &bucket, Prefix: aws.String(prefix), StartAfter: startAfter}
+ }
+ return inputs
+}
+
+// bucketPaginator pages through several listings in turn, as if they were one.
+type bucketPaginator struct {
+ client *s3.Client
+ // inputs[0] is the listing in progress, and the rest are still to come.
+ inputs []*s3.ListObjectsV2Input
+ pages *s3.ListObjectsV2Paginator
+}
+
+// newBucketPaginator starts with the first of inputs, which must not be empty.
+func newBucketPaginator(client *s3.Client, inputs []*s3.ListObjectsV2Input) *bucketPaginator {
+ return &bucketPaginator{client: client, inputs: inputs, pages: s3.NewListObjectsV2Paginator(client, inputs[0])}
+}
+
+// HasMorePages reports whether any listing has pages left, moving on to the next listing when the
+// current one is done.
+func (p *bucketPaginator) HasMorePages() bool {
+ for !p.pages.HasMorePages() {
+ if len(p.inputs) == 1 {
+ return false
+ }
+ p.inputs = p.inputs[1:]
+ p.pages = s3.NewListObjectsV2Paginator(p.client, p.inputs[0])
+ }
+ return true
+}
+
+// NextPage returns the next page of the current listing.
+func (p *bucketPaginator) NextPage(ctx context.Context) (*s3.ListObjectsV2Output, error) {
+ page, err := p.pages.NextPage(ctx)
+ if err != nil && p.inputs[0].Prefix != nil {
+ // A policy can allow listing some prefixes and not others, so say which one failed.
+ err = fmt.Errorf("could not list prefix %q: %w", *p.inputs[0].Prefix, err)
+ }
+ return page, err
+}
diff --git a/pkg/sources/s3/paginator_test.go b/pkg/sources/s3/paginator_test.go
new file mode 100644
index 000000000..b305802d4
--- /dev/null
+++ b/pkg/sources/s3/paginator_test.go
@@ -0,0 +1,249 @@
+package s3
+
+import (
+ "slices"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/go-logr/logr/funcr"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
+)
+
+func TestSource_ListInputs(t *testing.T) {
+ startAfter := "infra/main.tf"
+
+ tests := []struct {
+ name string
+ bucket string
+ prefixes []string
+ wantPrefixes []string
+ }{
+ {name: "no include prefixes", bucket: "bucket", wantPrefixes: []string{""}},
+ {
+ name: "one listing per include prefix, in key order",
+ bucket: "bucket",
+ prefixes: []string{"src/", "infra/"},
+ wantPrefixes: []string{"infra/", "src/"},
+ },
+ {
+ name: "directory bucket",
+ bucket: "data--usw2-az1--x-s3",
+ prefixes: []string{"src"},
+ wantPrefixes: []string{""},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Source{objectFilter: newTestObjectFilter(t, tt.prefixes, nil, nil, nil)}
+
+ var gotPrefixes []string
+ for _, input := range s.listInputs(tt.bucket, &startAfter) {
+ assert.Equal(t, tt.bucket, aws.ToString(input.Bucket))
+ assert.Equal(t, startAfter, aws.ToString(input.StartAfter), "every listing resumes at the checkpoint")
+ gotPrefixes = append(gotPrefixes, aws.ToString(input.Prefix))
+ }
+ assert.Equal(t, tt.wantPrefixes, gotPrefixes)
+ })
+ }
+}
+
+// prefixTestObjects has keys under the include prefixes the tests below use, one of them nested in
+// another, and keys between and around them that no include prefix covers.
+func prefixTestObjects() map[string]string {
+ objects := make(map[string]string)
+ for _, key := range []string{
+ "configs/app.yaml",
+ "infra/main.tf", "infra/vars.tf",
+ "logs/1.log", "logs/2.log", "logs/3.log", "logs/4.log",
+ "src/main.go", "src/vendor/dep.go",
+ "tests/main_test.go",
+ } {
+ objects[key] = "contents of " + key
+ }
+ return objects
+}
+
+func withIncludePrefixes(prefixes ...string) func(*sourcespb.S3) {
+ return func(conn *sourcespb.S3) { conn.IncludePrefixes = prefixes }
+}
+
+// scannedKeys counts the chunks reported for each object.
+func scannedKeys(chunks []sources.Chunk) map[string]int {
+ keys := make(map[string]int)
+ for _, chunk := range chunks {
+ keys[chunk.SourceMetadata.GetS3().GetFile()]++
+ }
+ return keys
+}
+
+func TestSource_ChunkUnit_ListsOnlyIncludePrefixes(t *testing.T) {
+ fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: prefixTestObjects()}}}
+ s := newFakeS3Source(t, fake, withIncludePrefixes("src/", "infra/", "src/vendor/"))
+
+ reporter := sourcestest.TestReporter{}
+ require.NoError(t, s.ChunkUnit(context.Background(), S3SourceUnit{Bucket: "bucket"}, &reporter))
+
+ want := map[string]int{"infra/main.tf": 1, "infra/vars.tf": 1, "src/main.go": 1, "src/vendor/dep.go": 1}
+ assert.Equal(t, want, scannedKeys(reporter.Chunks), "each object under a prefix is scanned once")
+ // infra/ and src/ fit a page each, where listing the whole bucket takes six.
+ assert.EqualValues(t, 2, fake.listCalls.Load())
+}
+
+// A checkpoint is the last key the scan finished, whether it lies under an include prefix or not, as
+// one left by a scan that listed the whole bucket may.
+func TestSource_ChunkUnit_ResumesAcrossIncludePrefixes(t *testing.T) {
+ tests := []struct {
+ name string
+ startAfter string
+ want []string
+ }{
+ {
+ name: "before every prefix",
+ startAfter: "a",
+ want: []string{"infra/main.tf", "infra/vars.tf", "src/main.go", "src/vendor/dep.go"},
+ },
+ {
+ name: "within a prefix",
+ startAfter: "infra/main.tf",
+ want: []string{"infra/vars.tf", "src/main.go", "src/vendor/dep.go"},
+ },
+ {name: "between prefixes", startAfter: "logs/3.log", want: []string{"src/main.go", "src/vendor/dep.go"}},
+ {name: "before a nested prefix", startAfter: "src/main.go", want: []string{"src/vendor/dep.go"}},
+ {name: "after every prefix", startAfter: "tests/main_test.go"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: prefixTestObjects()}}}
+ s := newFakeS3Source(t, fake, withIncludePrefixes("infra/", "src/", "src/vendor/"))
+
+ unit := S3SourceUnit{Bucket: "bucket"}
+ unitID, _ := unit.SourceUnitID()
+ s.SetEncodedResumeInfoFor(unitID, tt.startAfter)
+
+ reporter := sourcestest.TestReporter{}
+ require.NoError(t, s.ChunkUnit(context.Background(), unit, &reporter))
+
+ want := make(map[string]int)
+ for _, key := range tt.want {
+ want[key] = 1
+ }
+ assert.Equal(t, want, scannedKeys(reporter.Chunks))
+ })
+ }
+}
+
+// S3 applies the include prefixes itself, so a mistyped one leaves no object for the filter to
+// exclude. The scan must still say it found nothing, or it looks like a clean scan of an empty bucket,
+// but only when the prefixes truly listed nothing.
+func TestSource_ChunkUnit_ReportsIncludePrefixesListingNothing(t *testing.T) {
+ tests := []struct {
+ name string
+ prefix string
+ startAfter string
+ want bool
+ }{
+ {name: "mistyped prefix", prefix: "infa/", want: true},
+ {name: "resumed after every prefix", prefix: "infra/", startAfter: "infra/vars.tf"},
+ {name: "listed objects the scan skips", prefix: "empty"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ objects := prefixTestObjects()
+ objects["empty.txt"] = ""
+ fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: objects}}}
+ s := newFakeS3Source(t, fake, withIncludePrefixes(tt.prefix))
+
+ unit := S3SourceUnit{Bucket: "bucket"}
+ unitID, _ := unit.SourceUnitID()
+ s.SetEncodedResumeInfoFor(unitID, tt.startAfter)
+
+ var mu sync.Mutex
+ var logged []string
+ ctx := context.WithLogger(context.Background(), funcr.New(func(_, args string) {
+ mu.Lock()
+ defer mu.Unlock()
+ logged = append(logged, args)
+ }, funcr.Options{}))
+
+ require.NoError(t, s.ChunkUnit(ctx, unit, &sourcestest.TestReporter{}))
+
+ assert.Equal(t, tt.want, slices.ContainsFunc(logged, func(line string) bool {
+ return strings.Contains(line, "Found no objects under the include prefixes")
+ }))
+ })
+ }
+}
+
+// A policy can allow listing some prefixes and not others, so a failed listing must say which.
+func TestBucketPaginator_ErrorNamesPrefix(t *testing.T) {
+ fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: prefixTestObjects(), listPrefix: "infra/"}}}
+ s := newFakeS3Source(t, fake, withIncludePrefixes("infra/", "src/"))
+ client, err := s.newClient(context.Background(), s.defaultRegion(), "")
+ require.NoError(t, err)
+
+ paginator := newBucketPaginator(client, s.listInputs("bucket", nil))
+ var listErr error
+ for paginator.HasMorePages() && listErr == nil {
+ _, listErr = paginator.NextPage(context.Background())
+ }
+
+ assert.ErrorContains(t, listErr, `could not list prefix "src/"`)
+}
+
+func TestSource_ChunkUnit_CountsOnlyIncludePrefixes(t *testing.T) {
+ enableUnitProgress(t)
+ fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: prefixTestObjects()}}}
+ s := newFakeS3Source(t, fake, withIncludePrefixes("infra/", "src/"))
+
+ unit := S3SourceUnit{Bucket: "bucket"}
+ require.NoError(t, s.ChunkUnit(context.Background(), unit, &sourcestest.TestReporter{}))
+
+ unitID, _ := unit.SourceUnitID()
+ got, _ := s.GetUnitProgressFor(unitID)
+ assert.Equal(t, sources.UnitProgressReady, got.State)
+ assert.EqualValues(t, 4, got.ItemsTotal)
+ assert.EqualValues(t, 4, fake.listCalls.Load(), "the count lists the same two prefixes as the scan")
+}
+
+// A policy can limit listing to some prefixes. Validate must list the way the scan does, or it
+// rejects access the scan has.
+func TestSource_Validate_ListsIncludePrefix(t *testing.T) {
+ tests := []struct {
+ name string
+ prefixes []string
+ wantErr bool
+ }{
+ {name: "whole bucket", wantErr: true},
+ {name: "include prefix", prefixes: []string{"infra/"}},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ fake := &fakeS3{buckets: map[string]fakeBucket{
+ "bucket": {objects: prefixTestObjects(), listPrefix: "infra/"},
+ }}
+ s := newFakeS3Source(t, fake, withIncludePrefixes(tt.prefixes...), func(conn *sourcespb.S3) {
+ conn.Buckets = []string{"bucket"}
+ })
+
+ errs := s.Validate(context.Background())
+
+ if tt.wantErr {
+ assert.NotEmpty(t, errs)
+ } else {
+ assert.Empty(t, errs)
+ }
+ })
+ }
+}
diff --git a/pkg/sources/s3/progress.go b/pkg/sources/s3/progress.go
index 870277e64..aa2d11a32 100644
--- a/pkg/sources/s3/progress.go
+++ b/pkg/sources/s3/progress.go
@@ -111,9 +111,9 @@ func (s *Source) startCount(
}
}
-// countBucket lists the whole bucket once to add up the objects the scan downloads and their size,
-// since S3 has no API that reports either. Keys at or before startAfter were finished by an earlier
-// run, so they count as done too. Nothing is kept per object.
+// countBucket lists the bucket from the start, the same way the scan does, to add up the objects the
+// scan downloads and their size, since S3 has no API that reports either. Keys at or before startAfter
+// were finished by an earlier run, so they count as done too. Nothing is kept per object.
func (s *Source) countBucket(
ctx context.Context,
client *s3.Client,
@@ -121,7 +121,7 @@ func (s *Source) countBucket(
startAfter *string,
progress *unitProgress,
) {
- paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{Bucket: &bucket})
+ paginator := newBucketPaginator(client, s.listInputs(bucket, nil))
for paginator.HasMorePages() {
page, err := paginator.NextPage(ctx)
if err != nil {
diff --git a/pkg/sources/s3/progress_test.go b/pkg/sources/s3/progress_test.go
index 96f63b821..525450adb 100644
--- a/pkg/sources/s3/progress_test.go
+++ b/pkg/sources/s3/progress_test.go
@@ -133,6 +133,8 @@ type fakeBucket struct {
// countDelay holds back listings that start from the beginning of the bucket, which only the count
// makes when the scan is resumed, so a test can make the scan finish first.
countDelay time.Duration
+ // listPrefix denies listings outside it, as an IAM policy on s3:prefix does.
+ listPrefix string
}
// fakeS3 serves the parts of the S3 API a unit scan uses, path style, two keys per page.
@@ -171,6 +173,12 @@ func (f *fakeS3) list(w http.ResponseWriter, r *http.Request, name string, bucke
const pageSize = 2
query := r.URL.Query()
+ prefix := query.Get("prefix")
+ if !strings.HasPrefix(prefix, bucket.listPrefix) {
+ http.Error(w, "access denied", http.StatusForbidden)
+ return
+ }
+
after := query.Get("continuation-token")
if after == "" {
after = query.Get("start-after")
@@ -181,7 +189,7 @@ func (f *fakeS3) list(w http.ResponseWriter, r *http.Request, name string, bucke
keys := make([]string, 0, len(bucket.objects))
for key := range bucket.objects {
- if key > after {
+ if strings.HasPrefix(key, prefix) && key > after {
keys = append(keys, key)
}
}
@@ -220,16 +228,21 @@ func sleep(r *http.Request, d time.Duration) bool {
}
}
-// newFakeS3Source starts fake as an S3-compatible endpoint and returns a source initialized against it.
-func newFakeS3Source(t *testing.T, fake *fakeS3) *Source {
+// newFakeS3Source starts fake as an S3-compatible endpoint and returns a source initialized against it,
+// after configure has adjusted the connection.
+func newFakeS3Source(t *testing.T, fake *fakeS3, configure ...func(*sourcespb.S3)) *Source {
t.Helper()
server := httptest.NewServer(fake)
t.Cleanup(server.Close)
- conn, err := anypb.New(&sourcespb.S3{
+ s3Conn := &sourcespb.S3{
Credential: &sourcespb.S3_Unauthenticated{},
Endpoint: server.URL,
- })
+ }
+ for _, f := range configure {
+ f(s3Conn)
+ }
+ conn, err := anypb.New(s3Conn)
require.NoError(t, err)
s := &Source{}
diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go
index 1165b28f0..9c8a821da 100644
--- a/pkg/sources/s3/s3.go
+++ b/pkg/sources/s3/s3.go
@@ -460,7 +460,8 @@ func (s *Source) scanBucket(
return 0
}
- // scannedWholeBucket is set once the listing below has gone from the start of the bucket to the end.
+ // scannedWholeBucket is set once the listing below has gone from start to end, so it has seen every
+ // object the scan can include.
var scannedWholeBucket bool
if progress != nil {
stopCount := s.startCount(ctx, regionalClient, bucket, startAfter, progress)
@@ -472,14 +473,10 @@ func (s *Source) scanBucket(
errorCount := sync.Map{}
- input := &s3.ListObjectsV2Input{Bucket: &bucket}
- if startAfter != nil {
- input.StartAfter = startAfter
- }
-
pageNumber := 1
- paginator := s3.NewListObjectsV2Paginator(regionalClient, input)
+ paginator := newBucketPaginator(regionalClient, s.listInputs(bucket, startAfter))
var objectCount, filteredCount uint64
+ var listed int
listFailed := false
for paginator.HasMorePages() {
output, err := paginator.NextPage(ctx)
@@ -515,6 +512,7 @@ func (s *Source) scanBucket(
}
s.pageChunker(ctx, pageMetadata, processingState, reporter, checkpointer)
+ listed += len(output.Contents)
pageNumber++
}
scannedWholeBucket = startAfter == nil && !listFailed && ctx.Err() == nil
@@ -524,6 +522,11 @@ func (s *Source) scanBucket(
if objectCount == 0 && filteredCount > 0 {
ctx.Logger().Info("Scanned no objects in bucket", "excluded_by_object_filter", filteredCount)
}
+ // The same goes for include prefixes, which S3 applies itself: a mistyped one lists nothing rather than
+ // excluding anything.
+ if prefixes := s.objectFilter.listPrefixes(); len(prefixes) > 0 && listed == 0 && scannedWholeBucket {
+ ctx.Logger().Info("Found no objects under the include prefixes", "include_prefixes", prefixes)
+ }
return objectCount
}
@@ -770,7 +773,8 @@ func (s *Source) validateBucketAccess(ctx context.Context, client *s3.Client, ro
continue
}
- _, err = regionalClient.ListObjectsV2(ctx, &s3.ListObjectsV2Input{Bucket: &bucket})
+ // Make the scan's first listing, so access limited to the include prefixes passes.
+ _, err = regionalClient.ListObjectsV2(ctx, s.listInputs(bucket, nil)[0])
if err == nil {
wasAbleToListAnyBucket = true
} else if shouldHaveAccessToAllBuckets {