Skip to content

Find, verify, and analyze leaked credentials

10.8 KB
Raw Blame
package s3

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"slices"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"google.golang.org/protobuf/types/known/anypb"

	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
	"github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest"
)

func TestUnitProgress_CountThenScan(t *testing.T) {
	var progress sources.Progress
	p := newUnitProgress("unit", &progress)

	p.setState(sources.UnitProgressCounting)
	p.counted(4, 4000, 1, 1000)

	got, ok := progress.GetUnitProgressFor("unit")
	require.True(t, ok)
	assert.Equal(t, sources.UnitProgress{
		State:        sources.UnitProgressCounting,
		ItemsDone:    1,
		ItemsTotal:   4,
		BytesDone:    1000,
		BytesTotal:   4000,
		ResumedBytes: 1000,
	}, got)
	assert.Zero(t, got.Percent(), "no percent while the total is still being counted")

	p.setState(sources.UnitProgressReady)
	p.objectDone(1000)

	got, _ = progress.GetUnitProgressFor("unit")
	assert.EqualValues(t, 2, got.ItemsDone)
	assert.EqualValues(t, 2000, got.BytesDone)
	assert.EqualValues(t, 50, got.Percent())
}

func TestUnitProgress_Finish(t *testing.T) {
	tests := []struct {
		name               string
		state              sources.UnitProgressState
		itemsTotal         uint64
		itemsDone          uint64
		scannedWholeBucket bool
		wantState          sources.UnitProgressState
		wantItemsTotal     uint64
	}{
		{
			name:  "counted, and objects added during the scan",
			state: sources.UnitProgressReady, itemsTotal: 4, itemsDone: 5, scannedWholeBucket: true,
			wantState: sources.UnitProgressReady, wantItemsTotal: 5,
		},
		{
			name:  "counted, and the scan listing broke off",
			state: sources.UnitProgressReady, itemsTotal: 4, itemsDone: 2,
			wantState: sources.UnitProgressReady, wantItemsTotal: 4,
		},
		{
			name:  "scan listed the whole bucket before the count finished",
			state: sources.UnitProgressCounting, itemsTotal: 1, itemsDone: 3, scannedWholeBucket: true,
			wantState: sources.UnitProgressReady, wantItemsTotal: 3,
		},
		{
			name:  "count failed but the scan listed the whole bucket",
			state: sources.UnitProgressUnavailable, itemsTotal: 1, itemsDone: 3, scannedWholeBucket: true,
			wantState: sources.UnitProgressReady, wantItemsTotal: 3,
		},
		{
			name:  "resumed scan finished before the count",
			state: sources.UnitProgressCounting, itemsTotal: 1, itemsDone: 3,
			wantState: sources.UnitProgressUnavailable, wantItemsTotal: 1,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			var progress sources.Progress
			p := newUnitProgress("unit", &progress)
			p.setState(tt.state)
			p.counted(tt.itemsTotal, tt.itemsTotal*100, 0, 0)
			for range tt.itemsDone {
				p.objectDone(100)
			}

			p.finish(tt.scannedWholeBucket)

			got, _ := progress.GetUnitProgressFor("unit")
			assert.Equal(t, tt.wantState, got.State)
			assert.Equal(t, tt.wantItemsTotal, got.ItemsTotal)
			assert.Equal(t, tt.wantItemsTotal*100, got.BytesTotal)
			assert.Equal(t, tt.itemsDone, got.ItemsDone)
		})
	}
}

func TestUnitProgress_IgnoresUpdatesAfterFinish(t *testing.T) {
	var progress sources.Progress
	p := newUnitProgress("unit", &progress)
	p.objectDone(100)
	p.finish(true)
	want, _ := progress.GetUnitProgressFor("unit")

	p.objectDone(100)
	p.counted(5, 500, 0, 0)
	p.setState(sources.UnitProgressUnavailable)

	got, _ := progress.GetUnitProgressFor("unit")
	assert.Equal(t, want, got)
}

func TestUnitProgress_NilIsANoOp(t *testing.T) {
	var p *unitProgress
	assert.NotPanics(t, func() { p.objectDone(100) })
}

// fakeBucket is a bucket served by fakeS3.
type fakeBucket struct {
	objects map[string]string
	// countDelay holds back listings that start from the beginning of the bucket, which only the count
	// makes when the scan is resumed, so a test can make the scan finish first.
	countDelay time.Duration
	// listPrefix denies listings outside it, as an IAM policy on s3:prefix does.
	listPrefix string
}

// fakeS3 serves the parts of the S3 API a unit scan uses, path style, two keys per page.
type fakeS3 struct {
	buckets   map[string]fakeBucket
	getDelay  time.Duration
	listCalls atomic.Int32
}

func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	name, key, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), "/")
	bucket, ok := f.buckets[name]
	if !ok {
		http.Error(w, "no such bucket", http.StatusNotFound)
		return
	}

	if key == "" {
		f.list(w, r, name, bucket)
		return
	}

	if !sleep(r, f.getDelay) {
		return
	}
	body, ok := bucket.objects[key]
	if !ok {
		http.Error(w, "no such key", http.StatusNotFound)
		return
	}
	_, _ = w.Write([]byte(body))
}

func (f *fakeS3) list(w http.ResponseWriter, r *http.Request, name string, bucket fakeBucket) {
	f.listCalls.Add(1)
	const pageSize = 2

	query := r.URL.Query()
	prefix := query.Get("prefix")
	if !strings.HasPrefix(prefix, bucket.listPrefix) {
		http.Error(w, "access denied", http.StatusForbidden)
		return
	}

	after := query.Get("continuation-token")
	if after == "" {
		after = query.Get("start-after")
		if after == "" && !sleep(r, bucket.countDelay) {
			return
		}
	}

	keys := make([]string, 0, len(bucket.objects))
	for key := range bucket.objects {
		if strings.HasPrefix(key, prefix) && key > after {
			keys = append(keys, key)
		}
	}
	slices.Sort(keys)
	truncated := len(keys) > pageSize
	if truncated {
		keys = keys[:pageSize]
	}

	var b strings.Builder
	fmt.Fprintf(&b, `<?xml version="1.0" encoding="UTF-8"?><ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`+
		`<Name>%s</Name><KeyCount>%d</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>%t</IsTruncated>`, name, len(keys), truncated)
	for _, key := range keys {
		fmt.Fprintf(&b, `<Contents><Key>%s</Key><LastModified>2024-01-01T00:00:00.000Z</LastModified><Size>%d</Size>`+
			`<StorageClass>STANDARD</StorageClass></Contents>`, key, len(bucket.objects[key]))
	}
	if truncated {
		fmt.Fprintf(&b, `<NextContinuationToken>%s</NextContinuationToken>`, keys[len(keys)-1])
	}
	b.WriteString(`</ListBucketResult>`)

	w.Header().Set("Content-Type", "application/xml")
	_, _ = w.Write([]byte(b.String()))
}

// sleep waits for d unless the request is cancelled first, and reports whether it waited the whole time.
func sleep(r *http.Request, d time.Duration) bool {
	if d == 0 {
		return true
	}
	select {
	case <-time.After(d):
		return true
	case <-r.Context().Done():
		return false
	}
}

// newFakeS3Source starts fake as an S3-compatible endpoint and returns a source initialized against it,
// after configure has adjusted the connection.
func newFakeS3Source(t *testing.T, fake *fakeS3, configure ...func(*sourcespb.S3)) *Source {
	t.Helper()
	server := httptest.NewServer(fake)
	t.Cleanup(server.Close)

	s3Conn := &sourcespb.S3{
		Credential: &sourcespb.S3_Unauthenticated{},
		Endpoint:   server.URL,
	}
	for _, f := range configure {
		f(s3Conn)
	}
	conn, err := anypb.New(s3Conn)
	require.NoError(t, err)

	s := &Source{}
	require.NoError(t, s.Init(context.Background(), "s3 test source", 0, 0, false, conn, 1))
	return s
}

func enableUnitProgress(t *testing.T) {
	t.Helper()
	feature.EnableS3UnitProgress.Store(true)
	t.Cleanup(func() { feature.EnableS3UnitProgress.Store(false) })
}

// testObjects are three objects of 10, 20 and 30 bytes, and one the scan skips as empty.
func testObjects() map[string]string {
	return map[string]string{
		"a.txt":     strings.Repeat("a", 10),
		"b.txt":     strings.Repeat("b", 20),
		"c.txt":     strings.Repeat("c", 30),
		"empty.txt": "",
	}
}

func TestSource_ChunkUnit_UnitProgress(t *testing.T) {
	enableUnitProgress(t)
	fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: testObjects()}}}
	s := newFakeS3Source(t, fake)

	unit := S3SourceUnit{Bucket: "bucket"}
	reporter := sourcestest.TestReporter{}
	require.NoError(t, s.ChunkUnit(context.Background(), unit, &reporter))

	unitID, _ := unit.SourceUnitID()
	got, ok := s.GetUnitProgressFor(unitID)
	require.True(t, ok)
	assert.Equal(t, sources.UnitProgress{
		State:      sources.UnitProgressReady,
		ItemsDone:  3,
		ItemsTotal: 3,
		BytesDone:  60,
		BytesTotal: 60,
	}, got)
	assert.EqualValues(t, 99, got.Percent())
	assert.NotEmpty(t, reporter.Chunks)
}

func TestSource_ChunkUnit_ResumedUnitProgress(t *testing.T) {
	enableUnitProgress(t)
	// Slow downloads let the count finish first.
	fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: testObjects()}}, getDelay: 200 * time.Millisecond}
	s := newFakeS3Source(t, fake)

	unit := S3SourceUnit{Bucket: "bucket"}
	unitID, _ := unit.SourceUnitID()
	s.SetEncodedResumeInfoFor(unitID, "a.txt")

	require.NoError(t, s.ChunkUnit(context.Background(), unit, &sourcestest.TestReporter{}))

	got, _ := s.GetUnitProgressFor(unitID)
	assert.Equal(t, sources.UnitProgress{
		State:        sources.UnitProgressReady,
		ItemsDone:    3,
		ItemsTotal:   3,
		BytesDone:    60,
		BytesTotal:   60,
		ResumedBytes: 10,
	}, got, "the object an earlier run finished counts as done")
}

// With one progress shared by every bucket, a count cut short in one bucket left the others at 0%.
// Each unit now has its own, so one bucket's count cannot affect another's.
func TestSource_ChunkUnit_UnitsDoNotShareProgress(t *testing.T) {
	enableUnitProgress(t)
	fake := &fakeS3{buckets: map[string]fakeBucket{
		"resumed": {objects: testObjects(), countDelay: 10 * time.Second},
		"fresh":   {objects: testObjects()},
	}}
	s := newFakeS3Source(t, fake)

	// The resumed scan finishes long before its count, which is cancelled.
	resumed := S3SourceUnit{Bucket: "resumed"}
	resumedID, _ := resumed.SourceUnitID()
	s.SetEncodedResumeInfoFor(resumedID, "a.txt")
	require.NoError(t, s.ChunkUnit(context.Background(), resumed, &sourcestest.TestReporter{}))

	fresh := S3SourceUnit{Bucket: "fresh"}
	require.NoError(t, s.ChunkUnit(context.Background(), fresh, &sourcestest.TestReporter{}))

	got, _ := s.GetUnitProgressFor(resumedID)
	assert.Equal(t, sources.UnitProgressUnavailable, got.State, "a resumed bucket with no count has no total")
	assert.EqualValues(t, 2, got.ItemsDone)
	assert.Zero(t, got.Percent())

	freshID, _ := fresh.SourceUnitID()
	got, _ = s.GetUnitProgressFor(freshID)
	assert.Equal(t, sources.UnitProgressReady, got.State)
	assert.EqualValues(t, 60, got.BytesTotal)
	assert.EqualValues(t, 99, got.Percent())
}

func TestSource_ChunkUnit_NoUnitProgressUnlessEnabled(t *testing.T) {
	fake := &fakeS3{buckets: map[string]fakeBucket{"bucket": {objects: testObjects()}}}
	s := newFakeS3Source(t, fake)

	unit := S3SourceUnit{Bucket: "bucket"}
	require.NoError(t, s.ChunkUnit(context.Background(), unit, &sourcestest.TestReporter{}))

	unitID, _ := unit.SourceUnitID()
	_, ok := s.GetUnitProgressFor(unitID)
	assert.False(t, ok)
	assert.EqualValues(t, 2, fake.listCalls.Load(), "only the scan lists the bucket")
}