Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[chore] Add SourceUnitEnumChunker filesystem tests (#1873)
Miccah03dc7cb68db1
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.goindex 2e7b71785..64eca62ce 100644--- a/pkg/sources/filesystem/filesystem.go+++ b/pkg/sources/filesystem/filesystem.go@@ -39,8 +39,7 @@ type Source struct {// Ensure the Source satisfies the interfaces at compile timevar _ sources.Source = (*Source)(nil)var _ sources.SourceUnitUnmarshaller = (*Source)(nil)-var _ sources.SourceUnitEnumerator = (*Source)(nil)-var _ sources.SourceUnitChunker = (*Source)(nil)+var _ sources.SourceUnitEnumChunker = (*Source)(nil)// Type returns the type of source.// It is used for matching source types in configuration and job input.diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.goindex ef5dfbdd3..e6925fafe 100644--- a/pkg/sources/filesystem/filesystem_test.go+++ b/pkg/sources/filesystem/filesystem_test.go@@ -15,6 +15,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sources"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sourcestest")func TestSource_Scan(t *testing.T) {@@ -84,23 +85,15 @@ func TestSource_Scan(t *testing.T) {}func TestScanFile(t *testing.T) {- tmpfile, err := os.CreateTemp("", "example.txt")- if err != nil {- t.Fatal(err)- }- defer os.Remove(tmpfile.Name())-chunkSize := sources.ChunkSizesecretPart1 := "SECRET"secretPart2 := "SPLIT"// Split the secret into two parts and pad the rest of the chunk with A's.data := strings.Repeat("A", chunkSize-len(secretPart1)) + secretPart1 + secretPart2 + strings.Repeat("A", chunkSize-len(secretPart2))- _, err = tmpfile.Write([]byte(data))- assert.Nil(t, err)-- err = tmpfile.Close()+ tmpfile, cleanup, err := createTempFile("", data)assert.Nil(t, err)+ defer cleanup()source := &Source{}chunksChan := make(chan *sources.Chunk, 2)@@ -120,3 +113,195 @@ func TestScanFile(t *testing.T) {assert.Contains(t, foundSecret, secretPart1+secretPart2)}++func TestEnumerate(t *testing.T) {+ t.Parallel()+ ctx := context.Background()++ // Setup the connection to test enumeration.+ units := []string{+ "/one", "/two", "/three",+ "/path/to/dir/", "/path/to/another/dir/",+ }+ conn, err := anypb.New(&sourcespb.Filesystem{+ Paths: units[0:3],+ Directories: units[3:],+ })+ assert.NoError(t, err)++ // Initialize the source.+ s := Source{}+ err = s.Init(ctx, "test enumerate", 0, 0, true, conn, 1)+ assert.NoError(t, err)++ reporter := sourcestest.TestReporter{}+ err = s.Enumerate(ctx, &reporter)+ assert.NoError(t, err)++ assert.Equal(t, len(units), len(reporter.Units))+ assert.Equal(t, 0, len(reporter.UnitErrs))+ for _, unit := range reporter.Units {+ assert.Contains(t, units, unit.SourceUnitID())+ }+ for _, unit := range units {+ assert.Contains(t, reporter.Units, sources.CommonSourceUnit{ID: unit})+ }+}++func TestChunkUnit(t *testing.T) {+ t.Parallel()+ ctx := context.Background()++ // Setup test file to chunk.+ fileContents := "TestChunkUnit"+ tmpfile, cleanup, err := createTempFile("", fileContents)+ assert.NoError(t, err)+ defer cleanup()++ tmpdir, cleanup, err := createTempDir("", "foo", "bar", "baz")+ assert.NoError(t, err)+ defer cleanup()++ conn, err := anypb.New(&sourcespb.Filesystem{})+ assert.NoError(t, err)++ // Initialize the source.+ s := Source{}+ err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)+ assert.NoError(t, err)++ // Happy path single file.+ reporter := sourcestest.TestReporter{}+ err = s.ChunkUnit(ctx, sources.CommonSourceUnit{+ ID: tmpfile.Name(),+ }, &reporter)+ assert.NoError(t, err)++ // Happy path directory.+ err = s.ChunkUnit(ctx, sources.CommonSourceUnit{+ ID: tmpdir,+ }, &reporter)+ assert.NoError(t, err)++ // Error path.+ err = s.ChunkUnit(ctx, sources.CommonSourceUnit{+ ID: "/file/not/found",+ }, &reporter)+ assert.NoError(t, err)++ assert.Equal(t, 4, len(reporter.Chunks))+ assert.Equal(t, 1, len(reporter.ChunkErrs))+ dataFound := make(map[string]struct{}, 4)+ for _, chunk := range reporter.Chunks {+ dataFound[string(chunk.Data)] = struct{}{}+ }+ assert.Contains(t, dataFound, fileContents)+ assert.Contains(t, dataFound, "foo")+ assert.Contains(t, dataFound, "bar")+ assert.Contains(t, dataFound, "baz")+}++func TestEnumerateReporterErr(t *testing.T) {+ t.Parallel()+ ctx := context.Background()++ // Setup the connection to test enumeration.+ units := []string{+ "/one", "/two", "/three",+ "/path/to/dir/", "/path/to/another/dir/",+ }+ conn, err := anypb.New(&sourcespb.Filesystem{+ Paths: units[0:3],+ Directories: units[3:],+ })+ assert.NoError(t, err)++ // Initialize the source.+ s := Source{}+ err = s.Init(ctx, "test enumerate", 0, 0, true, conn, 1)+ assert.NoError(t, err)++ // Enumerate should always return an error if the reporter returns an+ // error.+ reporter := sourcestest.ErrReporter{}+ err = s.Enumerate(ctx, &reporter)+ assert.Error(t, err)+}++func TestChunkUnitReporterErr(t *testing.T) {+ t.Parallel()+ ctx := context.Background()++ // Setup test file to chunk.+ tmpfile, err := os.CreateTemp("", "example.txt")+ if err != nil {+ t.Fatal(err)+ }+ defer os.Remove(tmpfile.Name())++ fileContents := []byte("TestChunkUnit")+ _, err = tmpfile.Write(fileContents)+ assert.NoError(t, err)+ assert.NoError(t, tmpfile.Close())++ conn, err := anypb.New(&sourcespb.Filesystem{})+ assert.NoError(t, err)++ // Initialize the source.+ s := Source{}+ err = s.Init(ctx, "test chunk unit", 0, 0, true, conn, 1)+ assert.NoError(t, err)++ // Happy path. ChunkUnit should always return an error if the reporter+ // returns an error.+ reporter := sourcestest.ErrReporter{}+ err = s.ChunkUnit(ctx, sources.CommonSourceUnit{+ ID: tmpfile.Name(),+ }, &reporter)+ assert.Error(t, err)++ // Error path. ChunkUnit should always return an error if the reporter+ // returns an error.+ err = s.ChunkUnit(ctx, sources.CommonSourceUnit{+ ID: "/file/not/found",+ }, &reporter)+ assert.Error(t, err)+}++// createTempFile is a helper function to create a temporary file in the given+// directory with the provided contents. If dir is "", the operating system's+// temp directory is used.+func createTempFile(dir string, contents string) (*os.File, func(), error) {+ tmpfile, err := os.CreateTemp(dir, "trufflehogtest")+ if err != nil {+ return nil, nil, err+ }++ if _, err := tmpfile.Write([]byte(contents)); err != nil {+ _ = os.Remove(tmpfile.Name())+ return nil, nil, err+ }+ if err := tmpfile.Close(); err != nil {+ _ = os.Remove(tmpfile.Name())+ return nil, nil, err+ }+ return tmpfile, func() { _ = os.Remove(tmpfile.Name()) }, nil+}++// createTempDir is a helper function to create a temporary directory in the+// given directory with files containing the provided contents. If dir is "",+// the operating system's temp directory is used.+func createTempDir(dir string, contents ...string) (string, func(), error) {+ tmpdir, err := os.MkdirTemp(dir, "trufflehogtest")+ if err != nil {+ return "", nil, err+ }++ for _, content := range contents {+ if _, _, err := createTempFile(tmpdir, content); err != nil {+ _ = os.RemoveAll(tmpdir)+ return "", nil, err+ }+ }+ return tmpdir, func() { _ = os.RemoveAll(tmpdir) }, nil+}diff --git a/pkg/sourcestest/sourcestest.go b/pkg/sourcestest/sourcestest.gonew file mode 100644index 000000000..6631ad540--- /dev/null+++ b/pkg/sourcestest/sourcestest.go@@ -0,0 +1,61 @@+package sourcestest++import (+ "fmt"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++type reporter interface {+ sources.UnitReporter+ sources.ChunkReporter+}++var (+ _ reporter = (*TestReporter)(nil)+ _ reporter = (*ErrReporter)(nil)+)++// TestReporter is a helper struct that implements both UnitReporter and+// ChunkReporter by simply recording the values passed in the methods.+type TestReporter struct {+ Units []sources.SourceUnit+ UnitErrs []error+ Chunks []sources.Chunk+ ChunkErrs []error+}++func (t *TestReporter) UnitOk(_ context.Context, unit sources.SourceUnit) error {+ t.Units = append(t.Units, unit)+ return nil+}+func (t *TestReporter) UnitErr(_ context.Context, err error) error {+ t.UnitErrs = append(t.UnitErrs, err)+ return nil+}+func (t *TestReporter) ChunkOk(_ context.Context, chunk sources.Chunk) error {+ t.Chunks = append(t.Chunks, chunk)+ return nil+}+func (t *TestReporter) ChunkErr(_ context.Context, err error) error {+ t.ChunkErrs = append(t.ChunkErrs, err)+ return nil+}++// ErrReporter implements UnitReporter and ChunkReporter but always returns an+// error.+type ErrReporter struct{}++func (ErrReporter) UnitOk(context.Context, sources.SourceUnit) error {+ return fmt.Errorf("ErrReporter: UnitOk error")+}+func (ErrReporter) UnitErr(context.Context, error) error {+ return fmt.Errorf("ErrReporter: UnitErr error")+}+func (ErrReporter) ChunkOk(context.Context, sources.Chunk) error {+ return fmt.Errorf("ErrReporter: ChunkOk error")+}+func (ErrReporter) ChunkErr(context.Context, error) error {+ return fmt.Errorf("ErrReporter: ChunkErr error")+}