Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[feat] - Streamlined File Handling with BufferedReaderSeeker (#3041)
ahravf8654820258c
diff --git a/pkg/handlers/ar_test.go b/pkg/handlers/ar_test.goindex 59285ca47..430fc4e73 100644--- a/pkg/handlers/ar_test.go+++ b/pkg/handlers/ar_test.go@@ -20,7 +20,6 @@ func TestHandleARFile(t *testing.T) {rdr, err := newFileReader(file)assert.NoError(t, err)- defer rdr.Close()handler := newARHandler()archiveChan, err := handler.HandleFile(context.AddLogger(ctx), rdr)diff --git a/pkg/handlers/archive.go b/pkg/handlers/archive.goindex 77677634d..925153acc 100644--- a/pkg/handlers/archive.go+++ b/pkg/handlers/archive.go@@ -87,15 +87,14 @@ func (h *archiveHandler) openArchive(ctx logContext.Context, depth int, reader freturn ErrMaxDepthReached}- arReader := reader.BufferedFileReaderif reader.format == nil && depth > 0 {- return h.handleNonArchiveContent(ctx, arReader, archiveChan)+ return h.handleNonArchiveContent(ctx, reader, archiveChan)}switch archive := reader.format.(type) {case archiver.Decompressor:// Decompress tha archive and feed the decompressed data back into the archive handler to extract any nested archives.- compReader, err := archive.OpenReader(arReader)+ compReader, err := archive.OpenReader(reader)if err != nil {return fmt.Errorf("error opening decompressor with format: %s %w", reader.format.Name(), err)}@@ -109,11 +108,10 @@ func (h *archiveHandler) openArchive(ctx logContext.Context, depth int, reader f}return fmt.Errorf("error creating custom reader: %w", err)}- defer rdr.Close()return h.openArchive(ctx, depth+1, rdr, archiveChan)case archiver.Extractor:- err := archive.Extract(logContext.WithValue(ctx, depthKey, depth+1), arReader, nil, h.extractorHandler(archiveChan))+ err := archive.Extract(logContext.WithValue(ctx, depthKey, depth+1), reader, nil, h.extractorHandler(archiveChan))if err != nil {return fmt.Errorf("error extracting archive with format: %s: %w", reader.format.Name(), err)}@@ -193,7 +191,6 @@ func (h *archiveHandler) extractorHandler(archiveChan chan []byte) func(context.}return fmt.Errorf("error creating custom reader: %w", err)}- defer rdr.Close()h.metrics.incFilesProcessed()h.metrics.observeFileSize(fileSize)diff --git a/pkg/handlers/archive_test.go b/pkg/handlers/archive_test.goindex 1217fd017..0bde50470 100644--- a/pkg/handlers/archive_test.go+++ b/pkg/handlers/archive_test.go@@ -119,7 +119,6 @@ func TestOpenInvalidArchive(t *testing.T) {rdr, err := newFileReader(io.NopCloser(reader))assert.NoError(t, err)- defer rdr.Close()archiveChan := make(chan []byte)diff --git a/pkg/handlers/default_test.go b/pkg/handlers/default_test.goindex 3d071ad6f..b4aa15333 100644--- a/pkg/handlers/default_test.go+++ b/pkg/handlers/default_test.go@@ -20,7 +20,6 @@ func TestHandleNonArchiveFile(t *testing.T) {rdr, err := newFileReader(file)assert.NoError(t, err)- defer rdr.Close()handler := newDefaultHandler(defaultHandlerType)archiveChan, err := handler.HandleFile(context.AddLogger(ctx), rdr)diff --git a/pkg/handlers/handlers.go b/pkg/handlers/handlers.goindex aa810a2f2..ceaf84fad 100644--- a/pkg/handlers/handlers.go+++ b/pkg/handlers/handlers.go@@ -9,7 +9,7 @@ import ("github.com/mholt/archiver/v4"logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"- "github.com/trufflesecurity/trufflehog/v3/pkg/readers"+ "github.com/trufflesecurity/trufflehog/v3/pkg/iobuf""github.com/trufflesecurity/trufflehog/v3/pkg/sources")@@ -29,63 +29,58 @@ import (// promotes a more cohesive and maintainable codebase. It also embeds a BufferedFileReader to provide efficient// random access to the file content.type fileReader struct {- format archiver.Format- mimeType mimeType- *readers.BufferedFileReader+ format archiver.Format+ mimeType mimeTypeisGenericArchive bool++ *iobuf.BufferedReadSeeker}var ErrEmptyReader = errors.New("reader is empty")-func newFileReader(r io.ReadCloser) (fileReader, error) {- defer r.Close()+// newFileReader creates a fileReader from an io.Reader, optionally using BufferedFileWriter for certain formats.+func newFileReader(r io.Reader) (fileReader, error) {+ var reader fileReader++ bufReader := iobuf.NewBufferedReaderSeeker(r)- var (- reader fileReader- rdr *readers.BufferedFileReader- err error- )- rdr, err = readers.NewBufferedFileReader(r)+ mime, err := mimetype.DetectReader(bufReader)if err != nil {- return reader, fmt.Errorf("error creating random access reader: %w", err)+ return reader, fmt.Errorf("unable to detect MIME type: %w", err)}- reader.BufferedFileReader = rdr+ reader.mimeType = mimeType(mime.String())- // Ensure the reader is closed if an error occurs after the reader is created.- // During non-error conditions, the caller is responsible for closing the reader.- defer func() {- if err != nil && rdr != nil {- _ = rdr.Close()- }- }()-- // Check if the reader is empty.- if rdr.Size() == 0 {- return reader, ErrEmptyReader+ // Reset the reader to the beginning because DetectReader consumes the reader.+ if _, err := bufReader.Seek(0, io.SeekStart); err != nil {+ return reader, fmt.Errorf("error resetting reader after MIME detection: %w", err)}- format, arReader, err := archiver.Identify("", rdr)+ format, _, err := archiver.Identify("", bufReader)switch {- case err == nil: // Archive detected+ case err == nil:reader.isGenericArchive = truereader.mimeType = mimeType(format.Name())reader.format = format+case errors.Is(err, archiver.ErrNoMatch):- // Not an archive handled by archiver, try to detect MIME type.- // This will occur for un-supported archive types and non-archive files. (ex: .deb, .rpm, .txt)- mimeT, err := mimetype.DetectReader(arReader)- if err != nil {- return reader, fmt.Errorf("error detecting MIME type: %w", err)- }- reader.mimeType = mimeType(mimeT.String())- default: // Error identifying archive+ // Not an archive handled by archiver.+ // Continue with the default reader.+ default:return reader, fmt.Errorf("error identifying archive: %w", err)}- if _, err = rdr.Seek(0, io.SeekStart); err != nil {- return reader, fmt.Errorf("error seeking to start of file: %w", err)+ // Reset the reader to the beginning again to allow the handler to read from the start.+ // This is necessary because Identify consumes the reader.+ if _, err := bufReader.Seek(0, io.SeekStart); err != nil {+ return reader, fmt.Errorf("error resetting reader after archive identification: %w", err)}+ // Disable buffering after initial reads.+ // This optimization ensures we don't continue writing to the buffer after the initial reads.+ bufReader.DisableBuffering()++ reader.BufferedReadSeeker = bufReader+return reader, nil}@@ -168,7 +163,7 @@ func selectHandler(file fileReader) FileHandler {// the function will skip processing the file and return nil.func HandleFile(ctx logContext.Context,- reader io.ReadCloser,+ reader io.Reader,chunkSkel *sources.Chunk,reporter sources.ChunkReporter,options ...func(*fileHandlingConfig),@@ -185,7 +180,6 @@ func HandleFile(}return fmt.Errorf("error creating custom reader: %w", err)}- defer rdr.Close()config := newFileHandlingConfig(options...)if config.skipArchives && rdr.isGenericArchive {diff --git a/pkg/handlers/handlers_test.go b/pkg/handlers/handlers_test.goindex cfa23ab7c..31821dfee 100644--- a/pkg/handlers/handlers_test.go+++ b/pkg/handlers/handlers_test.go@@ -1,6 +1,7 @@package handlersimport (+ "io""net/http""os""strings"@@ -42,11 +43,13 @@ func TestHandleFile(t *testing.T) {}func BenchmarkHandleFile(b *testing.B) {+ file, err := os.Open("testdata/test.tgz")+ assert.Nil(b, err)+ defer file.Close()++ b.ReportAllocs()for i := 0; i < b.N; i++ {sourceChan := make(chan *sources.Chunk, 1)- file, err := os.Open("testdata/test.tgz")- assert.Nil(b, err)-b.StartTimer()go func() {defer close(sourceChan)@@ -57,6 +60,9 @@ func BenchmarkHandleFile(b *testing.B) {for range sourceChan {}b.StopTimer()++ _, err = file.Seek(0, io.SeekStart)+ assert.NoError(b, err)}}@@ -201,6 +207,31 @@ func TestHandleFileAR(t *testing.T) {assert.Equal(t, wantChunkCount, len(reporter.Ch))}+func BenchmarkHandleAR(b *testing.B) {+ file, err := os.Open("testdata/test.deb")+ assert.Nil(b, err)+ defer file.Close()++ b.ReportAllocs()+ for i := 0; i < b.N; i++ {+ sourceChan := make(chan *sources.Chunk, 1)++ b.StartTimer()+ go func() {+ defer close(sourceChan)+ err := HandleFile(context.Background(), file, &sources.Chunk{}, sources.ChanReporter{Ch: sourceChan})+ assert.NoError(b, err)+ }()++ for range sourceChan {+ }+ b.StopTimer()++ _, err = file.Seek(0, io.SeekStart)+ assert.NoError(b, err)+ }+}+func TestHandleFileNonArchive(t *testing.T) {wantChunkCount := 6reporter := sources.ChanReporter{Ch: make(chan *sources.Chunk, wantChunkCount)}@@ -220,7 +251,7 @@ func TestExtractTarContentWithEmptyFile(t *testing.T) {file, err := os.Open("testdata/testdir.zip")assert.Nil(t, err)- chunkCh := make(chan *sources.Chunk)+ chunkCh := make(chan *sources.Chunk, 1)go func() {defer close(chunkCh)err := HandleFile(logContext.Background(), file, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh})@@ -234,3 +265,48 @@ func TestExtractTarContentWithEmptyFile(t *testing.T) {}assert.Equal(t, wantCount, count)}++func TestHandleTar(t *testing.T) {+ file, err := os.Open("testdata/test.tar")+ assert.Nil(t, err)+ defer file.Close()++ chunkCh := make(chan *sources.Chunk, 1)+ go func() {+ defer close(chunkCh)+ err := HandleFile(logContext.Background(), file, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh})+ assert.NoError(t, err)+ }()++ wantCount := 1+ count := 0+ for range chunkCh {+ count+++ }+ assert.Equal(t, wantCount, count)+}++func BenchmarkHandleTar(b *testing.B) {+ file, err := os.Open("testdata/test.tar")+ assert.Nil(b, err)+ defer file.Close()++ b.ReportAllocs()+ for i := 0; i < b.N; i++ {+ sourceChan := make(chan *sources.Chunk, 1)++ b.StartTimer()+ go func() {+ defer close(sourceChan)+ err := HandleFile(context.Background(), file, &sources.Chunk{}, sources.ChanReporter{Ch: sourceChan})+ assert.NoError(b, err)+ }()++ for range sourceChan {+ }+ b.StopTimer()++ _, err = file.Seek(0, io.SeekStart)+ assert.NoError(b, err)+ }+}diff --git a/pkg/handlers/rpm_test.go b/pkg/handlers/rpm_test.goindex f90d7b672..7ef1a1c82 100644--- a/pkg/handlers/rpm_test.go+++ b/pkg/handlers/rpm_test.go@@ -20,7 +20,6 @@ func TestHandleRPMFile(t *testing.T) {rdr, err := newFileReader(file)assert.NoError(t, err)- defer rdr.Close()handler := newRPMHandler()archiveChan, err := handler.HandleFile(context.AddLogger(ctx), rdr)diff --git a/pkg/handlers/testdata/test.tar b/pkg/handlers/testdata/test.tarnew file mode 100644index 000000000..fdbeb0d5bBinary files /dev/null and b/pkg/handlers/testdata/test.tar differdiff --git a/pkg/iobuf/bufferedreaderseeker.go b/pkg/iobuf/bufferedreaderseeker.gonew file mode 100644index 000000000..9db909fc3--- /dev/null+++ b/pkg/iobuf/bufferedreaderseeker.go@@ -0,0 +1,198 @@+package iobuf++import (+ "bytes"+ "errors"+ "fmt"+ "io"+)++// BufferedReadSeeker provides a buffered reading interface with seeking capabilities.+// It wraps an io.Reader and optionally an io.Seeker, allowing for efficient+// reading and seeking operations, even on non-seekable underlying readers.+type BufferedReadSeeker struct {+ reader io.Reader+ seeker io.Seeker // If the reader supports seeking, it's stored here for direct access++ buffer *bytes.Buffer // Internal buffer to store read bytes for non-seekable readers++ bytesRead int64 // Total number of bytes read from the underlying reader+ index int64 // Current position in the virtual stream++ // Flag to control buffering. This flag is used to indicate whether buffering is active.+ // Buffering is enabled during initial reads (e.g., for MIME type detection and format identification).+ // Once these operations are done, buffering should be disabled to prevent further writes to the buffer+ // and to optimize subsequent reads directly from the underlying reader. This helps avoid excessive+ // memory usage while still providing the necessary functionality for initial detection operations.+ activeBuffering bool+}++// NewBufferedReaderSeeker creates and initializes a BufferedReadSeeker.+// It takes an io.Reader and checks if it supports seeking.+// If the reader supports seeking, it is stored in the seeker field.+func NewBufferedReaderSeeker(r io.Reader) *BufferedReadSeeker {+ var (+ seeker io.Seeker+ buffer *bytes.Buffer+ activeBuffering = true+ )+ if s, ok := r.(io.Seeker); ok {+ seeker = s+ activeBuffering = false+ }++ const mimeTypeBufferSize = 3072 // Approx buffer size for MIME type detection++ if seeker == nil {+ buffer = bytes.NewBuffer(make([]byte, 0, mimeTypeBufferSize))+ }++ return &BufferedReadSeeker{+ reader: r,+ seeker: seeker,+ buffer: buffer,+ bytesRead: 0,+ index: 0,+ activeBuffering: activeBuffering,+ }+}++// Read reads len(out) bytes from the reader starting at the current index.+// It handles both seekable and non-seekable underlying readers efficiently.+func (br *BufferedReadSeeker) Read(out []byte) (int, error) {+ // For seekable readers, read directly from the underlying reader.+ if br.seeker != nil {+ n, err := br.reader.Read(out)+ br.index += int64(n)+ br.bytesRead = max(br.bytesRead, br.index)+ return n, err+ }++ // For non-seekable readers, use buffered reading.+ outLen := int64(len(out))+ if outLen == 0 {+ return 0, nil+ }++ // If the current read position (br.index) is within the buffer's valid data range,+ // read from the buffer. This ensures previously read data (e.g., for mime type detection)+ // is included in subsequent reads, providing a consistent view of the reader's content.+ if br.index < int64(br.buffer.Len()) {+ n := copy(out, br.buffer.Bytes()[br.index:])+ br.index += int64(n)+ return n, nil+ }++ if !br.activeBuffering {+ // If buffering is not active, read directly from the underlying reader.+ n, err := br.reader.Read(out)+ br.index += int64(n)+ br.bytesRead = max(br.bytesRead, br.index)+ return n, err+ }++ // Ensure there are enough bytes in the buffer to read from.+ if outLen+br.index > int64(br.buffer.Len()) {+ bytesToRead := int(outLen + br.index - int64(br.buffer.Len()))+ readerBytes := make([]byte, bytesToRead)+ n, err := br.reader.Read(readerBytes)+ br.buffer.Write(readerBytes[:n])+ br.bytesRead += int64(n)++ if err != nil {+ return n, err+ }+ }++ // Ensure the read does not exceed the buffer length.+ endIndex := br.index + outLen+ bufLen := int64(br.buffer.Len())+ if endIndex > bufLen {+ endIndex = bufLen+ }++ if br.index >= bufLen {+ return 0, io.EOF+ }++ n := copy(out, br.buffer.Bytes()[br.index:endIndex])+ br.index += int64(n)+ return n, nil+}++// Seek sets the offset for the next Read or Write to offset.+// It supports both seekable and non-seekable underlying readers.+func (br *BufferedReadSeeker) Seek(offset int64, whence int) (int64, error) {+ if br.seeker != nil {+ // Use the underlying Seeker if available.+ newIndex, err := br.seeker.Seek(offset, whence)+ if err != nil {+ return 0, fmt.Errorf("error seeking in reader: %w", err)+ }+ if newIndex > br.bytesRead {+ br.bytesRead = newIndex+ }+ return newIndex, nil+ }++ // Manual seeking for non-seekable readers.+ newIndex := br.index++ const bufferSize = 64 * 1024 // 64KB chunk size for reading++ switch whence {+ case io.SeekStart:+ newIndex = offset+ case io.SeekCurrent:+ newIndex += offset+ case io.SeekEnd:+ // Read the entire reader to determine its length+ buffer := make([]byte, bufferSize)+ for {+ n, err := br.reader.Read(buffer)+ if n > 0 {+ if br.activeBuffering {+ br.buffer.Write(buffer[:n])+ }+ br.bytesRead += int64(n)+ }+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ return 0, err+ }+ }+ newIndex = min(br.bytesRead+offset, br.bytesRead)+ default:+ return 0, errors.New("invalid whence value")+ }++ if newIndex < 0 {+ return 0, errors.New("can not seek to before start of reader")+ }++ br.index = newIndex++ return newIndex, nil+}++// ReadAt reads len(out) bytes into out starting at offset off in the underlying input source.+// It uses Seek and Read to implement random access reading.+func (br *BufferedReadSeeker) ReadAt(out []byte, offset int64) (int, error) {+ startIndex, err := br.Seek(offset, io.SeekStart)+ if err != nil {+ return 0, err+ }++ if startIndex != offset {+ return 0, io.EOF+ }++ return br.Read(out)+}++// DisableBuffering stops the buffering process.+// This is useful after initial reads (e.g., for MIME type detection and format identification)+// to prevent further writes to the buffer, optimizing subsequent reads.+func (br *BufferedReadSeeker) DisableBuffering() { br.activeBuffering = false }diff --git a/pkg/iobuf/bufferedreaderseeker_test.go b/pkg/iobuf/bufferedreaderseeker_test.gonew file mode 100644index 000000000..50b067b28--- /dev/null+++ b/pkg/iobuf/bufferedreaderseeker_test.go@@ -0,0 +1,362 @@+package iobuf++import (+ "bytes"+ "io"+ "strings"+ "testing"++ "github.com/stretchr/testify/assert"+)++func TestBufferedReaderSeekerRead(t *testing.T) {+ tests := []struct {+ name string+ reader io.Reader+ activeBuffering bool+ reads []int+ expectedReads []int+ expectedBytes [][]byte+ expectedBytesRead int64+ expectedIndex int64+ expectedBuffer []byte+ expectedError error+ }{+ {+ name: "read from seekable reader",+ reader: strings.NewReader("test data"),+ activeBuffering: true,+ reads: []int{4},+ expectedReads: []int{4},+ expectedBytes: [][]byte{[]byte("test")},+ expectedBytesRead: 4,+ expectedIndex: 4,+ },+ {+ name: "read from non-seekable reader with buffering",+ reader: bytes.NewBufferString("test data"),+ activeBuffering: true,+ reads: []int{4},+ expectedReads: []int{4},+ expectedBytes: [][]byte{[]byte("test")},+ expectedBytesRead: 4,+ expectedIndex: 4,+ expectedBuffer: []byte("test"),+ },+ {+ name: "read from non-seekable reader without buffering",+ reader: bytes.NewBufferString("test data"),+ activeBuffering: false,+ reads: []int{4},+ expectedReads: []int{4},+ expectedBytes: [][]byte{[]byte("test")},+ expectedBytesRead: 4,+ expectedIndex: 4,+ },+ {+ name: "read beyond buffer",+ reader: strings.NewReader("test data"),+ activeBuffering: true,+ reads: []int{10},+ expectedReads: []int{9},+ expectedBytes: [][]byte{[]byte("test data")},+ expectedBytesRead: 9,+ expectedIndex: 9,+ },+ {+ name: "read with empty reader",+ reader: strings.NewReader(""),+ activeBuffering: true,+ reads: []int{4},+ expectedReads: []int{0},+ expectedBytes: [][]byte{[]byte("")},+ expectedBytesRead: 0,+ expectedIndex: 0,+ expectedError: io.EOF,+ },+ {+ name: "read exact buffer size",+ reader: strings.NewReader("test"),+ activeBuffering: true,+ reads: []int{4},+ expectedReads: []int{4},+ expectedBytes: [][]byte{[]byte("test")},+ expectedBytesRead: 4,+ expectedIndex: 4,+ },+ {+ name: "read less than buffer size",+ reader: strings.NewReader("te"),+ activeBuffering: true,+ reads: []int{4},+ expectedReads: []int{2},+ expectedBytes: [][]byte{[]byte("te")},+ expectedBytesRead: 2,+ expectedIndex: 2,+ },+ {+ name: "read more than buffer size without buffering",+ reader: bytes.NewBufferString("test data"),+ activeBuffering: false,+ reads: []int{4},+ expectedReads: []int{4},+ expectedBytes: [][]byte{[]byte("test")},+ expectedBytesRead: 4,+ expectedIndex: 4,+ },+ {+ name: "multiple reads with buffering",+ reader: bytes.NewBufferString("test data"),+ activeBuffering: true,+ reads: []int{4, 5},+ expectedReads: []int{4, 5},+ expectedBytes: [][]byte{[]byte("test"), []byte(" data")},+ expectedBytesRead: 9,+ expectedIndex: 9,+ expectedBuffer: []byte("test data"),+ },+ {+ name: "multiple reads without buffering",+ reader: bytes.NewBufferString("test data"),+ activeBuffering: false,+ reads: []int{4, 5},+ expectedReads: []int{4, 5},+ expectedBytes: [][]byte{[]byte("test"), []byte(" data")},+ expectedBytesRead: 9,+ expectedIndex: 9,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ t.Parallel()++ brs := NewBufferedReaderSeeker(tt.reader)+ brs.activeBuffering = tt.activeBuffering++ for i, readSize := range tt.reads {+ buf := make([]byte, readSize)+ n, err := brs.Read(buf)++ assert.Equal(t, tt.expectedReads[i], n, "read %d: unexpected number of bytes read", i+1)+ assert.Equal(t, tt.expectedBytes[i], buf[:n], "read %d: unexpected bytes", i+1)++ if i == len(tt.reads)-1 {+ if tt.expectedError != nil {+ assert.ErrorIs(t, err, tt.expectedError)+ } else {+ assert.NoError(t, err)+ }+ }+ }++ assert.Equal(t, tt.expectedBytesRead, brs.bytesRead)+ assert.Equal(t, tt.expectedIndex, brs.index)++ if brs.buffer != nil && len(tt.expectedBuffer) > 0 {+ assert.Equal(t, tt.expectedBuffer, brs.buffer.Bytes())+ } else {+ assert.Nil(t, tt.expectedBuffer)+ }+ })+ }+}++func TestBufferedReaderSeekerSeek(t *testing.T) {+ tests := []struct {+ name string+ reader io.Reader+ offset int64+ whence int+ expectedPos int64+ expectedErr bool+ expectedRead []byte+ }{+ {+ name: "seek on seekable reader with SeekStart",+ reader: strings.NewReader("test data"),+ offset: 4,+ whence: io.SeekStart,+ expectedPos: 4,+ expectedErr: false,+ expectedRead: []byte(" dat"),+ },+ {+ name: "seek on seekable reader with SeekCurrent",+ reader: strings.NewReader("test data"),+ offset: 4,+ whence: io.SeekCurrent,+ expectedPos: 4,+ expectedErr: false,+ expectedRead: []byte(" dat"),+ },+ {+ name: "seek on seekable reader with SeekEnd",+ reader: strings.NewReader("test data"),+ offset: -4,+ whence: io.SeekEnd,+ expectedPos: 5,+ expectedErr: false,+ expectedRead: []byte("data"),+ },+ {+ name: "seek on non-seekable reader with SeekStart",+ reader: bytes.NewBufferString("test data"),+ offset: 4,+ whence: io.SeekStart,+ expectedPos: 4,+ expectedErr: false,+ expectedRead: []byte{},+ },+ {+ name: "seek on non-seekable reader with SeekCurrent",+ reader: bytes.NewBufferString("test data"),+ offset: 4,+ whence: io.SeekCurrent,+ expectedPos: 4,+ expectedErr: false,+ expectedRead: []byte{},+ },+ {+ name: "seek on non-seekable reader with SeekEnd",+ reader: bytes.NewBufferString("test data"),+ offset: -4,+ whence: io.SeekEnd,+ expectedPos: 5,+ expectedErr: false,+ expectedRead: []byte{},+ },+ {+ name: "seek to negative position",+ reader: strings.NewReader("test data"),+ offset: -1,+ whence: io.SeekStart,+ expectedPos: 0,+ expectedErr: true,+ expectedRead: nil,+ },+ {+ name: "seek beyond EOF on non-seekable reader",+ reader: bytes.NewBufferString("test data"),+ offset: 20,+ whence: io.SeekEnd,+ expectedPos: 9,+ expectedErr: false,+ expectedRead: []byte{},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ t.Parallel()++ brs := NewBufferedReaderSeeker(tt.reader)+ pos, err := brs.Seek(tt.offset, tt.whence)+ if tt.expectedErr {+ assert.Error(t, err)+ return+ }++ assert.NoError(t, err)+ assert.Equal(t, tt.expectedPos, pos)+ if len(tt.expectedRead) > 0 {+ buf := make([]byte, len(tt.expectedRead))+ nn, err := brs.Read(buf)+ assert.NoError(t, err)+ assert.Equal(t, len(tt.expectedRead), nn)+ assert.Equal(t, tt.expectedRead, buf[:nn])+ }+ })+ }+}++func TestBufferedReaderSeekerReadAt(t *testing.T) {+ tests := []struct {+ name string+ reader io.Reader+ offset int64+ length int+ expectedN int+ expectErr bool+ expectedOut []byte+ }{+ {+ name: "read within buffer on seekable reader",+ reader: strings.NewReader("test data"),+ offset: 5,+ length: 4,+ expectedN: 4,+ expectedOut: []byte("data"),+ },+ {+ name: "read within buffer on non-seekable reader",+ reader: bytes.NewBufferString("test data"),+ offset: 5,+ length: 4,+ expectedN: 4,+ expectedOut: []byte("data"),+ },+ {+ name: "read beyond buffer",+ reader: strings.NewReader("test data"),+ offset: 9,+ length: 1,+ expectedN: 0,+ expectErr: true,+ expectedOut: []byte{},+ },+ {+ name: "read at start",+ reader: strings.NewReader("test data"),+ offset: 0,+ length: 4,+ expectedN: 4,+ expectedOut: []byte("test"),+ },+ {+ name: "read with zero length",+ reader: strings.NewReader("test data"),+ offset: 0,+ length: 0,+ expectedN: 0,+ expectedOut: []byte{},+ },+ {+ name: "read negative offset",+ reader: strings.NewReader("test data"),+ offset: -1,+ length: 4,+ expectedN: 0,+ expectErr: true,+ expectedOut: []byte{},+ },+ {+ name: "read beyond end on non-seekable reader",+ reader: bytes.NewBufferString("test data"),+ offset: 20,+ length: 4,+ expectedN: 0,+ expectErr: true,+ expectedOut: []byte{},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ t.Parallel()++ brs := NewBufferedReaderSeeker(tt.reader)++ out := make([]byte, tt.length)+ n, err := brs.ReadAt(out, tt.offset)+ if tt.expectErr {+ assert.Error(t, err)+ return+ }++ assert.NoError(t, err)+ assert.Equal(t, tt.expectedN, n)+ assert.Equal(t, tt.expectedOut, out[:n])+ })+ }+}diff --git a/pkg/readers/bufferedfilereader.go b/pkg/readers/bufferedfilereader.godeleted file mode 100644index 44c2bb303..000000000--- a/pkg/readers/bufferedfilereader.go+++ /dev/null@@ -1,84 +0,0 @@-package readers--import (- "fmt"- "io"-- bufferedfilewriter "github.com/trufflesecurity/trufflehog/v3/pkg/writers/buffered_file_writer"-)--// Compile time check to ensure that BufferedFileReader implements io.ReaderAt.-var _ io.ReaderAt = (*BufferedFileReader)(nil)--// BufferedFileReader provides random access read, seek, and close capabilities on top of the BufferedFileWriter.-// It combines the functionality of BufferedFileWriter for buffered writing, io.ReadSeekCloser for-// random access reading and seeking.-type BufferedFileReader struct {- bufWriter *bufferedfilewriter.BufferedFileWriter- reader io.ReadSeekCloser-}--// NewBufferedFileReader initializes a BufferedFileReader from an io.Reader by using-// the BufferedFileWriter's functionality to read and store data, then setting up a io.ReadSeekCloser-// for random access reading and seeking.-// Close should be called when the BufferedFileReader is no longer needed.-// It returns the initialized BufferedFileReader and any error encountered during the process.-func NewBufferedFileReader(r io.Reader) (*BufferedFileReader, error) {- writer, err := bufferedfilewriter.NewFromReader(r)- if err != nil {- return nil, fmt.Errorf("error creating BufferedFileReader: %w", err)- }-- // Ensure that the BufferedFileWriter is in read-only mode.- if err := writer.CloseForWriting(); err != nil {- return nil, err- }-- rdr, err := writer.ReadSeekCloser()- if err != nil {- return nil, err- }-- return &BufferedFileReader{writer, rdr}, nil-}--// Close the BufferedFileReader.-// It should be called when the BufferedFileReader is no longer needed.-// Note that closing the BufferedFileReader does not affect the underlying bytes.Reader,-// which can still be used for reading, seeking, and reading at specific positions.-// Close is a no-op for the bytes.Reader.-func (b *BufferedFileReader) Close() error { return b.reader.Close() }--// Read reads up to len(p) bytes into p from the underlying reader.-// It returns the number of bytes read and any error encountered.-// If the reader reaches the end of the available data, Read returns 0, io.EOF.-// It implements the io.Reader interface.-func (b *BufferedFileReader) Read(p []byte) (int, error) { return b.reader.Read(p) }--// Seek sets the offset for the next Read operation on the underlying reader.-// The offset is interpreted according to the whence parameter:-// - io.SeekStart means relative to the start of the file-// - io.SeekCurrent means relative to the current offset-// - io.SeekEnd means relative to the end of the file-//-// Seek returns the new offset and any error encountered.-// It implements the io.Seeker interface.-func (b *BufferedFileReader) Seek(offset int64, whence int) (int64, error) {- return b.reader.Seek(offset, whence)-}--// ReadAt reads len(p) bytes from the underlying io.ReadSeekCloser starting at byte offset off.-// It returns the number of bytes read and any error encountered.-// If the io.ReadSeekCloser reaches the end of the available data before len(p) bytes are read,-// ReadAt returns the number of bytes read and io.EOF.-// It implements the io.ReaderAt interface.-func (b *BufferedFileReader) ReadAt(p []byte, off int64) (n int, err error) {- _, err = b.reader.Seek(off, io.SeekStart)- if err != nil {- return 0, err- }- return b.reader.Read(p)-}--// Size returns the total size of the data stored in the BufferedFileReader.-func (b *BufferedFileReader) Size() int { return b.bufWriter.Len() }diff --git a/pkg/readers/bufferedfilereader_test.go b/pkg/readers/bufferedfilereader_test.godeleted file mode 100644index a50d2e1b0..000000000--- a/pkg/readers/bufferedfilereader_test.go+++ /dev/null@@ -1,98 +0,0 @@-package readers--import (- "bytes"- "io"- "testing"-- "github.com/stretchr/testify/assert"-)--func TestBufferedFileReader(t *testing.T) {- t.Parallel()-- data := []byte("Hello, World!")-- bufferReadSeekCloser, err := NewBufferedFileReader(bytes.NewReader(data))- assert.NoError(t, err)-- // Test Read.- buffer := make([]byte, len(data))- n, err := bufferReadSeekCloser.Read(buffer)- assert.NoError(t, err)- assert.Equal(t, len(data), n)- assert.Equal(t, data, buffer)-- // Test Seek.- offset := 7- seekPos, err := bufferReadSeekCloser.Seek(int64(offset), io.SeekStart)- assert.NoError(t, err)- assert.Equal(t, int64(offset), seekPos)-- // Test ReadAt.- buffer = make([]byte, len(data)-offset)- n, err = bufferReadSeekCloser.ReadAt(buffer, int64(offset))- assert.NoError(t, err)- assert.Equal(t, len(data)-offset, n)- assert.Equal(t, data[offset:], buffer)-- // Test Close.- err = bufferReadSeekCloser.Close()- assert.NoError(t, err)-}--func TestBufferedFileReaderClose(t *testing.T) {- t.Parallel()-- data := []byte("Hello, World!")-- bufferReadSeekCloser, err := NewBufferedFileReader(bytes.NewReader(data))- assert.NoError(t, err)-- err = bufferReadSeekCloser.Close()- assert.NoError(t, err)-- // Read should NOT return any data after closing the reader.- buffer := make([]byte, len(data))- n, err := bufferReadSeekCloser.Read(buffer)- assert.ErrorIs(t, err, io.EOF)- assert.Equal(t, 0, n)-}--func TestBufferedFileReaderReadFromFile(t *testing.T) {- t.Parallel()-- // Create a large byte slice to simulate data exceeding the threshold.- largeData := make([]byte, 1024*1024) // 1 MB- for i := range largeData {- largeData[i] = byte(i % 256)- }-- bufferReadSeekCloser, err := NewBufferedFileReader(bytes.NewReader(largeData))- assert.NoError(t, err)- defer bufferReadSeekCloser.Close()-- // Test Read.- buffer := make([]byte, len(largeData))- n, err := bufferReadSeekCloser.Read(buffer)- assert.NoError(t, err)- assert.Equal(t, len(largeData), n)- assert.Equal(t, largeData, buffer)-- // Test Seek.- offset := 512 * 1024 // 512 KB- seekPos, err := bufferReadSeekCloser.Seek(int64(offset), io.SeekStart)- assert.NoError(t, err)- assert.Equal(t, int64(offset), seekPos)-- // Test ReadAt.- buffer = make([]byte, len(largeData)-offset)- n, err = bufferReadSeekCloser.ReadAt(buffer, int64(offset))- assert.NoError(t, err)- assert.Equal(t, len(largeData)-offset, n)- assert.Equal(t, largeData[offset:], buffer)-- // Test Close.- err = bufferReadSeekCloser.Close()- assert.NoError(t, err)-}diff --git a/pkg/sources/chunker.go b/pkg/sources/chunker.goindex 347d88446..7a9731530 100644--- a/pkg/sources/chunker.go+++ b/pkg/sources/chunker.go@@ -92,7 +92,7 @@ func createReaderFn(config *chunkReaderConfig) ChunkReader {}func readInChunks(ctx context.Context, reader io.Reader, config *chunkReaderConfig) <-chan ChunkResult {- const channelSize = 1+ const channelSize = 64chunkReader := bufio.NewReaderSize(reader, config.chunkSize)chunkResultChan := make(chan ChunkResult, channelSize)diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.goindex ff50b1b46..74ae9bcae 100644--- a/pkg/sources/filesystem/filesystem.go+++ b/pkg/sources/filesystem/filesystem.go@@ -166,6 +166,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *souif err != nil {return fmt.Errorf("unable to open file: %w", err)}+ defer inputFile.Close()logger.V(3).Info("scanning file")diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 6a529b5e5..19a1b6aae 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -1238,29 +1238,47 @@ func (s *Git) handleBinary(ctx context.Context, gitDir string, reporter sources.}cmd := exec.Command("git", "-C", gitDir, "cat-file", "blob", commitHash.String()+":"+path)+ stdout, err := s.executeCatFileCmd(cmd)+ if err != nil {+ return err+ }++ done := make(chan error, 1)+ // Read from stdout to prevent the pipe buffer from filling up and causing the command to hang.+ // This allows us to stream the file contents to the handler.+ go func() {+ defer close(done)+ done <- handlers.HandleFile(ctx, stdout, chunkSkel, reporter, handlers.WithSkipArchives(s.skipArchives))+ }()++ // Close to signal that we are done writing to the pipe, which allows the reading goroutine to finish.+ if closeErr := stdout.Close(); closeErr != nil && !errors.Is(closeErr, os.ErrClosed) {+ ctx.Logger().Error(fmt.Errorf("error closing stdout: %w", closeErr), "closing stdout failed")+ }+ if waitErr := cmd.Wait(); waitErr != nil {+ return fmt.Errorf("error waiting for git cat-file: %w", waitErr)+ }++ // Wait for the command to finish and the handler to complete.+ // Capture any error from the file handling process.+ return <-done+}++func (s *Git) executeCatFileCmd(cmd *exec.Cmd) (io.ReadCloser, error) {var stderr bytes.Buffercmd.Stderr = &stderrstdout, err := cmd.StdoutPipe()if err != nil {- return fmt.Errorf("error running git cat-file: %w\n%s", err, stderr.Bytes())+ return nil, fmt.Errorf("error running git cat-file: %w\n%s", err, stderr.Bytes())}- defer func() {- if err = cmd.Wait(); err != nil {- ctx.Logger().Error(fmt.Errorf(- "error waiting for command: command=%s, stderr=%s, commit=%s: %w",- cmd.String(), stderr.String(), commitHash.String(), err,- ), "waiting for command failed")- }- }()-if err := cmd.Start(); err != nil {- return fmt.Errorf("error starting git cat-file: %w\n%s", err, stderr.Bytes())+ return nil, fmt.Errorf("error starting git cat-file: %w\n%s", err, stderr.Bytes())}- return handlers.HandleFile(fileCtx, stdout, chunkSkel, reporter, handlers.WithSkipArchives(s.skipArchives))+ return stdout, nil}func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.goindex 63ab995cd..ebd39a102 100644--- a/pkg/sources/s3/s3.go+++ b/pkg/sources/s3/s3.go@@ -355,6 +355,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan}return nil}+ defer res.Body.Close()email := "Unknown"if obj.Owner != nil {