Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[bug] - Improve seekability check for stdout pipes in BufferedReadSeeker (#3189)
ahrav26ae7cb09de0
diff --git a/pkg/handlers/archive.go b/pkg/handlers/archive.goindex 484b18236..ed663c258 100644--- a/pkg/handlers/archive.go+++ b/pkg/handlers/archive.go@@ -78,6 +78,9 @@ var ErrMaxDepthReached = errors.New("max archive depth reached")// it either decompresses or extracts the contents directly, sending data to the provided channel.// Returns an error if the archive cannot be processed due to issues like exceeding maximum depth or unsupported formats.func (h *archiveHandler) openArchive(ctx logContext.Context, depth int, reader fileReader, archiveChan chan []byte) error {+ ctx.Logger().V(4).Info("Starting archive processing", "depth", depth)+ defer ctx.Logger().V(4).Info("Finished archive processing", "depth", depth)+if common.IsDone(ctx) {return ctx.Err()}@@ -200,6 +203,7 @@ func (h *archiveHandler) extractorHandler(archiveChan chan []byte) func(context.h.metrics.incFilesProcessed()h.metrics.observeFileSize(fileSize)+ lCtx.Logger().V(4).Info("Processed file successfully", "filename", file.Name(), "size", file.Size())return h.openArchive(lCtx, depth, rdr, archiveChan)}}diff --git a/pkg/handlers/handlers_test.go b/pkg/handlers/handlers_test.goindex 09efd9228..e9e0e5f73 100644--- a/pkg/handlers/handlers_test.go+++ b/pkg/handlers/handlers_test.go@@ -4,6 +4,7 @@ import ("io""net/http""os"+ "os/exec""strings""testing""time"@@ -417,3 +418,60 @@ func TestHandleLargeHTTPJson(t *testing.T) {}assert.Equal(t, wantCount, count)}++func TestHandlePipe(t *testing.T) {+ r, w := io.Pipe()++ go func() {+ defer w.Close()+ file, err := os.Open("testdata/test.tar")+ assert.NoError(t, err)+ defer file.Close()+ _, err = io.Copy(w, file)+ assert.NoError(t, err)+ }()++ chunkCh := make(chan *sources.Chunk, 1)+ go func() {+ defer close(chunkCh)+ err := HandleFile(logContext.Background(), r, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh})+ assert.NoError(t, err)+ }()++ wantCount := 1+ count := 0+ for range chunkCh {+ count+++ }+ assert.Equal(t, wantCount, count)+}++func TestHandleZipCommandStdoutPipe(t *testing.T) {+ cmd := exec.Command("zip", "-j", "-", "testdata/nested-dirs.zip")+ stdout, err := cmd.StdoutPipe()+ assert.NoError(t, err)++ err = cmd.Start()+ assert.NoError(t, err)++ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()++ chunkCh := make(chan *sources.Chunk, 1)+ go func() {+ defer close(chunkCh)+ err := HandleFile(ctx, stdout, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh})+ assert.NoError(t, err)+ }()++ err = cmd.Wait()+ assert.NoError(t, err)++ wantCount := 8+ count := 0+ for range chunkCh {+ count+++ }++ assert.Equal(t, wantCount, count)+}diff --git a/pkg/iobuf/bufferedreaderseeker.go b/pkg/iobuf/bufferedreaderseeker.goindex d5818a79a..47ba5119e 100644--- a/pkg/iobuf/bufferedreaderseeker.go+++ b/pkg/iobuf/bufferedreaderseeker.go@@ -54,15 +54,34 @@ type BufferedReadSeeker struct {sizeKnown bool // Whether the total size of the reader is known}+// asSeeker checks if a reader reliably supports seeking operations.+// Some types, like os.File when used as a pipe, may implement io.Seeker+// but do not actually support seeking.+func asSeeker(r io.Reader) io.Seeker {+ seeker, ok := r.(io.Seeker)+ if !ok {+ return nil+ }++ _, err := seeker.Seek(0, io.SeekCurrent)+ if err != nil {+ return nil+ }+ return seeker+}+// NewBufferedReaderSeeker creates and initializes a BufferedReadSeeker.// It takes an io.Reader and checks if it supports seeking.// If the reader supports seeking, it is stored in the seeker field.func NewBufferedReaderSeeker(r io.Reader) *BufferedReadSeeker {const defaultThreshold = 1 << 24 // 16MB threshold for switching to file buffering- seeker, _ := r.(io.Seeker)+ var (+ buf *buffer.Buffer+ seeker io.Seeker+ )- var buf *buffer.Buffer+ seeker = asSeeker(r)if seeker == nil {buf = defaultBufferPool.Get()}@@ -101,6 +120,10 @@ func (br *BufferedReadSeeker) Read(out []byte) (int, error) {return n, err}+ if br.buf == nil {+ br.buf = br.bufPool.Get()+ }+var (totalBytesRead interr error@@ -302,12 +325,13 @@ func (br *BufferedReadSeeker) createTempFile() error {}func (br *BufferedReadSeeker) flushBufferToDisk() error {- if _, err := br.buf.WriteTo(br.tempFile); err != nil {+ n, err := br.buf.WriteTo(br.tempFile)+ if err != nil {return err}- br.diskBufferSize = int64(br.buf.Len())+ br.diskBufferSize += n- return nil+ return err}// ReadAt reads len(out) bytes into out starting at offset off in the underlying input source.@@ -322,7 +346,6 @@ func (br *BufferedReadSeeker) ReadAt(out []byte, offset int64) (int, error) {return br.Read(out)}- // For non-seekable readers, use our buffering logic.currentIndex := br.indexif _, err := br.Seek(offset, io.SeekStart); err != nil {diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 23a69f6cf..0a83664d8 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -654,9 +654,9 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path stringcommitHash := plumbing.NewHash(fullHash)if err := s.handleBinary(ctx, gitDir, reporter, chunkSkel, commitHash, fileName); err != nil {- logger.V(1).Info(+ logger.Error(+ err,"error handling binary file",- "error", err,"filename", fileName,"commit", commitHash,"file", diff.PathB,@@ -877,7 +877,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,Verify: s.verify,}if err := s.handleBinary(ctx, gitDir, reporter, chunkSkel, commitHash, fileName); err != nil {- logger.V(1).Info("error handling binary file", "error", err, "filename", fileName)+ logger.Error(err, "error handling binary file", "filename", fileName)}continue}@@ -1246,26 +1246,11 @@ func (s *Git) handleBinary(ctx context.Context, gitDir string, reporter sources.return err}- done := make(chan error, 1)- // Read from stdout to prevent the pipe buffer from filling up and causing the command to hang.- // This allows us to stream the file contents to the handler.- go func() {- defer close(done)- done <- handlers.HandleFile(ctx, stdout, chunkSkel, reporter, handlers.WithSkipArchives(s.skipArchives))- }()-- // Close to signal that we are done writing to the pipe, which allows the reading goroutine to finish.- if closeErr := stdout.Close(); closeErr != nil && !errors.Is(closeErr, os.ErrClosed) {- ctx.Logger().Error(fmt.Errorf("error closing stdout: %w", closeErr), "closing stdout failed")- }-- if waitErr := cmd.Wait(); waitErr != nil {- return fmt.Errorf("error waiting for git cat-file: %w", waitErr)+ if err = handlers.HandleFile(ctx, stdout, chunkSkel, reporter, handlers.WithSkipArchives(s.skipArchives)); err != nil {+ return err}- // Wait for the command to finish and the handler to complete.- // Capture any error from the file handling process.- return <-done+ return cmd.Wait()}func (s *Git) executeCatFileCmd(cmd *exec.Cmd) (io.ReadCloser, error) {