Skip to content

Find, verify, and analyze leaked credentials

[bug] - Improve seekability check for stdout pipes in BufferedReadSeeker (#3189)

ahrav26ae7cb09de0
diff --git a/pkg/handlers/archive.go b/pkg/handlers/archive.go
index 484b18236..ed663c258 100644
--- a/pkg/handlers/archive.go
+++ b/pkg/handlers/archive.go
@@ -78,6 +78,9 @@ var ErrMaxDepthReached = errors.New("max archive depth reached")
// it either decompresses or extracts the contents directly, sending data to the provided channel.
// Returns an error if the archive cannot be processed due to issues like exceeding maximum depth or unsupported formats.
func (h *archiveHandler) openArchive(ctx logContext.Context, depth int, reader fileReader, archiveChan chan []byte) error {
+ ctx.Logger().V(4).Info("Starting archive processing", "depth", depth)
+ defer ctx.Logger().V(4).Info("Finished archive processing", "depth", depth)
+
if common.IsDone(ctx) {
return ctx.Err()
}
@@ -200,6 +203,7 @@ func (h *archiveHandler) extractorHandler(archiveChan chan []byte) func(context.
h.metrics.incFilesProcessed()
h.metrics.observeFileSize(fileSize)
+ lCtx.Logger().V(4).Info("Processed file successfully", "filename", file.Name(), "size", file.Size())
return h.openArchive(lCtx, depth, rdr, archiveChan)
}
}
diff --git a/pkg/handlers/handlers_test.go b/pkg/handlers/handlers_test.go
index 09efd9228..e9e0e5f73 100644
--- a/pkg/handlers/handlers_test.go
+++ b/pkg/handlers/handlers_test.go
@@ -4,6 +4,7 @@ import (
"io"
"net/http"
"os"
+ "os/exec"
"strings"
"testing"
"time"
@@ -417,3 +418,60 @@ func TestHandleLargeHTTPJson(t *testing.T) {
}
assert.Equal(t, wantCount, count)
}
+
+func TestHandlePipe(t *testing.T) {
+ r, w := io.Pipe()
+
+ go func() {
+ defer w.Close()
+ file, err := os.Open("testdata/test.tar")
+ assert.NoError(t, err)
+ defer file.Close()
+ _, err = io.Copy(w, file)
+ assert.NoError(t, err)
+ }()
+
+ chunkCh := make(chan *sources.Chunk, 1)
+ go func() {
+ defer close(chunkCh)
+ err := HandleFile(logContext.Background(), r, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh})
+ assert.NoError(t, err)
+ }()
+
+ wantCount := 1
+ count := 0
+ for range chunkCh {
+ count++
+ }
+ assert.Equal(t, wantCount, count)
+}
+
+func TestHandleZipCommandStdoutPipe(t *testing.T) {
+ cmd := exec.Command("zip", "-j", "-", "testdata/nested-dirs.zip")
+ stdout, err := cmd.StdoutPipe()
+ assert.NoError(t, err)
+
+ err = cmd.Start()
+ assert.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ chunkCh := make(chan *sources.Chunk, 1)
+ go func() {
+ defer close(chunkCh)
+ err := HandleFile(ctx, stdout, &sources.Chunk{}, sources.ChanReporter{Ch: chunkCh})
+ assert.NoError(t, err)
+ }()
+
+ err = cmd.Wait()
+ assert.NoError(t, err)
+
+ wantCount := 8
+ count := 0
+ for range chunkCh {
+ count++
+ }
+
+ assert.Equal(t, wantCount, count)
+}
diff --git a/pkg/iobuf/bufferedreaderseeker.go b/pkg/iobuf/bufferedreaderseeker.go
index d5818a79a..47ba5119e 100644
--- a/pkg/iobuf/bufferedreaderseeker.go
+++ b/pkg/iobuf/bufferedreaderseeker.go
@@ -54,15 +54,34 @@ type BufferedReadSeeker struct {
sizeKnown bool // Whether the total size of the reader is known
}
+// asSeeker checks if a reader reliably supports seeking operations.
+// Some types, like os.File when used as a pipe, may implement io.Seeker
+// but do not actually support seeking.
+func asSeeker(r io.Reader) io.Seeker {
+ seeker, ok := r.(io.Seeker)
+ if !ok {
+ return nil
+ }
+
+ _, err := seeker.Seek(0, io.SeekCurrent)
+ if err != nil {
+ return nil
+ }
+ return seeker
+}
+
// NewBufferedReaderSeeker creates and initializes a BufferedReadSeeker.
// It takes an io.Reader and checks if it supports seeking.
// If the reader supports seeking, it is stored in the seeker field.
func NewBufferedReaderSeeker(r io.Reader) *BufferedReadSeeker {
const defaultThreshold = 1 << 24 // 16MB threshold for switching to file buffering
- seeker, _ := r.(io.Seeker)
+ var (
+ buf *buffer.Buffer
+ seeker io.Seeker
+ )
- var buf *buffer.Buffer
+ seeker = asSeeker(r)
if seeker == nil {
buf = defaultBufferPool.Get()
}
@@ -101,6 +120,10 @@ func (br *BufferedReadSeeker) Read(out []byte) (int, error) {
return n, err
}
+ if br.buf == nil {
+ br.buf = br.bufPool.Get()
+ }
+
var (
totalBytesRead int
err error
@@ -302,12 +325,13 @@ func (br *BufferedReadSeeker) createTempFile() error {
}
func (br *BufferedReadSeeker) flushBufferToDisk() error {
- if _, err := br.buf.WriteTo(br.tempFile); err != nil {
+ n, err := br.buf.WriteTo(br.tempFile)
+ if err != nil {
return err
}
- br.diskBufferSize = int64(br.buf.Len())
+ br.diskBufferSize += n
- return nil
+ return err
}
// ReadAt reads len(out) bytes into out starting at offset off in the underlying input source.
@@ -322,7 +346,6 @@ func (br *BufferedReadSeeker) ReadAt(out []byte, offset int64) (int, error) {
return br.Read(out)
}
- // For non-seekable readers, use our buffering logic.
currentIndex := br.index
if _, err := br.Seek(offset, io.SeekStart); err != nil {
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 23a69f6cf..0a83664d8 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -654,9 +654,9 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
commitHash := plumbing.NewHash(fullHash)
if err := s.handleBinary(ctx, gitDir, reporter, chunkSkel, commitHash, fileName); err != nil {
- logger.V(1).Info(
+ logger.Error(
+ err,
"error handling binary file",
- "error", err,
"filename", fileName,
"commit", commitHash,
"file", diff.PathB,
@@ -877,7 +877,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
Verify: s.verify,
}
if err := s.handleBinary(ctx, gitDir, reporter, chunkSkel, commitHash, fileName); err != nil {
- logger.V(1).Info("error handling binary file", "error", err, "filename", fileName)
+ logger.Error(err, "error handling binary file", "filename", fileName)
}
continue
}
@@ -1246,26 +1246,11 @@ func (s *Git) handleBinary(ctx context.Context, gitDir string, reporter sources.
return err
}
- done := make(chan error, 1)
- // Read from stdout to prevent the pipe buffer from filling up and causing the command to hang.
- // This allows us to stream the file contents to the handler.
- go func() {
- defer close(done)
- done <- handlers.HandleFile(ctx, stdout, chunkSkel, reporter, handlers.WithSkipArchives(s.skipArchives))
- }()
-
- // Close to signal that we are done writing to the pipe, which allows the reading goroutine to finish.
- if closeErr := stdout.Close(); closeErr != nil && !errors.Is(closeErr, os.ErrClosed) {
- ctx.Logger().Error(fmt.Errorf("error closing stdout: %w", closeErr), "closing stdout failed")
- }
-
- if waitErr := cmd.Wait(); waitErr != nil {
- return fmt.Errorf("error waiting for git cat-file: %w", waitErr)
+ if err = handlers.HandleFile(ctx, stdout, chunkSkel, reporter, handlers.WithSkipArchives(s.skipArchives)); err != nil {
+ return err
}
- // Wait for the command to finish and the handler to complete.
- // Capture any error from the file handling process.
- return <-done
+ return cmd.Wait()
}
func (s *Git) executeCatFileCmd(cmd *exec.Cmd) (io.ReadCloser, error) {