Skip to content

Find, verify, and analyze leaked credentials

fix: scan files with lines exceeding bufio's default 64 KB token limit (#5022)

Muneeb Ullah Khan90a3efa36b6d
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index 53489b2e0..ccf7980ba 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -750,7 +750,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
email := commit.Author
when := commit.Date.UTC().Format("2006-01-02 15:04:05 -0700")
-
if fullHash != lastCommitHash {
depth++
lastCommitHash = fullHash
@@ -901,6 +900,14 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, email
defer func() { _ = reader.Close() }()
originalChunk := bufio.NewScanner(reader)
+ // Default bufio max token size (64 KB) is too small for files with long lines
+ // (e.g. minified JS, base64 blobs). Raise the cap to 10 MB so those lines
+ // are still scanned; the oversize-line path below will chunk them correctly.
+ // The initial buffer starts at 4 KB (same as bufio's default) and grows only
+ // when a line actually exceeds the current size, keeping allocations cheap for
+ // the common case of small diffs.
+ const maxScanTokenSize = 10 * 1024 * 1024
+ originalChunk.Buffer(make([]byte, 4096), maxScanTokenSize)
newChunkBuffer := bytes.Buffer{}
lastOffset := 0
for offset := 0; originalChunk.Scan(); offset++ {
@@ -967,6 +974,9 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, email
ctx.Logger().Error(err, "error writing to chunk buffer", "filename", fileName, "commit", hash, "file", diff.PathB)
}
}
+ if err := originalChunk.Err(); err != nil {
+ ctx.Logger().Error(err, "error scanning chunk", "filename", fileName, "commit", hash, "file", diff.PathB)
+ }
// Send anything still in the new chunk buffer
if newChunkBuffer.Len() > 0 {
metadata := s.sourceMetadataFunc(SourceMetadataInfo{
diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go
index 7b8ccf357..57ade12a8 100644
--- a/pkg/sources/git/git_test.go
+++ b/pkg/sources/git/git_test.go
@@ -8,6 +8,7 @@ import (
"path/filepath"
"runtime"
"strings"
+ "sync"
"testing"
"time"
@@ -1163,3 +1164,47 @@ func TestPrepareRepoWithNormalizationBare(t *testing.T) {
})
}
}
+
+// TestGitChunk_LongLine verifies that files containing lines longer than
+// bufio's default 64 KB token limit are still scanned. Before the fix,
+// bufio.Scanner would silently stop on the first oversized line and produce
+// zero chunks for that file.
+func TestGitChunk_LongLine(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+
+ repoPath := setupTestRepo(t, "long-line-repo")
+
+ // Build a single line that is 100 KB — well above the old 64 KB cap.
+ longLine := strings.Repeat("a", 100*1024)
+ addTestFileAndCommit(t, repoPath, "long_line.txt", longLine)
+
+ conn, err := anypb.New(&sourcespb.Git{
+ Credential: &sourcespb.Git_Unauthenticated{
+ Unauthenticated: &credentialspb.Unauthenticated{},
+ },
+ Repositories: []string{"file://" + repoPath},
+ })
+ assert.NoError(t, err)
+
+ s := Source{}
+ assert.NoError(t, s.Init(ctx, "test long line", 0, 0, false, conn, 1))
+
+ chunksCh := make(chan *sources.Chunk, 64)
+ var count int
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for range chunksCh {
+ count++
+ }
+ }()
+
+ assert.NoError(t, s.Chunks(ctx, chunksCh))
+ close(chunksCh)
+ wg.Wait()
+ // ensure the goroutine has finished writing to count before we read it
+ // one chunk for the commit/file metadata, and at least one chunk for the file content
+ assert.Equal(t, 2, count, "expected two chunks from a file with a 100 KB line")
+}