Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
fix: scan files with lines exceeding bufio's default 64 KB token limit (#5022)
Muneeb Ullah Khan90a3efa36b6d
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 53489b2e0..ccf7980ba 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -750,7 +750,6 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path stringemail := commit.Authorwhen := commit.Date.UTC().Format("2006-01-02 15:04:05 -0700")-if fullHash != lastCommitHash {depth++lastCommitHash = fullHash@@ -901,6 +900,14 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, emaildefer func() { _ = reader.Close() }()originalChunk := bufio.NewScanner(reader)+ // Default bufio max token size (64 KB) is too small for files with long lines+ // (e.g. minified JS, base64 blobs). Raise the cap to 10 MB so those lines+ // are still scanned; the oversize-line path below will chunk them correctly.+ // The initial buffer starts at 4 KB (same as bufio's default) and grows only+ // when a line actually exceeds the current size, keeping allocations cheap for+ // the common case of small diffs.+ const maxScanTokenSize = 10 * 1024 * 1024+ originalChunk.Buffer(make([]byte, 4096), maxScanTokenSize)newChunkBuffer := bytes.Buffer{}lastOffset := 0for offset := 0; originalChunk.Scan(); offset++ {@@ -967,6 +974,9 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, emailctx.Logger().Error(err, "error writing to chunk buffer", "filename", fileName, "commit", hash, "file", diff.PathB)}}+ if err := originalChunk.Err(); err != nil {+ ctx.Logger().Error(err, "error scanning chunk", "filename", fileName, "commit", hash, "file", diff.PathB)+ }// Send anything still in the new chunk bufferif newChunkBuffer.Len() > 0 {metadata := s.sourceMetadataFunc(SourceMetadataInfo{diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.goindex 7b8ccf357..57ade12a8 100644--- a/pkg/sources/git/git_test.go+++ b/pkg/sources/git/git_test.go@@ -8,6 +8,7 @@ import ("path/filepath""runtime""strings"+ "sync""testing""time"@@ -1163,3 +1164,47 @@ func TestPrepareRepoWithNormalizationBare(t *testing.T) {})}}++// TestGitChunk_LongLine verifies that files containing lines longer than+// bufio's default 64 KB token limit are still scanned. Before the fix,+// bufio.Scanner would silently stop on the first oversized line and produce+// zero chunks for that file.+func TestGitChunk_LongLine(t *testing.T) {+ t.Parallel()+ ctx := context.Background()++ repoPath := setupTestRepo(t, "long-line-repo")++ // Build a single line that is 100 KB — well above the old 64 KB cap.+ longLine := strings.Repeat("a", 100*1024)+ addTestFileAndCommit(t, repoPath, "long_line.txt", longLine)++ conn, err := anypb.New(&sourcespb.Git{+ Credential: &sourcespb.Git_Unauthenticated{+ Unauthenticated: &credentialspb.Unauthenticated{},+ },+ Repositories: []string{"file://" + repoPath},+ })+ assert.NoError(t, err)++ s := Source{}+ assert.NoError(t, s.Init(ctx, "test long line", 0, 0, false, conn, 1))++ chunksCh := make(chan *sources.Chunk, 64)+ var count int+ var wg sync.WaitGroup+ wg.Add(1)+ go func() {+ defer wg.Done()+ for range chunksCh {+ count+++ }+ }()++ assert.NoError(t, s.Chunks(ctx, chunksCh))+ close(chunksCh)+ wg.Wait()+ // ensure the goroutine has finished writing to count before we read it+ // one chunk for the commit/file metadata, and at least one chunk for the file content+ assert.Equal(t, 2, count, "expected two chunks from a file with a 100 KB line")+}