Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add max commit size (#1079)
Bill Richb37080e6a58c
diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.goindex c2b6d6b58..8ed374b49 100644--- a/pkg/gitparse/gitparse.go+++ b/pkg/gitparse/gitparse.go@@ -194,7 +194,11 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan chvar currentDiff *Diffdefer common.RecoverWithExit(ctx)+ defer close(commitChan)for {+ if common.IsDone(ctx) {+ break+ }line, err := outReader.ReadBytes([]byte("\n")[0])if err != nil && len(line) == 0 {break@@ -233,6 +237,21 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch}if currentDiff != nil && currentDiff.Content.Len() > 0 {currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)+ // If the currentDiff is over 1GB, drop it into the channel so it isn't held in memory waiting for more commits.+ totalSize := 0+ for _, diff := range currentCommit.Diffs {+ totalSize += diff.Content.Len()+ }+ if totalSize > c.maxCommitSize {+ commitChan <- *currentCommit+ currentCommit = &Commit{+ Hash: currentCommit.Hash,+ Author: currentCommit.Author,+ Date: currentCommit.Date,+ Message: currentCommit.Message,+ Diffs: []Diff{},+ }+ }}currentDiff = &Diff{}case isModeLine(line):@@ -278,13 +297,16 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan chbreak}}+ cleanupParse(currentCommit, currentDiff, commitChan)+}++func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit) {if currentDiff != nil && currentDiff.Content.Len() > 0 {currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)}if currentCommit != nil {commitChan <- *currentCommit}- close(commitChan)}// Date: Tue Aug 10 15:20:40 2021 +0100diff --git a/pkg/gitparse/gitparse_test.go b/pkg/gitparse/gitparse_test.goindex 2f3a0bdff..8967d264c 100644--- a/pkg/gitparse/gitparse_test.go+++ b/pkg/gitparse/gitparse_test.go@@ -125,7 +125,7 @@ func TestSingleCommitSingleDiff(t *testing.T) {}func TestMultiCommitContextDiff(t *testing.T) {- r := bytes.NewReader([]byte(singleCommitContextDiff))+ r := bytes.NewReader([]byte(multiCommitContextDiff))parser := NewParser()commitChan := make(chan Commit)dateOne, _ := time.Parse(parser.dateFormat, "Mon Mar 15 23:27:16 2021 -0700")@@ -208,6 +208,26 @@ func TestMaxDiffSize(t *testing.T) {}+func TestMaxCommitSize(t *testing.T) {+ parser := NewParser(WithMaxCommitSize(1))+ commitText := bytes.Buffer{}+ commitText.WriteString(singleCommitMultiDiff)+ commitChan := make(chan Commit)+ ctx, cancel := context.WithTimeout(context.Background(), time.Duration(1)*time.Second)+ defer cancel()+ go func() {+ parser.fromReader(ctx, &commitText, commitChan)+ }()+ commitCount := 0+ for range commitChan {+ commitCount+++ }+ if commitCount != 2 {+ t.Errorf("Commit count does not match. Got: %d, expected: %d", commitCount, 2)+ }++}+const singleCommitSingleDiff = `commit 70001020fab32b1fcf2f1f0e5c66424eae649826 (HEAD -> master, origin/master, origin/HEAD)Author: Dustin Decker <[email protected]>Date: Mon Mar 15 23:27:16 2021 -0700@@ -241,7 +261,7 @@ aws_secret_access_key = Tg0pz8Jii8hkLx4+PnUisM8GmKs3a2DK+9qz/lieoutput = jsonregion = us-east-2`-const singleCommitContextDiff = `commit 70001020fab32b1fcf2f1f0e5c66424eae649826 (HEAD -> master, origin/master, origin/HEAD)+const multiCommitContextDiff = `commit 70001020fab32b1fcf2f1f0e5c66424eae649826 (HEAD -> master, origin/master, origin/HEAD)Author: Dustin Decker <[email protected]>Date: Mon Mar 15 23:27:16 2021 -0700@@ -288,6 +308,47 @@ index 239b415..2ee133b 100644+okay thank you bye`+const singleCommitMultiDiff = `commit 70001020fab32b1fcf2f1f0e5c66424eae649826 (HEAD -> master, origin/master, origin/HEAD)+Author: Dustin Decker <[email protected]>+Date: Mon Mar 15 23:27:16 2021 -0700++ Update aws++diff --git a/aws b/aws+index 2ee133b..12b4843 100644+--- a/aws++++ b/aws+@@ -1,7 +1,5 @@+-blah blaj+-+-this is the secret: [Default]+-Access key Id: AKIAILE3JG6KMS3HZGCA+-Secret Access Key: 6GKmgiS3EyIBJbeSp7sQ+0PoJrPZjPUg8SF6zYz7+-+-okay thank you bye++[default]++aws_access_key_id = AKIAXYZDQCEN4B6JSJQI++aws_secret_access_key = Tg0pz8Jii8hkLx4+PnUisM8GmKs3a2DK+9qz/lie++output = json++region = us-east-2++diff --git a/aws b/aws+index 239b415..2ee133b 100644+--- a/aws++++ b/aws+@@ -1,5 +1,7 @@+ blah blaj++-this is the secret: AKIA2E0A8F3B244C9986++this is the secret: [Default]++Access key Id: AKIAILE3JG6KMS3HZGCA++Secret Access Key: 6GKmgiS3EyIBJbeSp7sQ+0PoJrPZjPUg8SF6zYz7++-okay thank you bye+\ No newline at end of file++okay thank you bye+`+const singleCommitContextDiffMessageOne = `Update aws`