Skip to content

Find, verify, and analyze leaked credentials

Add max commit size (#1079)

Bill Richb37080e6a58c
diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.go
index c2b6d6b58..8ed374b49 100644
--- a/pkg/gitparse/gitparse.go
+++ b/pkg/gitparse/gitparse.go
@@ -194,7 +194,11 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch
var currentDiff *Diff
defer common.RecoverWithExit(ctx)
+ defer close(commitChan)
for {
+ if common.IsDone(ctx) {
+ break
+ }
line, err := outReader.ReadBytes([]byte("\n")[0])
if err != nil && len(line) == 0 {
break
@@ -233,6 +237,21 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch
}
if currentDiff != nil && currentDiff.Content.Len() > 0 {
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+ // If the currentDiff is over 1GB, drop it into the channel so it isn't held in memory waiting for more commits.
+ totalSize := 0
+ for _, diff := range currentCommit.Diffs {
+ totalSize += diff.Content.Len()
+ }
+ if totalSize > c.maxCommitSize {
+ commitChan <- *currentCommit
+ currentCommit = &Commit{
+ Hash: currentCommit.Hash,
+ Author: currentCommit.Author,
+ Date: currentCommit.Date,
+ Message: currentCommit.Message,
+ Diffs: []Diff{},
+ }
+ }
}
currentDiff = &Diff{}
case isModeLine(line):
@@ -278,13 +297,16 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch
break
}
}
+ cleanupParse(currentCommit, currentDiff, commitChan)
+}
+
+func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit) {
if currentDiff != nil && currentDiff.Content.Len() > 0 {
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
}
if currentCommit != nil {
commitChan <- *currentCommit
}
- close(commitChan)
}
// Date: Tue Aug 10 15:20:40 2021 +0100
diff --git a/pkg/gitparse/gitparse_test.go b/pkg/gitparse/gitparse_test.go
index 2f3a0bdff..8967d264c 100644
--- a/pkg/gitparse/gitparse_test.go
+++ b/pkg/gitparse/gitparse_test.go
@@ -125,7 +125,7 @@ func TestSingleCommitSingleDiff(t *testing.T) {
}
func TestMultiCommitContextDiff(t *testing.T) {
- r := bytes.NewReader([]byte(singleCommitContextDiff))
+ r := bytes.NewReader([]byte(multiCommitContextDiff))
parser := NewParser()
commitChan := make(chan Commit)
dateOne, _ := time.Parse(parser.dateFormat, "Mon Mar 15 23:27:16 2021 -0700")
@@ -208,6 +208,26 @@ func TestMaxDiffSize(t *testing.T) {
}
+func TestMaxCommitSize(t *testing.T) {
+ parser := NewParser(WithMaxCommitSize(1))
+ commitText := bytes.Buffer{}
+ commitText.WriteString(singleCommitMultiDiff)
+ commitChan := make(chan Commit)
+ ctx, cancel := context.WithTimeout(context.Background(), time.Duration(1)*time.Second)
+ defer cancel()
+ go func() {
+ parser.fromReader(ctx, &commitText, commitChan)
+ }()
+ commitCount := 0
+ for range commitChan {
+ commitCount++
+ }
+ if commitCount != 2 {
+ t.Errorf("Commit count does not match. Got: %d, expected: %d", commitCount, 2)
+ }
+
+}
+
const singleCommitSingleDiff = `commit 70001020fab32b1fcf2f1f0e5c66424eae649826 (HEAD -> master, origin/master, origin/HEAD)
Author: Dustin Decker <[email protected]>
Date: Mon Mar 15 23:27:16 2021 -0700
@@ -241,7 +261,7 @@ aws_secret_access_key = Tg0pz8Jii8hkLx4+PnUisM8GmKs3a2DK+9qz/lie
output = json
region = us-east-2
`
-const singleCommitContextDiff = `commit 70001020fab32b1fcf2f1f0e5c66424eae649826 (HEAD -> master, origin/master, origin/HEAD)
+const multiCommitContextDiff = `commit 70001020fab32b1fcf2f1f0e5c66424eae649826 (HEAD -> master, origin/master, origin/HEAD)
Author: Dustin Decker <[email protected]>
Date: Mon Mar 15 23:27:16 2021 -0700
@@ -288,6 +308,47 @@ index 239b415..2ee133b 100644
+okay thank you bye
`
+const singleCommitMultiDiff = `commit 70001020fab32b1fcf2f1f0e5c66424eae649826 (HEAD -> master, origin/master, origin/HEAD)
+Author: Dustin Decker <[email protected]>
+Date: Mon Mar 15 23:27:16 2021 -0700
+
+ Update aws
+
+diff --git a/aws b/aws
+index 2ee133b..12b4843 100644
+--- a/aws
++++ b/aws
+@@ -1,7 +1,5 @@
+-blah blaj
+-
+-this is the secret: [Default]
+-Access key Id: AKIAILE3JG6KMS3HZGCA
+-Secret Access Key: 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
+-
+-okay thank you bye
++[default]
++aws_access_key_id = AKIAXYZDQCEN4B6JSJQI
++aws_secret_access_key = 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
++output = json
++region = us-east-2
+
+diff --git a/aws b/aws
+index 239b415..2ee133b 100644
+--- a/aws
++++ b/aws
+@@ -1,5 +1,7 @@
+ blah blaj
+
+-this is the secret: AKIA2E0A8F3B244C9986
++this is the secret: [Default]
++Access key Id: AKIAILE3JG6KMS3HZGCA
++Secret Access Key: 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
+
+-okay thank you bye
+\ No newline at end of file
++okay thank you bye
+`
+
const singleCommitContextDiffMessageOne = `Update aws
`