Skip to content

Find, verify, and analyze leaked credentials

Add base64 detectors. (#414)

ahrav3b1cd654473d
diff --git a/pkg/decoders/base64.go b/pkg/decoders/base64.go
new file mode 100644
index 000000000..6dd0c43d9
--- /dev/null
+++ b/pkg/decoders/base64.go
@@ -0,0 +1,78 @@
+package decoders
+
+import (
+ "bytes"
+ "encoding/base64"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+type Base64 struct{}
+
+var (
+ b64Charset = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=")
+ b64EndChars = "+/="
+)
+
+func getSubstringsOfCharacterSet(data []byte, charset []byte, threshold int) []string {
+ count := 0
+ substrings := []string{}
+ letters := strings.Builder{}
+ if len(data) == 0 {
+ return nil
+ }
+ for _, char := range string(data) {
+ if bytes.ContainsRune(charset, char) {
+ letters.WriteRune(char)
+ count++
+ } else {
+ if count > threshold {
+ substrings = appendB64Substring(letters, substrings)
+ }
+ letters.Reset()
+ count = 0
+ }
+ }
+
+ if count > threshold && len(letters.String()) > 0 {
+ substrings = appendB64Substring(letters, substrings)
+ }
+
+ return substrings
+}
+
+func appendB64Substring(letters strings.Builder, substrings []string) []string {
+
+ substring := strings.TrimLeft(letters.String(), b64EndChars)
+ // handle key=value
+ if strings.Contains(strings.TrimRight(substring, b64EndChars), "=") {
+ split := strings.SplitN(substring, "=", 2)
+ substrings = append(substrings, split[len(split)-1])
+ } else {
+ substrings = append(substrings, substring)
+ }
+ return substrings
+}
+
+func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
+
+ encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, b64Charset, 20)
+ decodedSubstrings := map[string][]byte{}
+
+ for _, str := range encodedSubstrings {
+ dec, err := base64.StdEncoding.DecodeString(str)
+ if err == nil && len(dec) > 0 {
+ decodedSubstrings[str] = dec
+ }
+ }
+
+ if len(decodedSubstrings) > 0 {
+ for substring, dec := range decodedSubstrings {
+ chunk.Data = bytes.Replace(chunk.Data, []byte(substring), dec, 1)
+ }
+ return chunk
+ }
+
+ return nil
+}
diff --git a/pkg/decoders/base64_test.go b/pkg/decoders/base64_test.go
new file mode 100644
index 000000000..f64160c90
--- /dev/null
+++ b/pkg/decoders/base64_test.go
@@ -0,0 +1,102 @@
+package decoders
+
+import (
+ "testing"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestBase64_FromChunk(t *testing.T) {
+ tests := []struct {
+ name string
+ chunk *sources.Chunk
+ want *sources.Chunk
+ }{
+ {
+ name: "only b64 chunk",
+ chunk: &sources.Chunk{
+ Data: []byte(`bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q=`),
+ },
+ want: &sources.Chunk{
+ Data: []byte(`longer-encoded-secret-test`),
+ },
+ },
+ {
+ name: "mixed content",
+ chunk: &sources.Chunk{
+ Data: []byte(`token: bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q=`),
+ },
+ want: &sources.Chunk{
+ Data: []byte(`token: longer-encoded-secret-test`),
+ },
+ },
+ {
+ name: "no chunk",
+ chunk: &sources.Chunk{
+ Data: []byte(``),
+ },
+ want: nil,
+ },
+ {
+ name: "env var (looks like all b64 decodable but has `=` in the middle)",
+ chunk: &sources.Chunk{
+ Data: []byte(`some-encoded-secret=dGVzdHNlY3JldA==`),
+ },
+ want: &sources.Chunk{
+ Data: []byte(`some-encoded-secret=testsecret`),
+ },
+ },
+ {
+ name: "has longer b64 inside",
+ chunk: &sources.Chunk{
+ Data: []byte(`some-encoded-secret="bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q="`),
+ },
+ want: &sources.Chunk{
+ Data: []byte(`some-encoded-secret="longer-encoded-secret-test"`),
+ },
+ },
+ {
+ name: "many possible substrings",
+ chunk: &sources.Chunk{
+ Data: []byte(`Many substrings in this slack message could be base64 decoded
+ but only dGhpcyBlbmNhcHN1bGF0ZWQgc2VjcmV0 should be decoded.`),
+ },
+ want: &sources.Chunk{
+ Data: []byte(`Many substrings in this slack message could be base64 decoded
+ but only this encapsulated secret should be decoded.`),
+ },
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ d := &Base64{}
+ got := d.FromChunk(tt.chunk)
+ if tt.want != nil {
+ if got == nil {
+ t.Fatal("got nil, did not want nil")
+ }
+ if diff := pretty.Compare(string(got.Data), string(tt.want.Data)); diff != "" {
+ t.Errorf("Base64FromChunk() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ } else {
+ if got != nil {
+ t.Error("Expected nil chunk")
+ }
+ }
+ })
+ }
+}
+
+func BenchmarkFromChunk(benchmark *testing.B) {
+ d := Base64{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ d.FromChunk(&sources.Chunk{Data: data})
+ }
+ })
+ }
+}
diff --git a/pkg/decoders/decoders.go b/pkg/decoders/decoders.go
index 078933456..3a20e1d9b 100644
--- a/pkg/decoders/decoders.go
+++ b/pkg/decoders/decoders.go
@@ -7,6 +7,7 @@ import (
func DefaultDecoders() []Decoder {
return []Decoder{
&Plain{},
+ &Base64{},
}
}