Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add base64 detectors. (#414)
ahrav3b1cd654473d
diff --git a/pkg/decoders/base64.go b/pkg/decoders/base64.gonew file mode 100644index 000000000..6dd0c43d9--- /dev/null+++ b/pkg/decoders/base64.go@@ -0,0 +1,78 @@+package decoders++import (+ "bytes"+ "encoding/base64"+ "strings"++ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++type Base64 struct{}++var (+ b64Charset = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=")+ b64EndChars = "+/="+)++func getSubstringsOfCharacterSet(data []byte, charset []byte, threshold int) []string {+ count := 0+ substrings := []string{}+ letters := strings.Builder{}+ if len(data) == 0 {+ return nil+ }+ for _, char := range string(data) {+ if bytes.ContainsRune(charset, char) {+ letters.WriteRune(char)+ count+++ } else {+ if count > threshold {+ substrings = appendB64Substring(letters, substrings)+ }+ letters.Reset()+ count = 0+ }+ }++ if count > threshold && len(letters.String()) > 0 {+ substrings = appendB64Substring(letters, substrings)+ }++ return substrings+}++func appendB64Substring(letters strings.Builder, substrings []string) []string {++ substring := strings.TrimLeft(letters.String(), b64EndChars)+ // handle key=value+ if strings.Contains(strings.TrimRight(substring, b64EndChars), "=") {+ split := strings.SplitN(substring, "=", 2)+ substrings = append(substrings, split[len(split)-1])+ } else {+ substrings = append(substrings, substring)+ }+ return substrings+}++func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {++ encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, b64Charset, 20)+ decodedSubstrings := map[string][]byte{}++ for _, str := range encodedSubstrings {+ dec, err := base64.StdEncoding.DecodeString(str)+ if err == nil && len(dec) > 0 {+ decodedSubstrings[str] = dec+ }+ }++ if len(decodedSubstrings) > 0 {+ for substring, dec := range decodedSubstrings {+ chunk.Data = bytes.Replace(chunk.Data, []byte(substring), dec, 1)+ }+ return chunk+ }++ return nil+}diff --git a/pkg/decoders/base64_test.go b/pkg/decoders/base64_test.gonew file mode 100644index 000000000..f64160c90--- /dev/null+++ b/pkg/decoders/base64_test.go@@ -0,0 +1,102 @@+package decoders++import (+ "testing"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++func TestBase64_FromChunk(t *testing.T) {+ tests := []struct {+ name string+ chunk *sources.Chunk+ want *sources.Chunk+ }{+ {+ name: "only b64 chunk",+ chunk: &sources.Chunk{+ Data: []byte(`bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q=`),+ },+ want: &sources.Chunk{+ Data: []byte(`longer-encoded-secret-test`),+ },+ },+ {+ name: "mixed content",+ chunk: &sources.Chunk{+ Data: []byte(`token: bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q=`),+ },+ want: &sources.Chunk{+ Data: []byte(`token: longer-encoded-secret-test`),+ },+ },+ {+ name: "no chunk",+ chunk: &sources.Chunk{+ Data: []byte(``),+ },+ want: nil,+ },+ {+ name: "env var (looks like all b64 decodable but has `=` in the middle)",+ chunk: &sources.Chunk{+ Data: []byte(`some-encoded-secret=dGVzdHNlY3JldA==`),+ },+ want: &sources.Chunk{+ Data: []byte(`some-encoded-secret=testsecret`),+ },+ },+ {+ name: "has longer b64 inside",+ chunk: &sources.Chunk{+ Data: []byte(`some-encoded-secret="bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q="`),+ },+ want: &sources.Chunk{+ Data: []byte(`some-encoded-secret="longer-encoded-secret-test"`),+ },+ },+ {+ name: "many possible substrings",+ chunk: &sources.Chunk{+ Data: []byte(`Many substrings in this slack message could be base64 decoded+ but only dGhpcyBlbmNhcHN1bGF0ZWQgc2VjcmV0 should be decoded.`),+ },+ want: &sources.Chunk{+ Data: []byte(`Many substrings in this slack message could be base64 decoded+ but only this encapsulated secret should be decoded.`),+ },+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ d := &Base64{}+ got := d.FromChunk(tt.chunk)+ if tt.want != nil {+ if got == nil {+ t.Fatal("got nil, did not want nil")+ }+ if diff := pretty.Compare(string(got.Data), string(tt.want.Data)); diff != "" {+ t.Errorf("Base64FromChunk() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ } else {+ if got != nil {+ t.Error("Expected nil chunk")+ }+ }+ })+ }+}++func BenchmarkFromChunk(benchmark *testing.B) {+ d := Base64{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ d.FromChunk(&sources.Chunk{Data: data})+ }+ })+ }+}diff --git a/pkg/decoders/decoders.go b/pkg/decoders/decoders.goindex 078933456..3a20e1d9b 100644--- a/pkg/decoders/decoders.go+++ b/pkg/decoders/decoders.go@@ -7,6 +7,7 @@ import (func DefaultDecoders() []Decoder {return []Decoder{&Plain{},+ &Base64{},}}