Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[INS-470] Add Tly detector to defaults.go, gate it behind feat flag and update its verification logic (#5006)
Muneeb Ullah Khanc5ebff41ee79
diff --git a/main.go b/main.goindex 820953f87..56bbf9b85 100644--- a/main.go+++ b/main.go@@ -534,6 +534,7 @@ func run(state overseer.State, logSync func() error) {feature.GitLabOAuthDetectorEnabled.Store(true)feature.EnigmaDetectorEnabled.Store(true)feature.DatadogApiKeyDetectorEnabled.Store(true)+ feature.TlyDetectorEnabled.Store(true)conf := &config.Config{}if *configFilename != "" {diff --git a/pkg/detectors/tly/tly.go b/pkg/detectors/tly/tly.goindex d756125e4..8ca17235d 100644--- a/pkg/detectors/tly/tly.go+++ b/pkg/detectors/tly/tly.go@@ -2,6 +2,8 @@ package tlyimport ("context"+ "fmt"+ "io""net/http""strings"@@ -31,38 +33,76 @@ func (s Scanner) Keywords() []string {}// FromData will find and optionally verify TLy secrets in a given set of bytes.-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {- dataStr := string(data)+func (s Scanner) FromData(+ ctx context.Context,+ verify bool,+ data []byte,+) (results []detectors.Result, err error) {- matches := keyPat.FindAllStringSubmatch(dataStr, -1)+ dataStr := string(data)- for _, match := range matches {- resMatch := strings.TrimSpace(match[1])+ uniqueKeys := make(map[string]struct{})+ for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {+ uniqueKeys[strings.TrimSpace(match[1])] = struct{}{}+ }- s1 := detectors.Result{+ for key := range uniqueKeys {+ result := detectors.Result{DetectorType: detector_typepb.DetectorType_TLy,- Raw: []byte(resMatch),- SecretParts: map[string]string{"key": resMatch},+ Raw: []byte(key),+ SecretParts: map[string]string{+ "key": key,+ },}if verify {- req, err := http.NewRequestWithContext(ctx, "GET", "https://t.ly/api/v1/link/stats?api_token="+resMatch+"&short_url=https://t.ly/h9YS", nil)- if err != nil {- continue- }- res, err := client.Do(req)- if err == nil {- defer func() { _ = res.Body.Close() }()- if res.StatusCode >= 200 && res.StatusCode < 300 {- s1.Verified = true- }- }+ verified, verificationErr := verifyTLyKey(ctx, client, key)+ result.SetVerificationError(verificationErr, key)+ result.Verified = verified}- results = append(results, s1)+ results = append(results, result)+ }++ return+}++func verifyTLyKey(+ ctx context.Context,+ client *http.Client,+ key string,+) (bool, error) {++ req, err := http.NewRequestWithContext(+ ctx,+ http.MethodGet,+ "https://api.t.ly/api/v1/link/list",+ http.NoBody,+ )+ if err != nil {+ return false, err}- return results, nil+ req.Header.Set("Authorization", "Bearer "+key)+ req.Header.Set("Accept", "application/json")++ res, err := client.Do(req)+ if err != nil {+ return false, err+ }+ defer func() {+ _, _ = io.Copy(io.Discard, res.Body)+ _ = res.Body.Close()+ }()++ switch res.StatusCode {+ case http.StatusOK:+ return true, nil+ case http.StatusUnauthorized, http.StatusForbidden:+ return false, nil+ default:+ return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)+ }}func (s Scanner) Type() detector_typepb.DetectorType {diff --git a/pkg/detectors/tly/tly_integration_test.go b/pkg/detectors/tly/tly_integration_test.goindex 502170071..65e831ef2 100644--- a/pkg/detectors/tly/tly_integration_test.go+++ b/pkg/detectors/tly/tly_integration_test.go@@ -9,8 +9,8 @@ import ("testing""time"- "github.com/kylelemons/godebug/pretty"-+ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"@@ -96,8 +96,19 @@ func TestTLy_FromChunk(t *testing.T) {}got[i].Raw = nil}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("TLy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ ignoreOpts := cmpopts.IgnoreFields(+ detectors.Result{},+ "ExtraData",+ "verificationError",+ "primarySecret",+ "SecretParts",+ "chunkOffset",+ "chunkOffsetSet",+ )++ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Tly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}})}diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.goindex a19e2040b..c3de0f556 100644--- a/pkg/engine/defaults/defaults.go+++ b/pkg/engine/defaults/defaults.go@@ -773,6 +773,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timecamp""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timezoneapi""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tineswebhook"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tly""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tmetric""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/todoist""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tokeet"@@ -1671,6 +1672,7 @@ func buildDetectorList() []detectors.Detector {&timecamp.Scanner{},&timezoneapi.Scanner{},&tineswebhook.Scanner{},+ &tly.Scanner{},&tmetric.Scanner{},&todoist.Scanner{},// &toggltrack.Scanner{},@@ -1784,6 +1786,8 @@ func buildDetectorList() []detectors.Detector {return !feature.EnigmaDetectorEnabled.Load()case *datadogapikey.Scanner:return !feature.DatadogApiKeyDetectorEnabled.Load()+ case *tly.Scanner:+ return !feature.TlyDetectorEnabled.Load()default:return false}diff --git a/pkg/engine/defaults/defaults_test.go b/pkg/engine/defaults/defaults_test.goindex ef6641a29..f52219ebb 100644--- a/pkg/engine/defaults/defaults_test.go+++ b/pkg/engine/defaults/defaults_test.go@@ -122,7 +122,6 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{detector_typepb.DetectorType_IPInfo: {},detector_typepb.DetectorType_Lob: {},detector_typepb.DetectorType_Rev: {},- detector_typepb.DetectorType_TLy: {},detector_typepb.DetectorType_Tru: {},detector_typepb.DetectorType_User: {},detector_typepb.DetectorType_Wit: {},@@ -134,6 +133,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{detector_typepb.DetectorType_Enigma: {},detector_typepb.DetectorType_GitLabOauth2: {},detector_typepb.DetectorType_Pinecone: {},+ detector_typepb.DetectorType_TLy: {},// Reserved / special types.detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()diff --git a/pkg/feature/feature.go b/pkg/feature/feature.goindex a1ff5aa90..80271d4e8 100644--- a/pkg/feature/feature.go+++ b/pkg/feature/feature.go@@ -21,6 +21,7 @@ var (GitLabOAuthDetectorEnabled atomic.BoolEnigmaDetectorEnabled atomic.BoolDatadogApiKeyDetectorEnabled atomic.Bool+ TlyDetectorEnabled atomic.Bool)type AtomicString struct {