Skip to content

Find, verify, and analyze leaked credentials

[INS-470] Add Tly detector to defaults.go, gate it behind feat flag and update its verification logic (#5006)

Muneeb Ullah Khanc5ebff41ee79
diff --git a/main.go b/main.go
index 820953f87..56bbf9b85 100644
--- a/main.go
+++ b/main.go
@@ -534,6 +534,7 @@ func run(state overseer.State, logSync func() error) {
feature.GitLabOAuthDetectorEnabled.Store(true)
feature.EnigmaDetectorEnabled.Store(true)
feature.DatadogApiKeyDetectorEnabled.Store(true)
+ feature.TlyDetectorEnabled.Store(true)
conf := &config.Config{}
if *configFilename != "" {
diff --git a/pkg/detectors/tly/tly.go b/pkg/detectors/tly/tly.go
index d756125e4..8ca17235d 100644
--- a/pkg/detectors/tly/tly.go
+++ b/pkg/detectors/tly/tly.go
@@ -2,6 +2,8 @@ package tly
import (
"context"
+ "fmt"
+ "io"
"net/http"
"strings"
@@ -31,38 +33,76 @@ func (s Scanner) Keywords() []string {
}
// FromData will find and optionally verify TLy secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
- dataStr := string(data)
+func (s Scanner) FromData(
+ ctx context.Context,
+ verify bool,
+ data []byte,
+) (results []detectors.Result, err error) {
- matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+ dataStr := string(data)
- for _, match := range matches {
- resMatch := strings.TrimSpace(match[1])
+ uniqueKeys := make(map[string]struct{})
+ for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+ uniqueKeys[strings.TrimSpace(match[1])] = struct{}{}
+ }
- s1 := detectors.Result{
+ for key := range uniqueKeys {
+ result := detectors.Result{
DetectorType: detector_typepb.DetectorType_TLy,
- Raw: []byte(resMatch),
- SecretParts: map[string]string{"key": resMatch},
+ Raw: []byte(key),
+ SecretParts: map[string]string{
+ "key": key,
+ },
}
if verify {
- req, err := http.NewRequestWithContext(ctx, "GET", "https://t.ly/api/v1/link/stats?api_token="+resMatch+"&short_url=https://t.ly/h9YS", nil)
- if err != nil {
- continue
- }
- res, err := client.Do(req)
- if err == nil {
- defer func() { _ = res.Body.Close() }()
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- s1.Verified = true
- }
- }
+ verified, verificationErr := verifyTLyKey(ctx, client, key)
+ result.SetVerificationError(verificationErr, key)
+ result.Verified = verified
}
- results = append(results, s1)
+ results = append(results, result)
+ }
+
+ return
+}
+
+func verifyTLyKey(
+ ctx context.Context,
+ client *http.Client,
+ key string,
+) (bool, error) {
+
+ req, err := http.NewRequestWithContext(
+ ctx,
+ http.MethodGet,
+ "https://api.t.ly/api/v1/link/list",
+ http.NoBody,
+ )
+ if err != nil {
+ return false, err
}
- return results, nil
+ req.Header.Set("Authorization", "Bearer "+key)
+ req.Header.Set("Accept", "application/json")
+
+ res, err := client.Do(req)
+ if err != nil {
+ return false, err
+ }
+ defer func() {
+ _, _ = io.Copy(io.Discard, res.Body)
+ _ = res.Body.Close()
+ }()
+
+ switch res.StatusCode {
+ case http.StatusOK:
+ return true, nil
+ case http.StatusUnauthorized, http.StatusForbidden:
+ return false, nil
+ default:
+ return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+ }
}
func (s Scanner) Type() detector_typepb.DetectorType {
diff --git a/pkg/detectors/tly/tly_integration_test.go b/pkg/detectors/tly/tly_integration_test.go
index 502170071..65e831ef2 100644
--- a/pkg/detectors/tly/tly_integration_test.go
+++ b/pkg/detectors/tly/tly_integration_test.go
@@ -9,8 +9,8 @@ import (
"testing"
"time"
- "github.com/kylelemons/godebug/pretty"
-
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
@@ -96,8 +96,19 @@ func TestTLy_FromChunk(t *testing.T) {
}
got[i].Raw = nil
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("TLy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ ignoreOpts := cmpopts.IgnoreFields(
+ detectors.Result{},
+ "ExtraData",
+ "verificationError",
+ "primarySecret",
+ "SecretParts",
+ "chunkOffset",
+ "chunkOffsetSet",
+ )
+
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Tly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go
index a19e2040b..c3de0f556 100644
--- a/pkg/engine/defaults/defaults.go
+++ b/pkg/engine/defaults/defaults.go
@@ -773,6 +773,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timecamp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/timezoneapi"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tineswebhook"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tly"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tmetric"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/todoist"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tokeet"
@@ -1671,6 +1672,7 @@ func buildDetectorList() []detectors.Detector {
&timecamp.Scanner{},
&timezoneapi.Scanner{},
&tineswebhook.Scanner{},
+ &tly.Scanner{},
&tmetric.Scanner{},
&todoist.Scanner{},
// &toggltrack.Scanner{},
@@ -1784,6 +1786,8 @@ func buildDetectorList() []detectors.Detector {
return !feature.EnigmaDetectorEnabled.Load()
case *datadogapikey.Scanner:
return !feature.DatadogApiKeyDetectorEnabled.Load()
+ case *tly.Scanner:
+ return !feature.TlyDetectorEnabled.Load()
default:
return false
}
diff --git a/pkg/engine/defaults/defaults_test.go b/pkg/engine/defaults/defaults_test.go
index ef6641a29..f52219ebb 100644
--- a/pkg/engine/defaults/defaults_test.go
+++ b/pkg/engine/defaults/defaults_test.go
@@ -122,7 +122,6 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
detector_typepb.DetectorType_IPInfo: {},
detector_typepb.DetectorType_Lob: {},
detector_typepb.DetectorType_Rev: {},
- detector_typepb.DetectorType_TLy: {},
detector_typepb.DetectorType_Tru: {},
detector_typepb.DetectorType_User: {},
detector_typepb.DetectorType_Wit: {},
@@ -134,6 +133,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{
detector_typepb.DetectorType_Enigma: {},
detector_typepb.DetectorType_GitLabOauth2: {},
detector_typepb.DetectorType_Pinecone: {},
+ detector_typepb.DetectorType_TLy: {},
// Reserved / special types.
detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()
diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go
index a1ff5aa90..80271d4e8 100644
--- a/pkg/feature/feature.go
+++ b/pkg/feature/feature.go
@@ -21,6 +21,7 @@ var (
GitLabOAuthDetectorEnabled atomic.Bool
EnigmaDetectorEnabled atomic.Bool
DatadogApiKeyDetectorEnabled atomic.Bool
+ TlyDetectorEnabled atomic.Bool
)
type AtomicString struct {