Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
more detectors
Dustin Decker5596025b0b64
diff --git a/pkg/detectors/abbysale/abbysale_test.go b/pkg/detectors/abbysale/abbysale_test.goindex 2ac4e119f..0a55d131a 100644--- a/pkg/detectors/abbysale/abbysale_test.go+++ b/pkg/detectors/abbysale/abbysale_test.go@@ -16,7 +16,7 @@ import (func TestAbbysale_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/abstract/abstract_test.go b/pkg/detectors/abstract/abstract_test.goindex c937303b0..c34e44674 100644--- a/pkg/detectors/abstract/abstract_test.go+++ b/pkg/detectors/abstract/abstract_test.go@@ -16,7 +16,7 @@ import (func TestAbstract_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/abuseipdb/abuseipdb_test.go b/pkg/detectors/abuseipdb/abuseipdb_test.goindex ff3bc5d48..9772b6ed5 100644--- a/pkg/detectors/abuseipdb/abuseipdb_test.go+++ b/pkg/detectors/abuseipdb/abuseipdb_test.go@@ -16,7 +16,7 @@ import (func TestAbuseIPDB_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/accuweather/accuweather_test.go b/pkg/detectors/accuweather/accuweather_test.goindex 269ffcd00..c12bcc231 100644--- a/pkg/detectors/accuweather/accuweather_test.go+++ b/pkg/detectors/accuweather/accuweather_test.go@@ -16,7 +16,7 @@ import (func TestAccuweather_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/adafruitio/adafruitio_test.go b/pkg/detectors/adafruitio/adafruitio_test.goindex 028f8dd10..4a119c575 100644--- a/pkg/detectors/adafruitio/adafruitio_test.go+++ b/pkg/detectors/adafruitio/adafruitio_test.go@@ -16,7 +16,7 @@ import (func TestAdafruitIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/adobeio/adobeio_test.go b/pkg/detectors/adobeio/adobeio_test.goindex 5309903ec..031a69e33 100644--- a/pkg/detectors/adobeio/adobeio_test.go+++ b/pkg/detectors/adobeio/adobeio_test.go@@ -16,7 +16,7 @@ import (func TestAdobeIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/adzuna/adzuna_test.go b/pkg/detectors/adzuna/adzuna_test.goindex 123c27c6f..0e2598191 100644--- a/pkg/detectors/adzuna/adzuna_test.go+++ b/pkg/detectors/adzuna/adzuna_test.go@@ -16,7 +16,7 @@ import (func TestAdzuna_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/agora/agora.go b/pkg/detectors/agora/agora.gonew file mode 100644index 000000000..1c6054d99--- /dev/null+++ b/pkg/detectors/agora/agora.go@@ -0,0 +1,84 @@+package agora++import (+ "context"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"agora"}) + `\b([a-z0-9]{32})\b`)+ secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"agora"}) + `\b([a-z0-9]{32})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"agora"}+}++// FromData will find and optionally verify Agora secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)+ secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }++ resMatch := strings.TrimSpace(match[1])++ for _, secret := range secretMatches {+ if len(secret) != 2 {+ continue+ }+ resSecret := strings.TrimSpace(secret[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_Agora,+ Raw: []byte(resMatch),+ }++ if verify {+ req, _ := http.NewRequest("GET", "https://api.agora.io/dev/v1/projects", nil)+ req.SetBasicAuth(resSecret, resMatch)+ res, err := client.Do(req)++ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/agora/agora_test.go b/pkg/detectors/agora/agora_test.gonew file mode 100644index 000000000..d558c4113--- /dev/null+++ b/pkg/detectors/agora/agora_test.go@@ -0,0 +1,114 @@+package agora++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestAgora_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ id := testSecrets.MustGetField("AGORA")+ secret := testSecrets.MustGetField("AGORA_SECRET")+ inactiveSecret := testSecrets.MustGetField("AGORA_SECRET_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but not valid ", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Agora.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Agora.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.goindex 6ce727e8d..f4240b80c 100644--- a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go+++ b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go@@ -16,7 +16,7 @@ import (func TestAirbrakeProjectKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.goindex 965f0e25d..57b2fa40d 100644--- a/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go+++ b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go@@ -16,7 +16,7 @@ import (func TestAirbrakeUserKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/airship/airship_test.go b/pkg/detectors/airship/airship_test.goindex 778d337fe..dbc66fbbc 100644--- a/pkg/detectors/airship/airship_test.go+++ b/pkg/detectors/airship/airship_test.go@@ -16,7 +16,7 @@ import (func TestAirship_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/airtableapikey/airtableapikey_test.go b/pkg/detectors/airtableapikey/airtableapikey_test.goindex 01030a1ad..85ea10403 100644--- a/pkg/detectors/airtableapikey/airtableapikey_test.go+++ b/pkg/detectors/airtableapikey/airtableapikey_test.go@@ -16,7 +16,7 @@ import (func TestAirtableApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/airvisual/airvisual_test.go b/pkg/detectors/airvisual/airvisual_test.goindex 97a45bcc2..9ed306dcc 100644--- a/pkg/detectors/airvisual/airvisual_test.go+++ b/pkg/detectors/airvisual/airvisual_test.go@@ -16,7 +16,7 @@ import (func TestAirVisual_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/alconost/alconost_test.go b/pkg/detectors/alconost/alconost_test.goindex baa98acae..45dc7b3bc 100644--- a/pkg/detectors/alconost/alconost_test.go+++ b/pkg/detectors/alconost/alconost_test.go@@ -16,7 +16,7 @@ import (func TestAlconost_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/alegra/alegra_test.go b/pkg/detectors/alegra/alegra_test.goindex 5c73d054a..44a0ceaa3 100644--- a/pkg/detectors/alegra/alegra_test.go+++ b/pkg/detectors/alegra/alegra_test.go@@ -16,7 +16,7 @@ import (func TestAlegra_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/aletheiaapi/aletheiaapi_test.go b/pkg/detectors/aletheiaapi/aletheiaapi_test.goindex 7e1a8fb01..1526f203a 100644--- a/pkg/detectors/aletheiaapi/aletheiaapi_test.go+++ b/pkg/detectors/aletheiaapi/aletheiaapi_test.go@@ -16,7 +16,7 @@ import (func TestAletheiaApi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey.go b/pkg/detectors/algoliaadminkey/algoliaadminkey.gonew file mode 100644index 000000000..fd7c079ab--- /dev/null+++ b/pkg/detectors/algoliaadminkey/algoliaadminkey.go@@ -0,0 +1,81 @@+package algoliaadminkey++import (+ "context"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia"}) + `\b([a-zA-Z0-9]{32})\b`)+ idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia"}) + `\b([A-Z0-9]{10})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"algolia"}+}++// FromData will find and optionally verify AlgoliaAdminKey secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)+ idMatches := idPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])+ for _, idMatch := range idMatches {+ if len(idMatch) != 2 {+ continue+ }+ resIdMatch := strings.TrimSpace(idMatch[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,+ Raw: []byte(resMatch),+ }++ if verify {+ req, _ := http.NewRequest("GET", "https://"+resIdMatch+"-dsn.algolia.net/1/keys", nil)+ req.Header.Add("X-Algolia-Application-Id", resIdMatch)+ req.Header.Add("X-Algolia-API-Key", resMatch)+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }+ }+ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go b/pkg/detectors/algoliaadminkey/algoliaadminkey_test.gonew file mode 100644index 000000000..5fc7b25b3--- /dev/null+++ b/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go@@ -0,0 +1,114 @@+package algoliaadminkey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestAlgoliaAdminKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ALGOLIAADMINKEY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("ALGOLIAADMINKEY_INACTIVE")+ id := testSecrets.MustGetField("ALGOLIAADMINKEY_APPID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AlgoliaAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AlgoliaAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/alibaba/alibaba.go b/pkg/detectors/alibaba/alibaba.gonew file mode 100644index 000000000..3cd8a3e5e--- /dev/null+++ b/pkg/detectors/alibaba/alibaba.go@@ -0,0 +1,84 @@+package alibaba++import (+ "context"+ "regexp"+ "time"++ "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs"+ log "github.com/sirupsen/logrus"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ keyPat = regexp.MustCompile(`\b(LTAI[a-zA-Z0-9]{17,21})[\"' ;\s]*`)+ secretPat = regexp.MustCompile(`\b([a-zA-Z0-9]{30})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"LTAI"}+}++// FromData will find and optionally verify Alibaba secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)+ for _, match := range matches {+ //Plausible key pat found, look for secrets match+ secMatches := secretPat.FindAllStringSubmatch(dataStr, -1)++ for _, secMatch := range secMatches {++ if len(match) != 2 {+ continue+ }++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Alibaba,+ Raw: []byte(match[1]),+ Redacted: match[1],+ }++ if verify {+ ecsClient, err := ecs.NewClientWithAccessKey(+ "us-east-1", // your region ID+ match[1], // your AccessKey ID+ secMatch[1]) // your AccessKey Secret+ if err != nil {+ log.WithError(err).Debug("error creating alibaba client, skipping")+ continue+ }+ // Create an API request and set parameters+ request := ecs.CreateDescribeInstancesRequest()+ request.ConnectTimeout = (time.Duration(5) * time.Second)+ request.Scheme = "https"+ request.Domain = "ecs.aliyuncs.com"+ // Initiate the request and handle exceptions+ _, err = ecsClient.DescribeInstances(request)+ if err != nil {+ s.Verified = false++ } else {+ s.Verified = true+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }+ }+ return+}diff --git a/pkg/detectors/alibaba/alibaba_test.go b/pkg/detectors/alibaba/alibaba_test.gonew file mode 100644index 000000000..2413d86de--- /dev/null+++ b/pkg/detectors/alibaba/alibaba_test.go@@ -0,0 +1,103 @@+package alibaba++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestAlibaba_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ALIBABA_SECRET")+ secretInactive := testSecrets.MustGetField("ALIBABA_SECRET_INACTIVE")+ id := testSecrets.MustGetField("ALIBABA_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("Alibaba keys are here: %s\n %s within", id, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Alibaba,+ Verified: true,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("Alibaba keys are here: %s\n %s within", id, secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Alibaba,+ Verified: false,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Alibaba.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Alibaba.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}diff --git a/pkg/detectors/alienvault/alienvault_test.go b/pkg/detectors/alienvault/alienvault_test.goindex c698a4583..d045210a1 100644--- a/pkg/detectors/alienvault/alienvault_test.go+++ b/pkg/detectors/alienvault/alienvault_test.go@@ -16,7 +16,7 @@ import (func TestAlienVault_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/allsports/allsports_test.go b/pkg/detectors/allsports/allsports_test.goindex 6c62ff83c..b2acfe937 100644--- a/pkg/detectors/allsports/allsports_test.go+++ b/pkg/detectors/allsports/allsports_test.go@@ -16,7 +16,7 @@ import (func TestAllsports_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/amadeus/amadeus_test.go b/pkg/detectors/amadeus/amadeus_test.goindex 63e389409..699d6c334 100644--- a/pkg/detectors/amadeus/amadeus_test.go+++ b/pkg/detectors/amadeus/amadeus_test.go@@ -16,7 +16,7 @@ import (func TestAmadeus_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ambee/ambee_test.go b/pkg/detectors/ambee/ambee_test.goindex 962070bd1..e5293dcd6 100644--- a/pkg/detectors/ambee/ambee_test.go+++ b/pkg/detectors/ambee/ambee_test.go@@ -16,7 +16,7 @@ import (func TestAmbee_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go b/pkg/detectors/amplitudeapikey/amplitudeapikey_test.goindex da8560a4b..3dd7bcf4b 100644--- a/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go+++ b/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go@@ -16,7 +16,7 @@ import (func TestAmplitudeApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/anypoint/anypoint_test.go b/pkg/detectors/anypoint/anypoint_test.goindex 99a8ec51e..ca1e6b58d 100644--- a/pkg/detectors/anypoint/anypoint_test.go+++ b/pkg/detectors/anypoint/anypoint_test.go@@ -16,7 +16,7 @@ import (func TestAnypoint_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/apacta/apacta_test.go b/pkg/detectors/apacta/apacta_test.goindex 5aadfd0f0..238bce55c 100644--- a/pkg/detectors/apacta/apacta_test.go+++ b/pkg/detectors/apacta/apacta_test.go@@ -16,7 +16,7 @@ import (func TestApacta_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/api2cart/api2cart_test.go b/pkg/detectors/api2cart/api2cart_test.goindex a1a58e1fb..36d894d75 100644--- a/pkg/detectors/api2cart/api2cart_test.go+++ b/pkg/detectors/api2cart/api2cart_test.go@@ -16,7 +16,7 @@ import (func TestApi2Cart_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/apideck/apideck_test.go b/pkg/detectors/apideck/apideck_test.goindex 539a12221..48f67af43 100644--- a/pkg/detectors/apideck/apideck_test.go+++ b/pkg/detectors/apideck/apideck_test.go@@ -16,7 +16,7 @@ import (func TestApiDeck_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/apiflash/apiflash_test.go b/pkg/detectors/apiflash/apiflash_test.goindex 8ce3c4b0c..f916c9cae 100644--- a/pkg/detectors/apiflash/apiflash_test.go+++ b/pkg/detectors/apiflash/apiflash_test.go@@ -16,7 +16,7 @@ import (func TestApiflash_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/apifonica/apifonica_test.go b/pkg/detectors/apifonica/apifonica_test.goindex 9c798f358..bfc616e09 100644--- a/pkg/detectors/apifonica/apifonica_test.go+++ b/pkg/detectors/apifonica/apifonica_test.go@@ -16,7 +16,7 @@ import (func TestApifonica_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/apify/apify_test.go b/pkg/detectors/apify/apify_test.goindex e4ff1b219..9d5ebecc2 100644--- a/pkg/detectors/apify/apify_test.go+++ b/pkg/detectors/apify/apify_test.go@@ -16,7 +16,7 @@ import (func TestApify_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/apimatic/apimatic_test.go b/pkg/detectors/apimatic/apimatic_test.goindex aa4e707f4..b2cb41c7a 100644--- a/pkg/detectors/apimatic/apimatic_test.go+++ b/pkg/detectors/apimatic/apimatic_test.go@@ -16,7 +16,7 @@ import (func TestAPIMatic_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/apiscience/apiscience_test.go b/pkg/detectors/apiscience/apiscience_test.goindex 52709f350..91cdf6b7f 100644--- a/pkg/detectors/apiscience/apiscience_test.go+++ b/pkg/detectors/apiscience/apiscience_test.go@@ -16,7 +16,7 @@ import (func TestApiScience_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/apollo/apollo_test.go b/pkg/detectors/apollo/apollo_test.goindex 832cfcf98..eefab358a 100644--- a/pkg/detectors/apollo/apollo_test.go+++ b/pkg/detectors/apollo/apollo_test.go@@ -16,7 +16,7 @@ import (func TestApollo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/appcues/appcues_test.go b/pkg/detectors/appcues/appcues_test.goindex 927d2dd5e..0083e9d68 100644--- a/pkg/detectors/appcues/appcues_test.go+++ b/pkg/detectors/appcues/appcues_test.go@@ -16,7 +16,7 @@ import (func TestAppcues_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/appfollow/appfollow_test.go b/pkg/detectors/appfollow/appfollow_test.goindex 8816f9fd1..b9e4fca6b 100644--- a/pkg/detectors/appfollow/appfollow_test.go+++ b/pkg/detectors/appfollow/appfollow_test.go@@ -16,7 +16,7 @@ import (func TestAppfollow_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/appsynergy/appsynergy_test.go b/pkg/detectors/appsynergy/appsynergy_test.goindex 8105a54e4..5be9a53e3 100644--- a/pkg/detectors/appsynergy/appsynergy_test.go+++ b/pkg/detectors/appsynergy/appsynergy_test.go@@ -16,7 +16,7 @@ import (func TestAppSynergy_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/apptivo/apptivo_test.go b/pkg/detectors/apptivo/apptivo_test.goindex 4965e5edb..78c4d6b90 100644--- a/pkg/detectors/apptivo/apptivo_test.go+++ b/pkg/detectors/apptivo/apptivo_test.go@@ -16,7 +16,7 @@ import (func TestApptivo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/artifactory/artifactory_test.go b/pkg/detectors/artifactory/artifactory_test.goindex db674bae1..f5cf1ee29 100644--- a/pkg/detectors/artifactory/artifactory_test.go+++ b/pkg/detectors/artifactory/artifactory_test.go@@ -16,7 +16,7 @@ import (func TestArtifactory_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/artsy/artsy_test.go b/pkg/detectors/artsy/artsy_test.goindex c84675b41..7fe3d077b 100644--- a/pkg/detectors/artsy/artsy_test.go+++ b/pkg/detectors/artsy/artsy_test.go@@ -16,7 +16,7 @@ import (func TestArtsy_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/asanaoauth/asanaoauth_test.go b/pkg/detectors/asanaoauth/asanaoauth_test.goindex 3bb5def32..24850a13a 100644--- a/pkg/detectors/asanaoauth/asanaoauth_test.go+++ b/pkg/detectors/asanaoauth/asanaoauth_test.go@@ -16,7 +16,7 @@ import (func TestAsanaOauth_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.goindex 31d0bbb34..0cea9f33a 100644--- a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go+++ b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go@@ -16,7 +16,7 @@ import (func TestAsanaPersonalAccessToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/assemblyai/assemblyai_test.go b/pkg/detectors/assemblyai/assemblyai_test.goindex 4f43f7730..4c6072093 100644--- a/pkg/detectors/assemblyai/assemblyai_test.go+++ b/pkg/detectors/assemblyai/assemblyai_test.go@@ -16,7 +16,7 @@ import (func TestAssemblyai_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/audd/audd_test.go b/pkg/detectors/audd/audd_test.goindex 2b0d2eced..f9d23f685 100644--- a/pkg/detectors/audd/audd_test.go+++ b/pkg/detectors/audd/audd_test.go@@ -16,7 +16,7 @@ import (func TestAudd_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.goindex 97586c913..0143cbdd7 100644--- a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go+++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go@@ -16,7 +16,7 @@ import (func TestAuth0ManagementApiToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/auth0oauth/auth0oauth_test.go b/pkg/detectors/auth0oauth/auth0oauth_test.goindex 87dfd6d4d..68fd96352 100644--- a/pkg/detectors/auth0oauth/auth0oauth_test.go+++ b/pkg/detectors/auth0oauth/auth0oauth_test.go@@ -16,7 +16,7 @@ import (func TestAuth0oauth_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/autodesk/autodesk_test.go b/pkg/detectors/autodesk/autodesk_test.goindex 08c368c44..a14b4585e 100644--- a/pkg/detectors/autodesk/autodesk_test.go+++ b/pkg/detectors/autodesk/autodesk_test.go@@ -16,7 +16,7 @@ import (func TestAutodesk_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/autoklose/autoklose_test.go b/pkg/detectors/autoklose/autoklose_test.goindex 5afd2baeb..a1684ec79 100644--- a/pkg/detectors/autoklose/autoklose_test.go+++ b/pkg/detectors/autoklose/autoklose_test.go@@ -16,7 +16,7 @@ import (func TestAutoklose_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/autopilot/autopilot_test.go b/pkg/detectors/autopilot/autopilot_test.goindex b4af56675..ba92646d9 100644--- a/pkg/detectors/autopilot/autopilot_test.go+++ b/pkg/detectors/autopilot/autopilot_test.go@@ -16,7 +16,7 @@ import (func TestAutoPilot_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/aviationstack/aviationstack_test.go b/pkg/detectors/aviationstack/aviationstack_test.goindex ef6c4c4cb..b16c85a53 100644--- a/pkg/detectors/aviationstack/aviationstack_test.go+++ b/pkg/detectors/aviationstack/aviationstack_test.go@@ -16,7 +16,7 @@ import (func TestAviationStack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.gonew file mode 100644index 000000000..c6590f3ba--- /dev/null+++ b/pkg/detectors/aws/aws.go@@ -0,0 +1,96 @@+package aws++import (+ "context"+ "regexp"+ "strings"++ "github.com/aws/aws-sdk-go-v2/credentials"+ "github.com/aws/aws-sdk-go-v2/service/sts"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ // TODO: Support other key types https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids+ keyPat = regexp.MustCompile(`\b(AKIA[0-9A-Z]{16})\b`)+ secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"AKI"}+}++func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {+ dataStr := string(data)+ var results []detectors.Result++ keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)+ secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)++ for _, keyMatch := range keyMatches {+ if len(keyMatch) != 2 {+ continue+ }++ key := strings.TrimSpace(keyMatch[1])++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_AWS,+ Raw: []byte(key),+ Redacted: key,+ }+ // TODO: Remove possible matches if they verify positive.+ if verify {+ for _, secretMatch := range secretMatches {+ if len(secretMatch) != 2 {+ continue+ }++ secret := strings.TrimSpace(secretMatch[1])++ result, err := callerIdentity(ctx, key, secret)+ if err != nil {+ // It also errors for signature mismatches on the client side before sending, and it's quite noisy.+ continue+ }+ if result != nil && result.Account != nil {+ s.Verified = true+ break+ }+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ if len(secretMatches) > 0 {+ results = append(results, s)+ }+ }++ return detectors.CleanResults(results), nil+}++func callerIdentity(ctx context.Context, key, secret string) (*sts.GetCallerIdentityOutput, error) {+ svc := sts.New(sts.Options{+ HTTPClient: common.SaneHttpClient(),+ Logger: nil,+ Region: "us-west-2",+ Credentials: credentials.NewStaticCredentialsProvider(key, secret, ""),+ })+ result, err := svc.GetCallerIdentity(ctx, nil)+ return result, err+}diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.gonew file mode 100644index 000000000..0afbd1fdf--- /dev/null+++ b/pkg/detectors/aws/aws_test.go@@ -0,0 +1,167 @@+package aws++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/aws/aws-sdk-go-v2/service/sts"+ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestScanner_FromChunk(t *testing.T) {++ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AWS_SECRET")+ secretInactive := testSecrets.MustGetField("AWS_INACTIVE")+ id := testSecrets.MustGetField("AWS")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ args args+ wantSecrets []detectors.Result+ wantErr bool+ }{+ {+ name: "live key",+ args: args{+ ctx: ctx,+ data: []byte(fmt.Sprintf("You can find a aws secret %s within awsId %s", secret, id)),++ verify: true,+ },+ wantSecrets: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: true,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "dead key",+ args: args{+ ctx: ctx,+ data: []byte(fmt.Sprintf("You can find a aws secret %s within awsId %s", secretInactive, id)),+ verify: true,+ },+ wantSecrets: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ args: args{+ ctx: ctx,+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ wantSecrets: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Scanner.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.wantSecrets); diff != "" {+ t.Errorf("%s: Scanner.FromData() diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}++func Test_callerIdentity(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AWS_SECRET")+ secretInactive := testSecrets.MustGetField("AWS_INACTIVE")+ id := testSecrets.MustGetField("AWS")++ type args struct {+ key string+ secret string+ ctx context.Context+ }+ tests := []struct {+ name string+ args args+ want *sts.GetCallerIdentityOutput+ wantErr bool+ }{+ {+ name: "invalid",+ args: args{+ key: id,+ secret: secretInactive,+ ctx: context.Background(),+ },+ wantErr: true,+ },+ {+ name: "valid",+ args: args{+ key: id,+ secret: secret,+ ctx: context.Background(),+ },+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ _, err := callerIdentity(tt.args.ctx, tt.args.key, tt.args.secret)+ if (err != nil) != tt.wantErr {+ t.Errorf("callerIdentity() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ })+ }+}diff --git a/pkg/detectors/axonaut/axonaut_test.go b/pkg/detectors/axonaut/axonaut_test.goindex 1b9a54739..03eaaebeb 100644--- a/pkg/detectors/axonaut/axonaut_test.go+++ b/pkg/detectors/axonaut/axonaut_test.go@@ -16,7 +16,7 @@ import (func TestAxonaut_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/aylien/aylien_test.go b/pkg/detectors/aylien/aylien_test.goindex b4822aa0b..65f22c650 100644--- a/pkg/detectors/aylien/aylien_test.go+++ b/pkg/detectors/aylien/aylien_test.go@@ -16,7 +16,7 @@ import (func TestAylien_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ayrshare/ayrshare_test.go b/pkg/detectors/ayrshare/ayrshare_test.goindex dd682c1a6..6fd26ad8f 100644--- a/pkg/detectors/ayrshare/ayrshare_test.go+++ b/pkg/detectors/ayrshare/ayrshare_test.go@@ -16,7 +16,7 @@ import (func TestAyrshare_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/azure/azure.go b/pkg/detectors/azure/azure.gonew file mode 100644index 000000000..097b8f2a1--- /dev/null+++ b/pkg/detectors/azure/azure.go@@ -0,0 +1,87 @@+package azure++import (+ "context"+ "fmt"+ "regexp"+ "strings"++ "github.com/Azure/go-autorest/autorest/azure/auth"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++func mustFmtPat(id, pat string) *regexp.Regexp {+ combinedID := strings.ReplaceAll(id, "_", "") + "|" + id+ return regexp.MustCompile(fmt.Sprintf(pat, combinedID))+}++var (+ // TODO: Azure storage access keys and investigate other types of creds.++ // Azure App Oauth+ idPatFmt = `(?i)(%s).{0,20}([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})`+ clientIDPat = mustFmtPat("client_id", idPatFmt)+ tenantIDPat = mustFmtPat("tenant_id", idPatFmt)++ // TODO: support old patterns+ secretPatFmt = `(?i)(%s).{0,20}([a-z0-9_\.\-~]{34})`+ clientSecretPat = mustFmtPat("client_secret", secretPatFmt)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"azure"}+}++// FromData will find and optionally verify Azure secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ clientSecretMatches := clientSecretPat.FindAllStringSubmatch(dataStr, -1)+ for _, clientSecret := range clientSecretMatches {+ tenantIDMatches := tenantIDPat.FindAllStringSubmatch(dataStr, -1)+ for _, tenantID := range tenantIDMatches {+ clientIDMatches := clientIDPat.FindAllStringSubmatch(dataStr, -1)+ for _, clientID := range clientIDMatches {+ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Azure,+ Raw: []byte(clientSecret[2]),+ Redacted: clientID[2],+ }++ if verify {+ cred := auth.NewClientCredentialsConfig(clientID[2], clientSecret[2], tenantID[2])+ token, err := cred.ServicePrincipalToken()+ if err != nil {+ continue+ }+ err = token.Refresh()+ if err == nil {+ s.Verified = true+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(s.Redacted, detectors.DefaultFalsePositives, true) {+ continue+ }+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }+ }+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/azure/azure_test.go b/pkg/detectors/azure/azure_test.gonew file mode 100644index 000000000..8a5d7207d--- /dev/null+++ b/pkg/detectors/azure/azure_test.go@@ -0,0 +1,110 @@+package azure++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestAzure_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AZURE_SECRET")+ secretInactive := testSecrets.MustGetField("AZURE_INACTIVE")+ id := testSecrets.MustGetField("AZURE_ID")+ tenantId := testSecrets.MustGetField("AZURE_TENANT_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`+ tenant_id=%s+ client_id=%s+ client_secret=%s+ client_secret=%s+ `, tenantId, id, secretInactive, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Azure,+ Redacted: id,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`+ tenant_id=%s+ client_id=%s+ client_secret=%s+ `, tenantId, id, secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Azure,+ Redacted: id,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Azure.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Azure.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}diff --git a/pkg/detectors/bannerbear/bannerbear_test.go b/pkg/detectors/bannerbear/bannerbear_test.goindex 9f9c5a022..625c8ee23 100644--- a/pkg/detectors/bannerbear/bannerbear_test.go+++ b/pkg/detectors/bannerbear/bannerbear_test.go@@ -16,7 +16,7 @@ import (func TestBannerbear_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/baremetrics/baremetrics_test.go b/pkg/detectors/baremetrics/baremetrics_test.goindex bf1654fb7..1397b86dc 100644--- a/pkg/detectors/baremetrics/baremetrics_test.go+++ b/pkg/detectors/baremetrics/baremetrics_test.go@@ -16,7 +16,7 @@ import (func TestBaremetrics_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/baseapiio/baseapiio_test.go b/pkg/detectors/baseapiio/baseapiio_test.goindex fb988ac8d..591d167c9 100644--- a/pkg/detectors/baseapiio/baseapiio_test.go+++ b/pkg/detectors/baseapiio/baseapiio_test.go@@ -16,7 +16,7 @@ import (func TestBaseApiIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/beamer/beamer_test.go b/pkg/detectors/beamer/beamer_test.goindex 81e5eed6d..18617452c 100644--- a/pkg/detectors/beamer/beamer_test.go+++ b/pkg/detectors/beamer/beamer_test.go@@ -16,7 +16,7 @@ import (func TestBeamer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/beebole/beebole_test.go b/pkg/detectors/beebole/beebole_test.goindex 6de959947..0c6132142 100644--- a/pkg/detectors/beebole/beebole_test.go+++ b/pkg/detectors/beebole/beebole_test.go@@ -16,7 +16,7 @@ import (func TestBeebole_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/besttime/besttime_test.go b/pkg/detectors/besttime/besttime_test.goindex 847db3fc1..9e5eae6d7 100644--- a/pkg/detectors/besttime/besttime_test.go+++ b/pkg/detectors/besttime/besttime_test.go@@ -16,7 +16,7 @@ import (func TestBesttime_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/billomat/billomat_test.go b/pkg/detectors/billomat/billomat_test.goindex 9193922ac..421bcd0e2 100644--- a/pkg/detectors/billomat/billomat_test.go+++ b/pkg/detectors/billomat/billomat_test.go@@ -16,7 +16,7 @@ import (func TestBillomat_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/bitbar/bitbar_test.go b/pkg/detectors/bitbar/bitbar_test.goindex 7fd158e7a..50b631ee3 100644--- a/pkg/detectors/bitbar/bitbar_test.go+++ b/pkg/detectors/bitbar/bitbar_test.go@@ -16,7 +16,7 @@ import (func TestBitbar_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go b/pkg/detectors/bitcoinaverage/bitcoinaverage_test.goindex 4dafb76a3..ccd97104c 100644--- a/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go+++ b/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go@@ -16,7 +16,7 @@ import (func TestBitcoinAverage_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/bitfinex/bitfinex_test.go b/pkg/detectors/bitfinex/bitfinex_test.goindex 13d53af61..1d8bc5b6f 100644--- a/pkg/detectors/bitfinex/bitfinex_test.go+++ b/pkg/detectors/bitfinex/bitfinex_test.go@@ -16,7 +16,7 @@ import (func TestBitfinex_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.goindex 140087708..6e46c8fe1 100644--- a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go+++ b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go@@ -16,7 +16,7 @@ import (func TestBitLyAccessToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/bitmex/bitmex_test.go b/pkg/detectors/bitmex/bitmex_test.goindex 2a5b1f54b..ee06131d0 100644--- a/pkg/detectors/bitmex/bitmex_test.go+++ b/pkg/detectors/bitmex/bitmex_test.go@@ -16,7 +16,7 @@ import (func TestBitmex_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/blablabus/blablabus_test.go b/pkg/detectors/blablabus/blablabus_test.goindex 3724355be..3eca42cfc 100644--- a/pkg/detectors/blablabus/blablabus_test.go+++ b/pkg/detectors/blablabus/blablabus_test.go@@ -16,7 +16,7 @@ import (func TestBlablabus_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/blazemeter/blazemeter_test.go b/pkg/detectors/blazemeter/blazemeter_test.goindex 5cec03e47..87c48b2a8 100644--- a/pkg/detectors/blazemeter/blazemeter_test.go+++ b/pkg/detectors/blazemeter/blazemeter_test.go@@ -16,7 +16,7 @@ import (func TestBlazemeter_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/blitapp/blitapp_test.go b/pkg/detectors/blitapp/blitapp_test.goindex 3583ef810..9a8c8cff8 100644--- a/pkg/detectors/blitapp/blitapp_test.go+++ b/pkg/detectors/blitapp/blitapp_test.go@@ -16,7 +16,7 @@ import (func TestBlitApp_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/blogger/blogger_test.go b/pkg/detectors/blogger/blogger_test.goindex fecd80aeb..241ee722b 100644--- a/pkg/detectors/blogger/blogger_test.go+++ b/pkg/detectors/blogger/blogger_test.go@@ -16,7 +16,7 @@ import (func TestBlogger_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/bombbomb/bombbomb_test.go b/pkg/detectors/bombbomb/bombbomb_test.goindex 8fb198a5c..ee4def006 100644--- a/pkg/detectors/bombbomb/bombbomb_test.go+++ b/pkg/detectors/bombbomb/bombbomb_test.go@@ -16,7 +16,7 @@ import (func TestBombBomb_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/brandfetch/brandfetch_test.go b/pkg/detectors/brandfetch/brandfetch_test.goindex 3e87ac047..8c524bc41 100644--- a/pkg/detectors/brandfetch/brandfetch_test.go+++ b/pkg/detectors/brandfetch/brandfetch_test.go@@ -16,7 +16,7 @@ import (func TestBrandfetch_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/browshot/browshot_test.go b/pkg/detectors/browshot/browshot_test.goindex 207216d40..4e60b5392 100644--- a/pkg/detectors/browshot/browshot_test.go+++ b/pkg/detectors/browshot/browshot_test.go@@ -16,7 +16,7 @@ import (func TestBrowshot_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/buddyns/buddyns_test.go b/pkg/detectors/buddyns/buddyns_test.goindex 085c566bc..ca6a187d8 100644--- a/pkg/detectors/buddyns/buddyns_test.go+++ b/pkg/detectors/buddyns/buddyns_test.go@@ -16,7 +16,7 @@ import (func TestBuddyns_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/bugherd/bugherd_test.go b/pkg/detectors/bugherd/bugherd_test.goindex edb2d9d12..e5a41aded 100644--- a/pkg/detectors/bugherd/bugherd_test.go+++ b/pkg/detectors/bugherd/bugherd_test.go@@ -16,7 +16,7 @@ import (func TestBugherd_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/bugsnag/bugsnag_test.go b/pkg/detectors/bugsnag/bugsnag_test.goindex 89891fa07..f5043aaff 100644--- a/pkg/detectors/bugsnag/bugsnag_test.go+++ b/pkg/detectors/bugsnag/bugsnag_test.go@@ -16,7 +16,7 @@ import (func TestBugsnag_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/buildkite/buildkite_test.go b/pkg/detectors/buildkite/buildkite_test.goindex 7fb826623..0a951607a 100644--- a/pkg/detectors/buildkite/buildkite_test.go+++ b/pkg/detectors/buildkite/buildkite_test.go@@ -16,7 +16,7 @@ import (func TestBuildkite_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/bulbul/bulbul_test.go b/pkg/detectors/bulbul/bulbul_test.goindex cd0bf8f83..92af8317d 100644--- a/pkg/detectors/bulbul/bulbul_test.go+++ b/pkg/detectors/bulbul/bulbul_test.go@@ -16,7 +16,7 @@ import (func TestBulbul_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/buttercms/buttercms_test.go b/pkg/detectors/buttercms/buttercms_test.goindex c0d4060b4..1ee550292 100644--- a/pkg/detectors/buttercms/buttercms_test.go+++ b/pkg/detectors/buttercms/buttercms_test.go@@ -16,7 +16,7 @@ import (func TestButterCMS_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/caflou/caflou_test.go b/pkg/detectors/caflou/caflou_test.goindex 5c8659f93..fd415b925 100644--- a/pkg/detectors/caflou/caflou_test.go+++ b/pkg/detectors/caflou/caflou_test.go@@ -16,7 +16,7 @@ import (func TestCaflou_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/calendarific/calendarific_test.go b/pkg/detectors/calendarific/calendarific_test.goindex f3bed174c..27cd9d4e5 100644--- a/pkg/detectors/calendarific/calendarific_test.go+++ b/pkg/detectors/calendarific/calendarific_test.go@@ -16,7 +16,7 @@ import (func TestCalendarific_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/calendlyapikey/calendlyapikey_test.go b/pkg/detectors/calendlyapikey/calendlyapikey_test.goindex 5babe5e55..eb6c95932 100644--- a/pkg/detectors/calendlyapikey/calendlyapikey_test.go+++ b/pkg/detectors/calendlyapikey/calendlyapikey_test.go@@ -16,7 +16,7 @@ import (func TestCalendlyApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/calorieninja/calorieninja_test.go b/pkg/detectors/calorieninja/calorieninja_test.goindex 3b7c2733c..e58e4fd27 100644--- a/pkg/detectors/calorieninja/calorieninja_test.go+++ b/pkg/detectors/calorieninja/calorieninja_test.go@@ -16,7 +16,7 @@ import (func TestCalorieninja_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/campayn/campayn_test.go b/pkg/detectors/campayn/campayn_test.goindex 0a43f0987..a92199053 100644--- a/pkg/detectors/campayn/campayn_test.go+++ b/pkg/detectors/campayn/campayn_test.go@@ -16,7 +16,7 @@ import (func TestCampayn_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cannyio/cannyio_test.go b/pkg/detectors/cannyio/cannyio_test.goindex 2688f7cc6..e42ff851c 100644--- a/pkg/detectors/cannyio/cannyio_test.go+++ b/pkg/detectors/cannyio/cannyio_test.go@@ -16,7 +16,7 @@ import (func TestCannyIo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/capsulecrm/capsulecrm_test.go b/pkg/detectors/capsulecrm/capsulecrm_test.goindex 134c3aa6f..e36edb7c2 100644--- a/pkg/detectors/capsulecrm/capsulecrm_test.go+++ b/pkg/detectors/capsulecrm/capsulecrm_test.go@@ -16,7 +16,7 @@ import (func TestCapsuleCRM_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/carboninterface/carboninterface_test.go b/pkg/detectors/carboninterface/carboninterface_test.goindex 3b117130f..19515acdb 100644--- a/pkg/detectors/carboninterface/carboninterface_test.go+++ b/pkg/detectors/carboninterface/carboninterface_test.go@@ -16,7 +16,7 @@ import (func TestCarbonInterface_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cashboard/cashboard_test.go b/pkg/detectors/cashboard/cashboard_test.goindex aa2f4375e..6272bec34 100644--- a/pkg/detectors/cashboard/cashboard_test.go+++ b/pkg/detectors/cashboard/cashboard_test.go@@ -16,7 +16,7 @@ import (func TestCashboard_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/censys/censys_test.go b/pkg/detectors/censys/censys_test.goindex 7a9d2f6ce..e6adfe563 100644--- a/pkg/detectors/censys/censys_test.go+++ b/pkg/detectors/censys/censys_test.go@@ -16,7 +16,7 @@ import (func TestCensys_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/centralstationcrm/centralstationcrm_test.go b/pkg/detectors/centralstationcrm/centralstationcrm_test.goindex e04ebf527..fdc48dd5d 100644--- a/pkg/detectors/centralstationcrm/centralstationcrm_test.go+++ b/pkg/detectors/centralstationcrm/centralstationcrm_test.go@@ -16,7 +16,7 @@ import (func TestCentralStationCRM_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/chatbot/chatbot_test.go b/pkg/detectors/chatbot/chatbot_test.goindex ee902c42c..a27c07bd0 100644--- a/pkg/detectors/chatbot/chatbot_test.go+++ b/pkg/detectors/chatbot/chatbot_test.go@@ -16,7 +16,7 @@ import (func TestChatbot_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/checio/checio_test.go b/pkg/detectors/checio/checio_test.goindex 649f61c74..832b67876 100644--- a/pkg/detectors/checio/checio_test.go+++ b/pkg/detectors/checio/checio_test.go@@ -16,7 +16,7 @@ import (func TestChecIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/checklyhq/checklyhq_test.go b/pkg/detectors/checklyhq/checklyhq_test.goindex 52511ac12..cbb674591 100644--- a/pkg/detectors/checklyhq/checklyhq_test.go+++ b/pkg/detectors/checklyhq/checklyhq_test.go@@ -16,7 +16,7 @@ import (func TestChecklyHQ_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/checkout/checkout_test.go b/pkg/detectors/checkout/checkout_test.goindex 567e3e91e..77c2a29e0 100644--- a/pkg/detectors/checkout/checkout_test.go+++ b/pkg/detectors/checkout/checkout_test.go@@ -16,7 +16,7 @@ import (func TestCheckout_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cicero/cicero_test.go b/pkg/detectors/cicero/cicero_test.goindex 3fdbb601a..277bdd895 100644--- a/pkg/detectors/cicero/cicero_test.go+++ b/pkg/detectors/cicero/cicero_test.go@@ -16,7 +16,7 @@ import (func TestCicero_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/circleci/circleci.go b/pkg/detectors/circleci/circleci.gonew file mode 100644index 000000000..5fb9da4ad--- /dev/null+++ b/pkg/detectors/circleci/circleci.go@@ -0,0 +1,69 @@+package circleci++import (+ "context"+ "net/http"+ "regexp"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"circle"}) + `([a-fA-F0-9]{40})`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"circle"}+}++// FromData will find and optionally verify Circle secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {++ token := match[1]++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Circle,+ Raw: []byte(token),+ }++ if verify {+ client := common.RetryableHttpClient()+ // https://circleci.com/docs/api/#authentication+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://circleci.com/api/v2/me", nil)+ req.Header.Add("Accept", "application/json;")+ req.Header.Add("Circle-Token", token)+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ }+ if res != nil && res.StatusCode >= 200 && res.StatusCode < 300 {+ s.Verified = true+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/circleci/circleci_test.go b/pkg/detectors/circleci/circleci_test.gonew file mode 100644index 000000000..573506c4a--- /dev/null+++ b/pkg/detectors/circleci/circleci_test.go@@ -0,0 +1,115 @@+package circleci++import (+ "context"+ "fmt"+ "os"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestCircleCI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CIRCLECI")+ secretInactive := testSecrets.MustGetField("CIRCLECI_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a circle secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Circle,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a circle secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Circle,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CircleCI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ if os.Getenv("FORCE_PASS_DIFF") == "true" {+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CircleCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/clarifai/clarifai_test.go b/pkg/detectors/clarifai/clarifai_test.goindex 72f1bdf62..cadf6d5da 100644--- a/pkg/detectors/clarifai/clarifai_test.go+++ b/pkg/detectors/clarifai/clarifai_test.go@@ -16,7 +16,7 @@ import (func TestClarifai_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/clearbit/clearbit_test.go b/pkg/detectors/clearbit/clearbit_test.goindex 5fb4c3548..350ab194b 100644--- a/pkg/detectors/clearbit/clearbit_test.go+++ b/pkg/detectors/clearbit/clearbit_test.go@@ -16,7 +16,7 @@ import (func TestClearbit_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/clickhelp/clickhelp_test.go b/pkg/detectors/clickhelp/clickhelp_test.goindex 6eb6ceec5..4a61b43b9 100644--- a/pkg/detectors/clickhelp/clickhelp_test.go+++ b/pkg/detectors/clickhelp/clickhelp_test.go@@ -16,7 +16,7 @@ import (func TestClickhelp_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/clicksendsms/clicksendsms_test.go b/pkg/detectors/clicksendsms/clicksendsms_test.goindex eadd04145..18ed1cf24 100644--- a/pkg/detectors/clicksendsms/clicksendsms_test.go+++ b/pkg/detectors/clicksendsms/clicksendsms_test.go@@ -16,7 +16,7 @@ import (func TestClickSendsms_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.goindex 1ea4bc3a7..ff2a64a4c 100644--- a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go+++ b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go@@ -16,7 +16,7 @@ import (func TestClickupPersonalToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/clinchpad/clinchpad_test.go b/pkg/detectors/clinchpad/clinchpad_test.goindex 4abf634f3..ccfdc8c34 100644--- a/pkg/detectors/clinchpad/clinchpad_test.go+++ b/pkg/detectors/clinchpad/clinchpad_test.go@@ -16,7 +16,7 @@ import (func TestClinchpad_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/clockify/clockify_test.go b/pkg/detectors/clockify/clockify_test.goindex 01e9e336a..16187fad3 100644--- a/pkg/detectors/clockify/clockify_test.go+++ b/pkg/detectors/clockify/clockify_test.go@@ -16,7 +16,7 @@ import (func TestClockify_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/clockworksms/clockworksms_test.go b/pkg/detectors/clockworksms/clockworksms_test.goindex 106f9eca1..b53d0a051 100644--- a/pkg/detectors/clockworksms/clockworksms_test.go+++ b/pkg/detectors/clockworksms/clockworksms_test.go@@ -16,7 +16,7 @@ import (func TestClockworksms_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/closecrm/close_test.go b/pkg/detectors/closecrm/close_test.goindex cb081abd7..41cb42002 100644--- a/pkg/detectors/closecrm/close_test.go+++ b/pkg/detectors/closecrm/close_test.go@@ -16,7 +16,7 @@ import (func TestClose_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cloudelements/cloudelements_test.go b/pkg/detectors/cloudelements/cloudelements_test.goindex 755691ca4..24fa7daa0 100644--- a/pkg/detectors/cloudelements/cloudelements_test.go+++ b/pkg/detectors/cloudelements/cloudelements_test.go@@ -16,7 +16,7 @@ import (func TestCloudElements_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.goindex d7d43faa2..4ab326639 100644--- a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go+++ b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go@@ -16,7 +16,7 @@ import (func TestCloudflareApiToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go b/pkg/detectors/cloudflarecakey/cloudflarecakey_test.goindex 207ce0440..799ada693 100644--- a/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go+++ b/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go@@ -16,7 +16,7 @@ import (func TestCloudflareCaKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go b/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.goindex 8434dacb4..f6e7ece54 100644--- a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go+++ b/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go@@ -16,7 +16,7 @@ import (func TestCloudflareGlobalApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cloudmersive/cloudmersive_test.go b/pkg/detectors/cloudmersive/cloudmersive_test.goindex c82b64479..58da6f1e5 100644--- a/pkg/detectors/cloudmersive/cloudmersive_test.go+++ b/pkg/detectors/cloudmersive/cloudmersive_test.go@@ -16,7 +16,7 @@ import (func TestCloudmersive_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cloudplan/cloudplan_test.go b/pkg/detectors/cloudplan/cloudplan_test.goindex 2ffd68a96..55cbe4383 100644--- a/pkg/detectors/cloudplan/cloudplan_test.go+++ b/pkg/detectors/cloudplan/cloudplan_test.go@@ -16,7 +16,7 @@ import (func TestCloudplan_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cloverly/cloverly_test.go b/pkg/detectors/cloverly/cloverly_test.goindex e49a98042..65272dea6 100644--- a/pkg/detectors/cloverly/cloverly_test.go+++ b/pkg/detectors/cloverly/cloverly_test.go@@ -16,7 +16,7 @@ import (func TestCloverly_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/clustdoc/clustdoc_test.go b/pkg/detectors/clustdoc/clustdoc_test.goindex 0aa2e8e77..83cc00876 100644--- a/pkg/detectors/clustdoc/clustdoc_test.go+++ b/pkg/detectors/clustdoc/clustdoc_test.go@@ -16,7 +16,7 @@ import (func TestClustDoc_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/codacy/codacy_test.go b/pkg/detectors/codacy/codacy_test.goindex 59a37429c..a8fe6dcdb 100644--- a/pkg/detectors/codacy/codacy_test.go+++ b/pkg/detectors/codacy/codacy_test.go@@ -16,7 +16,7 @@ import (func TestCodacy_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/coinapi/coinapi_test.go b/pkg/detectors/coinapi/coinapi_test.goindex 0ee75e690..e30cbd142 100644--- a/pkg/detectors/coinapi/coinapi_test.go+++ b/pkg/detectors/coinapi/coinapi_test.go@@ -16,7 +16,7 @@ import (func TestCoinApi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/coinbase/coinbase_test.go b/pkg/detectors/coinbase/coinbase_test.goindex c8d77af63..289b81ee2 100644--- a/pkg/detectors/coinbase/coinbase_test.go+++ b/pkg/detectors/coinbase/coinbase_test.go@@ -16,7 +16,7 @@ package coinbase// func TestCoinbase_FromChunk(t *testing.T) {// ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)// defer cancel()-// testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+// testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")// if err != nil {// t.Fatalf("could not get test secrets from GCP: %s", err)// }diff --git a/pkg/detectors/coinlayer/coinlayer_test.go b/pkg/detectors/coinlayer/coinlayer_test.goindex d0d832b24..7c83a87e6 100644--- a/pkg/detectors/coinlayer/coinlayer_test.go+++ b/pkg/detectors/coinlayer/coinlayer_test.go@@ -16,7 +16,7 @@ import (func TestCoinlayer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/coinlib/coinlib_test.go b/pkg/detectors/coinlib/coinlib_test.goindex bcd64cb54..d6b15f389 100644--- a/pkg/detectors/coinlib/coinlib_test.go+++ b/pkg/detectors/coinlib/coinlib_test.go@@ -16,7 +16,7 @@ import (func TestCoinlib_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/commercejs/commercejs_test.go b/pkg/detectors/commercejs/commercejs_test.goindex 3950336a1..50f9e1e93 100644--- a/pkg/detectors/commercejs/commercejs_test.go+++ b/pkg/detectors/commercejs/commercejs_test.go@@ -16,7 +16,7 @@ import (func TestCommerceJS_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/commodities/commodities_test.go b/pkg/detectors/commodities/commodities_test.goindex f0021a333..3f83e518a 100644--- a/pkg/detectors/commodities/commodities_test.go+++ b/pkg/detectors/commodities/commodities_test.go@@ -16,7 +16,7 @@ import (func TestCommodities_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/companyhub/companyhub_test.go b/pkg/detectors/companyhub/companyhub_test.goindex db3c32361..1588ae321 100644--- a/pkg/detectors/companyhub/companyhub_test.go+++ b/pkg/detectors/companyhub/companyhub_test.go@@ -16,7 +16,7 @@ import (func TestCompanyHub_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/confluent/confluent_test.go b/pkg/detectors/confluent/confluent_test.goindex f55467ab1..a5a634acf 100644--- a/pkg/detectors/confluent/confluent_test.go+++ b/pkg/detectors/confluent/confluent_test.go@@ -16,7 +16,7 @@ import (func TestConfluent_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.goindex a06fdb186..f03585a26 100644--- a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go+++ b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go@@ -16,7 +16,7 @@ import (func TestContentfulPersonalAccessToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/convertkit/convertkit_test.go b/pkg/detectors/convertkit/convertkit_test.goindex cd1cb4c49..4405c2bc3 100644--- a/pkg/detectors/convertkit/convertkit_test.go+++ b/pkg/detectors/convertkit/convertkit_test.go@@ -16,7 +16,7 @@ import (func TestConvertkit_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/copper/copper_test.go b/pkg/detectors/copper/copper_test.goindex 00210b860..b3363304d 100644--- a/pkg/detectors/copper/copper_test.go+++ b/pkg/detectors/copper/copper_test.go@@ -16,7 +16,7 @@ import (func TestCopper_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/countrylayer/countrylayer_test.go b/pkg/detectors/countrylayer/countrylayer_test.goindex de26723f7..3641e69b4 100644--- a/pkg/detectors/countrylayer/countrylayer_test.go+++ b/pkg/detectors/countrylayer/countrylayer_test.go@@ -16,7 +16,7 @@ import (func TestCountryLayer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/courier/courier_test.go b/pkg/detectors/courier/courier_test.goindex c210d49b7..4f130ab26 100644--- a/pkg/detectors/courier/courier_test.go+++ b/pkg/detectors/courier/courier_test.go@@ -16,7 +16,7 @@ import (func TestCourier_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/coveralls/coveralls_test.go b/pkg/detectors/coveralls/coveralls_test.goindex ae165f359..69da7122c 100644--- a/pkg/detectors/coveralls/coveralls_test.go+++ b/pkg/detectors/coveralls/coveralls_test.go@@ -16,7 +16,7 @@ import (func TestCoveralls_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/crowdin/crowdin_test.go b/pkg/detectors/crowdin/crowdin_test.goindex 24c2d4122..a1516452f 100644--- a/pkg/detectors/crowdin/crowdin_test.go+++ b/pkg/detectors/crowdin/crowdin_test.go@@ -16,7 +16,7 @@ import (func TestCrowdin_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/cryptocompare/cryptocompare_test.go b/pkg/detectors/cryptocompare/cryptocompare_test.goindex 01dc4d024..4459935f3 100644--- a/pkg/detectors/cryptocompare/cryptocompare_test.go+++ b/pkg/detectors/cryptocompare/cryptocompare_test.go@@ -16,7 +16,7 @@ import (func TestCryptoCompare_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/currencycloud/currencycloud_test.go b/pkg/detectors/currencycloud/currencycloud_test.goindex e2c40b056..710e2b596 100644--- a/pkg/detectors/currencycloud/currencycloud_test.go+++ b/pkg/detectors/currencycloud/currencycloud_test.go@@ -16,7 +16,7 @@ import (func TestCurrencycloud_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/currencyfreaks/currencyfreaks_test.go b/pkg/detectors/currencyfreaks/currencyfreaks_test.goindex 17d0b0449..8583abe0c 100644--- a/pkg/detectors/currencyfreaks/currencyfreaks_test.go+++ b/pkg/detectors/currencyfreaks/currencyfreaks_test.go@@ -16,7 +16,7 @@ import (func TestCurrencyfreaks_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/currencylayer/currencylayer_test.go b/pkg/detectors/currencylayer/currencylayer_test.goindex 19771d2e5..13706c143 100644--- a/pkg/detectors/currencylayer/currencylayer_test.go+++ b/pkg/detectors/currencylayer/currencylayer_test.go@@ -16,7 +16,7 @@ import (func TestCurrencylayer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/currencyscoop/currencyscoop_test.go b/pkg/detectors/currencyscoop/currencyscoop_test.goindex 8c3d02514..d18a68bf4 100644--- a/pkg/detectors/currencyscoop/currencyscoop_test.go+++ b/pkg/detectors/currencyscoop/currencyscoop_test.go@@ -16,7 +16,7 @@ import (func TestCurrencyscoop_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/currentsapi/currentsapi_test.go b/pkg/detectors/currentsapi/currentsapi_test.goindex 403a788f4..63c607fe5 100644--- a/pkg/detectors/currentsapi/currentsapi_test.go+++ b/pkg/detectors/currentsapi/currentsapi_test.go@@ -16,7 +16,7 @@ import (func TestCurrentsAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/customerguru/customerguru_test.go b/pkg/detectors/customerguru/customerguru_test.goindex 1a9334efe..05cc9f319 100644--- a/pkg/detectors/customerguru/customerguru_test.go+++ b/pkg/detectors/customerguru/customerguru_test.go@@ -16,7 +16,7 @@ import (func TestCustomerGuru_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/customerio/customerio_test.go b/pkg/detectors/customerio/customerio_test.goindex acaa3a557..e9531ae0d 100644--- a/pkg/detectors/customerio/customerio_test.go+++ b/pkg/detectors/customerio/customerio_test.go@@ -16,7 +16,7 @@ import (func TestCustomerIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/d7network/d7network_test.go b/pkg/detectors/d7network/d7network_test.goindex f578454be..abfa04b48 100644--- a/pkg/detectors/d7network/d7network_test.go+++ b/pkg/detectors/d7network/d7network_test.go@@ -16,7 +16,7 @@ import (func TestD7Network_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/dailyco/dailyco_test.go b/pkg/detectors/dailyco/dailyco_test.goindex dd81bde4b..b20a656e6 100644--- a/pkg/detectors/dailyco/dailyco_test.go+++ b/pkg/detectors/dailyco/dailyco_test.go@@ -16,7 +16,7 @@ import (func TestDailyCO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/dandelion/dandelion_test.go b/pkg/detectors/dandelion/dandelion_test.goindex 6da14a140..013b8d05f 100644--- a/pkg/detectors/dandelion/dandelion_test.go+++ b/pkg/detectors/dandelion/dandelion_test.go@@ -16,7 +16,7 @@ import (func TestDandelion_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/datadogtoken/datadogtoken_test.go b/pkg/detectors/datadogtoken/datadogtoken_test.goindex 4fc6ba9b7..4bd76c87a 100644--- a/pkg/detectors/datadogtoken/datadogtoken_test.go+++ b/pkg/detectors/datadogtoken/datadogtoken_test.go@@ -16,7 +16,7 @@ import (func TestDatadogToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/datafire/datafire_test.go b/pkg/detectors/datafire/datafire_test.goindex afa3cd417..354e93a24 100644--- a/pkg/detectors/datafire/datafire_test.go+++ b/pkg/detectors/datafire/datafire_test.go@@ -16,7 +16,7 @@ import (func TestDataFire_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/datagov/datagov.go b/pkg/detectors/datagov/datagov.gonew file mode 100644index 000000000..e45930ec4--- /dev/null+++ b/pkg/detectors/datagov/datagov.go@@ -0,0 +1,73 @@+package datagov++import (+ "context"+ "fmt"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"data.gov"}) + `\b([a-zA-Z0-9]{40})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"data.gov"}+}++// FromData will find and optionally verify DataGov secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_DataGov,+ Raw: []byte(resMatch),+ }++ if verify {+ req, _ := http.NewRequest("GET", fmt.Sprintf("https://api.ers.usda.gov/data/arms/state?api_key=%s", resMatch), nil)+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/datagov/datagov_test.go b/pkg/detectors/datagov/datagov_test.gonew file mode 100644index 000000000..7fe5d6b8e--- /dev/null+++ b/pkg/detectors/datagov/datagov_test.go@@ -0,0 +1,113 @@+package datagov++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestDataGov_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("DATAGOV")+ inactiveSecret := testSecrets.MustGetField("DATAGOV_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a data.gov secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_DataGov,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a data.gov secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_DataGov,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("DataGov.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("DataGov.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/debounce/debounce_test.go b/pkg/detectors/debounce/debounce_test.goindex 166535b67..d0d03f438 100644--- a/pkg/detectors/debounce/debounce_test.go+++ b/pkg/detectors/debounce/debounce_test.go@@ -16,7 +16,7 @@ import (func TestDebounce_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/deepai/deepai_test.go b/pkg/detectors/deepai/deepai_test.goindex 2ccb662bf..e83d5ea21 100644--- a/pkg/detectors/deepai/deepai_test.go+++ b/pkg/detectors/deepai/deepai_test.go@@ -16,7 +16,7 @@ import (func TestDeepAI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/deepgram/deepgram_test.go b/pkg/detectors/deepgram/deepgram_test.goindex 9370b1847..21bba55bd 100644--- a/pkg/detectors/deepgram/deepgram_test.go+++ b/pkg/detectors/deepgram/deepgram_test.go@@ -16,7 +16,7 @@ import (func TestDeepgram_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/delighted/delighted_test.go b/pkg/detectors/delighted/delighted_test.goindex 66262fe54..d72756a13 100644--- a/pkg/detectors/delighted/delighted_test.go+++ b/pkg/detectors/delighted/delighted_test.go@@ -16,7 +16,7 @@ import (func TestDelighted_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/deputy/deputy_test.go b/pkg/detectors/deputy/deputy_test.goindex 268a490b8..fe9881d91 100644--- a/pkg/detectors/deputy/deputy_test.go+++ b/pkg/detectors/deputy/deputy_test.go@@ -16,7 +16,7 @@ import (func TestDeputy_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/detectlanguage/detectlanguage_test.go b/pkg/detectors/detectlanguage/detectlanguage_test.goindex 81e429a22..78470889e 100644--- a/pkg/detectors/detectlanguage/detectlanguage_test.go+++ b/pkg/detectors/detectlanguage/detectlanguage_test.go@@ -16,7 +16,7 @@ import (func TestDetectLanguage_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.goindex 5cc1fb7f4..cd3097ce5 100644--- a/pkg/detectors/detectors.go+++ b/pkg/detectors/detectors.go@@ -2,6 +2,11 @@ package detectorsimport ("context"+ "os"+ "path/filepath"+ "runtime"+ "strings"+ "unicode""github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb""github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb"@@ -61,11 +66,11 @@ func CleanResults(results []Result) []Result {return results}- var cleaned = make([]Result, 0)+ var cleaned = make(map[string]Result, 0)for _, s := range results {if s.Verified {- cleaned = append(cleaned, s)+ cleaned[s.Redacted] = s}}@@ -73,5 +78,53 @@ func CleanResults(results []Result) []Result {return results[:1]}- return cleaned+ results = results[:0]+ for _, r := range cleaned {+ results = append(results, r)+ }++ return results+}++// Prefix regex ensures that at least one of the given keywords is within+// 20 characters of the capturing group that follows.+// This can help prevent false positives.+func PrefixRegex(keywords []string) string {+ pre := `(?i)(?:`+ middle := strings.Join(keywords, "|")+ post := `).{0,40}`+ return pre + middle + post+}++//KeyIsRandom is a Low cost check to make sure that 'keys' include a number to reduce FPs.+//Golang doesnt support regex lookaheads, so must be done in seperate calls.+//TODO improve checks. Shannon entropy did not work well.+func KeyIsRandom(key string) bool {+ for _, ch := range key {+ if unicode.IsDigit(ch) {+ return true+ }+ }++ return false+}++func MustGetBenchmarkData() map[string][]byte {+ _, filename, _, _ := runtime.Caller(0)+ dir := filepath.Dir(filename)+ small := make([]byte, 0)+ medium, err := os.ReadFile(filepath.Join(dir, "detectors.go"))+ if err != nil {+ panic(err)+ }+ big := make([]byte, 0)+ for i := 0; i < 25; i++ {+ big = append(big, medium...)+ }++ return map[string][]byte{+ "small": small,+ "medium": medium,+ "big": big,+ }}diff --git a/pkg/detectors/dfuse/dfuse_test.go b/pkg/detectors/dfuse/dfuse_test.goindex fdce7e7f6..82481742d 100644--- a/pkg/detectors/dfuse/dfuse_test.go+++ b/pkg/detectors/dfuse/dfuse_test.go@@ -16,7 +16,7 @@ import (func TestDfuse_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/digitaloceantoken/digitaloceantoken_test.go b/pkg/detectors/digitaloceantoken/digitaloceantoken_test.goindex febacde97..1acb821fc 100644--- a/pkg/detectors/digitaloceantoken/digitaloceantoken_test.go+++ b/pkg/detectors/digitaloceantoken/digitaloceantoken_test.go@@ -16,7 +16,7 @@ import (func TestDigitalOceanToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/discordbottoken/discordbottoken_test.go b/pkg/detectors/discordbottoken/discordbottoken_test.goindex aadd33e4f..3c0a8c7a2 100644--- a/pkg/detectors/discordbottoken/discordbottoken_test.go+++ b/pkg/detectors/discordbottoken/discordbottoken_test.go@@ -16,7 +16,7 @@ import (func TestDiscordBotToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/discordwebhook/discordwebhook_test.go b/pkg/detectors/discordwebhook/discordwebhook_test.goindex 806354ccb..c6ce7a1f0 100644--- a/pkg/detectors/discordwebhook/discordwebhook_test.go+++ b/pkg/detectors/discordwebhook/discordwebhook_test.go@@ -16,7 +16,7 @@ import (func TestDiscordWebhook_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ditto/ditto_test.go b/pkg/detectors/ditto/ditto_test.goindex accc3037e..13f8ee7e8 100644--- a/pkg/detectors/ditto/ditto_test.go+++ b/pkg/detectors/ditto/ditto_test.go@@ -16,7 +16,7 @@ import (func TestDitto_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/dnscheck/dnscheck_test.go b/pkg/detectors/dnscheck/dnscheck_test.goindex c94b540e7..6da82e114 100644--- a/pkg/detectors/dnscheck/dnscheck_test.go+++ b/pkg/detectors/dnscheck/dnscheck_test.go@@ -16,7 +16,7 @@ import (func TestDnscheck_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/documo/documo_test.go b/pkg/detectors/documo/documo_test.goindex 89bc2498c..41a3c7442 100644--- a/pkg/detectors/documo/documo_test.go+++ b/pkg/detectors/documo/documo_test.go@@ -16,7 +16,7 @@ import (func TestDocumo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/doppler/doppler_test.go b/pkg/detectors/doppler/doppler_test.goindex 981280dfe..a51b4baf9 100644--- a/pkg/detectors/doppler/doppler_test.go+++ b/pkg/detectors/doppler/doppler_test.go@@ -16,7 +16,7 @@ import (func TestDoppler_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/dotmailer/dotmailer_test.go b/pkg/detectors/dotmailer/dotmailer_test.goindex 569b44f3b..17df982e5 100644--- a/pkg/detectors/dotmailer/dotmailer_test.go+++ b/pkg/detectors/dotmailer/dotmailer_test.go@@ -16,7 +16,7 @@ import (func TestDotmailer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/dovico/dovico_test.go b/pkg/detectors/dovico/dovico_test.goindex e1c17de82..81145dcfd 100644--- a/pkg/detectors/dovico/dovico_test.go+++ b/pkg/detectors/dovico/dovico_test.go@@ -16,7 +16,7 @@ import (func TestDovico_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/dronahq/dronahq_test.go b/pkg/detectors/dronahq/dronahq_test.goindex 2768d50e6..e1a185005 100644--- a/pkg/detectors/dronahq/dronahq_test.go+++ b/pkg/detectors/dronahq/dronahq_test.go@@ -16,7 +16,7 @@ import (func TestDronaHQ_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/droneci/droneci_test.go b/pkg/detectors/droneci/droneci_test.goindex f78e5ef19..ea6fc8db9 100644--- a/pkg/detectors/droneci/droneci_test.go+++ b/pkg/detectors/droneci/droneci_test.go@@ -16,7 +16,7 @@ import (func TestDroneCI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/dropbox/dropbox.go b/pkg/detectors/dropbox/dropbox.gonew file mode 100644index 000000000..d3b5f6c00--- /dev/null+++ b/pkg/detectors/dropbox/dropbox.go@@ -0,0 +1,76 @@+package dropbox++import (+ "context"+ "fmt"+ "net/http"+ "regexp"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ keyPat = regexp.MustCompile(`\b(sl\.[A-Za-z0-9\-\_]{130,140})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"sl."}+}++// FromData will find and optionally verify Dropbox secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {++ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Dropbox,+ Raw: []byte(match[1]),+ }++ if verify {++ baseURL := "https://api.dropboxapi.com/2/users/get_current_account"++ client := common.RetryableHttpClient()++ req, _ := http.NewRequest("POST", baseURL, nil)+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))+ res, err := client.Do(req)+ if err != nil {+ return results, err+ }+ defer res.Body.Close()++ // 200 means good key for get current user+ // 400 is bad (malformed)+ // 403 bad scope+ if res.StatusCode == http.StatusOK {+ s.Verified = true+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/dropbox/dropbox_test.go b/pkg/detectors/dropbox/dropbox_test.gonew file mode 100644index 000000000..7d7060e9a--- /dev/null+++ b/pkg/detectors/dropbox/dropbox_test.go@@ -0,0 +1,108 @@+package dropbox++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestDropbox_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("DROPBOX")+ secretInactive := testSecrets.MustGetField("DROPBOX_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a dropbox secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Dropbox,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a dropbox secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Dropbox,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Dropbox.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Dropbox.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/dwolla/dwolla_test.go b/pkg/detectors/dwolla/dwolla_test.goindex 033b2378e..cf3e2c996 100644--- a/pkg/detectors/dwolla/dwolla_test.go+++ b/pkg/detectors/dwolla/dwolla_test.go@@ -16,7 +16,7 @@ import (func TestDwolla_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/dynalist/dynalist_test.go b/pkg/detectors/dynalist/dynalist_test.goindex 9aa303ad5..445bb537d 100644--- a/pkg/detectors/dynalist/dynalist_test.go+++ b/pkg/detectors/dynalist/dynalist_test.go@@ -16,7 +16,7 @@ import (func TestDynalist_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/dyspatch/dyspatch_test.go b/pkg/detectors/dyspatch/dyspatch_test.goindex 138813ceb..bba032d84 100644--- a/pkg/detectors/dyspatch/dyspatch_test.go+++ b/pkg/detectors/dyspatch/dyspatch_test.go@@ -16,7 +16,7 @@ import (func TestDyspatch_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/eagleeyenetworks/eagleeyenetworks_test.go b/pkg/detectors/eagleeyenetworks/eagleeyenetworks_test.goindex 9d715dbf8..7ab396558 100644--- a/pkg/detectors/eagleeyenetworks/eagleeyenetworks_test.go+++ b/pkg/detectors/eagleeyenetworks/eagleeyenetworks_test.go@@ -16,7 +16,7 @@ import (func TestEagleEyeNetworks_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/easyinsight/easyinsight_test.go b/pkg/detectors/easyinsight/easyinsight_test.goindex 31130b871..3a8d7cc53 100644--- a/pkg/detectors/easyinsight/easyinsight_test.go+++ b/pkg/detectors/easyinsight/easyinsight_test.go@@ -16,7 +16,7 @@ import (func TestEasyInsight_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/edamam/edamam_test.go b/pkg/detectors/edamam/edamam_test.goindex 5bac8bc70..9fc200a3c 100644--- a/pkg/detectors/edamam/edamam_test.go+++ b/pkg/detectors/edamam/edamam_test.go@@ -16,7 +16,7 @@ import (func TestEdamam_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/edenai/edenai_test.go b/pkg/detectors/edenai/edenai_test.goindex ca3b52e0d..054aeee86 100644--- a/pkg/detectors/edenai/edenai_test.go+++ b/pkg/detectors/edenai/edenai_test.go@@ -16,7 +16,7 @@ import (func TestEdenAI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/elasticemail/elasticemail_test.go b/pkg/detectors/elasticemail/elasticemail_test.goindex 4f3749658..2148b81c1 100644--- a/pkg/detectors/elasticemail/elasticemail_test.go+++ b/pkg/detectors/elasticemail/elasticemail_test.go@@ -16,7 +16,7 @@ import (func TestElasticEmail_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/enigma/enigma_test.go b/pkg/detectors/enigma/enigma_test.goindex 460e25b2d..c7bd4014d 100644--- a/pkg/detectors/enigma/enigma_test.go+++ b/pkg/detectors/enigma/enigma_test.go@@ -16,7 +16,7 @@ import (func TestEnigma_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ethplorer/ethplorer_test.go b/pkg/detectors/ethplorer/ethplorer_test.goindex 2fff1ad26..048ae1789 100644--- a/pkg/detectors/ethplorer/ethplorer_test.go+++ b/pkg/detectors/ethplorer/ethplorer_test.go@@ -16,7 +16,7 @@ import (func TestEthplorer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/etsyapikey/etsyapikey_test.go b/pkg/detectors/etsyapikey/etsyapikey_test.goindex fdc964dc7..afc8396fc 100644--- a/pkg/detectors/etsyapikey/etsyapikey_test.go+++ b/pkg/detectors/etsyapikey/etsyapikey_test.go@@ -16,7 +16,7 @@ import (func TestEtsyApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/everhour/everhour_test.go b/pkg/detectors/everhour/everhour_test.goindex 508be44e2..76c33c0b4 100644--- a/pkg/detectors/everhour/everhour_test.go+++ b/pkg/detectors/everhour/everhour_test.go@@ -16,7 +16,7 @@ import (func TestEverhour_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/exchangerateapi/exchangerateapi_test.go b/pkg/detectors/exchangerateapi/exchangerateapi_test.goindex 1b4af2022..010fe6905 100644--- a/pkg/detectors/exchangerateapi/exchangerateapi_test.go+++ b/pkg/detectors/exchangerateapi/exchangerateapi_test.go@@ -16,7 +16,7 @@ import (func TestExchangeRateAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/exchangeratesapi/exchangeratesapi_test.go b/pkg/detectors/exchangeratesapi/exchangeratesapi_test.goindex 238a497fa..ece1ceff5 100644--- a/pkg/detectors/exchangeratesapi/exchangeratesapi_test.go+++ b/pkg/detectors/exchangeratesapi/exchangeratesapi_test.go@@ -16,7 +16,7 @@ import (func TestExchangeRatesAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/facebookoauth/facebookoauth_test.go b/pkg/detectors/facebookoauth/facebookoauth_test.goindex f836ce641..1a43759bf 100644--- a/pkg/detectors/facebookoauth/facebookoauth_test.go+++ b/pkg/detectors/facebookoauth/facebookoauth_test.go@@ -16,7 +16,7 @@ import (func TestFacebookOAuth_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/faceplusplus/faceplusplus_test.go b/pkg/detectors/faceplusplus/faceplusplus_test.goindex ab087db52..03527f1a4 100644--- a/pkg/detectors/faceplusplus/faceplusplus_test.go+++ b/pkg/detectors/faceplusplus/faceplusplus_test.go@@ -16,7 +16,7 @@ import (func TestFaceplusplus_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fakejson/fakejson_test.go b/pkg/detectors/fakejson/fakejson_test.goindex 1e30d27cf..04fe4bccc 100644--- a/pkg/detectors/fakejson/fakejson_test.go+++ b/pkg/detectors/fakejson/fakejson_test.go@@ -16,7 +16,7 @@ import (func TestFakeJSON_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/falsepositives.go b/pkg/detectors/falsepositives.gonew file mode 100644index 000000000..b1ea5de67--- /dev/null+++ b/pkg/detectors/falsepositives.go@@ -0,0 +1,92 @@+package detectors++import (+ _ "embed"+ "strings"+ "unicode"+)++var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample"}++type FalsePositive string++//go:embed "badlist.txt"+var badList []byte++//go:embed "words.txt"+var wordList []byte++//go:embed "programmingbooks.txt"+var programmingBookWords []byte++type Wordlists struct {+ wordList []string+ badList []string+ programmingBookWords []string+}++var FalsePositiveWordlists = Wordlists{+ wordList: bytesToCleanWordList(wordList),+ badList: bytesToCleanWordList(badList),+ programmingBookWords: bytesToCleanWordList(programmingBookWords),+}++//IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met+//Currently that includes: No number, english word in key, or matches common example pattens.+//Only the secret key material should be passed into this function+func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) bool {++ for _, fp := range falsePositives {+ if strings.Contains(strings.ToLower(match), string(fp)) {+ return true+ }+ }++ if wordCheck {+ // check against common substring badlist+ if hasDictWord(FalsePositiveWordlists.badList, match) {+ return true+ }++ // check for dictionary word substrings+ if hasDictWord(FalsePositiveWordlists.wordList, match) {+ return true+ }++ // check for programming book token substrings+ if hasDictWord(FalsePositiveWordlists.programmingBookWords, match) {+ return true+ }+ }+ return false+}++func hasDictWord(wordList []string, token string) bool {+ lower := strings.ToLower(token)+ for _, word := range wordList {+ if strings.Contains(lower, word) {+ return true+ }+ }+ return false+}++func HasDigit(key string) bool {+ for _, ch := range key {+ if unicode.IsDigit(ch) {+ return true+ }+ }++ return false+}++func bytesToCleanWordList(data []byte) []string {+ words := []string{}+ for _, word := range strings.Split(string(data), "\n") {+ if strings.TrimSpace(word) != "" {+ words = append(words, strings.TrimSpace(strings.ToLower(word)))+ }+ }+ return words+}diff --git a/pkg/detectors/falsepositives_test.go b/pkg/detectors/falsepositives_test.gonew file mode 100644index 000000000..2375aeca0--- /dev/null+++ b/pkg/detectors/falsepositives_test.go@@ -0,0 +1,39 @@+package detectors++import "testing"++func TestIsFalsePositive(t *testing.T) {+ type args struct {+ match string+ falsePositives []FalsePositive+ }+ tests := []struct {+ name string+ args args+ want bool+ }{+ {+ name: "fp",+ args: args{+ match: "example",+ falsePositives: DefaultFalsePositives,+ },+ want: true,+ },+ {+ name: "not fp",+ args: args{+ match: "notafp123",+ falsePositives: DefaultFalsePositives,+ },+ want: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := IsKnownFalsePositive(tt.args.match, tt.args.falsePositives, false); got != tt.want {+ t.Errorf("IsKnownFalsePositive() = %v, want %v", got, tt.want)+ }+ })+ }+}diff --git a/pkg/detectors/fastforex/fastforex_test.go b/pkg/detectors/fastforex/fastforex_test.goindex e85aa4c6d..ef5d9e32d 100644--- a/pkg/detectors/fastforex/fastforex_test.go+++ b/pkg/detectors/fastforex/fastforex_test.go@@ -16,7 +16,7 @@ import (func TestFastForex_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.goindex 1f40893d3..f0f72750b 100644--- a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go+++ b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go@@ -16,7 +16,7 @@ import (func TestFastlyPersonalToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/feedier/feedier_test.go b/pkg/detectors/feedier/feedier_test.goindex 5f56f59ad..e66c72fed 100644--- a/pkg/detectors/feedier/feedier_test.go+++ b/pkg/detectors/feedier/feedier_test.go@@ -16,7 +16,7 @@ import (func TestFeedier_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fetchrss/fetchrss_test.go b/pkg/detectors/fetchrss/fetchrss_test.goindex 054b618fc..ff200feee 100644--- a/pkg/detectors/fetchrss/fetchrss_test.go+++ b/pkg/detectors/fetchrss/fetchrss_test.go@@ -16,7 +16,7 @@ import (func TestFetchrss_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.go b/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.goindex eee3e5e85..7f52ce4d8 100644--- a/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.go+++ b/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.go@@ -16,7 +16,7 @@ import (func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fileio/fileio_test.go b/pkg/detectors/fileio/fileio_test.goindex b30fbb097..99ed368ea 100644--- a/pkg/detectors/fileio/fileio_test.go+++ b/pkg/detectors/fileio/fileio_test.go@@ -16,7 +16,7 @@ import (func TestFileIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/finage/finage_test.go b/pkg/detectors/finage/finage_test.goindex 24607feea..63f17f52d 100644--- a/pkg/detectors/finage/finage_test.go+++ b/pkg/detectors/finage/finage_test.go@@ -16,7 +16,7 @@ import (func TestFinage_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/financialmodelingprep/financialmodelingprep_test.go b/pkg/detectors/financialmodelingprep/financialmodelingprep_test.goindex 8b2d9a9df..183377b44 100644--- a/pkg/detectors/financialmodelingprep/financialmodelingprep_test.go+++ b/pkg/detectors/financialmodelingprep/financialmodelingprep_test.go@@ -16,7 +16,7 @@ import (func TestFinancialModelingPrep_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/findl/findl_test.go b/pkg/detectors/findl/findl_test.goindex eb343f60e..b67ed0356 100644--- a/pkg/detectors/findl/findl_test.go+++ b/pkg/detectors/findl/findl_test.go@@ -16,7 +16,7 @@ import (func TestFindl_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/finnhub/finnhub_test.go b/pkg/detectors/finnhub/finnhub_test.goindex 9bd415a03..c5d211350 100644--- a/pkg/detectors/finnhub/finnhub_test.go+++ b/pkg/detectors/finnhub/finnhub_test.go@@ -16,7 +16,7 @@ import (func TestFinnhub_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fixerio/fixerio_test.go b/pkg/detectors/fixerio/fixerio_test.goindex 097a59c88..bfaa9c6b8 100644--- a/pkg/detectors/fixerio/fixerio_test.go+++ b/pkg/detectors/fixerio/fixerio_test.go@@ -16,7 +16,7 @@ import (func TestFixerIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/flatio/flatio_test.go b/pkg/detectors/flatio/flatio_test.goindex 0684dd875..6d900adcf 100644--- a/pkg/detectors/flatio/flatio_test.go+++ b/pkg/detectors/flatio/flatio_test.go@@ -16,7 +16,7 @@ import (func TestFlatIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fleetbase/fleetbase_test.go b/pkg/detectors/fleetbase/fleetbase_test.goindex 1c8da32e7..be301bc67 100644--- a/pkg/detectors/fleetbase/fleetbase_test.go+++ b/pkg/detectors/fleetbase/fleetbase_test.go@@ -16,7 +16,7 @@ import (func TestFleetbase_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/flickr/flickr_test.go b/pkg/detectors/flickr/flickr_test.goindex 96f9fdd44..31b41c99b 100644--- a/pkg/detectors/flickr/flickr_test.go+++ b/pkg/detectors/flickr/flickr_test.go@@ -16,7 +16,7 @@ import (func TestFlickr_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/flightapi/flightapi_test.go b/pkg/detectors/flightapi/flightapi_test.goindex 48fc8dbf8..c4b98e10d 100644--- a/pkg/detectors/flightapi/flightapi_test.go+++ b/pkg/detectors/flightapi/flightapi_test.go@@ -16,7 +16,7 @@ import (func TestFlightApi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/float/float_test.go b/pkg/detectors/float/float_test.goindex da89df290..101eefd89 100644--- a/pkg/detectors/float/float_test.go+++ b/pkg/detectors/float/float_test.go@@ -16,7 +16,7 @@ import (func TestFloat_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/flowflu/flowflu_test.go b/pkg/detectors/flowflu/flowflu_test.goindex b565a87b1..7f7e2c938 100644--- a/pkg/detectors/flowflu/flowflu_test.go+++ b/pkg/detectors/flowflu/flowflu_test.go@@ -16,7 +16,7 @@ import (func TestFlowFlu_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/flutterwave/flutterwave_test.go b/pkg/detectors/flutterwave/flutterwave_test.goindex e6b571e4d..a09b96239 100644--- a/pkg/detectors/flutterwave/flutterwave_test.go+++ b/pkg/detectors/flutterwave/flutterwave_test.go@@ -16,7 +16,7 @@ import (func TestFlutterwave_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fmfw/fmfw_test.go b/pkg/detectors/fmfw/fmfw_test.goindex 648e78893..73c18b051 100644--- a/pkg/detectors/fmfw/fmfw_test.go+++ b/pkg/detectors/fmfw/fmfw_test.go@@ -16,7 +16,7 @@ import (func TestFmfw_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/foursquare/foursquare_test.go b/pkg/detectors/foursquare/foursquare_test.goindex 237c18a79..38cdbb842 100644--- a/pkg/detectors/foursquare/foursquare_test.go+++ b/pkg/detectors/foursquare/foursquare_test.go@@ -16,7 +16,7 @@ import (func TestFoursquare_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/frameio/frameio_test.go b/pkg/detectors/frameio/frameio_test.goindex d202eb360..86cf0a4e3 100644--- a/pkg/detectors/frameio/frameio_test.go+++ b/pkg/detectors/frameio/frameio_test.go@@ -16,7 +16,7 @@ import (func TestFrameio_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/freshbooks/freshbooks_test.go b/pkg/detectors/freshbooks/freshbooks_test.goindex 2e06425d1..8c866f0a2 100644--- a/pkg/detectors/freshbooks/freshbooks_test.go+++ b/pkg/detectors/freshbooks/freshbooks_test.go@@ -16,7 +16,7 @@ import (func TestFreshbooks_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/freshdesk/freshdesk_test.go b/pkg/detectors/freshdesk/freshdesk_test.goindex eef4ed8e3..8835cb555 100644--- a/pkg/detectors/freshdesk/freshdesk_test.go+++ b/pkg/detectors/freshdesk/freshdesk_test.go@@ -16,7 +16,7 @@ import (func TestFreshdesk_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/front/front_test.go b/pkg/detectors/front/front_test.goindex 12a2241a7..273a549b1 100644--- a/pkg/detectors/front/front_test.go+++ b/pkg/detectors/front/front_test.go@@ -16,7 +16,7 @@ import (func TestFront_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fulcrum/fulcrum_test.go b/pkg/detectors/fulcrum/fulcrum_test.goindex 4ad178e63..f75e8e7d5 100644--- a/pkg/detectors/fulcrum/fulcrum_test.go+++ b/pkg/detectors/fulcrum/fulcrum_test.go@@ -16,7 +16,7 @@ import (func TestFulcrum_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fullstory/fullstory_test.go b/pkg/detectors/fullstory/fullstory_test.goindex 83f50835b..053d4f402 100644--- a/pkg/detectors/fullstory/fullstory_test.go+++ b/pkg/detectors/fullstory/fullstory_test.go@@ -16,7 +16,7 @@ import (func TestFullstory_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fusebill/fusebill_test.go b/pkg/detectors/fusebill/fusebill_test.goindex 36029356c..f8ea9f1cd 100644--- a/pkg/detectors/fusebill/fusebill_test.go+++ b/pkg/detectors/fusebill/fusebill_test.go@@ -16,7 +16,7 @@ import (func TestFusebill_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/fxmarket/fxmarket_test.go b/pkg/detectors/fxmarket/fxmarket_test.goindex e91b2ea2d..15d1e93b3 100644--- a/pkg/detectors/fxmarket/fxmarket_test.go+++ b/pkg/detectors/fxmarket/fxmarket_test.go@@ -16,7 +16,7 @@ import (func TestFxmarket_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/gcp/gcp.go b/pkg/detectors/gcp/gcp.gonew file mode 100644index 000000000..fdf86b791--- /dev/null+++ b/pkg/detectors/gcp/gcp.go@@ -0,0 +1,106 @@+package gcp++import (+ "context"+ "encoding/json"+ "regexp"+ "strings"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+ "golang.org/x/oauth2/google"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ keyPat = regexp.MustCompile(`\{[^{]+auth_provider_x509_cert_url[^}]+\}`)+)++type gcpKey struct {+ Type string `json:"type"`+ ProjectID string `json:"project_id"`+ PrivateKeyID string `json:"private_key_id"`+ PrivateKey string `json:"private_key"`+ ClientEmail string `json:"client_email"`+ ClientID string `json:"client_id"`+ AuthURI string `json:"auth_uri"`+ TokenURI string `json:"token_uri"`+ AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"`+ ClientX509CertURL string `json:"client_x509_cert_url"`+}++func trimCarrots(s string) string {+ s = strings.TrimPrefix(s, "<")+ s = strings.TrimSuffix(s, ">")+ return s+}++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"provider_x509"}+}++// FromData will find and optionally verify GCP secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllString(dataStr, -1)++ for _, match := range matches {+ key := match++ key = strings.ReplaceAll(key, `,\\n`, `\n`)+ key = strings.ReplaceAll(key, `\"\\n`, `\n`)+ key = strings.ReplaceAll(key, `\\"`, `"`)++ creds := gcpKey{}+ err := json.Unmarshal([]byte(key), &creds)+ if err != nil {+ continue+ }++ // for Slack mangling (mailto scheme and hyperlinks)+ if strings.Contains(creds.ClientEmail, `<mailto:`) {+ creds.ClientEmail = strings.Split(strings.Split(creds.ClientEmail, `<mailto:`)[1], `|`)[0]+ }+ creds.AuthProviderX509CertURL = trimCarrots(creds.AuthProviderX509CertURL)+ creds.AuthURI = trimCarrots(creds.AuthURI)+ creds.ClientX509CertURL = trimCarrots(creds.ClientX509CertURL)+ creds.TokenURI = trimCarrots(creds.TokenURI)++ // Not sure why this might happen, but we've observed this with a verified cred+ raw := []byte(creds.ClientEmail)+ if len(raw) == 0 {+ raw = []byte(key)+ }++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_GCP,+ Raw: raw,+ Redacted: creds.ClientEmail,+ }++ if verify {+ credBytes, _ := json.Marshal(creds)+ credentials, err := google.CredentialsFromJSON(ctx, credBytes, "https://www.googleapis.com/auth/cloud-platform")+ if err != nil {+ continue+ }+ if credentials != nil {+ _, err = credentials.TokenSource.Token()+ if err == nil {+ s.Verified = true+ }+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/gcp/gcp_test.go b/pkg/detectors/gcp/gcp_test.gonew file mode 100644index 000000000..685f3bd49--- /dev/null+++ b/pkg/detectors/gcp/gcp_test.go@@ -0,0 +1,113 @@+package gcp++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestGCP_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("GCP_SECRET")+ secretInactive := testSecrets.MustGetField("GCP_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a gcp secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_GCP,+ Verified: true,+ Redacted: "[email protected]",+ },+ },+ wantErr: false,+ },+ {+ name: "found, not verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a gcp secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_GCP,+ Verified: false,+ Redacted: "secretcom",+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("GCP.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("GCP.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/geckoboard/geckoboard_test.go b/pkg/detectors/geckoboard/geckoboard_test.goindex 6258bb129..d96c4bfa3 100644--- a/pkg/detectors/geckoboard/geckoboard_test.go+++ b/pkg/detectors/geckoboard/geckoboard_test.go@@ -16,7 +16,7 @@ import (func TestGeckoboard_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/generic/generic.go b/pkg/detectors/generic/generic.gonew file mode 100644index 000000000..d15d85edb--- /dev/null+++ b/pkg/detectors/generic/generic.go@@ -0,0 +1,139 @@+package generic++// cat scanner/pkg/secrets/generic/top-1000.txt | awk 'length($0)>5' > scanner/pkg/secrets/generic/words.txt++import (+ "context"+ "encoding/base64"+ "fmt"+ "regexp"+ "strings"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func New() Scanner {+ excludePatterns := []string{+ `[0-9A-Fa-f]{8}(?:-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}`, // UUID+ `[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}`, // UUIDv4+ `[A-Z]{2,6}\-[0-9]{2,6}`, // issue tracker+ `#[a-fA-F0-9]{6}\b`, // hex color code+ `\b[A-Fa-f0-9]{64}\b`, // hex encoded hash+ `\b[A-Fa-f0-9]{32}\b`, // hex encoded hash+ `https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)`, // http+ `\b([/]{0,1}([\w]+[/])+[\w\.]*)\b`, // filepath+ `([0-9A-F]{2}[:-]){5}([0-9A-F]{2})`, // MAC addr+ `\d{4}[-/]{1}([0]\d|1[0-2])[-/]{1}([0-2]\d|3[01])`, // date+ `[v|\-]\d\.\d`, //version+ `\d\.\d\.\d-`, //version+ `[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}`, // IPs and OIDs+ `\b[A-Fa-f0-9x]{6,99}\b`, // hex encoding+ `[A-Fa-f0-9x]{2}:[A-Fa-f0-9x]{2}:[A-Fa-f0-9x]{2}`, // hex encoding+ `[\w]+\([\w, ]+\)`, // function+ }++ excludeMatchers := []*regexp.Regexp{}+ for _, pat := range excludePatterns {+ excludeMatchers = append(excludeMatchers, regexp.MustCompile(pat))+ }++ return Scanner{+ excludeMatchers: excludeMatchers,+ }+}++type Scanner struct {+ wordList []string+ badList []string+ programmingBookWords []string+ excludeMatchers []*regexp.Regexp+}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var keywords = []string{"pass", "token", "cred", "secret", "key"}++var (+ // \x21-\x7e == ASCII 33 (0x21) and 126 (0x7e)+ keyPat = regexp.MustCompile(detectors.PrefixRegex(keywords) + `(\b[\x21-\x7e]{16,64}\b)`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return keywords+}++// FromData will find and optionally verify Generic secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {++ token := match[1]++ // Least expensive-> most expensive filters.+ // Substrings, then patterns.++ if detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {+ continue+ }++ // toss any that match regexes+ if hasReMatch(s.excludeMatchers, token) {+ continue+ }++ // clean up containment chars+ token = strings.Trim(token, fmt.Sprintf(`%s" '.,)(][}{`, "`"))++ // toss any that b64 decode+ // TODO: run them through again?+ _, err := base64.StdEncoding.DecodeString(token)+ if err == nil {+ continue+ }++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Generic,+ Raw: []byte(token),+ }++ results = append(results, s)+ }++ return+}++func hasReMatch(matchers []*regexp.Regexp, token string) bool {+ for _, m := range matchers {+ if m.MatchString(token) {+ return true+ }+ }+ return false+}++func hasDictWord(wordList []string, token string) bool {+ lower := strings.ToLower(token)+ for _, word := range wordList {+ if strings.Contains(lower, word) {+ return true+ }+ }+ return false+}++func bytesToCleanWordList(data []byte) []string {+ words := []string{}+ for _, word := range strings.Split(string(data), "\n") {+ if strings.TrimSpace(word) != "" {+ words = append(words, strings.TrimSpace(strings.ToLower(word)))+ }+ }+ return words+}diff --git a/pkg/detectors/generic/generic_test.go b/pkg/detectors/generic/generic_test.gonew file mode 100644index 000000000..5b27cf948--- /dev/null+++ b/pkg/detectors/generic/generic_test.go@@ -0,0 +1,57 @@+package generic++import (+ "context"+ "testing"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+)++func TestGeneric_FromChunk(t *testing.T) {+ ctx := context.Background()+ s := New()++ found := []string{+ "export CONFIG_SERVICE_PASSWORD=34t98hofi2309pr230",+ "CONFIG_SERVICE_PASSWORD: 34t98hofi2309pr230",+ "the secret is 34t98hofi2309pr230", // keyword within distance of cred-looking token+ }+ notFound := []string{+ "export MAGIC_VAR=piggymetrics201925", // key does not have keyword+ "export CONFIG_SERVICE_PASSWORD=testcredentials", // has test, cred+ "export CONFIG_SERVICE_PASSWORD=password123", // has pass+ "export CONFIG_SERVICE_PASSWORD=too-plain", // no digit+ "export CONFIG_SERVICE_PASSWORD=abcdefg123", // known FP+ "SECRET: mountain422", // excluded by word list+ "secret_guid=3fc0b7f7-da09-4ae7-a9c8-d69824b1819b", // excluded by matcher+ "secret_issue_key: BLAH-23490", // excluded by matcher+ }++ for _, data := range found {+ got, _ := s.FromData(ctx, false, []byte(data))+ if len(got) == 0 {+ t.Errorf("Generic.FromData() expected secret for data: %s", data)+ return+ }+ }++ for _, data := range notFound {+ got, _ := s.FromData(ctx, false, []byte(data))+ if len(got) != 0 {+ t.Errorf("Generic.FromData() expected no secret for data: %s", data)+ return+ }+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := New()+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/gengo/gengo.go b/pkg/detectors/gengo/gengo.gonew file mode 100644index 000000000..89c3b22c2--- /dev/null+++ b/pkg/detectors/gengo/gengo.go@@ -0,0 +1,118 @@+package gengo++import (+ "context"+ "crypto/hmac"+ "crypto/sha1"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "io/ioutil"+ "strconv"+ "time"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ //Removed bounds since there are some cases where the start and end of the token is a special character+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([ ]{0,1}[0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64}[ \r\n]{1})`)+ secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([ ]{0,1}[0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64}[ \r\n]{1})`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"gengo"}+}++// FromData will find and optionally verify Gengo secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)+ secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])++ for _, secretMatch := range secretMatches {+ if len(secretMatch) != 2 {+ continue+ }+ resSecretMatch := strings.TrimSpace(secretMatch[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_Gengo,+ Raw: []byte(resSecretMatch),+ }++ if verify {++ timestamp := strconv.FormatInt(time.Now().Unix(), 10)+ signature := getGengoSignature(timestamp, resSecretMatch)++ req, _ := http.NewRequest("GET", fmt.Sprintf("http://api.gengo.com/v2/account/me?ts=%s&api_key=%s&api_sig=%s", timestamp, resMatch, signature), nil)+ req.Header.Add("Accept", "application/json")+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ body, errBody := ioutil.ReadAll(res.Body)++ if errBody == nil {+ var response Response+ json.Unmarshal(body, &response)++ if res.StatusCode >= 200 && res.StatusCode < 300 && response.OpStat == "ok" {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }++ if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }+ }++ results = append(results, s1)+ }+ }++ return detectors.CleanResults(results), nil+}++type Response struct {+ OpStat string `json:"opstat"`+}++func getGengoSignature(timeStamp string, secret string) string {++ mac := hmac.New(sha1.New, []byte(secret))+ mac.Write([]byte(timeStamp))+ macsum := mac.Sum(nil)+ return hex.EncodeToString(macsum)+}diff --git a/pkg/detectors/gengo/gengo_test.go b/pkg/detectors/gengo/gengo_test.gonew file mode 100644index 000000000..4fff0ebb5--- /dev/null+++ b/pkg/detectors/gengo/gengo_test.go@@ -0,0 +1,115 @@+package gengo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestGengo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ key := testSecrets.MustGetField("GENGO_KEY")+ inactiveKey := testSecrets.MustGetField("GENGO_KEY_INACTIVE")+ secret := testSecrets.MustGetField("GENGO")+ inactiveSecret := testSecrets.MustGetField("GENGO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a gengo key %s with gengo secret %s within", key, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Gengo,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a gengo key %s with gengo secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Gengo,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Gengo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Gengo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/geoapify/geoapify_test.go b/pkg/detectors/geoapify/geoapify_test.goindex 61cad9426..c64d5358d 100644--- a/pkg/detectors/geoapify/geoapify_test.go+++ b/pkg/detectors/geoapify/geoapify_test.go@@ -16,7 +16,7 @@ import (func TestGeoapify_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/geocode/geocode_test.go b/pkg/detectors/geocode/geocode_test.goindex 904998d18..698c58730 100644--- a/pkg/detectors/geocode/geocode_test.go+++ b/pkg/detectors/geocode/geocode_test.go@@ -16,7 +16,7 @@ import (func TestGeocode_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/geocodify/geocodify_test.go b/pkg/detectors/geocodify/geocodify_test.goindex 943779374..e38ed2768 100644--- a/pkg/detectors/geocodify/geocodify_test.go+++ b/pkg/detectors/geocodify/geocodify_test.go@@ -16,7 +16,7 @@ import (func TestGeocodify_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/geocodio/geocodio_test.go b/pkg/detectors/geocodio/geocodio_test.goindex 38eb036f3..8adca4be9 100644--- a/pkg/detectors/geocodio/geocodio_test.go+++ b/pkg/detectors/geocodio/geocodio_test.go@@ -16,7 +16,7 @@ import (func TestGeocodio_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/geoipifi/geoipifi_test.go b/pkg/detectors/geoipifi/geoipifi_test.goindex 586a1c88b..1689ae691 100644--- a/pkg/detectors/geoipifi/geoipifi_test.go+++ b/pkg/detectors/geoipifi/geoipifi_test.go@@ -16,7 +16,7 @@ import (func TestGeoIpifi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/getemail/getemail_test.go b/pkg/detectors/getemail/getemail_test.goindex 4598d5e68..6c21905f3 100644--- a/pkg/detectors/getemail/getemail_test.go+++ b/pkg/detectors/getemail/getemail_test.go@@ -16,7 +16,7 @@ import (func TestGetEmail_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/getemails/getemails_test.go b/pkg/detectors/getemails/getemails_test.goindex ac4c30547..34853c49e 100644--- a/pkg/detectors/getemails/getemails_test.go+++ b/pkg/detectors/getemails/getemails_test.go@@ -16,7 +16,7 @@ import (func TestGetEmails_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/getgeoapi/getgeoapi_test.go b/pkg/detectors/getgeoapi/getgeoapi_test.goindex c70e3b835..66aeafc63 100644--- a/pkg/detectors/getgeoapi/getgeoapi_test.go+++ b/pkg/detectors/getgeoapi/getgeoapi_test.go@@ -16,7 +16,7 @@ import (func TestGetgeoapi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/getgist/getgist_test.go b/pkg/detectors/getgist/getgist_test.goindex 32b15455f..31ed98900 100644--- a/pkg/detectors/getgist/getgist_test.go+++ b/pkg/detectors/getgist/getgist_test.go@@ -16,7 +16,7 @@ import (func TestGetgist_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/getsandbox/getsandbox_test.go b/pkg/detectors/getsandbox/getsandbox_test.goindex 147b348ac..034caa30b 100644--- a/pkg/detectors/getsandbox/getsandbox_test.go+++ b/pkg/detectors/getsandbox/getsandbox_test.go@@ -16,7 +16,7 @@ import (func TestGetSandbox_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/github/github.go b/pkg/detectors/github/github.gonew file mode 100644index 000000000..45ebf9f9b--- /dev/null+++ b/pkg/detectors/github/github.go@@ -0,0 +1,96 @@+package github++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "regexp"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ // Oauth token+ // https://developer.github.com/v3/#oauth2-token-sent-in-a-header+ // Token type list:+ // https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/+ keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr)_[a-zA-Z0-9]{36,255}\b)`)++ //TODO: Oauth2 client_id and client_secret+ // https://developer.github.com/v3/#oauth2-keysecret+)++// TODO: Add secret context?? Information about access, ownership etc+type userRes struct {+ Login string `json:"login"`+ Type string `json:"type"`+ SiteAdmin bool `json:"site_admin"`+ Name string `json:"name"`+ Company string `json:"company"`+}++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"ghp_", "gho_", "ghu_", "ghs_", "ghr_"}+}++// FromData will find and optionally verify GitHub secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ // First match is entire regex, second is the first group.+ if len(match) != 2 {+ continue+ }++ token := match[1]++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Github,+ Raw: []byte(token),+ }++ if verify {+ client := common.RetryableHttpClient()+ // https://developer.github.com/v3/users/#get-the-authenticated-user+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)+ req.Header.Add("Content-Type", "application/json; charset=utf-8")+ req.Header.Add("Authorization", fmt.Sprintf("token %s", token))+ res, err := client.Do(req)+ if err != nil {+ break+ }+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ var userResponse userRes+ err = json.NewDecoder(res.Body).Decode(&userResponse)+ if err == nil {+ s.Verified = true+ }+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/github/github_test.go b/pkg/detectors/github/github_test.gonew file mode 100644index 000000000..c696ac417--- /dev/null+++ b/pkg/detectors/github/github_test.go@@ -0,0 +1,180 @@+package github++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestGitHub_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ unverifiedGhp := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHP")+ unverifiedGhpLong := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHP_LONG")+ unverifiedGho := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHO")+ unverifiedGhu := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHU")+ unverifiedGhs := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHS")+ unverifiedGhr := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHR")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, unverified ghp",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a github secret %s within", unverifiedGhp)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Github,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified gho",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a github secret %s within", unverifiedGho)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Github,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified ghu",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a github secret %s within", unverifiedGhu)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Github,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified ghs",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a github secret %s within", unverifiedGhs)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Github,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified ghr",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a github secret %s within", unverifiedGhr)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Github,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified ghp future length 255",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(unverifiedGhpLong),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Github,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("https://raw.github.com/k/d890e8640f20fba3215ba7be8e0ff145aeb8c17c/include/base64.js"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("GitHub.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("GitHub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/github_old/github_old.go b/pkg/detectors/github_old/github_old.gonew file mode 100644index 000000000..5abdf41ce--- /dev/null+++ b/pkg/detectors/github_old/github_old.go@@ -0,0 +1,92 @@+package github_old++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "regexp"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ // Oauth token+ // https://developer.github.com/v3/#oauth2-token-sent-in-a-header+ keyPat = regexp.MustCompile(`(?i)(?:github)[^\.].{0,40}[ =:'"]+([a-f0-9]{40})\b`)++ //TODO: Oauth2 client_id and client_secret+ // https://developer.github.com/v3/#oauth2-keysecret+)++// TODO: Add secret context?? Information about access, ownership etc+type userRes struct {+ Login string `json:"login"`+ Type string `json:"type"`+ SiteAdmin bool `json:"site_admin"`+ Name string `json:"name"`+ Company string `json:"company"`+}++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"github"}+}++// FromData will find and optionally verify GitHub secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ // First match is entire regex, second is the first group.+ if len(match) != 2 {+ continue+ }++ token := match[1]++ specificFPs := []detectors.FalsePositive{"github commit"}+ if detectors.IsKnownFalsePositive(token, specificFPs, false) {+ continue+ }++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Github,+ Raw: []byte(token),+ }++ if verify {+ client := common.RetryableHttpClient()+ // https://developer.github.com/v3/users/#get-the-authenticated-user+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)+ req.Header.Add("Content-Type", "application/json; charset=utf-8")+ req.Header.Add("Authorization", fmt.Sprintf("token %s", token))+ res, err := client.Do(req)+ if err != nil {+ break+ }+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ var userResponse userRes+ err = json.NewDecoder(res.Body).Decode(&userResponse)+ if err == nil {+ s.Verified = true+ }+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/github_old/github_old_test.go b/pkg/detectors/github_old/github_old_test.gonew file mode 100644index 000000000..97e5a5c82--- /dev/null+++ b/pkg/detectors/github_old/github_old_test.go@@ -0,0 +1,122 @@+package github_old++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestGitHub_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("GITHUB_OLD")+ secretInactive := testSecrets.MustGetField("GITHUB_OLD_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a github secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Github,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a github secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Github,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found url",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("https://raw.github.com/k/d890e8640f20fba3215ba7be8e0ff145aeb8c17c/include/base64.js"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ {+ name: "not found ref",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("https://github.com/lz4/lz4 @ dccf8826f1d76efcbdc655e63cc04cdbd1123619"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("GitHub.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("GitHub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/githubapp/githubapp_test.go b/pkg/detectors/githubapp/githubapp_test.goindex dad95c35f..8b3bc7e60 100644--- a/pkg/detectors/githubapp/githubapp_test.go+++ b/pkg/detectors/githubapp/githubapp_test.go@@ -17,7 +17,7 @@ import (func TestGitHubApp_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/gitlab/gitlab.go b/pkg/detectors/gitlab/gitlab.gonew file mode 100644index 000000000..9e232eb38--- /dev/null+++ b/pkg/detectors/gitlab/gitlab.go@@ -0,0 +1,82 @@+package gitlab++import (+ "context"+ "fmt"+ "net/http"+ "regexp"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b([a-zA-Z0-9\-=_]{20,22})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"gitlab"}+}++// FromData will find and optionally verify Gitlab secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }++ secret := detectors.Result{+ DetectorType: detectorspb.DetectorType_Gitlab,+ Raw: []byte(match[1]),+ }++ if verify {+ // there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry+ // they all grant access to different parts of the API. I couldn't find an endpoint that every+ // one of these scopes has access to, so we just check an example endpoint for each scope. If any+ // of them contain data, we know we have a valid key, but if they all fail, we don't+ baseURL := "https://gitlab.com/api/v4"++ client := common.RetryableHttpClient()++ // test `read_user` scope+ req, _ := http.NewRequest("GET", baseURL+"/user", nil)+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))+ res, err := client.Do(req)+ if err != nil {+ return results, err+ }+ defer res.Body.Close()++ // 200 means good key and has `read_user` scope+ // 403 means good key but not the right scope+ // 401 is bad key+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {+ secret.Verified = true+ }+ }++ if !secret.Verified {+ if detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, secret)+ }++ return+}diff --git a/pkg/detectors/gitlab/gitlab_test.go b/pkg/detectors/gitlab/gitlab_test.gonew file mode 100644index 000000000..f37eeb455--- /dev/null+++ b/pkg/detectors/gitlab/gitlab_test.go@@ -0,0 +1,111 @@+package gitlab++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestGitlab_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("GITLAB")+ secretInactive := testSecrets.MustGetField("GITLAB_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Gitlab,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Gitlab,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/gitlabv2/gitlab.go b/pkg/detectors/gitlabv2/gitlab.gonew file mode 100644index 000000000..265c4bf73--- /dev/null+++ b/pkg/detectors/gitlabv2/gitlab.go@@ -0,0 +1,82 @@+package gitlabv2++import (+ "context"+ "fmt"+ "net/http"+ "regexp"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"glpat-"}+}++// FromData will find and optionally verify Gitlab secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }++ secret := detectors.Result{+ DetectorType: detectorspb.DetectorType_Gitlab,+ Raw: []byte(match[1]),+ }++ if verify {+ // there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry+ // they all grant access to different parts of the API. I couldn't find an endpoint that every+ // one of these scopes has access to, so we just check an example endpoint for each scope. If any+ // of them contain data, we know we have a valid key, but if they all fail, we don't+ baseURL := "https://gitlab.com/api/v4"++ client := common.RetryableHttpClient()++ // test `read_user` scope+ req, _ := http.NewRequest("GET", baseURL+"/user", nil)+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))+ res, err := client.Do(req)+ if err != nil {+ return results, err+ }+ defer res.Body.Close()++ // 200 means good key and has `read_user` scope+ // 403 means good key but not the right scope+ // 401 is bad key+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {+ secret.Verified = true+ }+ }++ if !secret.Verified {+ if detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, secret)+ }++ return+}diff --git a/pkg/detectors/gitlabv2/gitlab_test.go b/pkg/detectors/gitlabv2/gitlab_test.gonew file mode 100644index 000000000..3782b5cdf--- /dev/null+++ b/pkg/detectors/gitlabv2/gitlab_test.go@@ -0,0 +1,107 @@+package gitlabv2++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestGitlab_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("GITLABV2")+ secretInactive := testSecrets.MustGetField("GITLABV2_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Gitlab,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within", secretInactive)),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/gitter/gitter_test.go b/pkg/detectors/gitter/gitter_test.goindex a93e703cd..a11ca3bb4 100644--- a/pkg/detectors/gitter/gitter_test.go+++ b/pkg/detectors/gitter/gitter_test.go@@ -16,7 +16,7 @@ import (func TestGitter_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/glassnode/glassnode_test.go b/pkg/detectors/glassnode/glassnode_test.goindex d92d7043c..e4e1f2075 100644--- a/pkg/detectors/glassnode/glassnode_test.go+++ b/pkg/detectors/glassnode/glassnode_test.go@@ -16,7 +16,7 @@ import (func TestGlassnode_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/gocardless/gocardless.go b/pkg/detectors/gocardless/gocardless.gonew file mode 100644index 000000000..a6da73df8--- /dev/null+++ b/pkg/detectors/gocardless/gocardless.go@@ -0,0 +1,77 @@+package gocardless++import (+ "context"+ "fmt"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ //Removed bounds at the end of the regex since there are some cases that the token ends with an underscore (_) or a dash (-)+ keyPat = regexp.MustCompile(`\b(live_[0-9A-Za-z\_\-]{40}[ "'\r\n]{1})`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"gocardless"}+}++// FromData will find and optionally verify GoCardless secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_GoCardless,+ Raw: []byte(resMatch),+ }++ if verify {+ req, _ := http.NewRequest("GET", "https://api.gocardless.com/customers/", nil)+ req.Header.Add("Content-Type", "application/json")+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))+ req.Header.Add("GoCardless-Version", "2015-07-06")+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/gocardless/gocardless_test.go b/pkg/detectors/gocardless/gocardless_test.gonew file mode 100644index 000000000..c384552d9--- /dev/null+++ b/pkg/detectors/gocardless/gocardless_test.go@@ -0,0 +1,113 @@+package gocardless++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestGoCardless_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("GOCARDLESS")+ inactiveSecret := testSecrets.MustGetField("GOCARDLESS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a gocardless secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_GoCardless,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a gocardless secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_GoCardless,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("GoCardless.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("GoCardless.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/goodday/goodday.go b/pkg/detectors/goodday/goodday.gonew file mode 100644index 000000000..fa081eaa3--- /dev/null+++ b/pkg/detectors/goodday/goodday.go@@ -0,0 +1,73 @@+package goodday++import (+ "context"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"goodday"}) + `\b([a-z0-9]{32})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"goodday"}+}++// FromData will find and optionally verify GoodDay secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_GoodDay,+ Raw: []byte(resMatch),+ }++ if verify {+ req, _ := http.NewRequest("GET", "https://api.goodday.work/2.0/users", nil)+ req.Header.Add("gd-api-token", resMatch)+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/goodday/goodday_test.go b/pkg/detectors/goodday/goodday_test.gonew file mode 100644index 000000000..3fb90bd48--- /dev/null+++ b/pkg/detectors/goodday/goodday_test.go@@ -0,0 +1,113 @@+package goodday++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestGoodDay_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("GOODDAY")+ inactiveSecret := testSecrets.MustGetField("GOODDAY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a goodday secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_GoodDay,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a goodday secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_GoodDay,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("GoodDay.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("GoodDay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/graphcms/graphcms_test.go b/pkg/detectors/graphcms/graphcms_test.goindex bc29c051f..7a0380ba3 100644--- a/pkg/detectors/graphcms/graphcms_test.go+++ b/pkg/detectors/graphcms/graphcms_test.go@@ -16,7 +16,7 @@ import (func TestGraphCMS_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/graphhopper/graphhopper_test.go b/pkg/detectors/graphhopper/graphhopper_test.goindex 8baa72743..f79aaa922 100644--- a/pkg/detectors/graphhopper/graphhopper_test.go+++ b/pkg/detectors/graphhopper/graphhopper_test.go@@ -16,7 +16,7 @@ import (func TestGraphhopper_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/groovehq/groovehq_test.go b/pkg/detectors/groovehq/groovehq_test.goindex be1d7ebff..befb0c5e6 100644--- a/pkg/detectors/groovehq/groovehq_test.go+++ b/pkg/detectors/groovehq/groovehq_test.go@@ -16,7 +16,7 @@ import (func TestGroovehq_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/guardianapi/guardianapi_test.go b/pkg/detectors/guardianapi/guardianapi_test.goindex fed330d15..cb4f53a91 100644--- a/pkg/detectors/guardianapi/guardianapi_test.go+++ b/pkg/detectors/guardianapi/guardianapi_test.go@@ -16,7 +16,7 @@ import (func TestGuardianapi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/guru/guru_test.go b/pkg/detectors/guru/guru_test.goindex 3fedf3bb0..54bad8df7 100644--- a/pkg/detectors/guru/guru_test.go+++ b/pkg/detectors/guru/guru_test.go@@ -16,7 +16,7 @@ import (func TestGuru_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/gyazo/gyazo_test.go b/pkg/detectors/gyazo/gyazo_test.goindex 1ed8fe1bc..621d330e5 100644--- a/pkg/detectors/gyazo/gyazo_test.go+++ b/pkg/detectors/gyazo/gyazo_test.go@@ -16,7 +16,7 @@ import (func TestGyazo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/happi/happi_test.go b/pkg/detectors/happi/happi_test.goindex 287b306b2..39f035404 100644--- a/pkg/detectors/happi/happi_test.go+++ b/pkg/detectors/happi/happi_test.go@@ -16,7 +16,7 @@ import (func TestHappi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/happyscribe/happyscribe_test.go b/pkg/detectors/happyscribe/happyscribe_test.goindex 6b19f0c1d..ec9a35c43 100644--- a/pkg/detectors/happyscribe/happyscribe_test.go+++ b/pkg/detectors/happyscribe/happyscribe_test.go@@ -16,7 +16,7 @@ import (func TestHappyscribe_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/harvest/harvest_test.go b/pkg/detectors/harvest/harvest_test.goindex 06bcc84d3..f4392b224 100644--- a/pkg/detectors/harvest/harvest_test.go+++ b/pkg/detectors/harvest/harvest_test.go@@ -16,7 +16,7 @@ import (func TestHarvest_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/hellosign/hellosign_test.go b/pkg/detectors/hellosign/hellosign_test.goindex f45246310..92f860c87 100644--- a/pkg/detectors/hellosign/hellosign_test.go+++ b/pkg/detectors/hellosign/hellosign_test.go@@ -16,7 +16,7 @@ import (func TestHelloSign_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/helpcrunch/helpcrunch_test.go b/pkg/detectors/helpcrunch/helpcrunch_test.goindex 806657d20..33c542c62 100644--- a/pkg/detectors/helpcrunch/helpcrunch_test.go+++ b/pkg/detectors/helpcrunch/helpcrunch_test.go@@ -16,7 +16,7 @@ import (func TestHelpCrunch_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/helpscout/helpscout_test.go b/pkg/detectors/helpscout/helpscout_test.goindex c9ac1e285..0ffef8fb1 100644--- a/pkg/detectors/helpscout/helpscout_test.go+++ b/pkg/detectors/helpscout/helpscout_test.go@@ -16,7 +16,7 @@ import (func TestHelpscout_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/hereapi/hereapi_test.go b/pkg/detectors/hereapi/hereapi_test.goindex c9b71c6ca..1567ea386 100644--- a/pkg/detectors/hereapi/hereapi_test.go+++ b/pkg/detectors/hereapi/hereapi_test.go@@ -16,7 +16,7 @@ import (func TestHereAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/heroku/heroku_test.go b/pkg/detectors/heroku/heroku_test.goindex 2ab4139a4..ce8604831 100644--- a/pkg/detectors/heroku/heroku_test.go+++ b/pkg/detectors/heroku/heroku_test.go@@ -16,7 +16,7 @@ import (func TestHeroku_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/hive/hive_test.go b/pkg/detectors/hive/hive_test.goindex 46f642615..f146fff1c 100644--- a/pkg/detectors/hive/hive_test.go+++ b/pkg/detectors/hive/hive_test.go@@ -16,7 +16,7 @@ import (func TestHive_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/hiveage/hiveage_test.go b/pkg/detectors/hiveage/hiveage_test.goindex b257d8b40..bcefbe78a 100644--- a/pkg/detectors/hiveage/hiveage_test.go+++ b/pkg/detectors/hiveage/hiveage_test.go@@ -16,7 +16,7 @@ import (func TestHiveage_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/holidayapi/holidayapi_test.go b/pkg/detectors/holidayapi/holidayapi_test.goindex 9510a0d65..55cb3afc0 100644--- a/pkg/detectors/holidayapi/holidayapi_test.go+++ b/pkg/detectors/holidayapi/holidayapi_test.go@@ -16,7 +16,7 @@ import (func TestHolidayAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/host/host_test.go b/pkg/detectors/host/host_test.goindex ff45f3e08..ac62e88ab 100644--- a/pkg/detectors/host/host_test.go+++ b/pkg/detectors/host/host_test.go@@ -16,7 +16,7 @@ import (func TestHost_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/html2pdf/html2pdf_test.go b/pkg/detectors/html2pdf/html2pdf_test.goindex 2ad8b1523..4a08a779c 100644--- a/pkg/detectors/html2pdf/html2pdf_test.go+++ b/pkg/detectors/html2pdf/html2pdf_test.go@@ -16,7 +16,7 @@ import (func TestHtml2Pdf_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/hubspotapikey/hubspotapikey_test.go b/pkg/detectors/hubspotapikey/hubspotapikey_test.goindex f5c4cf6e5..1a4261be4 100644--- a/pkg/detectors/hubspotapikey/hubspotapikey_test.go+++ b/pkg/detectors/hubspotapikey/hubspotapikey_test.go@@ -16,7 +16,7 @@ import (func TestHubSpotApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/humanity/humanity_test.go b/pkg/detectors/humanity/humanity_test.goindex 3ae82e41c..1ba17ad6f 100644--- a/pkg/detectors/humanity/humanity_test.go+++ b/pkg/detectors/humanity/humanity_test.go@@ -16,7 +16,7 @@ import (func TestHumanity_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/hunter/hunter_test.go b/pkg/detectors/hunter/hunter_test.goindex 1925a6899..1eecfd4fa 100644--- a/pkg/detectors/hunter/hunter_test.go+++ b/pkg/detectors/hunter/hunter_test.go@@ -16,7 +16,7 @@ import (func TestHunter_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}@@ -56,7 +56,7 @@ func TestHunter_FromChunk(t *testing.T) {s: Scanner{},args: args{ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a hunter secret %s within", secretInactive)),+ data: []byte(fmt.Sprintf("You can find a hunter secret %s within but not valid", secretInactive)), // the secret would satisfy the regex but not pass validationverify: true,},want: []detectors.Result{diff --git a/pkg/detectors/hypertrack/hypertrack_test.go b/pkg/detectors/hypertrack/hypertrack_test.goindex b4e83e94f..5fc157f86 100644--- a/pkg/detectors/hypertrack/hypertrack_test.go+++ b/pkg/detectors/hypertrack/hypertrack_test.go@@ -16,7 +16,7 @@ import (func TestHypertrack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ibmclouduserkey/ibmclouduserkey_test.go b/pkg/detectors/ibmclouduserkey/ibmclouduserkey_test.goindex 23b80d9d5..3860719ba 100644--- a/pkg/detectors/ibmclouduserkey/ibmclouduserkey_test.go+++ b/pkg/detectors/ibmclouduserkey/ibmclouduserkey_test.go@@ -16,7 +16,7 @@ import (func TestIbmCloudUserKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/iconfinder/iconfinder_test.go b/pkg/detectors/iconfinder/iconfinder_test.goindex 5ae4486c6..cf05df130 100644--- a/pkg/detectors/iconfinder/iconfinder_test.go+++ b/pkg/detectors/iconfinder/iconfinder_test.go@@ -16,7 +16,7 @@ import (func TestIconFinder_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/iexcloud/iexcloud_test.go b/pkg/detectors/iexcloud/iexcloud_test.goindex ad191ca57..2846dee42 100644--- a/pkg/detectors/iexcloud/iexcloud_test.go+++ b/pkg/detectors/iexcloud/iexcloud_test.go@@ -16,7 +16,7 @@ import (func TestIexcloud_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/imagekit/imagekit_test.go b/pkg/detectors/imagekit/imagekit_test.goindex 60b51d735..869c6f6f8 100644--- a/pkg/detectors/imagekit/imagekit_test.go+++ b/pkg/detectors/imagekit/imagekit_test.go@@ -16,7 +16,7 @@ import (func TestImagekit_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/imagga/imagga_test.go b/pkg/detectors/imagga/imagga_test.goindex 11d6c9539..ae938fdb2 100644--- a/pkg/detectors/imagga/imagga_test.go+++ b/pkg/detectors/imagga/imagga_test.go@@ -16,7 +16,7 @@ import (func TestImagga_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/impala/impala_test.go b/pkg/detectors/impala/impala_test.goindex 6bba820aa..4f76e1b0e 100644--- a/pkg/detectors/impala/impala_test.go+++ b/pkg/detectors/impala/impala_test.go@@ -16,7 +16,7 @@ import (func TestImpala_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/insightly/insightly_test.go b/pkg/detectors/insightly/insightly_test.goindex eff524871..361af36e3 100644--- a/pkg/detectors/insightly/insightly_test.go+++ b/pkg/detectors/insightly/insightly_test.go@@ -16,7 +16,7 @@ import (func TestInsightly_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/integromat/integromat_test.go b/pkg/detectors/integromat/integromat_test.goindex abc48cc29..40b30e30d 100644--- a/pkg/detectors/integromat/integromat_test.go+++ b/pkg/detectors/integromat/integromat_test.go@@ -16,7 +16,7 @@ import (func TestIntegromat_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/intercom/intercom_test.go b/pkg/detectors/intercom/intercom_test.goindex 39e283974..bb1e65e8d 100644--- a/pkg/detectors/intercom/intercom_test.go+++ b/pkg/detectors/intercom/intercom_test.go@@ -16,7 +16,7 @@ import (func TestInterCom_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/intrinio/intrinio_test.go b/pkg/detectors/intrinio/intrinio_test.goindex 844213a49..11f370a35 100644--- a/pkg/detectors/intrinio/intrinio_test.go+++ b/pkg/detectors/intrinio/intrinio_test.go@@ -16,7 +16,7 @@ import (func TestIntrinio_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/invoiceocean/invoiceocean_test.go b/pkg/detectors/invoiceocean/invoiceocean_test.goindex 779b6ceb1..262145663 100644--- a/pkg/detectors/invoiceocean/invoiceocean_test.go+++ b/pkg/detectors/invoiceocean/invoiceocean_test.go@@ -16,7 +16,7 @@ import (func TestInvoiceocean_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ipapi/ipapi_test.go b/pkg/detectors/ipapi/ipapi_test.goindex e4f8dd8e2..9aa5bd57b 100644--- a/pkg/detectors/ipapi/ipapi_test.go+++ b/pkg/detectors/ipapi/ipapi_test.go@@ -16,7 +16,7 @@ import (func TestIpapi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ipgeolocation/ipgeolocation_test.go b/pkg/detectors/ipgeolocation/ipgeolocation_test.goindex 58a21c72f..a22d18c85 100644--- a/pkg/detectors/ipgeolocation/ipgeolocation_test.go+++ b/pkg/detectors/ipgeolocation/ipgeolocation_test.go@@ -16,7 +16,7 @@ import (func TestIPGeolocation_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ipify/ipify_test.go b/pkg/detectors/ipify/ipify_test.goindex 4185afe75..87aa61af7 100644--- a/pkg/detectors/ipify/ipify_test.go+++ b/pkg/detectors/ipify/ipify_test.go@@ -16,7 +16,7 @@ import (func TestIpify_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ipinfodb/ipinfodb_test.go b/pkg/detectors/ipinfodb/ipinfodb_test.goindex 6a4fd2195..0cfed7661 100644--- a/pkg/detectors/ipinfodb/ipinfodb_test.go+++ b/pkg/detectors/ipinfodb/ipinfodb_test.go@@ -16,7 +16,7 @@ import (func TestIPinfoDB_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ipquality/ipquality_test.go b/pkg/detectors/ipquality/ipquality_test.goindex a58911a7b..90a62cb47 100644--- a/pkg/detectors/ipquality/ipquality_test.go+++ b/pkg/detectors/ipquality/ipquality_test.go@@ -16,7 +16,7 @@ import (func TestIpquality_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ipstack/ipstack_test.go b/pkg/detectors/ipstack/ipstack_test.goindex 2fe814816..5fc94b5f2 100644--- a/pkg/detectors/ipstack/ipstack_test.go+++ b/pkg/detectors/ipstack/ipstack_test.go@@ -16,7 +16,7 @@ import (func TestIpStack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.gonew file mode 100644index 000000000..d9be8e8d1--- /dev/null+++ b/pkg/detectors/jdbc/jdbc.go@@ -0,0 +1,74 @@+package jdbc++import (+ "context"+ "regexp"+ "strings"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ keyPat = regexp.MustCompile(`(?i)jdbc:[\w]{3,10}:\/\/\w[\s\S]{0,512}?password[=: \"']+(?P<pass>[^<{($]*?)[ \s'\"]+`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"jdbc"}+}++// FromData will find and optionally verify Jdbc secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)+ for _, match := range matches {+ if match[1] == "" {+ continue+ }+ token := match[0]+ password := match[1]++ //TODO if username and password are the same, username will also be redacted... I think this is probably correct.+ redact := strings.TrimSpace(strings.Replace(token, password, strings.Repeat("*", len(password)), -1))++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_JDBC,+ Raw: []byte(token),+ Redacted: redact,+ }++ if verify {+ //TODO can this be verified? Possibly. Could triage verification to other DBMS strings+ s.Verified = false+ // client := common.RetryableHttpClient()+ // req, _ := http.NewRequestWithContext(ctx, "GET", "https://jdbcci.com/api/v2/me", nil)+ // req.Header.Add("Accept", "application/json;")+ // req.Header.Add("Jdbc-Token", token)+ // res, err := client.Do(req)+ // if err != nil {+ // break+ // }+ // if res.StatusCode >= 200 && res.StatusCode < 300 {+ // s.Verified = true+ // }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/jdbc/jdbc_test.go b/pkg/detectors/jdbc/jdbc_test.gonew file mode 100644index 000000000..1dc8baec8--- /dev/null+++ b/pkg/detectors/jdbc/jdbc_test.go@@ -0,0 +1,89 @@+package jdbc++import (+ "context"+ "os"+ "testing"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestJdbc_FromChunk(t *testing.T) {+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(`jdbc connection string: jdbc:mysql://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=testpassword <-`),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_JDBC,+ Verified: false,+ Redacted: "jdbc:mysql://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=************",+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Jdbc.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ if os.Getenv("FORCE_PASS_DIFF") == "true" {+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/jiratoken/jiratoken_test.go b/pkg/detectors/jiratoken/jiratoken_test.goindex 01c96c4e6..c16f1228d 100644--- a/pkg/detectors/jiratoken/jiratoken_test.go+++ b/pkg/detectors/jiratoken/jiratoken_test.go@@ -16,7 +16,7 @@ import (func TestJiraToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/jotform/jotform_test.go b/pkg/detectors/jotform/jotform_test.goindex 6a93249d2..8793a391c 100644--- a/pkg/detectors/jotform/jotform_test.go+++ b/pkg/detectors/jotform/jotform_test.go@@ -16,7 +16,7 @@ import (func TestJotform_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/jumpcloud/jumpcloud_test.go b/pkg/detectors/jumpcloud/jumpcloud_test.goindex f1db1498b..28958e6e4 100644--- a/pkg/detectors/jumpcloud/jumpcloud_test.go+++ b/pkg/detectors/jumpcloud/jumpcloud_test.go@@ -16,7 +16,7 @@ import (func TestJumpcloud_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/juro/juro_test.go b/pkg/detectors/juro/juro_test.goindex fcc269b6f..4a831cc57 100644--- a/pkg/detectors/juro/juro_test.go+++ b/pkg/detectors/juro/juro_test.go@@ -16,7 +16,7 @@ import (func TestJuro_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/kanban/kanban_test.go b/pkg/detectors/kanban/kanban_test.goindex e8e2f6e26..f2ddecefc 100644--- a/pkg/detectors/kanban/kanban_test.go+++ b/pkg/detectors/kanban/kanban_test.go@@ -16,7 +16,7 @@ import (func TestKanban_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/karmacrm/karmacrm_test.go b/pkg/detectors/karmacrm/karmacrm_test.goindex c638a340e..a70f3dbc4 100644--- a/pkg/detectors/karmacrm/karmacrm_test.go+++ b/pkg/detectors/karmacrm/karmacrm_test.go@@ -16,7 +16,7 @@ import (func TestKarmaCRM_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/keenio/keenio_test.go b/pkg/detectors/keenio/keenio_test.goindex 8f5c68d36..82e0da4d7 100644--- a/pkg/detectors/keenio/keenio_test.go+++ b/pkg/detectors/keenio/keenio_test.go@@ -16,7 +16,7 @@ import (func TestKeenIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/kickbox/kickbox_test.go b/pkg/detectors/kickbox/kickbox_test.goindex f70f27d97..840e7fc9f 100644--- a/pkg/detectors/kickbox/kickbox_test.go+++ b/pkg/detectors/kickbox/kickbox_test.go@@ -16,7 +16,7 @@ import (func TestKickbox_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/kontent/kontent_test.go b/pkg/detectors/kontent/kontent_test.goindex a6106376d..61ad0f498 100644--- a/pkg/detectors/kontent/kontent_test.go+++ b/pkg/detectors/kontent/kontent_test.go@@ -16,7 +16,7 @@ import (func TestKontent_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/kraken/kraken_test.go b/pkg/detectors/kraken/kraken_test.goindex a2df32932..24c96f4d5 100644--- a/pkg/detectors/kraken/kraken_test.go+++ b/pkg/detectors/kraken/kraken_test.go@@ -16,7 +16,7 @@ import (func TestKraken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/kylas/kylas_test.go b/pkg/detectors/kylas/kylas_test.goindex 759353654..c532f8243 100644--- a/pkg/detectors/kylas/kylas_test.go+++ b/pkg/detectors/kylas/kylas_test.go@@ -16,7 +16,7 @@ import (func TestKylas_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/languagelayer/languagelayer_test.go b/pkg/detectors/languagelayer/languagelayer_test.goindex c94c412ea..2151f0ad0 100644--- a/pkg/detectors/languagelayer/languagelayer_test.go+++ b/pkg/detectors/languagelayer/languagelayer_test.go@@ -16,7 +16,7 @@ import (func TestLanguageLayer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/lastfm/lastfm_test.go b/pkg/detectors/lastfm/lastfm_test.goindex 79969d117..1117336a9 100644--- a/pkg/detectors/lastfm/lastfm_test.go+++ b/pkg/detectors/lastfm/lastfm_test.go@@ -16,7 +16,7 @@ import (func TestLastfm_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/launchdarkly/launchdarkly_test.go b/pkg/detectors/launchdarkly/launchdarkly_test.goindex a49d143e6..5189c6ab3 100644--- a/pkg/detectors/launchdarkly/launchdarkly_test.go+++ b/pkg/detectors/launchdarkly/launchdarkly_test.go@@ -16,7 +16,7 @@ import (func TestLaunchDarkly_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/leadfeeder/leadfeeder_test.go b/pkg/detectors/leadfeeder/leadfeeder_test.goindex a4b877b32..7f96c6a7c 100644--- a/pkg/detectors/leadfeeder/leadfeeder_test.go+++ b/pkg/detectors/leadfeeder/leadfeeder_test.go@@ -16,7 +16,7 @@ import (func TestLeadfeeder_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/lessannoyingcrm/lessannoyingcrm_test.go b/pkg/detectors/lessannoyingcrm/lessannoyingcrm_test.goindex 766d6ef7d..b988cdaa0 100644--- a/pkg/detectors/lessannoyingcrm/lessannoyingcrm_test.go+++ b/pkg/detectors/lessannoyingcrm/lessannoyingcrm_test.go@@ -16,7 +16,7 @@ import (func TestLessAnnoyingCRM_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/lexigram/lexigram_test.go b/pkg/detectors/lexigram/lexigram_test.goindex b9a649e63..a19a097d3 100644--- a/pkg/detectors/lexigram/lexigram_test.go+++ b/pkg/detectors/lexigram/lexigram_test.go@@ -16,7 +16,7 @@ import (func TestLexigram_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/linearapi/linearapi_test.go b/pkg/detectors/linearapi/linearapi_test.goindex 3980bd2ea..eebbb517c 100644--- a/pkg/detectors/linearapi/linearapi_test.go+++ b/pkg/detectors/linearapi/linearapi_test.go@@ -16,7 +16,7 @@ import (func TestLinearAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/linemessaging/linemessaging_test.go b/pkg/detectors/linemessaging/linemessaging_test.goindex efad6188c..aa40011ab 100644--- a/pkg/detectors/linemessaging/linemessaging_test.go+++ b/pkg/detectors/linemessaging/linemessaging_test.go@@ -16,7 +16,7 @@ import (func TestLineMessaging_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/linenotify/linenotify_test.go b/pkg/detectors/linenotify/linenotify_test.goindex 2e1583fc8..0e08a75dd 100644--- a/pkg/detectors/linenotify/linenotify_test.go+++ b/pkg/detectors/linenotify/linenotify_test.go@@ -16,7 +16,7 @@ import (func TestLineNotify_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/linkpreview/linkpreview_test.go b/pkg/detectors/linkpreview/linkpreview_test.goindex 9a2fd9dce..76bc42a78 100644--- a/pkg/detectors/linkpreview/linkpreview_test.go+++ b/pkg/detectors/linkpreview/linkpreview_test.go@@ -16,7 +16,7 @@ import (func TestLinkPreview_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/liveagent/liveagent_test.go b/pkg/detectors/liveagent/liveagent_test.goindex 705c15132..14084c4da 100644--- a/pkg/detectors/liveagent/liveagent_test.go+++ b/pkg/detectors/liveagent/liveagent_test.go@@ -16,7 +16,7 @@ import (func TestLiveAgent_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/lob/lob_test.go b/pkg/detectors/lob/lob_test.goindex e65fd9c8d..73cffba11 100644--- a/pkg/detectors/lob/lob_test.go+++ b/pkg/detectors/lob/lob_test.go@@ -16,7 +16,7 @@ import (func TestLob_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/locationiq/locationiq_test.go b/pkg/detectors/locationiq/locationiq_test.goindex 3504a64ab..368279972 100644--- a/pkg/detectors/locationiq/locationiq_test.go+++ b/pkg/detectors/locationiq/locationiq_test.go@@ -16,7 +16,7 @@ import (func TestLocationIQ_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/loginradius/loginradius_test.go b/pkg/detectors/loginradius/loginradius_test.goindex cbfe63956..1113da930 100644--- a/pkg/detectors/loginradius/loginradius_test.go+++ b/pkg/detectors/loginradius/loginradius_test.go@@ -16,7 +16,7 @@ import (func TestLoginradius_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/lokalisetoken/lokalisetoken_test.go b/pkg/detectors/lokalisetoken/lokalisetoken_test.goindex b95ccdd3a..58f0c3a2c 100644--- a/pkg/detectors/lokalisetoken/lokalisetoken_test.go+++ b/pkg/detectors/lokalisetoken/lokalisetoken_test.go@@ -16,7 +16,7 @@ import (func TestLokaliseToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/loyverse/loyverse_test.go b/pkg/detectors/loyverse/loyverse_test.goindex 21e1a41cd..b3d1b8de7 100644--- a/pkg/detectors/loyverse/loyverse_test.go+++ b/pkg/detectors/loyverse/loyverse_test.go@@ -16,7 +16,7 @@ import (func TestLoyverse_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/luno/luno_test.go b/pkg/detectors/luno/luno_test.goindex bd01f3410..4f5c8efc0 100644--- a/pkg/detectors/luno/luno_test.go+++ b/pkg/detectors/luno/luno_test.go@@ -16,7 +16,7 @@ import (func TestLuno_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/m3o/m3o_test.go b/pkg/detectors/m3o/m3o_test.goindex 288d82550..17afe1cc0 100644--- a/pkg/detectors/m3o/m3o_test.go+++ b/pkg/detectors/m3o/m3o_test.go@@ -16,7 +16,7 @@ import (func TestM3o_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/macaddress/macaddress_test.go b/pkg/detectors/macaddress/macaddress_test.goindex 95a1bd20f..2b6884cfc 100644--- a/pkg/detectors/macaddress/macaddress_test.go+++ b/pkg/detectors/macaddress/macaddress_test.go@@ -16,7 +16,7 @@ import (func TestMacaddress_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/magnetic/magnetic_test.go b/pkg/detectors/magnetic/magnetic_test.goindex 0698d5ff9..286417ff6 100644--- a/pkg/detectors/magnetic/magnetic_test.go+++ b/pkg/detectors/magnetic/magnetic_test.go@@ -16,7 +16,7 @@ import (func TestMagnetic_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mailboxlayer/mailboxlayer_test.go b/pkg/detectors/mailboxlayer/mailboxlayer_test.goindex 56ecb1f73..80c748512 100644--- a/pkg/detectors/mailboxlayer/mailboxlayer_test.go+++ b/pkg/detectors/mailboxlayer/mailboxlayer_test.go@@ -16,7 +16,7 @@ import (func TestMailboxplayer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mailchimp/mailchimp.go b/pkg/detectors/mailchimp/mailchimp.gonew file mode 100644index 000000000..2e1ec6131--- /dev/null+++ b/pkg/detectors/mailchimp/mailchimp.go@@ -0,0 +1,76 @@+package mailchimp++import (+ "context"+ "fmt"+ "net/http"+ "regexp"+ "strings"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ // TODO: Other country patterns?+ keyPat = regexp.MustCompile(`[0-9a-f]{32}-us[0-9]{1,2}`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"-us"}+}++// FromData will find and optionally verify Mailchimp secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ //pretty standard regex match+ matches := keyPat.FindAllString(dataStr, -1)++ for _, match := range matches {++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Mailchimp,+ Raw: []byte(match),+ }++ if verify {+ datacenter := strings.Split(match, "-")[1]++ client := common.RetryableHttpClient()+ // https://mailchimp.com/developer/guides/marketing-api-conventions/+ req, _ := http.NewRequest("GET", fmt.Sprintf("https://%s.api.mailchimp.com/3.0/", datacenter), nil)+ req.SetBasicAuth("anystring", match)+ res, err := client.Do(req)+ if err != nil {+ break+ }+ defer res.Body.Close()+ if res.StatusCode == 200 {+ s.Verified = true+ } else {+ s.Verified = false+ }++ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/mailchimp/mailchimp_test.go b/pkg/detectors/mailchimp/mailchimp_test.gonew file mode 100644index 000000000..5bd0e9aa6--- /dev/null+++ b/pkg/detectors/mailchimp/mailchimp_test.go@@ -0,0 +1,111 @@+package mailchimp++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestMailchimp_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("MAILCHIMP")+ secretInactive := testSecrets.MustGetField("MAILCHIMP_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a mailchimp secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Mailchimp,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a mailchimp secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Mailchimp,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Mailchimp.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Mailchimp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/mailerlite/mailerlite_test.go b/pkg/detectors/mailerlite/mailerlite_test.goindex e6157cfa9..ee9fa8ba9 100644--- a/pkg/detectors/mailerlite/mailerlite_test.go+++ b/pkg/detectors/mailerlite/mailerlite_test.go@@ -16,7 +16,7 @@ import (func TestMailerlite_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mailgun/mailgun_test.go b/pkg/detectors/mailgun/mailgun_test.goindex 1a6a95d57..6d105f13b 100644--- a/pkg/detectors/mailgun/mailgun_test.go+++ b/pkg/detectors/mailgun/mailgun_test.go@@ -18,7 +18,7 @@ import (func TestMailgun_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mailjetbasicauth/mailjetbasicauth_test.go b/pkg/detectors/mailjetbasicauth/mailjetbasicauth_test.goindex 693d25ab9..f1ed0d68e 100644--- a/pkg/detectors/mailjetbasicauth/mailjetbasicauth_test.go+++ b/pkg/detectors/mailjetbasicauth/mailjetbasicauth_test.go@@ -16,7 +16,7 @@ import (func TestMailJetBasicAuth_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mailjetsms/mailjetsms_test.go b/pkg/detectors/mailjetsms/mailjetsms_test.goindex 4aacce733..944b50480 100644--- a/pkg/detectors/mailjetsms/mailjetsms_test.go+++ b/pkg/detectors/mailjetsms/mailjetsms_test.go@@ -16,7 +16,7 @@ import (func TestMailJetSMS_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mailmodo/mailmodo_test.go b/pkg/detectors/mailmodo/mailmodo_test.goindex b76555b54..e3b3123f1 100644--- a/pkg/detectors/mailmodo/mailmodo_test.go+++ b/pkg/detectors/mailmodo/mailmodo_test.go@@ -16,7 +16,7 @@ import (func TestMailmodo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mailsac/mailsac_test.go b/pkg/detectors/mailsac/mailsac_test.goindex 8863d27a0..94c818958 100644--- a/pkg/detectors/mailsac/mailsac_test.go+++ b/pkg/detectors/mailsac/mailsac_test.go@@ -16,7 +16,7 @@ import (func TestMailsac_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mandrill/mandrill_test.go b/pkg/detectors/mandrill/mandrill_test.goindex 397ce58ea..43da79f76 100644--- a/pkg/detectors/mandrill/mandrill_test.go+++ b/pkg/detectors/mandrill/mandrill_test.go@@ -16,7 +16,7 @@ import (func TestMandrill_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/manifest/manifest_test.go b/pkg/detectors/manifest/manifest_test.goindex 258a752d8..95c920445 100644--- a/pkg/detectors/manifest/manifest_test.go+++ b/pkg/detectors/manifest/manifest_test.go@@ -16,7 +16,7 @@ import (func TestManifest_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mapbox/mapbox_test.go b/pkg/detectors/mapbox/mapbox_test.goindex 153f8825e..8a2c4bea8 100644--- a/pkg/detectors/mapbox/mapbox_test.go+++ b/pkg/detectors/mapbox/mapbox_test.go@@ -16,7 +16,7 @@ import (func TestMapBox_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mapquest/mapquest_test.go b/pkg/detectors/mapquest/mapquest_test.goindex fe1f3d83a..395b1ae40 100644--- a/pkg/detectors/mapquest/mapquest_test.go+++ b/pkg/detectors/mapquest/mapquest_test.go@@ -16,7 +16,7 @@ import (func TestMapquest_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/marketstack/marketstack_test.go b/pkg/detectors/marketstack/marketstack_test.goindex 44bab99f3..9b6babb68 100644--- a/pkg/detectors/marketstack/marketstack_test.go+++ b/pkg/detectors/marketstack/marketstack_test.go@@ -16,7 +16,7 @@ import (func TestMarketstack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken_test.go b/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken_test.goindex e961d410c..5704d3d4a 100644--- a/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken_test.go+++ b/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken_test.go@@ -16,7 +16,7 @@ import (func TestMattermostPersonalToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mavenlink/mavenlink_test.go b/pkg/detectors/mavenlink/mavenlink_test.goindex e473f1ff3..8d4fc97f9 100644--- a/pkg/detectors/mavenlink/mavenlink_test.go+++ b/pkg/detectors/mavenlink/mavenlink_test.go@@ -16,7 +16,7 @@ import (func TestMavenlink_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/maxmindlicense/maxmindlicense_test.go b/pkg/detectors/maxmindlicense/maxmindlicense_test.goindex 995a051ef..e8af4c92a 100644--- a/pkg/detectors/maxmindlicense/maxmindlicense_test.go+++ b/pkg/detectors/maxmindlicense/maxmindlicense_test.go@@ -16,7 +16,7 @@ import (func TestMaxMindLicense_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/meaningcloud/meaningcloud_test.go b/pkg/detectors/meaningcloud/meaningcloud_test.goindex 31b076735..6f8105b5a 100644--- a/pkg/detectors/meaningcloud/meaningcloud_test.go+++ b/pkg/detectors/meaningcloud/meaningcloud_test.go@@ -16,7 +16,7 @@ import (func TestMeaningCloud_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mediastack/mediastack_test.go b/pkg/detectors/mediastack/mediastack_test.goindex 0ae98f1ed..81fa09cb0 100644--- a/pkg/detectors/mediastack/mediastack_test.go+++ b/pkg/detectors/mediastack/mediastack_test.go@@ -16,7 +16,7 @@ import (func TestMediaStack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/meistertask/meistertask_test.go b/pkg/detectors/meistertask/meistertask_test.goindex c537e64b6..e7f71ecc7 100644--- a/pkg/detectors/meistertask/meistertask_test.go+++ b/pkg/detectors/meistertask/meistertask_test.go@@ -16,7 +16,7 @@ import (func TestMeistertask_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mesibo/mesibo_test.go b/pkg/detectors/mesibo/mesibo_test.goindex 2f33c22db..59097868c 100644--- a/pkg/detectors/mesibo/mesibo_test.go+++ b/pkg/detectors/mesibo/mesibo_test.go@@ -16,7 +16,7 @@ import (func TestMesibo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/messagebird/messagebird_test.go b/pkg/detectors/messagebird/messagebird_test.goindex 0e2467634..928eac459 100644--- a/pkg/detectors/messagebird/messagebird_test.go+++ b/pkg/detectors/messagebird/messagebird_test.go@@ -16,7 +16,7 @@ import (func TestMessageBird_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/metrilo/metrilo_test.go b/pkg/detectors/metrilo/metrilo_test.goindex 6bcf086b6..cbaf30194 100644--- a/pkg/detectors/metrilo/metrilo_test.go+++ b/pkg/detectors/metrilo/metrilo_test.go@@ -16,7 +16,7 @@ import (func TestMetrilo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_test.go b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_test.goindex db4b53c6a..79ea4dbd5 100644--- a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_test.go+++ b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_test.go@@ -16,7 +16,7 @@ import (func TestMicrosoftTeamsWebhook_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mindmeister/mindmeister_test.go b/pkg/detectors/mindmeister/mindmeister_test.goindex ce66487a5..74db9627c 100644--- a/pkg/detectors/mindmeister/mindmeister_test.go+++ b/pkg/detectors/mindmeister/mindmeister_test.go@@ -16,7 +16,7 @@ import (func TestMindmeister_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mite/mite_test.go b/pkg/detectors/mite/mite_test.goindex 836b0f9bb..a4959bf11 100644--- a/pkg/detectors/mite/mite_test.go+++ b/pkg/detectors/mite/mite_test.go@@ -16,7 +16,7 @@ import (func TestMite_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mixmax/mixmax_test.go b/pkg/detectors/mixmax/mixmax_test.goindex 065da4dce..a97f1cd67 100644--- a/pkg/detectors/mixmax/mixmax_test.go+++ b/pkg/detectors/mixmax/mixmax_test.go@@ -16,7 +16,7 @@ import (func TestMixmax_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mixpanel/mixpanel_test.go b/pkg/detectors/mixpanel/mixpanel_test.goindex 3c496849d..e70c56e3e 100644--- a/pkg/detectors/mixpanel/mixpanel_test.go+++ b/pkg/detectors/mixpanel/mixpanel_test.go@@ -16,7 +16,7 @@ import (func TestMixpanel_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/monday/monday_test.go b/pkg/detectors/monday/monday_test.goindex 7c0d91600..b12e425eb 100644--- a/pkg/detectors/monday/monday_test.go+++ b/pkg/detectors/monday/monday_test.go@@ -16,7 +16,7 @@ import (func TestMonday_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/moonclerk/moonclerk.go b/pkg/detectors/moonclerk/moonclerk.gonew file mode 100644index 000000000..07876807b--- /dev/null+++ b/pkg/detectors/moonclerk/moonclerk.go@@ -0,0 +1,74 @@+package moonclerk++import (+ "context"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"moonclerk"}) + `\b([0-9a-z]{32})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"moonclerk"}+}++// FromData will find and optionally verify Moonclerk secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_Moonclerk,+ Raw: []byte(resMatch),+ }++ if verify {+ req, _ := http.NewRequest("GET", "https://api.moonclerk.com/forms", nil)+ req.Header.Add("Accept", "application/vnd.moonclerk+json;version=1")+ req.Header.Add("Authorization", "Token token="+resMatch)+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/moonclerk/moonclerk_test.go b/pkg/detectors/moonclerk/moonclerk_test.gonew file mode 100644index 000000000..9a6caaffc--- /dev/null+++ b/pkg/detectors/moonclerk/moonclerk_test.go@@ -0,0 +1,113 @@+package moonclerk++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestMoonclerk_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("MOONCLERCK")+ inactiveSecret := testSecrets.MustGetField("MOONCLERCK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a moonclerk secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Moonclerk,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a moonclerk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Moonclerk,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Moonclerk.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Moonclerk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/moosend/moosend_test.go b/pkg/detectors/moosend/moosend_test.goindex c1a6b4a54..b8d1f43b0 100644--- a/pkg/detectors/moosend/moosend_test.go+++ b/pkg/detectors/moosend/moosend_test.go@@ -16,7 +16,7 @@ import (func TestMoosend_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/mrticktock/mrticktock_test.go b/pkg/detectors/mrticktock/mrticktock_test.goindex 1bd1fe064..4d3e1ed40 100644--- a/pkg/detectors/mrticktock/mrticktock_test.go+++ b/pkg/detectors/mrticktock/mrticktock_test.go@@ -16,7 +16,7 @@ import (func TestMrticktock_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/myfreshworks/myfreshworks_test.go b/pkg/detectors/myfreshworks/myfreshworks_test.goindex 7f4b162fe..20d0a3479 100644--- a/pkg/detectors/myfreshworks/myfreshworks_test.go+++ b/pkg/detectors/myfreshworks/myfreshworks_test.go@@ -16,7 +16,7 @@ import (func TestMyfreshworks_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/myintervals/myintervals_test.go b/pkg/detectors/myintervals/myintervals_test.goindex e1fd16a6f..6c560c3f2 100644--- a/pkg/detectors/myintervals/myintervals_test.go+++ b/pkg/detectors/myintervals/myintervals_test.go@@ -16,7 +16,7 @@ import (func TestMyintervals_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/nasdaqdatalink/nasdaqdatalink_test.go b/pkg/detectors/nasdaqdatalink/nasdaqdatalink_test.goindex e53973f70..cad4ab278 100644--- a/pkg/detectors/nasdaqdatalink/nasdaqdatalink_test.go+++ b/pkg/detectors/nasdaqdatalink/nasdaqdatalink_test.go@@ -16,7 +16,7 @@ import (func TestNasdaqDataLink_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/nethunt/nethunt_test.go b/pkg/detectors/nethunt/nethunt_test.goindex 1d2182cc3..efc864dc2 100644--- a/pkg/detectors/nethunt/nethunt_test.go+++ b/pkg/detectors/nethunt/nethunt_test.go@@ -16,7 +16,7 @@ import (func TestNethunt_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/netlify/netlify_test.go b/pkg/detectors/netlify/netlify_test.goindex 8b8c535aa..2366eb43e 100644--- a/pkg/detectors/netlify/netlify_test.go+++ b/pkg/detectors/netlify/netlify_test.go@@ -16,7 +16,7 @@ import (func TestNetlify_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/neutrinoapi/neutrinoapi_test.go b/pkg/detectors/neutrinoapi/neutrinoapi_test.goindex 07de39413..bd2f01925 100644--- a/pkg/detectors/neutrinoapi/neutrinoapi_test.go+++ b/pkg/detectors/neutrinoapi/neutrinoapi_test.go@@ -16,7 +16,7 @@ import (func TestNeutrinoApi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey_test.go b/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey_test.goindex 2c1f1a24b..a61e1719c 100644--- a/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey_test.go+++ b/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey_test.go@@ -16,7 +16,7 @@ import (func TestNewRelicPersonalApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/newsapi/newsapi_test.go b/pkg/detectors/newsapi/newsapi_test.goindex 21bec5b44..19b00a2bc 100644--- a/pkg/detectors/newsapi/newsapi_test.go+++ b/pkg/detectors/newsapi/newsapi_test.go@@ -16,7 +16,7 @@ import (func TestNewsapi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/newscatcher/newscatcher_test.go b/pkg/detectors/newscatcher/newscatcher_test.goindex 243a9b663..0f1acfbfb 100644--- a/pkg/detectors/newscatcher/newscatcher_test.go+++ b/pkg/detectors/newscatcher/newscatcher_test.go@@ -16,7 +16,7 @@ import (func TestNewscatcher_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/nexmoapikey/nexmoapikey_test.go b/pkg/detectors/nexmoapikey/nexmoapikey_test.goindex 1bebafc96..1cb369214 100644--- a/pkg/detectors/nexmoapikey/nexmoapikey_test.go+++ b/pkg/detectors/nexmoapikey/nexmoapikey_test.go@@ -16,7 +16,7 @@ import (func TestNexmoApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/nftport/nftport_test.go b/pkg/detectors/nftport/nftport_test.goindex f2e12a779..e2f7e3ae6 100644--- a/pkg/detectors/nftport/nftport_test.go+++ b/pkg/detectors/nftport/nftport_test.go@@ -16,7 +16,7 @@ import (func TestNftport_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/nicereply/nicereply_test.go b/pkg/detectors/nicereply/nicereply_test.goindex 35bdd9090..0a74f233d 100644--- a/pkg/detectors/nicereply/nicereply_test.go+++ b/pkg/detectors/nicereply/nicereply_test.go@@ -16,7 +16,7 @@ import (func TestNicereply_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/nimble/nimble_test.go b/pkg/detectors/nimble/nimble_test.goindex 0b68ed82d..b14f0a944 100644--- a/pkg/detectors/nimble/nimble_test.go+++ b/pkg/detectors/nimble/nimble_test.go@@ -16,7 +16,7 @@ import (func TestNimble_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/nitro/nitro_test.go b/pkg/detectors/nitro/nitro_test.goindex b6b891867..40b714577 100644--- a/pkg/detectors/nitro/nitro_test.go+++ b/pkg/detectors/nitro/nitro_test.go@@ -16,7 +16,7 @@ import (func TestNitro_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/noticeable/noticeable_test.go b/pkg/detectors/noticeable/noticeable_test.goindex 6c4fe8f37..5e2f03810 100644--- a/pkg/detectors/noticeable/noticeable_test.go+++ b/pkg/detectors/noticeable/noticeable_test.go@@ -16,7 +16,7 @@ import (func TestNoticeable_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/notion/notion_test.go b/pkg/detectors/notion/notion_test.goindex deb1ae217..1818796f5 100644--- a/pkg/detectors/notion/notion_test.go+++ b/pkg/detectors/notion/notion_test.go@@ -16,7 +16,7 @@ import (func TestNotion_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/numverify/numverify_test.go b/pkg/detectors/numverify/numverify_test.goindex c73d916a9..3f1d4ee8a 100644--- a/pkg/detectors/numverify/numverify_test.go+++ b/pkg/detectors/numverify/numverify_test.go@@ -16,7 +16,7 @@ import (func TestNumverify_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/nutritionix/nutritionix_test.go b/pkg/detectors/nutritionix/nutritionix_test.goindex ddb49d9ff..fb280086d 100644--- a/pkg/detectors/nutritionix/nutritionix_test.go+++ b/pkg/detectors/nutritionix/nutritionix_test.go@@ -16,7 +16,7 @@ import (func TestNutritionix_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/nylas/nylas_test.go b/pkg/detectors/nylas/nylas_test.goindex 439ea16f7..c101e9811 100644--- a/pkg/detectors/nylas/nylas_test.go+++ b/pkg/detectors/nylas/nylas_test.go@@ -16,7 +16,7 @@ import (func TestNylas_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/nytimes/nytimes_test.go b/pkg/detectors/nytimes/nytimes_test.goindex 8e789d18d..85f12de32 100644--- a/pkg/detectors/nytimes/nytimes_test.go+++ b/pkg/detectors/nytimes/nytimes_test.go@@ -16,7 +16,7 @@ import (func TestNytimes_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/oanda/oanda_test.go b/pkg/detectors/oanda/oanda_test.goindex 31c15e9c4..656768d2c 100644--- a/pkg/detectors/oanda/oanda_test.go+++ b/pkg/detectors/oanda/oanda_test.go@@ -16,7 +16,7 @@ import (func TestOanda_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.gonew file mode 100644index 000000000..625592b95--- /dev/null+++ b/pkg/detectors/okta/okta.go@@ -0,0 +1,82 @@+package okta++import (+ "context"+ "fmt"+ "net/http"+ "regexp"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)+ tokenPat = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)+ // TODO: Oauth client secrets+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"okta"}+}++// FromData will find and optionally verify Okta secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ for _, tokenMatch := range tokenPat.FindAll(data, -1) {+ token := string(tokenMatch)++ for _, domainMatch := range domainPat.FindAll(data, -1) {+ domain := string(domainMatch)++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Okta,+ Raw: []byte(token),+ }++ if verify {+ // curl -v -X GET \+ // -H "Accept: application/json" \+ // -H "Content-Type: application/json" \+ // -H "Authorization: Bearer token" \+ // "https://subdomain.okta.com/api/v1/groups?limit=1"+ //++ url := fmt.Sprintf("https://%s/api/v1/groups?limit=1", domain)+ req, err := http.NewRequest("GET", url, nil)+ if err != nil {+ return results, err+ }+ req.Header.Set("Accept", "application/json")+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Authorization", fmt.Sprintf("SSWS %s", token))++ resp, err := common.RetryableHttpClient().Do(req)+ if err != nil {+ return results, err+ }+ defer resp.Body.Close()+ if resp.StatusCode >= 200 && resp.StatusCode < 300 {+ s.Verified = true+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }+ }++ return+}diff --git a/pkg/detectors/okta/okta_test.go b/pkg/detectors/okta/okta_test.gonew file mode 100644index 000000000..e52970d7a--- /dev/null+++ b/pkg/detectors/okta/okta_test.go@@ -0,0 +1,113 @@+package okta++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestOkta_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("OKTA")+ secretInactive := testSecrets.MustGetField("OKTA_INACTIVE")+ domain := testSecrets.MustGetField("OKTA_DOMAIN")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found token, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a okta secret %s within oktaDomain %s", secret, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Okta,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a okta secret %s within oktaDomain %s", secretInactive, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Okta,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Okta.FromData) error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Okta.FromData) %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/omnisend/omnisend_test.go b/pkg/detectors/omnisend/omnisend_test.goindex 877bfa2a7..5d42751f1 100644--- a/pkg/detectors/omnisend/omnisend_test.go+++ b/pkg/detectors/omnisend/omnisend_test.go@@ -16,7 +16,7 @@ import (func TestOmnisend_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/onedesk/onedesk_test.go b/pkg/detectors/onedesk/onedesk_test.goindex 1eaabe729..e077b8ea2 100644--- a/pkg/detectors/onedesk/onedesk_test.go+++ b/pkg/detectors/onedesk/onedesk_test.go@@ -16,7 +16,7 @@ import (func TestOnedesk_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/onelogin/onelogin.go b/pkg/detectors/onelogin/onelogin.gonew file mode 100644index 000000000..87d8a2edc--- /dev/null+++ b/pkg/detectors/onelogin/onelogin.go@@ -0,0 +1,87 @@+package onelogin++import (+ "context"+ "fmt"+ "net/http"+ "regexp"+ "strings"+ "time"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ oauthClientIDPat = regexp.MustCompile(`(?i)id[a-zA-Z0-9_' "=]{0,20}([a-z0-9]{64})`)+ oauthClientSecretPat = regexp.MustCompile(`(?i)secret[a-zA-Z0-9_' "=]{0,20}([a-z0-9]{64})`)++ // TODO: Legacy API tokens++ falsePositives = []string{"example"}++ apiDomains = []string{"api.us.onelogin.com", "api.eu.onelogin.com"}++ client = http.Client{Timeout: time.Second * 5}+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"onelogin"}+}++// FromData will find and optionally verify Onelogin secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ for _, clientID := range oauthClientIDPat.FindAllStringSubmatch(dataStr, -1) {+ if len(clientID) != 2 {+ continue+ }+ for _, clientSecret := range oauthClientSecretPat.FindAllStringSubmatch(dataStr, -1) {+ if len(clientSecret) != 2 {+ continue+ }++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_OneLogin,+ Raw: []byte(clientID[1]),+ Redacted: clientID[1],+ }++ if verify {+ for _, domain := range apiDomains {+ tokenURL := fmt.Sprintf("https://%s/auth/oauth2/v2/token", domain)+ req, _ := http.NewRequestWithContext(ctx, "POST", tokenURL, strings.NewReader(`{"grant_type":"client_credentials"}`))+ req.Header.Add("Authorization", fmt.Sprintf("client_id:%s, client_secret:%s", clientID[1], clientSecret[1]))+ req.Header.Add("Content-Type", "application/json; charset=utf-8")+ res, err := client.Do(req)+ if err != nil {+ return results, err+ }+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s.Verified = true+ break+ }+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }+ }++ return+}diff --git a/pkg/detectors/onelogin/onelogin_test.go b/pkg/detectors/onelogin/onelogin_test.gonew file mode 100644index 000000000..2c4d391db--- /dev/null+++ b/pkg/detectors/onelogin/onelogin_test.go@@ -0,0 +1,112 @@+package onelogin++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestOnelogin_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ONELOGIN")+ secretInactive := testSecrets.MustGetField("ONELOGIN_INACTIVE")+ id := testSecrets.MustGetField("ONELOGIN_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a onelogin secret=%s within onelogin id %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_OneLogin,+ Verified: true,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a onelogin secret=%s within onelogin id %s", secretInactive, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_OneLogin,+ Verified: false,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Onelogin.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Onelogin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/onepagecrm/onepagecrm_test.go b/pkg/detectors/onepagecrm/onepagecrm_test.goindex 119844888..055f1b8e6 100644--- a/pkg/detectors/onepagecrm/onepagecrm_test.go+++ b/pkg/detectors/onepagecrm/onepagecrm_test.go@@ -16,7 +16,7 @@ import (func TestOnepageCRM_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/onwaterio/onwaterio_test.go b/pkg/detectors/onwaterio/onwaterio_test.goindex 3b420d55d..543615109 100644--- a/pkg/detectors/onwaterio/onwaterio_test.go+++ b/pkg/detectors/onwaterio/onwaterio_test.go@@ -16,7 +16,7 @@ import (func TestOnWaterIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/oopspam/oopspam_test.go b/pkg/detectors/oopspam/oopspam_test.goindex 1b5b75ece..d427fbc62 100644--- a/pkg/detectors/oopspam/oopspam_test.go+++ b/pkg/detectors/oopspam/oopspam_test.go@@ -16,7 +16,7 @@ import (func TestOOPSpam_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/opencagedata/opencagedata_test.go b/pkg/detectors/opencagedata/opencagedata_test.goindex a7db94e70..01b9e72f4 100644--- a/pkg/detectors/opencagedata/opencagedata_test.go+++ b/pkg/detectors/opencagedata/opencagedata_test.go@@ -16,7 +16,7 @@ import (func TestOpenCageData_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/opengraphr/opengraphr_test.go b/pkg/detectors/opengraphr/opengraphr_test.goindex ef47ca901..6e2104a75 100644--- a/pkg/detectors/opengraphr/opengraphr_test.go+++ b/pkg/detectors/opengraphr/opengraphr_test.go@@ -16,7 +16,7 @@ import (func TestOpengraphr_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/openuv/openuv_test.go b/pkg/detectors/openuv/openuv_test.goindex 0b997af30..e431805c5 100644--- a/pkg/detectors/openuv/openuv_test.go+++ b/pkg/detectors/openuv/openuv_test.go@@ -16,7 +16,7 @@ import (func TestOpenuv_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/openweather/openweather_test.go b/pkg/detectors/openweather/openweather_test.goindex 0ca781cb9..63428a00c 100644--- a/pkg/detectors/openweather/openweather_test.go+++ b/pkg/detectors/openweather/openweather_test.go@@ -16,7 +16,7 @@ import (func TestOpenWeather_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/optimizely/optimizely_test.go b/pkg/detectors/optimizely/optimizely_test.goindex 782c49315..84973f295 100644--- a/pkg/detectors/optimizely/optimizely_test.go+++ b/pkg/detectors/optimizely/optimizely_test.go@@ -16,7 +16,7 @@ import (func TestOptimizely_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/owlbot/owlbot_test.go b/pkg/detectors/owlbot/owlbot_test.goindex 99e9ca997..212ee321f 100644--- a/pkg/detectors/owlbot/owlbot_test.go+++ b/pkg/detectors/owlbot/owlbot_test.go@@ -16,7 +16,7 @@ import (func TestOwlbot_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.goindex 21f5d0128..83d8195d0 100644--- a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go+++ b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go@@ -16,7 +16,7 @@ import (func TestPagerDutyApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pandadoc/pandadoc_test.go b/pkg/detectors/pandadoc/pandadoc_test.goindex c3d1ec952..5b6fefd8a 100644--- a/pkg/detectors/pandadoc/pandadoc_test.go+++ b/pkg/detectors/pandadoc/pandadoc_test.go@@ -16,7 +16,7 @@ import (func TestPandadoc_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/paralleldots/paralleldots_test.go b/pkg/detectors/paralleldots/paralleldots_test.goindex 966ce6a42..8f826759d 100644--- a/pkg/detectors/paralleldots/paralleldots_test.go+++ b/pkg/detectors/paralleldots/paralleldots_test.go@@ -16,7 +16,7 @@ import (func TestParalleldots_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/partnerstack/partnerstack_test.go b/pkg/detectors/partnerstack/partnerstack_test.goindex 363984bbd..c2790b2e6 100644--- a/pkg/detectors/partnerstack/partnerstack_test.go+++ b/pkg/detectors/partnerstack/partnerstack_test.go@@ -16,7 +16,7 @@ import (func TestPartnerstack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/passbase/passbase_test.go b/pkg/detectors/passbase/passbase_test.goindex 9ed58a0ef..4ff11b18a 100644--- a/pkg/detectors/passbase/passbase_test.go+++ b/pkg/detectors/passbase/passbase_test.go@@ -16,7 +16,7 @@ import (func TestPassbase_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pastebin/pastebin_test.go b/pkg/detectors/pastebin/pastebin_test.goindex 6479ede7f..3a44f913a 100644--- a/pkg/detectors/pastebin/pastebin_test.go+++ b/pkg/detectors/pastebin/pastebin_test.go@@ -16,7 +16,7 @@ import (func TestPastebin_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/paymoapp/paymoapp_test.go b/pkg/detectors/paymoapp/paymoapp_test.goindex f306aa457..f1ed16c7e 100644--- a/pkg/detectors/paymoapp/paymoapp_test.go+++ b/pkg/detectors/paymoapp/paymoapp_test.go@@ -16,7 +16,7 @@ import (func TestPaymoapp_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/paymongo/paymongo.go b/pkg/detectors/paymongo/paymongo.gonew file mode 100644index 000000000..c6cc92ed6--- /dev/null+++ b/pkg/detectors/paymongo/paymongo.go@@ -0,0 +1,75 @@+package paymongo++import (+ "context"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"paymongo"}) + `\b([a-zA-Z0-9_]{32})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"paymongo"}+}++// FromData will find and optionally verify Paymongo secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_Paymongo,+ Raw: []byte(resMatch),+ }++ if verify {+ payload := strings.NewReader(`{"data":{"attributes":{"type":"paymaya"}}}`)+ req, _ := http.NewRequest("POST", "https://api.paymongo.com/v1/payment_methods", payload)+ req.Header.Add("Content-Type", "application/json")+ req.SetBasicAuth(resMatch, "")+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/paymongo/paymongo_test.go b/pkg/detectors/paymongo/paymongo_test.gonew file mode 100644index 000000000..c607e11ed--- /dev/null+++ b/pkg/detectors/paymongo/paymongo_test.go@@ -0,0 +1,113 @@+package paymongo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestPaymongo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PAYMONGO")+ inactiveSecret := testSecrets.MustGetField("PAYMONGO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paymongo secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paymongo,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paymongo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paymongo,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Paymongo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Paymongo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/paypaloauth/paypaloauth_test.go b/pkg/detectors/paypaloauth/paypaloauth_test.goindex ee8704333..468815ca6 100644--- a/pkg/detectors/paypaloauth/paypaloauth_test.go+++ b/pkg/detectors/paypaloauth/paypaloauth_test.go@@ -16,7 +16,7 @@ import (func TestPaypalOauth_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/paystack/paystack_test.go b/pkg/detectors/paystack/paystack_test.goindex de08aacd2..beded3b50 100644--- a/pkg/detectors/paystack/paystack_test.go+++ b/pkg/detectors/paystack/paystack_test.go@@ -16,7 +16,7 @@ import (func TestPaystack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pdflayer/pdflayer_test.go b/pkg/detectors/pdflayer/pdflayer_test.goindex cbbf9d3eb..100f40d64 100644--- a/pkg/detectors/pdflayer/pdflayer_test.go+++ b/pkg/detectors/pdflayer/pdflayer_test.go@@ -16,7 +16,7 @@ import (func TestPdfLayer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pdfshift/pdfshift_test.go b/pkg/detectors/pdfshift/pdfshift_test.goindex 009fdbe16..c56923401 100644--- a/pkg/detectors/pdfshift/pdfshift_test.go+++ b/pkg/detectors/pdfshift/pdfshift_test.go@@ -16,7 +16,7 @@ import (func TestPdfShift_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/peopledatalabs/peopledatalabs_test.go b/pkg/detectors/peopledatalabs/peopledatalabs_test.goindex a43a37e12..fe96d3d2f 100644--- a/pkg/detectors/peopledatalabs/peopledatalabs_test.go+++ b/pkg/detectors/peopledatalabs/peopledatalabs_test.go@@ -16,7 +16,7 @@ import (func TestPeopleDataLabs_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pepipost/pepipost_test.go b/pkg/detectors/pepipost/pepipost_test.goindex 5541196fd..5d77261b2 100644--- a/pkg/detectors/pepipost/pepipost_test.go+++ b/pkg/detectors/pepipost/pepipost_test.go@@ -16,7 +16,7 @@ import (func TestPepipost_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pipedrive/pipedrive_test.go b/pkg/detectors/pipedrive/pipedrive_test.goindex 090cb8a39..eca4fe0de 100644--- a/pkg/detectors/pipedrive/pipedrive_test.go+++ b/pkg/detectors/pipedrive/pipedrive_test.go@@ -16,7 +16,7 @@ import (func TestPipedrive_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pivotaltracker/pivotaltracker.go b/pkg/detectors/pivotaltracker/pivotaltracker.gonew file mode 100644index 000000000..6a86c268f--- /dev/null+++ b/pkg/detectors/pivotaltracker/pivotaltracker.go@@ -0,0 +1,77 @@+package pivotaltracker++import (+ "context"+ "net/http"+ "regexp"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ // Get token at https://www.pivotaltracker.com/profile+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pivotal"}) + `([a-z0-9]{32})`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"pivotal"}+}++// FromData will find and optionally verify PivotalTracker secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {++ // First match is entire regex, second is the first group.+ if len(match) != 2 {+ continue+ }++ token := match[1]++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_PivotalTracker,+ Raw: []byte(token),+ }++ if verify {+ client := common.RetryableHttpClient()+ // https://www.pivotaltracker.com/help/api/rest/v5#top+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://www.pivotaltracker.com/services/v5/me", nil)+ req.Header.Add("Content-Type", "application/json; charset=utf-8")+ req.Header.Add("X-TrackerToken", token)+ res, err := client.Do(req)+ if err != nil {+ break+ }+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s.Verified = true+ }++ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/pivotaltracker/pivotaltracker_test.go b/pkg/detectors/pivotaltracker/pivotaltracker_test.gonew file mode 100644index 000000000..dd2c9f50c--- /dev/null+++ b/pkg/detectors/pivotaltracker/pivotaltracker_test.go@@ -0,0 +1,112 @@+package pivotaltracker++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestPivotalTracker_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PIVOTALTRACKER")+ secretInactive := testSecrets.MustGetField("PIVOTALTRACKER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PivotalTracker,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PivotalTracker,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PivotalTracker.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PivotalTracker.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/pixabay/pixabay_test.go b/pkg/detectors/pixabay/pixabay_test.goindex 293f6c5f4..185ae0600 100644--- a/pkg/detectors/pixabay/pixabay_test.go+++ b/pkg/detectors/pixabay/pixabay_test.go@@ -16,7 +16,7 @@ import (func TestPixabay_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/plaidkey/plaidkey_test.go b/pkg/detectors/plaidkey/plaidkey_test.goindex 2e4654563..1904ff0b7 100644--- a/pkg/detectors/plaidkey/plaidkey_test.go+++ b/pkg/detectors/plaidkey/plaidkey_test.go@@ -16,7 +16,7 @@ import (func TestPlaidKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/planyo/planyo_test.go b/pkg/detectors/planyo/planyo_test.goindex 1dd4b3475..096ae1c15 100644--- a/pkg/detectors/planyo/planyo_test.go+++ b/pkg/detectors/planyo/planyo_test.go@@ -16,7 +16,7 @@ import (func TestPlanyo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/plivo/plivo_test.go b/pkg/detectors/plivo/plivo_test.goindex a669ff186..c62d0034c 100644--- a/pkg/detectors/plivo/plivo_test.go+++ b/pkg/detectors/plivo/plivo_test.go@@ -16,7 +16,7 @@ import (func TestPlivo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/poloniex/poloniex_test.go b/pkg/detectors/poloniex/poloniex_test.goindex 28886711d..6068e53c4 100644--- a/pkg/detectors/poloniex/poloniex_test.go+++ b/pkg/detectors/poloniex/poloniex_test.go@@ -16,7 +16,7 @@ import (func TestPoloniex_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/polygon/polygon.go b/pkg/detectors/polygon/polygon.gonew file mode 100644index 000000000..63c251806--- /dev/null+++ b/pkg/detectors/polygon/polygon.go@@ -0,0 +1,73 @@+package polygon++import (+ "context"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"polygon"}) + `\b([a-z0-9A-Z]{32})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"polygon"}+}++// FromData will find and optionally verify Polygon secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_Polygon,+ Raw: []byte(resMatch),+ }++ if verify {+ req, _ := http.NewRequest("GET", "https://api.polygon.io/v2/reference/locales?apiKey="+resMatch, nil)++ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/polygon/polygon_test.go b/pkg/detectors/polygon/polygon_test.gonew file mode 100644index 000000000..73d5df286--- /dev/null+++ b/pkg/detectors/polygon/polygon_test.go@@ -0,0 +1,113 @@+package polygon++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestPolygon_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("POLYGON_TOKEN")+ inactiveSecret := testSecrets.MustGetField("POLYGON_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a polygon secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Polygon,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a polygon secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Polygon,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Polygon.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Polygon.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/positionstack/positionstack_test.go b/pkg/detectors/positionstack/positionstack_test.goindex 9de12b5c9..be5edbe23 100644--- a/pkg/detectors/positionstack/positionstack_test.go+++ b/pkg/detectors/positionstack/positionstack_test.go@@ -16,7 +16,7 @@ import (func TestPositionStack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/postageapp/postageapp_test.go b/pkg/detectors/postageapp/postageapp_test.goindex ce36e75fe..ff4c0f97b 100644--- a/pkg/detectors/postageapp/postageapp_test.go+++ b/pkg/detectors/postageapp/postageapp_test.go@@ -16,7 +16,7 @@ import (func TestPostageApp_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/posthog/posthog_test.go b/pkg/detectors/posthog/posthog_test.goindex 6f80cd0de..9baac55c0 100644--- a/pkg/detectors/posthog/posthog_test.go+++ b/pkg/detectors/posthog/posthog_test.go@@ -16,7 +16,7 @@ import (func TestAppPosthog_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/postman/postman_test.go b/pkg/detectors/postman/postman_test.goindex aa9e2ecd2..4efcc3403 100644--- a/pkg/detectors/postman/postman_test.go+++ b/pkg/detectors/postman/postman_test.go@@ -16,7 +16,7 @@ import (func TestPostman_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/postmark/postmark_test.go b/pkg/detectors/postmark/postmark_test.goindex 025d91518..188876790 100644--- a/pkg/detectors/postmark/postmark_test.go+++ b/pkg/detectors/postmark/postmark_test.go@@ -16,7 +16,7 @@ import (func TestPostmark_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/powrbot/powrbot_test.go b/pkg/detectors/powrbot/powrbot_test.goindex a76567496..cbe38f9ad 100644--- a/pkg/detectors/powrbot/powrbot_test.go+++ b/pkg/detectors/powrbot/powrbot_test.go@@ -16,7 +16,7 @@ import (func TestPowrbot_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/privatekey/cracker.go b/pkg/detectors/privatekey/cracker.gonew file mode 100644index 000000000..80b4cf6f3--- /dev/null+++ b/pkg/detectors/privatekey/cracker.go@@ -0,0 +1,37 @@+package privatekey++import (+ "bytes"+ "crypto/x509"+ _ "embed"+ "errors"++ "golang.org/x/crypto/ssh"+)++//go:embed "rockyou-15.txt"+var rawCrackList []byte+var passphrases [][]byte++func init() {+ passphrases = bytes.Split(rawCrackList, []byte("\n"))+}++var (+ ErrUncrackable = errors.New("unable to crack encryption")+)++func crack(in []byte) (interface{}, error) {+ for _, passphrase := range passphrases {+ parsed, err := ssh.ParseRawPrivateKeyWithPassphrase(in, passphrase)+ if err != nil {+ if errors.Is(err, x509.IncorrectPasswordError) {+ continue+ } else {+ return nil, err+ }+ }+ return parsed, nil+ }+ return nil, ErrUncrackable+}diff --git a/pkg/detectors/privatekey/cracker_test.go b/pkg/detectors/privatekey/cracker_test.gonew file mode 100644index 000000000..196d87c1a--- /dev/null+++ b/pkg/detectors/privatekey/cracker_test.go@@ -0,0 +1,93 @@+package privatekey++import (+ _ "embed"+ "testing"++ "golang.org/x/crypto/ssh"+)++var (+ testEncryptedKeyCorrectPassword = []byte("123456")+ testEncryptedKeyIncorrectPassword = []byte("incorrect")+ testEncryptedKey = []byte(normalize(string(`-----BEGIN OPENSSH PRIVATE KEY-----+b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAjNIZuun+xgLkM8KuzfmQuRAAAAEAAAAAEAAAGXAAAAB3NzaC1yc2EAAAADAQABAAABgQDe3Al0EMPz+utVNk5DixaYrGMK56RqUoqGBinke6SWVWmqom1lBcJWzor6HlnMRPPr7YCEsJKL4IpuVwu+inRa5kdtNTyM7yyQTSR2xXCS0fUItNuq8pUktsH8VUggpMeew8hJv7rFA7tnIg3UXCl6iF+OLZKbDA5aa24idpcD8b1I9/RzTOB1fu0of5xd9vgODzGw5JvHQSJ0FaA42aNBMGwrDhDB3+sgnRNdWf6NNIh8KpXXMKJADf3klsyn6He8L2bPMp8a4wwys2YB35p5zQ0JURovsdewlOxH+NT7eP19eVf4dCreibxUmRUaob5DEoHEk8WrxjKWIYUuLeD6AfcW6oXyRU2Yy8Vrt6SqFl5+WAi47VMFTkDZYS/eCvG53q9UBHpCj7Qvb0vSkCZXBvBIhlw193F3PX4WvO1IXsMwvQ1D1X+lmomsItbqM0cJyKw6LU18QWiBHvE7BqcphaoL5E08W2ATTSRIMCp6rt4rptM7KyGK8rc6W+UYrCnWt6KlCA8AAAWQXk+lVx6bH5itIKKYmQr6cR/5xtZ2GHAxnYtvlW3xnGhU0MHv+lJ2+uoWlT2RXE5pdMUQj7rNWAMqkwifSKZs9wBfYeo1TaFDmC3nW7yHSN3XTuO78mPIW5JyvmE+Rj5qjsUn7fNmzECoAxnVERhwnF3KqUBEPzIAc6/7v/na9NTiiGaJPco9lvCoPWbVLN08WG+SuyU+0x5zc3ebzuPcYqu5/c5nmiGxhALrIhjIS0OV1mtAAFhvdMjMIHOijOzSKVCC7rRk5+kG9EMLNvOn/DUVSRHamw5gs2V3V+Zq2g5nYWfgq8aDSTB8XlIzOj1cz3HwfN6pfSNQ/3Qe+wOQfWfTWdO+JSL8aoBN5Wg8tDbgmvmbFrINsJfFfSm0wZgcHhC7Ul4U3v4c8PoNdK9HXwi+TKKzJ9nxLYb+vDh50cnkseu2gt0KwVpjIorxEqeK755mKPao3JmOMr6uFTQsb+g+ZNgPwl+nRHA4Igx+zADFj3twldnKIiRpBQ5J4acur3uQ+saanBTXgul1TiFiUGT2cnz+IiCsdPovg+TAMt868W5LmzpfH4Cy54JtaRC4/UuMnkTGbWgutVDnWj2stOAzsQ1YmhH5igUmc94mUL+W+8vQDCKpeI8n+quDS9zxTvy4L4H5Iz7OZlh0h6N13BDvCYXKcNF/ugkfxZbu8mZsZQQzXNR+wOrEtKoHc4AnXYNzsuHEoEyLyJxGfFRDSTLbyN9wFOS/c0k9Gjte+kQRZjBVGORE5sN6X3+akUnTF76RhbEc+LamrwM1h5340bwosRbR8I+UrsQdFfJBEj1ZSyMRJlMkFUNi6blt7bhyx+ea+Pm2A614nlYUBjw2KKzzn8N/0H2NpJjIptvDsbrx3BS/rKwOeJwavRrGnIlEzuAag4vx+Zb2TPVta45uz7fQP5IBl83b0BJKI5Zv/fniUeLI78W/UsZqb64YQbfRyBzFtI1T/SsCi0B+e0EyKMzbxtSceT1Mb8eJiVIq04Xpwez9fIUt5rSedZD8KPq8P6s0cGsR7Qmw6eXZ/dBR/a+s5vPhfIUmQawmnwAVuWNRdQQ79jUBSn5M+ZRVVTgEG+vFyvxr/bZqOo1JCoq5BmQhLWGRJ+Dk9TolbeFIVFrkuXkcu99a079ux7XSkON64oPzHrcsEzjPA1GPqs9CGBSO16wq/nI3zg+E+kcOCaurc9yHJJPwduem0+8WLX3WoGNfQRKurtQze2ppy8KarEtDhDd96sKkhYaqOg3GOX8+Yx827L4vuWSJSIqKuO2kH6kOCMUNO16piv0z/8u3CJxOGh9+4FZIop81fiFTKLhV3/gwLm+fzFY++KIZrLfZcUjzd80NNEja69F452Eb9HrI5BurN/PznDEi9bzM598Y7beyl4/kd4R2e+S7SW9/LOrGw5UgxtiU+kV8nPz1PdgxO4sRlnntSBEwkQBzMkLOpq2h2BuJ2TlMP/TWuwLQ+sDkv1Yk1pD0roGmtMzbujnURGxqRJ8gUmuIot4hpfyRSssvnRQQZ3lQCQCwHiE+HJxXWf5+c58zOMjW7o21tI8e13uUnbRoQVJM9XYqk1usPXIkYPYL9uOw3AW/Zn+cnDrsXvTK9ZxgGD+/90b1BNwVqMlUK+QggHNwl5qD8eoXK5cDvav66te+E+V7FYFQ06w3tytRVz8SjoaiChN02+muIjvl6G7Hoj1hObM2t/ZheN1EShS11z868hhS6Mx7GvIdtkXuvdiBYMiBLOshJQxB8Mzx+iug9W+Di3upLf0UMC1TqADGphsIHRU7RbmHQ8Rwp7dogswmDfpRSapPt9p0D+6Ad5VBzi3+f3BPXj76UBLMEJCrZR1P28vnAA7AyNHaLvMPlWDMG5v3V/UV+ugyFcoBAOyjiQgYST8F3e+Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=+-----END OPENSSH PRIVATE KEY-----`)))+)++func Test_crack(t *testing.T) {+ tests := []struct {+ name string+ in []byte+ wantErr bool+ }{+ {+ name: "crackable",+ wantErr: false,+ in: testEncryptedKey,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ _, err := crack(tt.in)+ if (err != nil) != tt.wantErr {+ t.Errorf("crack() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ })+ }+}++func BenchmarkParseWrongPassword(b *testing.B) {+ for n := 0; n < b.N; n++ {+ ssh.ParseRawPrivateKeyWithPassphrase(testEncryptedKey, testEncryptedKeyIncorrectPassword)+ }+}++func BenchmarkParseRightPassword(b *testing.B) {+ for n := 0; n < b.N; n++ {+ ssh.ParseRawPrivateKeyWithPassphrase(testEncryptedKey, testEncryptedKeyCorrectPassword)+ }+}++func BenchmarkCracker(b *testing.B) {+ for n := 0; n < b.N; n++ {+ crack(testEncryptedKey)+ }+}diff --git a/pkg/detectors/privatekey/fingerprint.go b/pkg/detectors/privatekey/fingerprint.gonew file mode 100644index 000000000..afb24f5d0--- /dev/null+++ b/pkg/detectors/privatekey/fingerprint.go@@ -0,0 +1,55 @@+package privatekey++import (+ "crypto/ecdsa"+ "crypto/ed25519"+ "crypto/rsa"+ "crypto/sha1"+ "crypto/x509"+ "encoding/hex"+ "errors"+ "strings"++ "golang.org/x/crypto/ssh"+)++var (+ ErrNotSupported = errors.New("key type not supported")+ ErrEncryptedKey = errors.New("key is encrypted")+)++func FingerprintPEMKey(in []byte) (string, error) {+ parsedKey, err := ssh.ParseRawPrivateKey(in)+ if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {+ parsedKey, err = crack(in)+ if err != nil {+ return "", err+ }+ } else if err != nil {+ return "", err+ }++ var pubKey interface{}+ switch privateKey := parsedKey.(type) {+ case *rsa.PrivateKey:+ pubKey = &privateKey.PublicKey+ case *ecdsa.PrivateKey:+ pubKey = &privateKey.PublicKey+ case ed25519.PrivateKey:+ pubKey = privateKey.Public()+ // No fingerprinting support for DSA+ // case *dsa.PrivateKey:+ // pubKey = privateKey.PublicKey+ default:+ return "", ErrNotSupported+ }++ publickeyBytes, err := x509.MarshalPKIXPublicKey(pubKey)+ if err != nil {+ return "", err+ }++ publicKeyFingerprint := sha1.Sum(publickeyBytes)+ publicKeyFingerprintInHex := hex.EncodeToString(publicKeyFingerprint[:])+ return publicKeyFingerprintInHex, nil+}diff --git a/pkg/detectors/privatekey/normalize.go b/pkg/detectors/privatekey/normalize.gonew file mode 100644index 000000000..cc8df87d0--- /dev/null+++ b/pkg/detectors/privatekey/normalize.go@@ -0,0 +1,33 @@+package privatekey++import (+ "strings"+)++const b64chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="++func normalize(in string) string {+ in = strings.ReplaceAll(in, `"`, "")+ in = strings.ReplaceAll(in, `'`, "")+ in = strings.ReplaceAll(in, "\t", "")+ in = strings.ReplaceAll(in, `\t`, "")+ in = strings.ReplaceAll(in, `\\t`, "")+ in = strings.ReplaceAll(in, `\n`, "\n")+ in = strings.ReplaceAll(in, `\\r\\n`, "\n")+ in = strings.ReplaceAll(in, `\r\n`, "\n")+ in = strings.ReplaceAll(in, "\r\n", "\n")+ in = strings.ReplaceAll(in, `\\r`, "\n")+ in = strings.ReplaceAll(in, "\r", "\n")+ in = strings.ReplaceAll(in, `\r`, "\n")+ in = strings.ReplaceAll(in, `\\n`, "\n")+ in = strings.ReplaceAll(in, `\n\n`, "\n")+ in = strings.ReplaceAll(in, "\n\n", "\n")+ in = strings.ReplaceAll(in, `\\`, "\n")++ cleaned := strings.Builder{}+ parts := strings.Split(in, "\n")+ for _, line := range parts {+ cleaned.WriteString(strings.TrimSpace(line) + "\n")+ }+ return cleaned.String()+}diff --git a/pkg/detectors/privatekey/privatekey.go b/pkg/detectors/privatekey/privatekey.gonew file mode 100644index 000000000..10e88af90--- /dev/null+++ b/pkg/detectors/privatekey/privatekey.go@@ -0,0 +1,146 @@+package privatekey++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "regexp"+ "time"++ log "github.com/sirupsen/logrus"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct {+ IncludeExpired bool+}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()+ keyPat = regexp.MustCompile(`(?i)-----\s*?BEGIN[ A-Z0-9_-]*?PRIVATE KEY\s*?-----[\s\S]*?----\s*?END[ A-Z0-9_-]*? PRIVATE KEY\s*?-----`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"private key"}+}++// FromData will find and optionally verify Privatekey secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {+ results := []detectors.Result{}+ dataStr := string(data)++ matches := keyPat.FindAllString(dataStr, -1)++ for _, match := range matches {++ token := normalize(match)++ if len(token) < 64 {+ continue+ }++ secret := detectors.Result{+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Raw: []byte(token),+ Redacted: token[0:64],+ }++ fingerprint, err := FingerprintPEMKey([]byte(token))+ if err != nil {+ continue+ }++ if verify {+ data, err := lookupFingerprint(fingerprint, s.IncludeExpired)+ if err != nil {+ log.Warn(err)+ return nil, err+ }+ secret.StructuredData = data+ if data != nil {+ secret.Verified = true+ }+ }++ results = append(results, secret)+ }++ return results, nil+}++func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (data *detectorspb.StructuredData, err error) {+ req, err := http.NewRequest("GET", fmt.Sprintf("https://keychecker.trufflesecurity.com/fingerprint/%s", publicKeyFingerprintInHex), nil)+ if err != nil {+ return+ }+ res, err := client.Do(req)+ if err != nil {+ return+ }+ defer res.Body.Close()++ results := DriftwoodResult{}+ err = json.NewDecoder(res.Body).Decode(&results)+ if err != nil {+ return+ }++ seen := map[string]struct{}{}+ for _, r := range results.CertificateResults {+ if _, ok := seen[r.CertificateFingerprint]; ok {+ continue+ }+ if !includeExpired && time.Since(r.ExpirationTimestamp) > 0 {+ continue+ }+ if data == nil {+ data = &detectorspb.StructuredData{}+ }+ if data.TlsPrivateKey == nil {+ data.TlsPrivateKey = make([]*detectorspb.TlsPrivateKey, 0)+ }+ data.TlsPrivateKey = append(data.TlsPrivateKey, &detectorspb.TlsPrivateKey{+ CertificateFingerprint: r.CertificateFingerprint,+ ExpirationTimestamp: r.ExpirationTimestamp.Unix(),+ VerificationUrl: fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint),+ })+ seen[r.CertificateFingerprint] = struct{}{}+ }++ for _, r := range results.GitHubSSHResults {+ if _, ok := seen[r.Username]; ok {+ continue+ }+ if data == nil {+ data = &detectorspb.StructuredData{}+ }+ if data.GithubSshKey == nil {+ data.GithubSshKey = make([]*detectorspb.GitHubSSHKey, 0)+ }+ data.GithubSshKey = append(data.GithubSshKey, &detectorspb.GitHubSSHKey{+ User: r.Username,+ })+ seen[r.Username] = struct{}{}+ }++ return+}++type DriftwoodResult struct {+ CertificateResults []struct {+ CertificateFingerprint string `json:"CertificateFingerprint"`+ ExpirationTimestamp time.Time `json:"ExpirationTimestamp"`+ } `json:"CertificateResults"`+ GitHubSSHResults []struct {+ Username string `json:"Username"`+ } `json:"GitHubSSHResults"`+}diff --git a/pkg/detectors/privatekey/privatekey_test.go b/pkg/detectors/privatekey/privatekey_test.gonew file mode 100644index 000000000..f0955f652--- /dev/null+++ b/pkg/detectors/privatekey/privatekey_test.go@@ -0,0 +1,263 @@+package privatekey++import (+ "context"+ "fmt"+ "os"+ "reflect"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestPrivatekey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secretTLS := testSecrets.MustGetField(("PRIVATEKEY_TLS"))+ secretGitHub := testSecrets.MustGetField(("PRIVATEKEY_GITHUB"))+ secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }++ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find privatekey secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Verified: false,+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYw",+ },+ {+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Verified: false,+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",+ },+ },+ wantErr: false,+ },+ {+ name: "found TLS private key, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(secretTLS),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Verified: true,+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",+ StructuredData: &detectorspb.StructuredData{+ TlsPrivateKey: []*detectorspb.TlsPrivateKey{+ {+ CertificateFingerprint: "1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",+ VerificationUrl: "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",+ ExpirationTimestamp: 1497337731,+ },+ {+ CertificateFingerprint: "0e9de31fb2ee16465a4d5d93b227d54f870326d1",+ VerificationUrl: "https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",+ ExpirationTimestamp: 1528192832,+ },+ },+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found GitHub SSH private key, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(secretGitHub),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Verified: true,+ Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",+ StructuredData: &detectorspb.StructuredData{+ GithubSshKey: []*detectorspb.GitHubSSHKey{+ {+ User: "thisisforgithub0",+ },+ },+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found encrypted GitHub SSH private key, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(`-----BEGIN OPENSSH PRIVATE KEY-----+ b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAjNIZuun+ xgLkM8KuzfmQuRAAAAEAAAAAEAAAGXAAAAB3NzaC1yc2EAAAADAQABAAABgQDe3Al0EMPz+ utVNk5DixaYrGMK56RqUoqGBinke6SWVWmqom1lBcJWzor6HlnMRPPr7YCEsJKL4IpuVwu+ inRa5kdtNTyM7yyQTSR2xXCS0fUItNuq8pUktsH8VUggpMeew8hJv7rFA7tnIg3UXCl6iF+ OLZKbDA5aa24idpcD8b1I9/RzTOB1fu0of5xd9vgODzGw5JvHQSJ0FaA42aNBMGwrDhDB3+ sgnRNdWf6NNIh8KpXXMKJADf3klsyn6He8L2bPMp8a4wwys2YB35p5zQ0JURovsdewlOxH+ NT7eP19eVf4dCreibxUmRUaob5DEoHEk8WrxjKWIYUuLeD6AfcW6oXyRU2Yy8Vrt6SqFl5+ WAi47VMFTkDZYS/eCvG53q9UBHpCj7Qvb0vSkCZXBvBIhlw193F3PX4WvO1IXsMwvQ1D1X+ lmomsItbqM0cJyKw6LU18QWiBHvE7BqcphaoL5E08W2ATTSRIMCp6rt4rptM7KyGK8rc6W+ UYrCnWt6KlCA8AAAWQXk+lVx6bH5itIKKYmQr6cR/5xtZ2GHAxnYtvlW3xnGhU0MHv+lJ2+ uoWlT2RXE5pdMUQj7rNWAMqkwifSKZs9wBfYeo1TaFDmC3nW7yHSN3XTuO78mPIW5JyvmE+ Rj5qjsUn7fNmzECoAxnVERhwnF3KqUBEPzIAc6/7v/na9NTiiGaJPco9lvCoPWbVLN08WG+ SuyU+0x5zc3ebzuPcYqu5/c5nmiGxhALrIhjIS0OV1mtAAFhvdMjMIHOijOzSKVCC7rRk5+ kG9EMLNvOn/DUVSRHamw5gs2V3V+Zq2g5nYWfgq8aDSTB8XlIzOj1cz3HwfN6pfSNQ/3Qe+ wOQfWfTWdO+JSL8aoBN5Wg8tDbgmvmbFrINsJfFfSm0wZgcHhC7Ul4U3v4c8PoNdK9HXwi+ TKKzJ9nxLYb+vDh50cnkseu2gt0KwVpjIorxEqeK755mKPao3JmOMr6uFTQsb+g+ZNgPwl+ nRHA4Igx+zADFj3twldnKIiRpBQ5J4acur3uQ+saanBTXgul1TiFiUGT2cnz+IiCsdPovg+ TAMt868W5LmzpfH4Cy54JtaRC4/UuMnkTGbWgutVDnWj2stOAzsQ1YmhH5igUmc94mUL+W+ 8vQDCKpeI8n+quDS9zxTvy4L4H5Iz7OZlh0h6N13BDvCYXKcNF/ugkfxZbu8mZsZQQzXNR+ wOrEtKoHc4AnXYNzsuHEoEyLyJxGfFRDSTLbyN9wFOS/c0k9Gjte+kQRZjBVGORE5sN6X3+ akUnTF76RhbEc+LamrwM1h5340bwosRbR8I+UrsQdFfJBEj1ZSyMRJlMkFUNi6blt7bhyx+ ea+Pm2A614nlYUBjw2KKzzn8N/0H2NpJjIptvDsbrx3BS/rKwOeJwavRrGnIlEzuAag4vx+ Zb2TPVta45uz7fQP5IBl83b0BJKI5Zv/fniUeLI78W/UsZqb64YQbfRyBzFtI1T/SsCi0B+ e0EyKMzbxtSceT1Mb8eJiVIq04Xpwez9fIUt5rSedZD8KPq8P6s0cGsR7Qmw6eXZ/dBR/a+ s5vPhfIUmQawmnwAVuWNRdQQ79jUBSn5M+ZRVVTgEG+vFyvxr/bZqOo1JCoq5BmQhLWGRJ+ Dk9TolbeFIVFrkuXkcu99a079ux7XSkON64oPzHrcsEzjPA1GPqs9CGBSO16wq/nI3zg+E+ kcOCaurc9yHJJPwduem0+8WLX3WoGNfQRKurtQze2ppy8KarEtDhDd96sKkhYaqOg3GOX8+ Yx827L4vuWSJSIqKuO2kH6kOCMUNO16piv0z/8u3CJxOGh9+4FZIop81fiFTKLhV3/gwLm+ fzFY++KIZrLfZcUjzd80NNEja69F452Eb9HrI5BurN/PznDEi9bzM598Y7beyl4/kd4R2e+ S7SW9/LOrGw5UgxtiU+kV8nPz1PdgxO4sRlnntSBEwkQBzMkLOpq2h2BuJ2TlMP/TWuwLQ+ sDkv1Yk1pD0roGmtMzbujnURGxqRJ8gUmuIot4hpfyRSssvnRQQZ3lQCQCwHiE+HJxXWf5+ c58zOMjW7o21tI8e13uUnbRoQVJM9XYqk1usPXIkYPYL9uOw3AW/Zn+cnDrsXvTK9ZxgGD+ /90b1BNwVqMlUK+QggHNwl5qD8eoXK5cDvav66te+E+V7FYFQ06w3tytRVz8SjoaiChN02+ muIjvl6G7Hoj1hObM2t/ZheN1EShS11z868hhS6Mx7GvIdtkXuvdiBYMiBLOshJQxB8Mzx+ iug9W+Di3upLf0UMC1TqADGphsIHRU7RbmHQ8Rwp7dogswmDfpRSapPt9p0D+6Ad5VBzi3+ f3BPXj76UBLMEJCrZR1P28vnAA7AyNHaLvMPlWDMG5v3V/UV+ugyFcoBAOyjiQgYST8F3e+ Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=+ -----END OPENSSH PRIVATE KEY-----+ `),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Verified: true,+ Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",+ StructuredData: &detectorspb.StructuredData{+ GithubSshKey: []*detectorspb.GitHubSSHKey{+ {+ User: "thisisforgithub0",+ },+ },+ },+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{IncludeExpired: true}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PrivatekeyCI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ if os.Getenv("FORCE_PASS_DIFF") == "true" {+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PrivatekeyCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}++func Test_lookupFingerprint(t *testing.T) {+ tests := []struct {+ name string+ publicKeyFingerprintInHex string+ wantFingerprints bool+ wantErr bool+ includeExpired bool+ }{+ {+ name: "got some",+ publicKeyFingerprintInHex: "4c5da06caa1c81df9c8e1abe43bac385de1bda76",+ wantFingerprints: true,+ wantErr: false,+ includeExpired: true,+ },+ {+ name: "got some",+ publicKeyFingerprintInHex: "none",+ wantFingerprints: false,+ wantErr: false,+ includeExpired: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ gotFingerprints, err := lookupFingerprint(tt.publicKeyFingerprintInHex, tt.includeExpired)+ if (err != nil) != tt.wantErr {+ t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.TlsPrivateKey) > 0, tt.wantFingerprints) {+ t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)+ }+ })+ }+}diff --git a/pkg/detectors/privatekey/rockyou-15.txt b/pkg/detectors/privatekey/rockyou-15.txtnew file mode 100644index 000000000..31140491c--- /dev/null+++ b/pkg/detectors/privatekey/rockyou-15.txt@@ -0,0 +1,249 @@+123456+12345+123456789+password+iloveyou+princess+1234567+12345678+abc123+nicole+daniel+babygirl+monkey+lovely+jessica+654321+michael+ashley+qwerty+111111+iloveu+000000+michelle+tigger+sunshine+chocolate+password1+soccer+anthony+friends+butterfly+purple+angel+jordan+liverpool+justin+loveme+fuckyou+123123+football+secret+andrea+carlos+jennifer+joshua+bubbles+1234567890+superman+hannah+amanda+loveyou+pretty+basketball+andrew+angels+tweety+flower+playboy+hello+elizabeth+hottie+tinkerbell+charlie+samantha+barbie+chelsea+lovers+teamo+jasmine+brandon+666666+shadow+melissa+eminem+matthew+robert+danielle+forever+family+jonathan+987654321+computer+whatever+dragon+vanessa+cookie+naruto+summer+sweety+spongebob+joseph+junior+softball+taylor+yellow+daniela+lauren+mickey+princesa+alexandra+alexis+jesus+estrella+miguel+william+thomas+beautiful+mylove+angela+poohbear+patrick+iloveme+sakura+adrian+alexander+destiny+christian+121212+sayang+america+dancer+monica+richard+112233+princess1+555555+diamond+carolina+steven+rangers+louise+orange+789456+999999+shorty+11111+nathan+snoopy+gabriel+hunter+cherry+killer+sandra+alejandro+buster+george+brittany+alejandra+patricia+rachel+tequiero+7777777+cheese+159753+arsenal+dolphin+antonio+heather+david+ginger+stephanie+peanut+blink182+sweetie+222222+beauty+987654+victoria+honey+00000+fernando+pokemon+maggie+corazon+chicken+pepper+cristina+rainbow+kisses+manuel+myspace+rebelde+angel1+ricardo+babygurl+heaven+55555+baseball+martin+greenday+november+alyssa+madison+mother+123321+123abc+mahalkita+batman+september+december+morgan+mariposa+maria+gabriela+iloveyou2+bailey+jeremy+pamela+kimberly+gemini+shannon+pictures+asshole+sophie+jessie+hellokitty+claudia+babygirl1+angelica+austin+mahalko+victor+horses+tiffany+mariana+eduardo+andres+courtney+booboo+kissme+harley+ronaldo+iloveyou1+precious+october+inuyasha+peaches+veronica+chris+888888+adriana+cutie+james+banana+prince+friend+jesus1+crystal+celtic\ No newline at end of filediff --git a/pkg/detectors/prospectcrm/prospectcrm_test.go b/pkg/detectors/prospectcrm/prospectcrm_test.goindex fed2d0ead..205537d89 100644--- a/pkg/detectors/prospectcrm/prospectcrm_test.go+++ b/pkg/detectors/prospectcrm/prospectcrm_test.go@@ -16,7 +16,7 @@ import (func TestProspectCRM_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/prospectio/prospectio_test.go b/pkg/detectors/prospectio/prospectio_test.goindex cb38ebacb..181ed7ae8 100644--- a/pkg/detectors/prospectio/prospectio_test.go+++ b/pkg/detectors/prospectio/prospectio_test.go@@ -16,7 +16,7 @@ import (func TestProspectIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/protocolsio/protocolsio.go b/pkg/detectors/protocolsio/protocolsio.gonew file mode 100644index 000000000..e5315f7b2--- /dev/null+++ b/pkg/detectors/protocolsio/protocolsio.go@@ -0,0 +1,74 @@+package protocolsio++import (+ "context"+ "fmt"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"protocols"}) + `\b([a-z0-9]{64})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"protocols"}+}++// FromData will find and optionally verify ProtocolsIO secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_ProtocolsIO,+ Raw: []byte(resMatch),+ }++ if verify {+ req, _ := http.NewRequest("GET", "https://www.protocols.io/api/v3/session/profile", nil)+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/protocolsio/protocolsio_test.go b/pkg/detectors/protocolsio/protocolsio_test.gonew file mode 100644index 000000000..0cabbfce4--- /dev/null+++ b/pkg/detectors/protocolsio/protocolsio_test.go@@ -0,0 +1,113 @@+package protocolsio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestProtocolsIO_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PROTOCOLSIO")+ inactiveSecret := testSecrets.MustGetField("PROTOCOLSIO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ProtocolsIO,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ProtocolsIO,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ProtocolsIO.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ProtocolsIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/proxycrawl/proxycrawl_test.go b/pkg/detectors/proxycrawl/proxycrawl_test.goindex 394976f3c..8ecea2bef 100644--- a/pkg/detectors/proxycrawl/proxycrawl_test.go+++ b/pkg/detectors/proxycrawl/proxycrawl_test.go@@ -16,7 +16,7 @@ import (func TestProxyCrawl_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.goindex d4a6c8a45..e2fbdc45d 100644--- a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go+++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go@@ -16,7 +16,7 @@ import (func TestPubNubPublishKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.goindex 820842f62..4f8bbfa43 100644--- a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go+++ b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go@@ -16,7 +16,7 @@ import (func TestPubNubSubscriptionKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/purestake/purestake_test.go b/pkg/detectors/purestake/purestake_test.goindex 47eab6ad9..d28fde2fc 100644--- a/pkg/detectors/purestake/purestake_test.go+++ b/pkg/detectors/purestake/purestake_test.go@@ -16,7 +16,7 @@ import (func TestPureStake_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go b/pkg/detectors/pushbulletapikey/pushbulletapikey_test.goindex f1d32a0f7..d786a495f 100644--- a/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go+++ b/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go@@ -16,7 +16,7 @@ import (func TestPushBulletApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go b/pkg/detectors/pusherchannelkey/pusherchannelkey_test.goindex 9fc558076..8752e5e12 100644--- a/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go+++ b/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go@@ -16,7 +16,7 @@ import (func TestPusherChannelKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/qualaroo/qualaroo_test.go b/pkg/detectors/qualaroo/qualaroo_test.goindex 020f80c19..1bb856092 100644--- a/pkg/detectors/qualaroo/qualaroo_test.go+++ b/pkg/detectors/qualaroo/qualaroo_test.go@@ -16,7 +16,7 @@ import (func TestQualaroo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/qubole/qubole_test.go b/pkg/detectors/qubole/qubole_test.goindex cd015dc8c..bad932f44 100644--- a/pkg/detectors/qubole/qubole_test.go+++ b/pkg/detectors/qubole/qubole_test.go@@ -16,7 +16,7 @@ import (func TestQubole_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/quickmetrics/quickmetrics_test.go b/pkg/detectors/quickmetrics/quickmetrics_test.goindex a6acfcd10..d40cdb613 100644--- a/pkg/detectors/quickmetrics/quickmetrics_test.go+++ b/pkg/detectors/quickmetrics/quickmetrics_test.go@@ -16,7 +16,7 @@ import (func TestQuickMetrics_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/rapidapi/rapidapi_test.go b/pkg/detectors/rapidapi/rapidapi_test.goindex 0724f1f5e..d64586337 100644--- a/pkg/detectors/rapidapi/rapidapi_test.go+++ b/pkg/detectors/rapidapi/rapidapi_test.go@@ -16,7 +16,7 @@ import (func TestRapidApi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/raven/raven_test.go b/pkg/detectors/raven/raven_test.goindex e214c6996..c83caaa5c 100644--- a/pkg/detectors/raven/raven_test.go+++ b/pkg/detectors/raven/raven_test.go@@ -16,7 +16,7 @@ import (func TestRaven_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/rawg/rawg_test.go b/pkg/detectors/rawg/rawg_test.goindex 9263627f2..90eb0ad44 100644--- a/pkg/detectors/rawg/rawg_test.go+++ b/pkg/detectors/rawg/rawg_test.go@@ -16,7 +16,7 @@ import (func TestRawg_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/razorpay/razorpay.go b/pkg/detectors/razorpay/razorpay.gonew file mode 100644index 000000000..2df80bf9d--- /dev/null+++ b/pkg/detectors/razorpay/razorpay.go@@ -0,0 +1,86 @@+package razorpay++import (+ "context"+ "regexp"++ log "github.com/sirupsen/logrus"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ razorpay "github.com/razorpay/razorpay-go"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++//The (`) character adds secondary encoding to parsed strings by Golang which also allows for escape sequences+var (+ keyPat = regexp.MustCompile(`(?i)\brzp_\w{2,6}_\w{10,20}\b`)+ secretPat = regexp.MustCompile(`(?:razor|secret|rzp|key)[-\w]*[\" :=']*([A-Za-z0-9]{20,50})`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"rzp_"}+}++// FromData will find and optionally verify RazorPay secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllString(dataStr, -1)++ for _, match := range matches {+ token := match++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_RazorPay,+ Raw: []byte(token),+ Redacted: token,+ }++ if verify {+ //https://dashboard.razorpay.com/#/access/signin+ //https://gitlab.com/trufflesec/trufflehog/-/blob/master/webapi/secrets/razorpay.py+ secMatches := secretPat.FindAllStringSubmatch(dataStr, -1)+ if len(secMatches) == 0 {+ //no secret keys were found. Declare unverified (This is how AWS secret handles the same logic)+ //TODO determine if key alone without secret is reportable+ s.Verified = false+ return+ }+ //we only want the secret, not its surrounding info - grabbing capture groups+ for _, secMatch := range secMatches {+ client := razorpay.NewClient(token, secMatch[1])+ resp, err := client.Order.All(nil, nil)+ //TODO Error handling is broken in SDK, fixed by https://github.com/razorpay/razorpay-go/pull/23+ //waiting to be reviewed and merged+ if resp == nil {+ continue+ }+ if err != nil {+ log.Debugf("Error verifying likely razorpay key/secret combo: %v", err)+ continue+ }+ //TODO debug with responses. could still be invalid at this stage++ s.Verified = true+ // fmt.Println(resp)+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/razorpay/razorpay_test.go b/pkg/detectors/razorpay/razorpay_test.gonew file mode 100644index 000000000..67c41d37b--- /dev/null+++ b/pkg/detectors/razorpay/razorpay_test.go@@ -0,0 +1,100 @@+package razorpay++import (+ "context"+ "fmt"+ "os"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestRazorPay_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secretInactive := testSecrets.MustGetField("RAZORPAY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("25:RAZORPAY_KEY='rzp_test_SnDTaP1ncfliDt'\n\"rzp_secret\" : \" %s\", ", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RazorPay,+ Redacted: "rzp_test_SnDTaP1ncfliDt",+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RazorPay.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ if os.Getenv("FORCE_PASS_DIFF") == "true" {+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RazorPay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/readme/readme_test.go b/pkg/detectors/readme/readme_test.goindex cf09fe71a..124938c13 100644--- a/pkg/detectors/readme/readme_test.go+++ b/pkg/detectors/readme/readme_test.go@@ -16,7 +16,7 @@ import (func TestReadMe_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go b/pkg/detectors/reallysimplesystems/reallysimplesystems_test.goindex f89322ed6..54652caa9 100644--- a/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go+++ b/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go@@ -16,7 +16,7 @@ import (func TestReallySimpleSystems_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/rebrandly/rebrandly_test.go b/pkg/detectors/rebrandly/rebrandly_test.goindex 7d27ee6a9..f76c6bde4 100644--- a/pkg/detectors/rebrandly/rebrandly_test.go+++ b/pkg/detectors/rebrandly/rebrandly_test.go@@ -16,7 +16,7 @@ import (func TestRebrandly_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/refiner/refiner_test.go b/pkg/detectors/refiner/refiner_test.goindex f7d19faec..202352caa 100644--- a/pkg/detectors/refiner/refiner_test.go+++ b/pkg/detectors/refiner/refiner_test.go@@ -16,7 +16,7 @@ import (func TestRefiner_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/restpack/restpack_test.go b/pkg/detectors/restpack/restpack_test.goindex b6a7621db..c0d07c0cc 100644--- a/pkg/detectors/restpack/restpack_test.go+++ b/pkg/detectors/restpack/restpack_test.go@@ -16,7 +16,7 @@ import (func TestRestpack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.goindex 3187ec0dd..b99fe92ac 100644--- a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go+++ b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go@@ -16,7 +16,7 @@ import (func TestRestpackHtmlToPdfAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.goindex a3a8a94a1..00beece9e 100644--- a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go+++ b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go@@ -16,7 +16,7 @@ import (func TestRestpackScreenshotAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/rev/rev_test.go b/pkg/detectors/rev/rev_test.goindex 5e5dd23f9..b83dd4e57 100644--- a/pkg/detectors/rev/rev_test.go+++ b/pkg/detectors/rev/rev_test.go@@ -16,7 +16,7 @@ import (func TestRev_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/revampcrm/revampcrm_test.go b/pkg/detectors/revampcrm/revampcrm_test.goindex 807cbba6b..9cbc4c5a8 100644--- a/pkg/detectors/revampcrm/revampcrm_test.go+++ b/pkg/detectors/revampcrm/revampcrm_test.go@@ -16,7 +16,7 @@ import (func TestRevampCRM_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ringcentral/ringcentral_test.go b/pkg/detectors/ringcentral/ringcentral_test.goindex 32205f0da..23a3b6d28 100644--- a/pkg/detectors/ringcentral/ringcentral_test.go+++ b/pkg/detectors/ringcentral/ringcentral_test.go@@ -16,7 +16,7 @@ import (func TestRingcentral_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ritekit/ritekit_test.go b/pkg/detectors/ritekit/ritekit_test.goindex fde353550..e17c253d5 100644--- a/pkg/detectors/ritekit/ritekit_test.go+++ b/pkg/detectors/ritekit/ritekit_test.go@@ -16,7 +16,7 @@ import (func TestRiteKit_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/roaring/roaring_test.go b/pkg/detectors/roaring/roaring_test.goindex 17ee0f19a..2323a5b23 100644--- a/pkg/detectors/roaring/roaring_test.go+++ b/pkg/detectors/roaring/roaring_test.go@@ -16,7 +16,7 @@ import (func TestRoaring_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/rocketreach/rocketreach_test.go b/pkg/detectors/rocketreach/rocketreach_test.goindex 7da7c8f5f..a34b7ccaa 100644--- a/pkg/detectors/rocketreach/rocketreach_test.go+++ b/pkg/detectors/rocketreach/rocketreach_test.go@@ -16,7 +16,7 @@ import (func TestRocketReach_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/roninapp/roninapp_test.go b/pkg/detectors/roninapp/roninapp_test.goindex 6fbd22995..9e99594e3 100644--- a/pkg/detectors/roninapp/roninapp_test.go+++ b/pkg/detectors/roninapp/roninapp_test.go@@ -16,7 +16,7 @@ import (func TestRoninApp_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/route4me/route4me_test.go b/pkg/detectors/route4me/route4me_test.goindex 8b98a68e4..2355b393c 100644--- a/pkg/detectors/route4me/route4me_test.go+++ b/pkg/detectors/route4me/route4me_test.go@@ -16,7 +16,7 @@ import (func TestRoute4me_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/rubygems/rubygems_test.go b/pkg/detectors/rubygems/rubygems_test.goindex ebd564af5..b8153f2c3 100644--- a/pkg/detectors/rubygems/rubygems_test.go+++ b/pkg/detectors/rubygems/rubygems_test.go@@ -16,7 +16,7 @@ import (func TestRubyGems_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/runrunit/runrunit_test.go b/pkg/detectors/runrunit/runrunit_test.goindex ae25716d1..f554ad35c 100644--- a/pkg/detectors/runrunit/runrunit_test.go+++ b/pkg/detectors/runrunit/runrunit_test.go@@ -16,7 +16,7 @@ import (func TestRunRunIt_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/salesblink/salesblink_test.go b/pkg/detectors/salesblink/salesblink_test.goindex 88a9a3127..b585007a6 100644--- a/pkg/detectors/salesblink/salesblink_test.go+++ b/pkg/detectors/salesblink/salesblink_test.go@@ -16,7 +16,7 @@ import (func TestSalesblink_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/salescookie/salescookie_test.go b/pkg/detectors/salescookie/salescookie_test.goindex e05c4bc6e..78749935a 100644--- a/pkg/detectors/salescookie/salescookie_test.go+++ b/pkg/detectors/salescookie/salescookie_test.go@@ -16,7 +16,7 @@ import (func TestSalescookie_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/salesflare/salesflare_test.go b/pkg/detectors/salesflare/salesflare_test.goindex 98a95a11b..7604af877 100644--- a/pkg/detectors/salesflare/salesflare_test.go+++ b/pkg/detectors/salesflare/salesflare_test.go@@ -16,7 +16,7 @@ import (func TestSalesflare_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/satismeterprojectkey/satismeterprojectkey_test.go b/pkg/detectors/satismeterprojectkey/satismeterprojectkey_test.goindex be7c60e8c..e6f02e603 100644--- a/pkg/detectors/satismeterprojectkey/satismeterprojectkey_test.go+++ b/pkg/detectors/satismeterprojectkey/satismeterprojectkey_test.go@@ -16,7 +16,7 @@ import (func TestSatismeterProjectkey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go b/pkg/detectors/satismeterwritekey/satismeterwritekey_test.goindex 957ae1c0b..394861ca7 100644--- a/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go+++ b/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go@@ -16,7 +16,7 @@ import (func TestSatismeterWritekey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/saucelabs/saucelabs_test.go b/pkg/detectors/saucelabs/saucelabs_test.goindex d394cd000..2445ce46a 100644--- a/pkg/detectors/saucelabs/saucelabs_test.go+++ b/pkg/detectors/saucelabs/saucelabs_test.go@@ -16,7 +16,7 @@ import (func TestSauceLabs_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/scalewaykey/scalewaykey_test.go b/pkg/detectors/scalewaykey/scalewaykey_test.goindex 4aff92aa7..573757f0c 100644--- a/pkg/detectors/scalewaykey/scalewaykey_test.go+++ b/pkg/detectors/scalewaykey/scalewaykey_test.go@@ -16,7 +16,7 @@ import (func TestScalewayKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/scrapeowl/scrapeowl_test.go b/pkg/detectors/scrapeowl/scrapeowl_test.goindex ce0e77f50..bc31f6103 100644--- a/pkg/detectors/scrapeowl/scrapeowl_test.go+++ b/pkg/detectors/scrapeowl/scrapeowl_test.go@@ -16,7 +16,7 @@ import (func TestScrapeowl_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/scraperapi/scraperapi_test.go b/pkg/detectors/scraperapi/scraperapi_test.goindex dd8368050..386240e10 100644--- a/pkg/detectors/scraperapi/scraperapi_test.go+++ b/pkg/detectors/scraperapi/scraperapi_test.go@@ -16,7 +16,7 @@ import (func TestScraperAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/scraperbox/scraperbox_test.go b/pkg/detectors/scraperbox/scraperbox_test.goindex a673ccb2f..1f96fb5a7 100644--- a/pkg/detectors/scraperbox/scraperbox_test.go+++ b/pkg/detectors/scraperbox/scraperbox_test.go@@ -16,7 +16,7 @@ import (func TestScraperBox_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/scrapersite/scrapersite_test.go b/pkg/detectors/scrapersite/scrapersite_test.goindex f92a967c5..f882ab3d0 100644--- a/pkg/detectors/scrapersite/scrapersite_test.go+++ b/pkg/detectors/scrapersite/scrapersite_test.go@@ -16,7 +16,7 @@ import (func TestScraperSite_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/scrapestack/scrapestack_test.go b/pkg/detectors/scrapestack/scrapestack_test.goindex 49f2867ee..b81893c01 100644--- a/pkg/detectors/scrapestack/scrapestack_test.go+++ b/pkg/detectors/scrapestack/scrapestack_test.go@@ -16,7 +16,7 @@ import (func TestScrapeStack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/scrapfly/scrapfly_test.go b/pkg/detectors/scrapfly/scrapfly_test.goindex bc0396e61..ff8e402c5 100644--- a/pkg/detectors/scrapfly/scrapfly_test.go+++ b/pkg/detectors/scrapfly/scrapfly_test.go@@ -16,7 +16,7 @@ import (func TestScrapfly_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/scrapingant/scrapingant_test.go b/pkg/detectors/scrapingant/scrapingant_test.goindex 41b37fb8d..7c228d924 100644--- a/pkg/detectors/scrapingant/scrapingant_test.go+++ b/pkg/detectors/scrapingant/scrapingant_test.go@@ -16,7 +16,7 @@ import (func TestScrapingAnt_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/scrapingbee/scrapingbee_test.go b/pkg/detectors/scrapingbee/scrapingbee_test.goindex c22b82dd6..7b257c573 100644--- a/pkg/detectors/scrapingbee/scrapingbee_test.go+++ b/pkg/detectors/scrapingbee/scrapingbee_test.go@@ -16,7 +16,7 @@ import (func TestScrapingBee_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/screenshotapi/screenshotapi_test.go b/pkg/detectors/screenshotapi/screenshotapi_test.goindex 0a2ad4015..9eb2861d0 100644--- a/pkg/detectors/screenshotapi/screenshotapi_test.go+++ b/pkg/detectors/screenshotapi/screenshotapi_test.go@@ -16,7 +16,7 @@ import (func TestScreenshotAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/screenshotlayer/screenshotlayer_test.go b/pkg/detectors/screenshotlayer/screenshotlayer_test.goindex 55f0e5452..0ae9c721a 100644--- a/pkg/detectors/screenshotlayer/screenshotlayer_test.go+++ b/pkg/detectors/screenshotlayer/screenshotlayer_test.go@@ -16,7 +16,7 @@ import (func TestScreenshotLayer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/securitytrails/securitytrails_test.go b/pkg/detectors/securitytrails/securitytrails_test.goindex a62219894..4cb4efcb6 100644--- a/pkg/detectors/securitytrails/securitytrails_test.go+++ b/pkg/detectors/securitytrails/securitytrails_test.go@@ -16,7 +16,7 @@ import (func TestSecurityTrails_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/segmentapikey/segmentapikey_test.go b/pkg/detectors/segmentapikey/segmentapikey_test.goindex 23901eb59..fd8c03dc3 100644--- a/pkg/detectors/segmentapikey/segmentapikey_test.go+++ b/pkg/detectors/segmentapikey/segmentapikey_test.go@@ -16,7 +16,7 @@ import (func TestSegmentApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/selectpdf/selectpdf_test.go b/pkg/detectors/selectpdf/selectpdf_test.goindex 082d38941..c1600228d 100644--- a/pkg/detectors/selectpdf/selectpdf_test.go+++ b/pkg/detectors/selectpdf/selectpdf_test.go@@ -16,7 +16,7 @@ import (func TestSelectPDF_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/semaphore/semaphore_test.go b/pkg/detectors/semaphore/semaphore_test.goindex 6a113e498..8c2915d26 100644--- a/pkg/detectors/semaphore/semaphore_test.go+++ b/pkg/detectors/semaphore/semaphore_test.go@@ -16,7 +16,7 @@ import (func TestSemaphore_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sendgrid/sendgrid.go b/pkg/detectors/sendgrid/sendgrid.gonew file mode 100644index 000000000..5c112b1bb--- /dev/null+++ b/pkg/detectors/sendgrid/sendgrid.go@@ -0,0 +1,83 @@+package sendgrid++import (+ "context"+ "fmt"+ "net/http"+ "regexp"+ "strings"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.RetryableHttpClient()++ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"SG."}+}++// FromData will find and optionally verify Sendgrid secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ res := strings.TrimSpace(match[1])++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_SendGrid,+ Raw: []byte(res),+ }++ if verify {+ // there are a few endpoints we can check, but templates seems the least sensitive.+ // 403 will be issued if the scope is wrong but the key is correct+ baseURL := "https://api.sendgrid.com/v3/templates"++ // test `read_user` scope+ req, _ := http.NewRequest("GET", baseURL, nil)+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", res))+ req.Header.Add("Content-Type", "application/json")+ res, err := client.Do(req)+ if err != nil {+ return results, err+ }+ defer res.Body.Close()++ // 200 means good key and has `templates` scope+ // 403 means good key but not the right scope+ // 401 is bad key+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {+ s.Verified = true+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/sendgrid/sendgrid_test.go b/pkg/detectors/sendgrid/sendgrid_test.gonew file mode 100644index 000000000..e567a746d--- /dev/null+++ b/pkg/detectors/sendgrid/sendgrid_test.go@@ -0,0 +1,108 @@+package sendgrid++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestSendgrid_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SENDGRID")+ secretInactive := testSecrets.MustGetField("SENDGRID_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendGrid,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendGrid,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/sendinbluev2/sendinbluev2_test.go b/pkg/detectors/sendinbluev2/sendinbluev2_test.goindex 0645a7e22..36e1e17a7 100644--- a/pkg/detectors/sendinbluev2/sendinbluev2_test.go+++ b/pkg/detectors/sendinbluev2/sendinbluev2_test.go@@ -16,7 +16,7 @@ import (func TestSendinBlueV2_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sentiment/sentiment_test.go b/pkg/detectors/sentiment/sentiment_test.goindex c616371c9..6f0ea0c46 100644--- a/pkg/detectors/sentiment/sentiment_test.go+++ b/pkg/detectors/sentiment/sentiment_test.go@@ -16,7 +16,7 @@ import (func TestSentiment_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sentrytoken/sentrytoken_test.go b/pkg/detectors/sentrytoken/sentrytoken_test.goindex 1c2fd74f6..27c1122b7 100644--- a/pkg/detectors/sentrytoken/sentrytoken_test.go+++ b/pkg/detectors/sentrytoken/sentrytoken_test.go@@ -16,7 +16,7 @@ import (func TestSentryToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/serphouse/serphouse_test.go b/pkg/detectors/serphouse/serphouse_test.goindex 52fa32959..fa90bbb02 100644--- a/pkg/detectors/serphouse/serphouse_test.go+++ b/pkg/detectors/serphouse/serphouse_test.go@@ -16,7 +16,7 @@ import (func TestSerphouse_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/serpstack/serpstack_test.go b/pkg/detectors/serpstack/serpstack_test.goindex 287779dd4..a12c91756 100644--- a/pkg/detectors/serpstack/serpstack_test.go+++ b/pkg/detectors/serpstack/serpstack_test.go@@ -16,7 +16,7 @@ import (func TestSerpStack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sheety/sheety_test.go b/pkg/detectors/sheety/sheety_test.goindex 2b2419546..5316e2cfc 100644--- a/pkg/detectors/sheety/sheety_test.go+++ b/pkg/detectors/sheety/sheety_test.go@@ -16,7 +16,7 @@ import (func TestSheety_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sherpadesk/sherpadesk_test.go b/pkg/detectors/sherpadesk/sherpadesk_test.goindex 31e7d0f53..295c7edcc 100644--- a/pkg/detectors/sherpadesk/sherpadesk_test.go+++ b/pkg/detectors/sherpadesk/sherpadesk_test.go@@ -16,7 +16,7 @@ import (func TestSherpadesk_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/shipday/shipday_test.go b/pkg/detectors/shipday/shipday_test.goindex 342df1d84..ef0a9b634 100644--- a/pkg/detectors/shipday/shipday_test.go+++ b/pkg/detectors/shipday/shipday_test.go@@ -16,7 +16,7 @@ import (func TestShipday_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/shodankey/shodankey_test.go b/pkg/detectors/shodankey/shodankey_test.goindex 750759e6c..2b995ff28 100644--- a/pkg/detectors/shodankey/shodankey_test.go+++ b/pkg/detectors/shodankey/shodankey_test.go@@ -16,7 +16,7 @@ import (func TestShodanKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/shortcut/shortcut_test.go b/pkg/detectors/shortcut/shortcut_test.goindex 2b159b6d0..37a2df726 100644--- a/pkg/detectors/shortcut/shortcut_test.go+++ b/pkg/detectors/shortcut/shortcut_test.go@@ -16,7 +16,7 @@ import (func TestShortcut_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/shotstack/shotstack_test.go b/pkg/detectors/shotstack/shotstack_test.goindex a798da854..351f0e733 100644--- a/pkg/detectors/shotstack/shotstack_test.go+++ b/pkg/detectors/shotstack/shotstack_test.go@@ -16,7 +16,7 @@ import (func TestShotstack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/shutterstock/shutterstock_test.go b/pkg/detectors/shutterstock/shutterstock_test.goindex 0ddb49b6a..6353f8c30 100644--- a/pkg/detectors/shutterstock/shutterstock_test.go+++ b/pkg/detectors/shutterstock/shutterstock_test.go@@ -16,7 +16,7 @@ import (func TestShutterstock_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go b/pkg/detectors/shutterstockoauth/shutterstockoauth_test.goindex 896b269c1..c5d07da93 100644--- a/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go+++ b/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go@@ -16,7 +16,7 @@ import (func TestShutterstockOAuth_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/signalwire/signalwire_test.go b/pkg/detectors/signalwire/signalwire_test.goindex e06dd792b..d82ed4a4b 100644--- a/pkg/detectors/signalwire/signalwire_test.go+++ b/pkg/detectors/signalwire/signalwire_test.go@@ -16,7 +16,7 @@ import (func TestSignalwire_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/signaturit/signaturit_test.go b/pkg/detectors/signaturit/signaturit_test.goindex 187ee3745..1c2a01714 100644--- a/pkg/detectors/signaturit/signaturit_test.go+++ b/pkg/detectors/signaturit/signaturit_test.go@@ -16,7 +16,7 @@ import (func TestSignaturit_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/signupgenius/signupgenius_test.go b/pkg/detectors/signupgenius/signupgenius_test.goindex 669bc059b..edfaf0bd9 100644--- a/pkg/detectors/signupgenius/signupgenius_test.go+++ b/pkg/detectors/signupgenius/signupgenius_test.go@@ -16,7 +16,7 @@ import (func TestSignupgenius_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sigopt/sigopt.go b/pkg/detectors/sigopt/sigopt.gonew file mode 100644index 000000000..76315fc83--- /dev/null+++ b/pkg/detectors/sigopt/sigopt.go@@ -0,0 +1,78 @@+package sigopt++import (+ "context"+ b64 "encoding/base64"+ "fmt"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sigopt"}) + `\b([A-Z0-9]{48})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"sigopt"}+}++// FromData will find and optionally verify Sigopt secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_Sigopt,+ Raw: []byte(resMatch),+ }++ if verify {+ data := fmt.Sprintf("%s:", resMatch)+ sEnc := b64.StdEncoding.EncodeToString([]byte(data))+ req, _ := http.NewRequest("GET", "https://api.sigopt.com/v1/experiments", nil)+ req.Header.Add("Content-Type", "application/json")+ req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/sigopt/sigopt_test.go b/pkg/detectors/sigopt/sigopt_test.gonew file mode 100644index 000000000..9215595b2--- /dev/null+++ b/pkg/detectors/sigopt/sigopt_test.go@@ -0,0 +1,113 @@+package sigopt++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestSigopt_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SIGOPT")+ inactiveSecret := testSecrets.MustGetField("SIGOPT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sigopt secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sigopt,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sigopt secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sigopt,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sigopt.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Sigopt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/simplesat/simplesat_test.go b/pkg/detectors/simplesat/simplesat_test.goindex 417810263..2857f3fa3 100644--- a/pkg/detectors/simplesat/simplesat_test.go+++ b/pkg/detectors/simplesat/simplesat_test.go@@ -16,7 +16,7 @@ import (func TestSimplesat_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/simplynoted/simplynoted_test.go b/pkg/detectors/simplynoted/simplynoted_test.goindex b0414d1d9..247b3d955 100644--- a/pkg/detectors/simplynoted/simplynoted_test.go+++ b/pkg/detectors/simplynoted/simplynoted_test.go@@ -16,7 +16,7 @@ import (func TestSimplyNoted_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/simvoly/simvoly_test.go b/pkg/detectors/simvoly/simvoly_test.goindex d6a9bd8ab..60be5d1a0 100644--- a/pkg/detectors/simvoly/simvoly_test.go+++ b/pkg/detectors/simvoly/simvoly_test.go@@ -16,7 +16,7 @@ import (func TestSimvoly_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sinchmessage/sinchmessage_test.go b/pkg/detectors/sinchmessage/sinchmessage_test.goindex 49a667275..ed428f829 100644--- a/pkg/detectors/sinchmessage/sinchmessage_test.go+++ b/pkg/detectors/sinchmessage/sinchmessage_test.go@@ -16,7 +16,7 @@ import (func TestSinchMessage_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/siteleaf/siteleaf_test.go b/pkg/detectors/siteleaf/siteleaf_test.goindex 3dc42f9be..32f027789 100644--- a/pkg/detectors/siteleaf/siteleaf_test.go+++ b/pkg/detectors/siteleaf/siteleaf_test.go@@ -16,7 +16,7 @@ import (func TestSiteleaf_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/skrappio/skrappio_test.go b/pkg/detectors/skrappio/skrappio_test.goindex aea509d00..410e494d1 100644--- a/pkg/detectors/skrappio/skrappio_test.go+++ b/pkg/detectors/skrappio/skrappio_test.go@@ -16,7 +16,7 @@ import (func TestSkrapio_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/skybiometry/skybiometry_test.go b/pkg/detectors/skybiometry/skybiometry_test.goindex da193275a..50973fa56 100644--- a/pkg/detectors/skybiometry/skybiometry_test.go+++ b/pkg/detectors/skybiometry/skybiometry_test.go@@ -16,7 +16,7 @@ import (func TestSkyBiometry_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/slack/slack.go b/pkg/detectors/slack/slack.gonew file mode 100644index 000000000..4556ffb97--- /dev/null+++ b/pkg/detectors/slack/slack.go@@ -0,0 +1,79 @@+package slack++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "regexp"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Check that the Slack scanner implements the SecretScanner interface at compile time.+var _ detectors.Detector = Scanner{}++var (+ tokenPats = map[string]*regexp.Regexp{+ "Slack Bot Token": regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-z0-9\-]*`),+ "Slack User Token": regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-z0-9\-]*`),+ "Slack Workspace Access Token": regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-z0-9\-]*`),+ "Slack Workspace Refresh Token": regexp.MustCompile(`xoxr\-[0-9]{10,13}\-[0-9]{10,13}[a-z0-9\-]*`),+ }+ verifyURL = "https://slack.com/api/auth.test"+)++type authRes struct {+ Ok bool `json:"ok"`+ URL string `json:"url"` // Workspace URL+ Team string `json:"team"` // Human friendly workspace name+ User string `json:"user"` // Username+ TeamID string `json:"team_id"`+ UserID string `json:"user_id"`+ BotID string `json:"bot_id"`+ Error string `json:"error"`+}++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"xoxb-", "xoxp-", "xoxa-", "xoxr-"}+}++// FromData will find and optionally verify Slack secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ for _, tokenPat := range tokenPats {+ tokens := tokenPat.FindAllString(dataStr, -1)++ for _, token := range tokens {+ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Slack,+ Raw: []byte(token),+ }+ if verify {+ client := common.RetryableHttpClient()+ req, _ := http.NewRequest("POST", verifyURL, nil)+ req.Header.Add("Content-Type", "application/json; charset=utf-8")+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ var authResponse authRes+ json.NewDecoder(res.Body).Decode(&authResponse)+ s.Verified = authResponse.Ok+ }+ }++ results = append(results, s)+ }+ }++ return+}diff --git a/pkg/detectors/slack/slack_test.go b/pkg/detectors/slack/slack_test.gonew file mode 100644index 000000000..b94f209c8--- /dev/null+++ b/pkg/detectors/slack/slack_test.go@@ -0,0 +1,87 @@+package slack++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestScanner_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SLACK")+ secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")+ tests := []struct {+ name string+ data []byte+ verify bool+ wantResults []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),+ verify: true,+ wantResults: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Slack,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found but unverified",+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),+ verify: true,+ wantResults: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Slack,+ Verified: false,+ },+ },+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(context.Background(), tt.verify, tt.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Scanner.FromData) error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.wantResults); diff != "" {+ t.Errorf("Scanner.FromData) %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/slackwebhook/slackwebhook_test.go b/pkg/detectors/slackwebhook/slackwebhook_test.goindex ce035f170..92e6ac6dc 100644--- a/pkg/detectors/slackwebhook/slackwebhook_test.go+++ b/pkg/detectors/slackwebhook/slackwebhook_test.go@@ -16,7 +16,7 @@ import (func TestSlackWebhook_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/smartsheets/smartsheets_test.go b/pkg/detectors/smartsheets/smartsheets_test.goindex bbf1f214f..07829f78c 100644--- a/pkg/detectors/smartsheets/smartsheets_test.go+++ b/pkg/detectors/smartsheets/smartsheets_test.go@@ -16,7 +16,7 @@ import (func TestSmartsheets_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/smartystreets/smartystreets_test.go b/pkg/detectors/smartystreets/smartystreets_test.goindex d5eaf08d0..cac5896f9 100644--- a/pkg/detectors/smartystreets/smartystreets_test.go+++ b/pkg/detectors/smartystreets/smartystreets_test.go@@ -16,7 +16,7 @@ import (func TestSmartyStreets_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/smooch/smooch_test.go b/pkg/detectors/smooch/smooch_test.goindex a74e35945..c1b0fd4bd 100644--- a/pkg/detectors/smooch/smooch_test.go+++ b/pkg/detectors/smooch/smooch_test.go@@ -16,7 +16,7 @@ import (func TestSmooch_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/snipcart/snipcart_test.go b/pkg/detectors/snipcart/snipcart_test.goindex cfbd86377..4e086c8ad 100644--- a/pkg/detectors/snipcart/snipcart_test.go+++ b/pkg/detectors/snipcart/snipcart_test.go@@ -16,7 +16,7 @@ import (func TestSnipcart_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/snykkey/snykkey_test.go b/pkg/detectors/snykkey/snykkey_test.goindex cbc1e2239..027da5fd2 100644--- a/pkg/detectors/snykkey/snykkey_test.go+++ b/pkg/detectors/snykkey/snykkey_test.go@@ -16,7 +16,7 @@ import (func TestSnykKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sparkpost/sparkpost_test.go b/pkg/detectors/sparkpost/sparkpost_test.goindex 8afaf237e..8a58d3f68 100644--- a/pkg/detectors/sparkpost/sparkpost_test.go+++ b/pkg/detectors/sparkpost/sparkpost_test.go@@ -16,7 +16,7 @@ import (func TestSparkpost_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.goindex 5225d09f2..7393e7253 100644--- a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go+++ b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go@@ -16,7 +16,7 @@ import (func TestSplunkObservabilityToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/spoonacular/spoonacular_test.go b/pkg/detectors/spoonacular/spoonacular_test.goindex 667d1e897..561df5149 100644--- a/pkg/detectors/spoonacular/spoonacular_test.go+++ b/pkg/detectors/spoonacular/spoonacular_test.go@@ -16,7 +16,7 @@ import (func TestSpoonacular_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sportsmonk/sportsmonk_test.go b/pkg/detectors/sportsmonk/sportsmonk_test.goindex a91ba6935..e1e55da16 100644--- a/pkg/detectors/sportsmonk/sportsmonk_test.go+++ b/pkg/detectors/sportsmonk/sportsmonk_test.go@@ -16,7 +16,7 @@ import (func TestSportsmonk_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/spotifykey/spotifykey_test.go b/pkg/detectors/spotifykey/spotifykey_test.goindex dca79f10e..04316a378 100644--- a/pkg/detectors/spotifykey/spotifykey_test.go+++ b/pkg/detectors/spotifykey/spotifykey_test.go@@ -16,7 +16,7 @@ import (func TestSpotifyKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/square/square.go b/pkg/detectors/square/square.gonew file mode 100644index 000000000..e2f215f14--- /dev/null+++ b/pkg/detectors/square/square.go@@ -0,0 +1,89 @@+package square++import (+ "context"+ "fmt"+ "net/http"+ "regexp"+ "strings"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ // more context to be added if this is too generic+ secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"square"}) + `(EAAA[a-zA-Z0-9\-\+\=]{60})`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"EAAA"}+}++// FromData will find and optionally verify Square secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ // Surprisingly there are still a lot of false positives! So, also doing substring check for square.+ if !strings.Contains(strings.ToLower(dataStr), "square") {+ return+ }++ secMatches := secretPat.FindAllStringSubmatch(dataStr, -1)+ for _, secMatch := range secMatches {+ if len(secMatch) != 2 {+ continue+ }+ res := strings.TrimSpace(secMatch[1])++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Square,+ Raw: []byte(res),+ }++ if verify {+ // there are a few endpoints we can check, but templates seems the least sensitive.+ // 403 will be issued if the scope is wrong but the key is correct+ baseURL := "https://connect.squareupsandbox.com/v2/merchants"++ client := common.RetryableHttpClient()++ // test `merchants` scope - its commonly allowed and low sensitivity+ req, _ := http.NewRequestWithContext(ctx, "GET", baseURL, nil)+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", res))+ req.Header.Add("Content-Type", "application/json")+ // unclear if this version needs to be set or matters, seems to work without, but docs want it+ //req.Header.Add("Square-Version", "2020-08-12")+ res, err := client.Do(req)+ if err != nil {+ return results, err+ }+ defer res.Body.Close()++ // 200 means good key and has `merchants` scope - default allowed by square+ // 401 is bad key+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {+ s.Verified = true+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/square/square_test.go b/pkg/detectors/square/square_test.gonew file mode 100644index 000000000..8b9778604--- /dev/null+++ b/pkg/detectors/square/square_test.go@@ -0,0 +1,112 @@+package square++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestSquare_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SQUARE_SECRET")+ secretInactive := testSecrets.MustGetField("SQUARE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a square secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Square,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unvierifed",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a square secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Square,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Square.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Square.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/squareapp/squareapp.go b/pkg/detectors/squareapp/squareapp.gonew file mode 100644index 000000000..fe2caedca--- /dev/null+++ b/pkg/detectors/squareapp/squareapp.go@@ -0,0 +1,96 @@+package squareapp++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "regexp"+ "strings"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ //possibly always `sq0csp` for secret+ // and `sq0idb` for app+ keyPat = regexp.MustCompile(`[\w\-]*sq0i[a-z]{2}-[0-9A-Za-z\-_]{22,43}`)+ secPat = regexp.MustCompile(`[\w\-]*sq0c[a-z]{2}-[0-9A-Za-z\-_]{40,50}`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"sq0i"}+}++// FromData will find and optionally verify SquareApp secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllString(dataStr, -1)+ secMatches := secPat.FindAllString(dataStr, -1)+ for _, match := range matches {++ for _, secMatch := range secMatches {++ if detectors.IsKnownFalsePositive(strings.ToLower(secMatch), detectors.DefaultFalsePositives, true) {+ continue+ }++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Square,+ Raw: []byte(match),+ Redacted: match,+ }++ if verify {+ baseURL := "https://connect.squareupsandbox.com/oauth2/revoke"++ client := common.RetryableHttpClient()+ reqData, err := json.Marshal(map[string]string{+ "client_id": match,+ "access_token": "fakeTruffleHogAccessTokenForVerification",+ })+ if err != nil {+ return results, err+ }++ req, _ := http.NewRequestWithContext(ctx, "POST", baseURL, bytes.NewReader(reqData))+ req.Header.Add("Authorization", fmt.Sprintf("Client %s", secMatch))+ req.Header.Add("Content-Type", "application/json")++ // unclear if this version needs to be set or matters, seems to work without, but docs want it+ //req.Header.Add("Square-Version", "2020-08-12")+ res, err := client.Do(req)+ if err != nil {+ return results, err+ }+ defer res.Body.Close()++ // 404 = Correct crentials. The fake access token should not be found+ if res.StatusCode == http.StatusNotFound {+ s.Verified = true+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }+ }+ return+}diff --git a/pkg/detectors/squareapp/squareapp_test.go b/pkg/detectors/squareapp/squareapp_test.gonew file mode 100644index 000000000..10d4a7e20--- /dev/null+++ b/pkg/detectors/squareapp/squareapp_test.go@@ -0,0 +1,114 @@+package squareapp++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestSquareApp_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SQUAREAPP_SECRET")+ secretInactive := testSecrets.MustGetField("SQUAREAPP_INACTIVE")+ id := testSecrets.MustGetField("SQUAREAPP_ID")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Square,+ Verified: true,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secretInactive, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Square,+ Verified: false,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SquareApp.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SquareApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/squarespace/squarespace_test.go b/pkg/detectors/squarespace/squarespace_test.goindex 2b3e5a3d5..2858518d4 100644--- a/pkg/detectors/squarespace/squarespace_test.go+++ b/pkg/detectors/squarespace/squarespace_test.go@@ -16,7 +16,7 @@ import (func TestSquarespace_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/squareup/squareup_test.go b/pkg/detectors/squareup/squareup_test.goindex 4594ff46a..796f27fe6 100644--- a/pkg/detectors/squareup/squareup_test.go+++ b/pkg/detectors/squareup/squareup_test.go@@ -16,7 +16,7 @@ import (func TestSquareup_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sslmate/sslmate_test.go b/pkg/detectors/sslmate/sslmate_test.goindex 0e40efec5..c70c7ec99 100644--- a/pkg/detectors/sslmate/sslmate_test.go+++ b/pkg/detectors/sslmate/sslmate_test.go@@ -16,7 +16,7 @@ import (func TestSslMate_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/stitchdata/stitchdata_test.go b/pkg/detectors/stitchdata/stitchdata_test.goindex a421a9d2d..4f6f591cf 100644--- a/pkg/detectors/stitchdata/stitchdata_test.go+++ b/pkg/detectors/stitchdata/stitchdata_test.go@@ -16,7 +16,7 @@ import (func TestStitchdata_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/stockdata/stockdata_test.go b/pkg/detectors/stockdata/stockdata_test.goindex 3b7812c12..0d6b52daa 100644--- a/pkg/detectors/stockdata/stockdata_test.go+++ b/pkg/detectors/stockdata/stockdata_test.go@@ -16,7 +16,7 @@ import (func TestStockdata_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/storecove/storecove_test.go b/pkg/detectors/storecove/storecove_test.goindex acadb3083..8b4f425b9 100644--- a/pkg/detectors/storecove/storecove_test.go+++ b/pkg/detectors/storecove/storecove_test.go@@ -16,7 +16,7 @@ import (func TestStorecove_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/stormglass/stormglass_test.go b/pkg/detectors/stormglass/stormglass_test.goindex 92775d2c3..e5519543b 100644--- a/pkg/detectors/stormglass/stormglass_test.go+++ b/pkg/detectors/stormglass/stormglass_test.go@@ -16,7 +16,7 @@ import (func TestStormglass_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/storyblok/storyblok_test.go b/pkg/detectors/storyblok/storyblok_test.goindex a20e46159..6b413e313 100644--- a/pkg/detectors/storyblok/storyblok_test.go+++ b/pkg/detectors/storyblok/storyblok_test.go@@ -16,7 +16,7 @@ import (func TestStoryblok_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/storychief/storychief_test.go b/pkg/detectors/storychief/storychief_test.goindex 6d0ba5ad0..2b395c98a 100644--- a/pkg/detectors/storychief/storychief_test.go+++ b/pkg/detectors/storychief/storychief_test.go@@ -16,7 +16,7 @@ import (func TestStorychief_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/strava/strava_test.go b/pkg/detectors/strava/strava_test.goindex fb78bb901..0364b11a6 100644--- a/pkg/detectors/strava/strava_test.go+++ b/pkg/detectors/strava/strava_test.go@@ -16,7 +16,7 @@ import (func TestStrava_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/streak/streak_test.go b/pkg/detectors/streak/streak_test.goindex 3d4540d95..dc1fa4bdd 100644--- a/pkg/detectors/streak/streak_test.go+++ b/pkg/detectors/streak/streak_test.go@@ -16,7 +16,7 @@ import (func TestStreak_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/stripe/stripe.go b/pkg/detectors/stripe/stripe.gonew file mode 100644index 000000000..af9c6b1af--- /dev/null+++ b/pkg/detectors/stripe/stripe.go@@ -0,0 +1,76 @@+package stripe++import (+ "context"+ "fmt"+ "net/http"+ "regexp"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ //doesnt include test keys with "sk_test"+ secretKey = regexp.MustCompile(`[rs]k_live_[a-zA-Z0-9]{20,30}`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"k_live"}+}++// FromData will find and optionally verify Stripe secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {++ dataStr := string(data)++ matches := secretKey.FindAllString(dataStr, -1)++ for _, match := range matches {++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Stripe,+ Raw: []byte(match),+ }++ if verify {++ baseURL := "https://api.stripe.com/v1/charges"++ client := common.RetryableHttpClient()++ // test `read_user` scope+ req, _ := http.NewRequestWithContext(ctx, "GET", baseURL, nil)+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match))+ req.Header.Add("Content-Type", "application/json")+ res, err := client.Do(req)+ if err != nil {+ return results, err+ }+ defer res.Body.Close()++ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {+ s.Verified = true+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/stripe/stripe_test.go b/pkg/detectors/stripe/stripe_test.gonew file mode 100644index 000000000..ad83d629c--- /dev/null+++ b/pkg/detectors/stripe/stripe_test.go@@ -0,0 +1,111 @@+package stripe++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestStripe_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STRIPE_SECRET")+ secretInactive := testSecrets.MustGetField("STRIPE_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stripe,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stripe,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Stripe.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Stripe.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/stytch/stytch_test.go b/pkg/detectors/stytch/stytch_test.goindex 6945e315b..2c1b0d1e9 100644--- a/pkg/detectors/stytch/stytch_test.go+++ b/pkg/detectors/stytch/stytch_test.go@@ -16,7 +16,7 @@ import (func TestStytch_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/sumologickey/sumologickey_test.go b/pkg/detectors/sumologickey/sumologickey_test.goindex c3c8ef5d4..768051b41 100644--- a/pkg/detectors/sumologickey/sumologickey_test.go+++ b/pkg/detectors/sumologickey/sumologickey_test.go@@ -16,7 +16,7 @@ import (func TestSumoLogicKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/surveyanyplace/surveyanyplace_test.go b/pkg/detectors/surveyanyplace/surveyanyplace_test.goindex 4085c3f92..a31ba1687 100644--- a/pkg/detectors/surveyanyplace/surveyanyplace_test.go+++ b/pkg/detectors/surveyanyplace/surveyanyplace_test.go@@ -16,7 +16,7 @@ import (func TestSurveyAnyplace_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/surveybot/surveybot_test.go b/pkg/detectors/surveybot/surveybot_test.goindex b2164f547..9ffe87c36 100644--- a/pkg/detectors/surveybot/surveybot_test.go+++ b/pkg/detectors/surveybot/surveybot_test.go@@ -16,7 +16,7 @@ import (func TestSurveyBot_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/surveysparrow/surveysparrow_test.go b/pkg/detectors/surveysparrow/surveysparrow_test.goindex a44889f50..40778b0bd 100644--- a/pkg/detectors/surveysparrow/surveysparrow_test.go+++ b/pkg/detectors/surveysparrow/surveysparrow_test.go@@ -16,7 +16,7 @@ import (func TestSurveySparrow_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/survicate/survicate_test.go b/pkg/detectors/survicate/survicate_test.goindex 78cd23a7f..388151dbe 100644--- a/pkg/detectors/survicate/survicate_test.go+++ b/pkg/detectors/survicate/survicate_test.go@@ -16,7 +16,7 @@ import (func TestSurvicate_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/swell/swell_test.go b/pkg/detectors/swell/swell_test.goindex 9922472eb..9a9ce655e 100644--- a/pkg/detectors/swell/swell_test.go+++ b/pkg/detectors/swell/swell_test.go@@ -16,7 +16,7 @@ import (func TestSwell_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/swiftype/swiftype_test.go b/pkg/detectors/swiftype/swiftype_test.goindex e5abab3ab..d9714abe3 100644--- a/pkg/detectors/swiftype/swiftype_test.go+++ b/pkg/detectors/swiftype/swiftype_test.go@@ -16,7 +16,7 @@ import (func TestSwiftype_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/tatumio/tatumio_test.go b/pkg/detectors/tatumio/tatumio_test.goindex 88f72e600..ccf01bf55 100644--- a/pkg/detectors/tatumio/tatumio_test.go+++ b/pkg/detectors/tatumio/tatumio_test.go@@ -16,7 +16,7 @@ import (func TestTatumIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/taxjar/taxjar_test.go b/pkg/detectors/taxjar/taxjar_test.goindex 4b7f20f73..94a88c925 100644--- a/pkg/detectors/taxjar/taxjar_test.go+++ b/pkg/detectors/taxjar/taxjar_test.go@@ -16,7 +16,7 @@ import (func TestTaxjar_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/teamgate/teamgate_test.go b/pkg/detectors/teamgate/teamgate_test.goindex 8a91aa723..d1bf87e50 100644--- a/pkg/detectors/teamgate/teamgate_test.go+++ b/pkg/detectors/teamgate/teamgate_test.go@@ -16,7 +16,7 @@ import (func TestTeamgate_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/technicalanalysisapi/technicalanalysisapi_test.go b/pkg/detectors/technicalanalysisapi/technicalanalysisapi_test.goindex c98d39e2e..ec84b4594 100644--- a/pkg/detectors/technicalanalysisapi/technicalanalysisapi_test.go+++ b/pkg/detectors/technicalanalysisapi/technicalanalysisapi_test.go@@ -16,7 +16,7 @@ import (func TestTechnicalAnalysisApi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/telegrambottoken/telegrambottoken_test.go b/pkg/detectors/telegrambottoken/telegrambottoken_test.goindex 618be7451..f420e307e 100644--- a/pkg/detectors/telegrambottoken/telegrambottoken_test.go+++ b/pkg/detectors/telegrambottoken/telegrambottoken_test.go@@ -16,7 +16,7 @@ import (func TestTelegramBotToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/telnyx/telnyx_test.go b/pkg/detectors/telnyx/telnyx_test.goindex 75097a41e..eb6cb4669 100644--- a/pkg/detectors/telnyx/telnyx_test.go+++ b/pkg/detectors/telnyx/telnyx_test.go@@ -16,7 +16,7 @@ import (func TestTelnyx_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken_test.go b/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken_test.goindex 070b193fa..f9ca6100a 100644--- a/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken_test.go+++ b/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken_test.go@@ -16,7 +16,7 @@ import (func TestTerraformCloudPersonalToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/text2data/text2data_test.go b/pkg/detectors/text2data/text2data_test.goindex c8d75037f..706597554 100644--- a/pkg/detectors/text2data/text2data_test.go+++ b/pkg/detectors/text2data/text2data_test.go@@ -16,7 +16,7 @@ import (func TestText2Data_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/textmagic/textmagic_test.go b/pkg/detectors/textmagic/textmagic_test.goindex 3ffefa840..f21e6fdfd 100644--- a/pkg/detectors/textmagic/textmagic_test.go+++ b/pkg/detectors/textmagic/textmagic_test.go@@ -16,7 +16,7 @@ import (func TestTextmagic_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/thinkific/thinkific_test.go b/pkg/detectors/thinkific/thinkific_test.goindex 57791f57c..8c5ca9585 100644--- a/pkg/detectors/thinkific/thinkific_test.go+++ b/pkg/detectors/thinkific/thinkific_test.go@@ -16,7 +16,7 @@ import (func TestThinkific_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/thousandeyes/thousandeyes_test.go b/pkg/detectors/thousandeyes/thousandeyes_test.goindex 41155a76f..a7fda41e4 100644--- a/pkg/detectors/thousandeyes/thousandeyes_test.go+++ b/pkg/detectors/thousandeyes/thousandeyes_test.go@@ -16,7 +16,7 @@ import (func TestThousandEyes_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ticketmaster/ticketmaster_test.go b/pkg/detectors/ticketmaster/ticketmaster_test.goindex be670d586..95435d318 100644--- a/pkg/detectors/ticketmaster/ticketmaster_test.go+++ b/pkg/detectors/ticketmaster/ticketmaster_test.go@@ -16,7 +16,7 @@ import (func TestTicketMaster_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/tiingo/tiingo.go b/pkg/detectors/tiingo/tiingo.gonew file mode 100644index 000000000..20e8af23b--- /dev/null+++ b/pkg/detectors/tiingo/tiingo.go@@ -0,0 +1,75 @@+package tiingo++import (+ "context"+ "fmt"++ // "log"+ "regexp"+ "strings"++ "net/http"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ client = common.SaneHttpClient()++ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"tiingo"}) + `\b([0-9a-z]{40})\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"tiingo"}+}++// FromData will find and optionally verify Tiingo secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ if len(match) != 2 {+ continue+ }+ resMatch := strings.TrimSpace(match[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_Tiingo,+ Raw: []byte(resMatch),+ }++ if verify {+ req, _ := http.NewRequest("GET", "https://api.tiingo.com/tiingo/fundamentals/definitions", nil)+ req.Header.Add("Content-Type", "application/json")+ req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s1.Verified = true+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue+ }+ }+ }+ }++ results = append(results, s1)+ }++ return detectors.CleanResults(results), nil+}diff --git a/pkg/detectors/tiingo/tiingo_test.go b/pkg/detectors/tiingo/tiingo_test.gonew file mode 100644index 000000000..4fa578f9a--- /dev/null+++ b/pkg/detectors/tiingo/tiingo_test.go@@ -0,0 +1,113 @@+package tiingo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestTiingo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("TIINGO")+ inactiveSecret := testSecrets.MustGetField("TIINGO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a tiingo secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Tiingo,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a tiingo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Tiingo,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Tiingo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Tiingo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/timezoneapi/timezoneapi_test.go b/pkg/detectors/timezoneapi/timezoneapi_test.goindex 2504ed2aa..63a0446be 100644--- a/pkg/detectors/timezoneapi/timezoneapi_test.go+++ b/pkg/detectors/timezoneapi/timezoneapi_test.go@@ -16,7 +16,7 @@ import (func TestTimezoneapi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/tly/tly_test.go b/pkg/detectors/tly/tly_test.goindex 08c8f5751..f96a43d12 100644--- a/pkg/detectors/tly/tly_test.go+++ b/pkg/detectors/tly/tly_test.go@@ -16,7 +16,7 @@ import (func TestTLy_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/tmetric/tmetric_test.go b/pkg/detectors/tmetric/tmetric_test.goindex 0753b4586..3e579a87a 100644--- a/pkg/detectors/tmetric/tmetric_test.go+++ b/pkg/detectors/tmetric/tmetric_test.go@@ -16,7 +16,7 @@ import (func TestTmetric_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/todoist/todoist_test.go b/pkg/detectors/todoist/todoist_test.goindex b48a8c975..aaa572c2b 100644--- a/pkg/detectors/todoist/todoist_test.go+++ b/pkg/detectors/todoist/todoist_test.go@@ -16,7 +16,7 @@ import (func TestTodoist_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/toggltrack/toggltrack_test.go b/pkg/detectors/toggltrack/toggltrack_test.goindex 8de05dfe8..4cd0e893a 100644--- a/pkg/detectors/toggltrack/toggltrack_test.go+++ b/pkg/detectors/toggltrack/toggltrack_test.go@@ -16,7 +16,7 @@ import (func TestTogglTrack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/tomorrowio/tomorrowio_test.go b/pkg/detectors/tomorrowio/tomorrowio_test.goindex b4cd59619..f86feb343 100644--- a/pkg/detectors/tomorrowio/tomorrowio_test.go+++ b/pkg/detectors/tomorrowio/tomorrowio_test.go@@ -16,7 +16,7 @@ import (func TestTomorrowIO_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/tomtom/tomtom_test.go b/pkg/detectors/tomtom/tomtom_test.goindex 9b2b2b2d7..0fe2dbd52 100644--- a/pkg/detectors/tomtom/tomtom_test.go+++ b/pkg/detectors/tomtom/tomtom_test.go@@ -16,7 +16,7 @@ import (func TestTomtom_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/tradier/tradier_test.go b/pkg/detectors/tradier/tradier_test.goindex fc69478a1..52abf9271 100644--- a/pkg/detectors/tradier/tradier_test.go+++ b/pkg/detectors/tradier/tradier_test.go@@ -16,7 +16,7 @@ import (func TestTradier_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/travelpayouts/travelpayouts_test.go b/pkg/detectors/travelpayouts/travelpayouts_test.goindex 8a1823566..c168a791d 100644--- a/pkg/detectors/travelpayouts/travelpayouts_test.go+++ b/pkg/detectors/travelpayouts/travelpayouts_test.go@@ -16,7 +16,7 @@ import (func TestTravelPayouts_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/travisci/travisci_test.go b/pkg/detectors/travisci/travisci_test.goindex 73289293b..df66b0346 100644--- a/pkg/detectors/travisci/travisci_test.go+++ b/pkg/detectors/travisci/travisci_test.go@@ -16,7 +16,7 @@ import (func TestTravisCI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/trelloapikey/trelloapikey_test.go b/pkg/detectors/trelloapikey/trelloapikey_test.goindex 2eb55b0bc..7f64f2c9f 100644--- a/pkg/detectors/trelloapikey/trelloapikey_test.go+++ b/pkg/detectors/trelloapikey/trelloapikey_test.go@@ -16,7 +16,7 @@ import (func TestTrelloApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/tru/tru_test.go b/pkg/detectors/tru/tru_test.goindex be04b152a..747a68e5a 100644--- a/pkg/detectors/tru/tru_test.go+++ b/pkg/detectors/tru/tru_test.go@@ -16,7 +16,7 @@ import (func TestTru_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/twelvedata/twelvedata_test.go b/pkg/detectors/twelvedata/twelvedata_test.goindex 91d698e48..08a2b64ef 100644--- a/pkg/detectors/twelvedata/twelvedata_test.go+++ b/pkg/detectors/twelvedata/twelvedata_test.go@@ -16,7 +16,7 @@ import (func TestTwelveData_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/twilio/twilio.go b/pkg/detectors/twilio/twilio.gonew file mode 100644index 000000000..403f02db0--- /dev/null+++ b/pkg/detectors/twilio/twilio.go@@ -0,0 +1,90 @@+package twilio++import (+ "context"+ "net/http"+ "net/url"+ "regexp"+ "strings"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.+ sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)+ keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"sid"}+}++// FromData will find and optionally verify Twilio secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ identifierMatches := identifierPat.FindAllString(dataStr, -1)++ if len(identifierMatches) == 0 {+ return+ }++ keyMatches := keyPat.FindAllString(dataStr, -1)+ sidMatches := sidPat.FindAllString(dataStr, -1)++ for _, sid := range sidMatches {++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Twilio,+ Raw: []byte(sid),+ Redacted: sid,+ }++ if verify {+ client := common.RetryableHttpClient()+ for _, key := range keyMatches {++ form := url.Values{}+ form.Add("FriendlyName", "MyServiceName")+ req, _ := http.NewRequestWithContext(+ ctx, "POST", "https://verify.twilio.com/v2/Services",+ strings.NewReader(form.Encode()),+ )+ req.Header.Add("Content-Type", "application/x-www-form-urlencoded")+ req.Header.Add("Accept", "*/*")+ req.SetBasicAuth(sid, key)+ res, err := client.Do(req)+ if err != nil {+ break+ }+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ s.Verified = true+ }+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ if len(keyMatches) > 0 {+ results = append(results, s)+ }+ }++ return+}diff --git a/pkg/detectors/twilio/twilio_test.go b/pkg/detectors/twilio/twilio_test.gonew file mode 100644index 000000000..17805006f--- /dev/null+++ b/pkg/detectors/twilio/twilio_test.go@@ -0,0 +1,115 @@+package twilio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestTwilio_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("TWILLIO_API")+ secretInactive := testSecrets.MustGetField("TWILLIO_API_INACTIVE")+ id := testSecrets.MustGetField("TWILLIO_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Twilio,+ Verified: true,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "found, not verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secretInactive, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Twilio,+ Verified: false,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Twilio.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/tyntec/tyntec_test.go b/pkg/detectors/tyntec/tyntec_test.goindex d0885e7b2..40968a91c 100644--- a/pkg/detectors/tyntec/tyntec_test.go+++ b/pkg/detectors/tyntec/tyntec_test.go@@ -16,7 +16,7 @@ import (func TestTyntec_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/typeform/typeform_test.go b/pkg/detectors/typeform/typeform_test.goindex 2d612f5d6..9cc14c102 100644--- a/pkg/detectors/typeform/typeform_test.go+++ b/pkg/detectors/typeform/typeform_test.go@@ -16,7 +16,7 @@ import (func TestTypeform_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/ubidots/ubidots_test.go b/pkg/detectors/ubidots/ubidots_test.goindex 27b3be212..9ad99cf6e 100644--- a/pkg/detectors/ubidots/ubidots_test.go+++ b/pkg/detectors/ubidots/ubidots_test.go@@ -16,7 +16,7 @@ import (func TestUbidots_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/unifyid/unifyid_test.go b/pkg/detectors/unifyid/unifyid_test.goindex c47974ba4..f5ebb8bd1 100644--- a/pkg/detectors/unifyid/unifyid_test.go+++ b/pkg/detectors/unifyid/unifyid_test.go@@ -16,7 +16,7 @@ import (func TestUnifyid_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/unplugg/unplugg_test.go b/pkg/detectors/unplugg/unplugg_test.goindex b80f3e204..269155fc7 100644--- a/pkg/detectors/unplugg/unplugg_test.go+++ b/pkg/detectors/unplugg/unplugg_test.go@@ -16,7 +16,7 @@ import (func TestUnplugg_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/unsplash/unsplash_test.go b/pkg/detectors/unsplash/unsplash_test.goindex a018d10fc..02d3fa71e 100644--- a/pkg/detectors/unsplash/unsplash_test.go+++ b/pkg/detectors/unsplash/unsplash_test.go@@ -16,7 +16,7 @@ import (func TestUnsplash_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/upcdatabase/upcdatabase_test.go b/pkg/detectors/upcdatabase/upcdatabase_test.goindex a7120aec7..37f30c851 100644--- a/pkg/detectors/upcdatabase/upcdatabase_test.go+++ b/pkg/detectors/upcdatabase/upcdatabase_test.go@@ -16,7 +16,7 @@ import (func TestUPCDatabase_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/uplead/uplead_test.go b/pkg/detectors/uplead/uplead_test.goindex 25fd1dfdc..a3bd36648 100644--- a/pkg/detectors/uplead/uplead_test.go+++ b/pkg/detectors/uplead/uplead_test.go@@ -16,7 +16,7 @@ import (func TestUplead_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/upwave/upwave_test.go b/pkg/detectors/upwave/upwave_test.goindex 4dceb74f3..b62bf1cdd 100644--- a/pkg/detectors/upwave/upwave_test.go+++ b/pkg/detectors/upwave/upwave_test.go@@ -16,7 +16,7 @@ import (func TestUpwave_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.gonew file mode 100644index 000000000..92bf1d08c--- /dev/null+++ b/pkg/detectors/uri/uri.go@@ -0,0 +1,134 @@+package uri++import (+ "bytes"+ "context"+ "encoding/json"+ "io/ioutil"+ "net/http"+ "net/url"+ "regexp"+ "strings"++ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+)++type Scanner struct {+ allowKnownTestSites bool+}++// Ensure the Scanner satisfies the interface at compile time+var _ detectors.Detector = (*Scanner)(nil)++var (+ keyPat = regexp.MustCompile(`\b[a-zA-Z]{1,10}:?\/\/[-.%\w{}]{1,50}:([-.%\S]{3,50})@[-.%\w\/:]+\b`)+)++type proxyRes struct {+ Verified bool `json:"verified"`+}++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"http"}+}++func allowlistedProtos(scheme string) bool {+ allowlisted := []string{"http", "https", "mongodb", "redis", "ftp"}+ for _, s := range allowlisted {+ if s == scheme {+ return true+ }+ }+ return false+}++// FromData will find and optionally verify URI secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ //Prevent SSRF "https://us-central1-ssrfproxy.cloudfunctions.net/ssrfproxy-6d96c399-74bb-4299-b49d-c1c870277eb9"+ //TODO add as config option+ //TODO extend to other http outbound calls in other areas of code+ ssrfProtectorURL := "https://us-central1-ssrfproxy.cloudfunctions.net/ssrfproxy-6d96c399-74bb-4299-b49d-c1c870277eb9"++ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {++ if !s.allowKnownTestSites {+ if strings.Contains(match[0], "httpbin.org") {+ continue+ }+ if strings.Contains(match[0], "httpwatch.com") {+ continue+ }+ }++ token := match[0]+ password := match[1]++ parsedURL, err := url.Parse(token)+ if err != nil {+ continue+ }+ if _, ok := parsedURL.User.Password(); !ok {+ continue+ }+ if !allowlistedProtos(parsedURL.Scheme) {+ continue+ }++ redact := strings.TrimSpace(strings.Replace(token, password, strings.Repeat("*", len(password)), -1))++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_URI,+ Raw: []byte(token),+ Redacted: redact,+ }++ if verify {+ client := common.SaneHttpClient()+ // whitelist protocols++ // Assume a 200 response is a valid credential+ postValues := map[string]string{"protocol": parsedURL.Scheme, "credentialed_uri": token}+ jsonValue, _ := json.Marshal(postValues)+ req, _ := http.NewRequestWithContext(ctx, "POST", ssrfProtectorURL, bytes.NewBuffer(jsonValue))+ req.Header.Add("Content-Type", "application/json")+ res, err := client.Do(req)+ if err != nil {+ // log.WithError(err).Warn("Error in http post to SSRF proxy")+ continue+ }+ defer res.Body.Close()+ result := proxyRes{}+ body, err := ioutil.ReadAll(res.Body)+ if len(body) == 0 || err != nil {+ continue+ }+ err = json.Unmarshal(body, &result)+ if err != nil {+ // log.WithField("body", string(body)).WithError(err).Debug("Error decoding SSRF proxy response")+ continue+ }+ if result.Verified {+ s.Verified = true+ }+ }++ if !s.Verified {+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {+ continue+ }+ }++ results = append(results, s)+ }++ return+}diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.gonew file mode 100644index 000000000..8630076eb--- /dev/null+++ b/pkg/detectors/uri/uri_test.go@@ -0,0 +1,113 @@+package uri++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"+)++func TestURI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("URI_SECRET")+ secretInactive := testSecrets.MustGetField("URI_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{++ {+ name: "found, unverified, wrong username",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a uri secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_URI,+ Verified: false,+ Redacted: "https://user:**********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx",+ },+ },+ wantErr: false,+ },+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a uri secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_URI,+ Verified: true,+ Redacted: "https://httpwatch:**********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx",+ },+ },+ wantErr: false,+ },+ {+ name: "bad proto",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("file://user:[email protected]:123/wh/at/ever"),+ verify: true,+ },+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{allowKnownTestSites: true}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ // if os.Getenv("FORCE_PASS_DIFF") == "true" {+ // return+ // }+ for i := range got {+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/urlscan/urlscan_test.go b/pkg/detectors/urlscan/urlscan_test.goindex 99bb99dbc..bccc3a37a 100644--- a/pkg/detectors/urlscan/urlscan_test.go+++ b/pkg/detectors/urlscan/urlscan_test.go@@ -16,7 +16,7 @@ import (func TestUrlscan_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/user/user_test.go b/pkg/detectors/user/user_test.goindex 59de722ba..b126135d2 100644--- a/pkg/detectors/user/user_test.go+++ b/pkg/detectors/user/user_test.go@@ -16,7 +16,7 @@ import (func TestUser_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/userstack/userstack_test.go b/pkg/detectors/userstack/userstack_test.goindex ed11cfea9..e45228731 100644--- a/pkg/detectors/userstack/userstack_test.go+++ b/pkg/detectors/userstack/userstack_test.go@@ -16,7 +16,7 @@ import (func TestUserStack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/vatlayer/vatlayer_test.go b/pkg/detectors/vatlayer/vatlayer_test.goindex 17a74ed97..be0079217 100644--- a/pkg/detectors/vatlayer/vatlayer_test.go+++ b/pkg/detectors/vatlayer/vatlayer_test.go@@ -16,7 +16,7 @@ import (func TestVatLayer_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/vercel/vercel_test.go b/pkg/detectors/vercel/vercel_test.goindex 67f541e8c..8f32e6aa9 100644--- a/pkg/detectors/vercel/vercel_test.go+++ b/pkg/detectors/vercel/vercel_test.go@@ -16,7 +16,7 @@ import (func TestVercel_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/verifier/verifier_test.go b/pkg/detectors/verifier/verifier_test.goindex c0ee0a7d9..85bf48d6a 100644--- a/pkg/detectors/verifier/verifier_test.go+++ b/pkg/detectors/verifier/verifier_test.go@@ -16,7 +16,7 @@ import (func TestVerifier_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/verimail/verimail_test.go b/pkg/detectors/verimail/verimail_test.goindex 758aad8e0..564062f5d 100644--- a/pkg/detectors/verimail/verimail_test.go+++ b/pkg/detectors/verimail/verimail_test.go@@ -16,7 +16,7 @@ import (func TestVerimail_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/veriphone/veriphone_test.go b/pkg/detectors/veriphone/veriphone_test.goindex 28b86985e..6309dd278 100644--- a/pkg/detectors/veriphone/veriphone_test.go+++ b/pkg/detectors/veriphone/veriphone_test.go@@ -16,7 +16,7 @@ import (func TestVeriphone_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/versioneye/versioneye_test.go b/pkg/detectors/versioneye/versioneye_test.goindex 6d915d9bb..614cecc73 100644--- a/pkg/detectors/versioneye/versioneye_test.go+++ b/pkg/detectors/versioneye/versioneye_test.go@@ -16,7 +16,7 @@ import (func TestVersionEye_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/viewneo/viewneo_test.go b/pkg/detectors/viewneo/viewneo_test.goindex c4feb49f4..8f2eaa938 100644--- a/pkg/detectors/viewneo/viewneo_test.go+++ b/pkg/detectors/viewneo/viewneo_test.go@@ -16,7 +16,7 @@ import (func TestViewneo_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/virustotal/virustotal_test.go b/pkg/detectors/virustotal/virustotal_test.goindex a6987111c..a38c72c23 100644--- a/pkg/detectors/virustotal/virustotal_test.go+++ b/pkg/detectors/virustotal/virustotal_test.go@@ -16,7 +16,7 @@ import (func TestVirusTotal_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/visualcrossing/visualcrossing_test.go b/pkg/detectors/visualcrossing/visualcrossing_test.goindex 34477b634..15f279f1c 100644--- a/pkg/detectors/visualcrossing/visualcrossing_test.go+++ b/pkg/detectors/visualcrossing/visualcrossing_test.go@@ -16,7 +16,7 @@ import (func TestVisualcrossing_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/voicegain/voicegain_test.go b/pkg/detectors/voicegain/voicegain_test.goindex 3c38cedfe..75467325f 100644--- a/pkg/detectors/voicegain/voicegain_test.go+++ b/pkg/detectors/voicegain/voicegain_test.go@@ -16,7 +16,7 @@ import (func TestVoicegain_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/vouchery/vouchery_test.go b/pkg/detectors/vouchery/vouchery_test.goindex aae9f8588..0926d12f4 100644--- a/pkg/detectors/vouchery/vouchery_test.go+++ b/pkg/detectors/vouchery/vouchery_test.go@@ -16,7 +16,7 @@ import (func TestVouchery_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/vpnapi/vpnapi_test.go b/pkg/detectors/vpnapi/vpnapi_test.goindex c7b83c6a8..17a3d9d31 100644--- a/pkg/detectors/vpnapi/vpnapi_test.go+++ b/pkg/detectors/vpnapi/vpnapi_test.go@@ -16,7 +16,7 @@ import (func TestVpnapi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/vultrapikey/vultrapikey_test.go b/pkg/detectors/vultrapikey/vultrapikey_test.goindex 4b784ee0c..2ffc5079c 100644--- a/pkg/detectors/vultrapikey/vultrapikey_test.go+++ b/pkg/detectors/vultrapikey/vultrapikey_test.go@@ -16,7 +16,7 @@ import (func TestVultrApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/vyte/vyte_test.go b/pkg/detectors/vyte/vyte_test.goindex 9d24c878b..a89a38620 100644--- a/pkg/detectors/vyte/vyte_test.go+++ b/pkg/detectors/vyte/vyte_test.go@@ -16,7 +16,7 @@ import (func TestVyte_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/walkscore/walkscore_test.go b/pkg/detectors/walkscore/walkscore_test.goindex d0b7671b2..be6881e66 100644--- a/pkg/detectors/walkscore/walkscore_test.go+++ b/pkg/detectors/walkscore/walkscore_test.go@@ -16,7 +16,7 @@ import (func TestWalkscore_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/weatherbit/weatherbit_test.go b/pkg/detectors/weatherbit/weatherbit_test.goindex cad278ae6..cce0ea7ea 100644--- a/pkg/detectors/weatherbit/weatherbit_test.go+++ b/pkg/detectors/weatherbit/weatherbit_test.go@@ -16,7 +16,7 @@ import (func TestWeatherbit_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/weatherstack/weatherstack_test.go b/pkg/detectors/weatherstack/weatherstack_test.goindex aa81491b8..5599e738c 100644--- a/pkg/detectors/weatherstack/weatherstack_test.go+++ b/pkg/detectors/weatherstack/weatherstack_test.go@@ -16,7 +16,7 @@ import (func TestWeatherstack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/webex/webex_test.go b/pkg/detectors/webex/webex_test.goindex 70a139aba..77f21a2db 100644--- a/pkg/detectors/webex/webex_test.go+++ b/pkg/detectors/webex/webex_test.go@@ -16,7 +16,7 @@ import (func TestWebex_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/webflow/webflow_test.go b/pkg/detectors/webflow/webflow_test.goindex c10c4ed86..455760ea9 100644--- a/pkg/detectors/webflow/webflow_test.go+++ b/pkg/detectors/webflow/webflow_test.go@@ -16,7 +16,7 @@ import (func TestWebflow_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/webscraper/webscraper_test.go b/pkg/detectors/webscraper/webscraper_test.goindex f8ce45dbf..e806d1b72 100644--- a/pkg/detectors/webscraper/webscraper_test.go+++ b/pkg/detectors/webscraper/webscraper_test.go@@ -16,7 +16,7 @@ import (func TestWebScraper_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/webscraping/webscraping_test.go b/pkg/detectors/webscraping/webscraping_test.goindex 8c3b12aa1..dcacdb434 100644--- a/pkg/detectors/webscraping/webscraping_test.go+++ b/pkg/detectors/webscraping/webscraping_test.go@@ -16,7 +16,7 @@ import (func TestWebscraping_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/wepay/wepay_test.go b/pkg/detectors/wepay/wepay_test.goindex df71bfcb8..c8a5709b7 100644--- a/pkg/detectors/wepay/wepay_test.go+++ b/pkg/detectors/wepay/wepay_test.go@@ -16,7 +16,7 @@ import (func TestWePay_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/whoxy/whoxy_test.go b/pkg/detectors/whoxy/whoxy_test.goindex adafc5a92..a044a3a4f 100644--- a/pkg/detectors/whoxy/whoxy_test.go+++ b/pkg/detectors/whoxy/whoxy_test.go@@ -16,7 +16,7 @@ import (func TestWhoxy_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/worksnaps/worksnaps_test.go b/pkg/detectors/worksnaps/worksnaps_test.goindex 46ed8a332..bf2895d80 100644--- a/pkg/detectors/worksnaps/worksnaps_test.go+++ b/pkg/detectors/worksnaps/worksnaps_test.go@@ -16,7 +16,7 @@ import (func TestWorksnaps_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/workstack/workstack_test.go b/pkg/detectors/workstack/workstack_test.goindex fc3049c9c..4db3c8b3e 100644--- a/pkg/detectors/workstack/workstack_test.go+++ b/pkg/detectors/workstack/workstack_test.go@@ -16,7 +16,7 @@ import (func TestWorkstack_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/worldcoinindex/worldcoinindex_test.go b/pkg/detectors/worldcoinindex/worldcoinindex_test.goindex 3746ee31c..f8e78eed1 100644--- a/pkg/detectors/worldcoinindex/worldcoinindex_test.go+++ b/pkg/detectors/worldcoinindex/worldcoinindex_test.go@@ -16,7 +16,7 @@ import (func TestWorldCoinIndex_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/worldweather/worldweather_test.go b/pkg/detectors/worldweather/worldweather_test.goindex 182fd2f96..a785243fa 100644--- a/pkg/detectors/worldweather/worldweather_test.go+++ b/pkg/detectors/worldweather/worldweather_test.go@@ -16,7 +16,7 @@ import (func TestWorldweather_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/wrike/wrike_test.go b/pkg/detectors/wrike/wrike_test.goindex d645c7436..62fa95655 100644--- a/pkg/detectors/wrike/wrike_test.go+++ b/pkg/detectors/wrike/wrike_test.go@@ -16,7 +16,7 @@ import (func TestWrike_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/yandex/yandex_test.go b/pkg/detectors/yandex/yandex_test.goindex cad8c877a..c7b09eb30 100644--- a/pkg/detectors/yandex/yandex_test.go+++ b/pkg/detectors/yandex/yandex_test.go@@ -16,7 +16,7 @@ import (func TestYandex_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/youneedabudget/youneedabudget_test.go b/pkg/detectors/youneedabudget/youneedabudget_test.goindex e34dba1c2..029426432 100644--- a/pkg/detectors/youneedabudget/youneedabudget_test.go+++ b/pkg/detectors/youneedabudget/youneedabudget_test.go@@ -16,7 +16,7 @@ import (func TestYouNeedABudget_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/yousign/yousign_test.go b/pkg/detectors/yousign/yousign_test.goindex cbd04534b..e78b89f39 100644--- a/pkg/detectors/yousign/yousign_test.go+++ b/pkg/detectors/yousign/yousign_test.go@@ -16,7 +16,7 @@ import (func TestYousign_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/youtubeapikey/youtubeapikey_test.go b/pkg/detectors/youtubeapikey/youtubeapikey_test.goindex 2a2326177..1ed4bae88 100644--- a/pkg/detectors/youtubeapikey/youtubeapikey_test.go+++ b/pkg/detectors/youtubeapikey/youtubeapikey_test.go@@ -16,7 +16,7 @@ import (func TestYoutubeApiKey_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/zapierwebhook/zapierwebhook_test.go b/pkg/detectors/zapierwebhook/zapierwebhook_test.goindex cc073f18b..3247128bd 100644--- a/pkg/detectors/zapierwebhook/zapierwebhook_test.go+++ b/pkg/detectors/zapierwebhook/zapierwebhook_test.go@@ -16,7 +16,7 @@ import (func TestZapierWebhook_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/zendeskapi/zendeskapi_test.go b/pkg/detectors/zendeskapi/zendeskapi_test.goindex f79c37843..e6e35d9b7 100644--- a/pkg/detectors/zendeskapi/zendeskapi_test.go+++ b/pkg/detectors/zendeskapi/zendeskapi_test.go@@ -16,7 +16,7 @@ import (func TestZendeskApi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/zenscrape/zenscrape_test.go b/pkg/detectors/zenscrape/zenscrape_test.goindex e052e4dbd..745d38df4 100644--- a/pkg/detectors/zenscrape/zenscrape_test.go+++ b/pkg/detectors/zenscrape/zenscrape_test.go@@ -16,7 +16,7 @@ import (func TestZenscrape_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/zenserp/zenserp_test.go b/pkg/detectors/zenserp/zenserp_test.goindex addddc955..6e08374c6 100644--- a/pkg/detectors/zenserp/zenserp_test.go+++ b/pkg/detectors/zenserp/zenserp_test.go@@ -16,7 +16,7 @@ import (func TestZenserp_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/zeplin/zeplin_test.go b/pkg/detectors/zeplin/zeplin_test.goindex 5b85c41bc..d9f3d9c3e 100644--- a/pkg/detectors/zeplin/zeplin_test.go+++ b/pkg/detectors/zeplin/zeplin_test.go@@ -16,7 +16,7 @@ import (func TestZeplin_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/zerobounce/zerobounce_test.go b/pkg/detectors/zerobounce/zerobounce_test.goindex 091eb8f72..d3aa61255 100644--- a/pkg/detectors/zerobounce/zerobounce_test.go+++ b/pkg/detectors/zerobounce/zerobounce_test.go@@ -16,7 +16,7 @@ import (func TestZerobounce_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/zipapi/zipapi_test.go b/pkg/detectors/zipapi/zipapi_test.goindex adc601d27..59589a9a4 100644--- a/pkg/detectors/zipapi/zipapi_test.go+++ b/pkg/detectors/zipapi/zipapi_test.go@@ -16,7 +16,7 @@ import (func TestZipapi_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/zipbooks/zipbooks_test.go b/pkg/detectors/zipbooks/zipbooks_test.goindex 1606dfc2d..5dadb1dd5 100644--- a/pkg/detectors/zipbooks/zipbooks_test.go+++ b/pkg/detectors/zipbooks/zipbooks_test.go@@ -16,7 +16,7 @@ import (func TestZipbooks_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/zipcodeapi/zipcodeapi_test.go b/pkg/detectors/zipcodeapi/zipcodeapi_test.goindex 8558c5f19..7d7ea87c7 100644--- a/pkg/detectors/zipcodeapi/zipcodeapi_test.go+++ b/pkg/detectors/zipcodeapi/zipcodeapi_test.go@@ -16,7 +16,7 @@ import (func TestZipCodeAPI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecrets(ctx)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/zonkafeedback/zonkafeedback_test.go b/pkg/detectors/zonkafeedback/zonkafeedback_test.goindex 35214c829..f1b2b88ff 100644--- a/pkg/detectors/zonkafeedback/zonkafeedback_test.go+++ b/pkg/detectors/zonkafeedback/zonkafeedback_test.go@@ -16,7 +16,7 @@ import (func TestZonkaFeedback_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}